diff --git a/classes/File.php b/classes/File.php index 4ecd3b959a..8c788c0079 100644 --- a/classes/File.php +++ b/classes/File.php @@ -67,7 +67,14 @@ class File extends Memcached_DataObject return $att; } - function saveNew($redir_data, $given_url) { + /** + * Save a new file record. + * + * @param array $redir_data lookup data eg from File_redirection::where() + * @param string $given_url + * @return File + */ + function saveNew(array $redir_data, $given_url) { $x = new File; $x->url = $given_url; if (!empty($redir_data['protected'])) $x->protected = $redir_data['protected']; @@ -77,19 +84,36 @@ class File extends Memcached_DataObject if (isset($redir_data['time']) && $redir_data['time'] > 0) $x->date = intval($redir_data['time']); $file_id = $x->insert(); + $x->saveOembed($redir_data, $given_url); + return $x; + } + + /** + * Save embedding information for this file, if applicable. + * + * Normally this won't need to be called manually, as File::saveNew() + * takes care of it. + * + * @param array $redir_data lookup data eg from File_redirection::where() + * @param string $given_url + * @return boolean success + */ + public function saveOembed($redir_data, $given_url) + { if (isset($redir_data['type']) && (('text/html' === substr($redir_data['type'], 0, 9) || 'application/xhtml+xml' === substr($redir_data['type'], 0, 21))) && ($oembed_data = File_oembed::_getOembed($given_url))) { - $fo = File_oembed::staticGet('file_id', $file_id); + $fo = File_oembed::staticGet('file_id', $this->id); if (empty($fo)) { - File_oembed::saveNew($oembed_data, $file_id); + File_oembed::saveNew($oembed_data, $this->id); + return true; } else { common_log(LOG_WARNING, "Strangely, a File_oembed object exists for new file $file_id", __FILE__); } } - return $x; + return false; } function processNew($given_url, $notice_id=null) { @@ -105,6 +129,7 @@ class File extends Memcached_DataObject $redir_url = $redir_data['url']; } elseif (is_string($redir_data)) { $redir_url = $redir_data; + $redir_data = array(); } else { throw new ServerException("Can't process url '$given_url'"); } diff --git a/classes/File_oembed.php b/classes/File_oembed.php index 11f160718e..041b447404 100644 --- a/classes/File_oembed.php +++ b/classes/File_oembed.php @@ -81,6 +81,12 @@ class File_oembed extends Memcached_DataObject } } + /** + * Save embedding info for a new file. + * + * @param object $data Services_oEmbed_Object_* + * @param int $file_id + */ function saveNew($data, $file_id) { $file_oembed = new File_oembed; $file_oembed->file_id = $file_id; diff --git a/classes/File_redirection.php b/classes/File_redirection.php index 08a6e8d8be..f128b3e07c 100644 --- a/classes/File_redirection.php +++ b/classes/File_redirection.php @@ -58,24 +58,30 @@ class File_redirection extends Memcached_DataObject return $request; } - function _redirectWhere_imp($short_url, $redirs = 10, $protected = false) { + /** + * Check if this URL is a redirect and return redir info. + * + * Most code should call File_redirection::where instead, to check if we + * already know that redirection and avoid extra hits to the web. + * + * The URL is hit and any redirects are followed, up to 10 levels or until + * a protected URL is reached. + * + * @param string $in_url + * @return mixed one of: + * string - target URL, if this is a direct link or can't be followed + * array - redirect info if this is an *unknown* redirect: + * associative array with the following elements: + * code: HTTP status code + * redirects: count of redirects followed + * url: URL string of final target + * type (optional): MIME type from Content-Type header + * size (optional): byte size from Content-Length header + * time (optional): timestamp from Last-Modified header + */ + public function lookupWhere($short_url, $redirs = 10, $protected = false) { if ($redirs < 0) return false; - // let's see if we know this... - $a = File::staticGet('url', $short_url); - - if (!empty($a)) { - // this is a direct link to $a->url - return $a->url; - } else { - $b = File_redirection::staticGet('url', $short_url); - if (!empty($b)) { - // this is a redirect to $b->file_id - $a = File::staticGet('id', $b->file_id); - return $a->url; - } - } - if(strpos($short_url,'://') === false){ return $short_url; } @@ -93,12 +99,13 @@ class File_redirection extends Memcached_DataObject } } catch (Exception $e) { // Invalid URL or failure to reach server + common_log(LOG_ERR, "Error while following redirects for $short_url: " . $e->getMessage()); return $short_url; } if ($response->getRedirectCount() && File::isProtected($response->getUrl())) { // Bump back up the redirect chain until we find a non-protected URL - return self::_redirectWhere_imp($short_url, $response->getRedirectCount() - 1, true); + return self::lookupWhere($short_url, $response->getRedirectCount() - 1, true); } $ret = array('code' => $response->getStatus() @@ -115,11 +122,60 @@ class File_redirection extends Memcached_DataObject return $ret; } - function where($in_url) { - $ret = File_redirection::_redirectWhere_imp($in_url); + /** + * Check if this URL is a redirect and return redir info. + * If a File record is present for this URL, it is not considered a redirect. + * If a File_redirection record is present for this URL, the recorded target is returned. + * + * If no File or File_redirect record is present, the URL is hit and any + * redirects are followed, up to 10 levels or until a protected URL is + * reached. + * + * @param string $in_url + * @return mixed one of: + * string - target URL, if this is a direct link or a known redirect + * array - redirect info if this is an *unknown* redirect: + * associative array with the following elements: + * code: HTTP status code + * redirects: count of redirects followed + * url: URL string of final target + * type (optional): MIME type from Content-Type header + * size (optional): byte size from Content-Length header + * time (optional): timestamp from Last-Modified header + */ + public function where($in_url) { + // let's see if we know this... + $a = File::staticGet('url', $in_url); + + if (!empty($a)) { + // this is a direct link to $a->url + return $a->url; + } else { + $b = File_redirection::staticGet('url', $in_url); + if (!empty($b)) { + // this is a redirect to $b->file_id + $a = File::staticGet('id', $b->file_id); + return $a->url; + } + } + + $ret = File_redirection::lookupWhere($in_url); return $ret; } + /** + * Shorten a URL with the current user's configured shortening + * options, if applicable. + * + * If it cannot be shortened or the "short" URL is longer than the + * original, the original is returned. + * + * If the referenced item has not been seen before, embedding data + * may be saved. + * + * @param string $long_url + * @return string + */ function makeShort($long_url) { $canon = File_redirection::_canonUrl($long_url); @@ -141,11 +197,20 @@ class File_redirection extends Memcached_DataObject // store it $file = File::staticGet('url', $long_url); if (empty($file)) { + // Check if the target URL is itself a redirect... $redir_data = File_redirection::where($long_url); - $file = File::saveNew($redir_data, $long_url); - $file_id = $file->id; - if (!empty($redir_data['oembed']['json'])) { - File_oembed::saveNew($redir_data['oembed']['json'], $file_id); + if (is_array($redir_data)) { + // We haven't seen the target URL before. + // Save file and embedding data about it! + $file = File::saveNew($redir_data, $long_url); + $file_id = $file->id; + if (!empty($redir_data['oembed']['json'])) { + File_oembed::saveNew($redir_data['oembed']['json'], $file_id); + } + } else if (is_string($redir_data)) { + // The file is a known redirect target. + $file = File::staticGet('url', $redir_data); + $file_id = $file->id; } } else { $file_id = $file->id; diff --git a/lib/httpclient.php b/lib/httpclient.php index 4c3af8d7dd..64a51353c7 100644 --- a/lib/httpclient.php +++ b/lib/httpclient.php @@ -120,6 +120,16 @@ class HTTPClient extends HTTP_Request2 { $this->config['max_redirs'] = 10; $this->config['follow_redirects'] = true; + + // We've had some issues with keepalive breaking with + // HEAD requests, such as to youtube which seems to be + // emitting chunked encoding info for an empty body + // instead of not emitting anything. This may be a + // bug on YouTube's end, but the upstream libray + // ought to be investigated to see if we can handle + // it gracefully in that case as well. + $this->config['protocol_version'] = '1.0'; + parent::__construct($url, $method, $config); $this->setHeader('User-Agent', $this->userAgent()); } diff --git a/plugins/OStatus/OStatusPlugin.php b/plugins/OStatus/OStatusPlugin.php index 38042803ff..b472ae2420 100644 --- a/plugins/OStatus/OStatusPlugin.php +++ b/plugins/OStatus/OStatusPlugin.php @@ -929,4 +929,41 @@ class OStatusPlugin extends Plugin return true; } + + /** + * Utility function to check if the given URL is a canonical group profile + * page, and if so return the ID number. + * + * @param string $url + * @return mixed int or false + */ + public static function localGroupFromUrl($url) + { + $template = common_local_url('groupbyid', array('id' => '31337')); + $template = preg_quote($template, '/'); + $template = str_replace('31337', '(\d+)', $template); + if (preg_match("/$template/", $url, $matches)) { + return intval($matches[1]); + } + return false; + } + + /** + * Utility function to check if the given URL is a canonical user profile + * page, and if so return the ID number. + * + * @param string $url + * @return mixed int or false + */ + public static function localProfileFromUrl($url) + { + $template = common_local_url('userbyid', array('id' => '31337')); + $template = preg_quote($template, '/'); + $template = str_replace('31337', '(\d+)', $template); + if (preg_match("/$template/", $url, $matches)) { + return intval($matches[1]); + } + return false; + } + } diff --git a/plugins/OStatus/classes/Ostatus_profile.php b/plugins/OStatus/classes/Ostatus_profile.php index abc8100cee..6ae8e4fd58 100644 --- a/plugins/OStatus/classes/Ostatus_profile.php +++ b/plugins/OStatus/classes/Ostatus_profile.php @@ -675,13 +675,10 @@ class Ostatus_profile extends Memcached_DataObject } // Is the recipient a local group? - // @fixme we need a uri on user_group + // @fixme uri on user_group isn't reliable yet // $group = User_group::staticGet('uri', $recipient); - $template = common_local_url('groupbyid', array('id' => '31337')); - $template = preg_quote($template, '/'); - $template = str_replace('31337', '(\d+)', $template); - if (preg_match("/$template/", $recipient, $matches)) { - $id = $matches[1]; + $id = OStatusPlugin::localGroupFromUrl($recipient); + if ($id) { $group = User_group::staticGet('id', $id); if ($group) { // Deliver to all members of this local group if allowed. @@ -992,7 +989,15 @@ class Ostatus_profile extends Memcached_DataObject if (!$homeuri) { common_log(LOG_DEBUG, __METHOD__ . " empty actor profile URI: " . var_export($activity, true)); - throw new ServerException("No profile URI"); + throw new Exception("No profile URI"); + } + + if (OStatusPlugin::localProfileFromUrl($homeuri)) { + throw new Exception("Local user can't be referenced as remote."); + } + + if (OStatusPlugin::localGroupFromUrl($homeuri)) { + throw new Exception("Local group can't be referenced as remote."); } if (array_key_exists('feedurl', $hints)) { diff --git a/plugins/OStatus/scripts/fixup-shadow.php b/plugins/OStatus/scripts/fixup-shadow.php new file mode 100644 index 0000000000..0171b77bc9 --- /dev/null +++ b/plugins/OStatus/scripts/fixup-shadow.php @@ -0,0 +1,69 @@ +#!/usr/bin/env php +. + */ + +define('INSTALLDIR', realpath(dirname(__FILE__) . '/../../..')); + +$longoptions = array('dry-run'); + +$helptext = << $marker)); +$encProfile = $oprofile->escape($profileTemplate, true); +$encProfile = str_replace($marker, '%', $encProfile); + +$groupTemplate = common_local_url('groupbyid', array('id' => $marker)); +$encGroup = $oprofile->escape($groupTemplate, true); +$encGroup = str_replace($marker, '%', $encGroup); + +$sql = "SELECT * FROM ostatus_profile WHERE uri LIKE '%s' OR uri LIKE '%s'"; +$oprofile->query(sprintf($sql, $encProfile, $encGroup)); + +echo "Found $oprofile->N bogus ostatus_profile entries:\n"; + +while ($oprofile->fetch()) { + echo "$oprofile->uri"; + + if ($dry) { + echo " (unchanged)\n"; + } else { + echo " deleting..."; + $evil = clone($oprofile); + $evil->delete(); + echo " ok\n"; + } +} + +echo "done.\n"; +