Merge branch 'testing' of git@gitorious.org:statusnet/mainline into 0.9.x

classes/File.php

@@ -67,7 +67,14 @@ class File extends Memcached_DataObject
         return $att;
     }
 
-    function saveNew($redir_data, $given_url) {
+    /**
+     * Save a new file record.
+     *
+     * @param array $redir_data lookup data eg from File_redirection::where()
+     * @param string $given_url
+     * @return File
+     */
+    function saveNew(array $redir_data, $given_url) {
         $x = new File;
         $x->url = $given_url;
         if (!empty($redir_data['protected'])) $x->protected = $redir_data['protected'];
@@ -77,19 +84,36 @@ class File extends Memcached_DataObject
         if (isset($redir_data['time']) && $redir_data['time'] > 0) $x->date = intval($redir_data['time']);
         $file_id = $x->insert();
 
+        $x->saveOembed($redir_data, $given_url);
+        return $x;
+    }
+
+    /**
+     * Save embedding information for this file, if applicable.
+     *
+     * Normally this won't need to be called manually, as File::saveNew()
+     * takes care of it.
+     *
+     * @param array $redir_data lookup data eg from File_redirection::where()
+     * @param string $given_url
+     * @return boolean success
+     */
+    public function saveOembed($redir_data, $given_url)
+    {
         if (isset($redir_data['type'])
             && (('text/html' === substr($redir_data['type'], 0, 9) || 'application/xhtml+xml' === substr($redir_data['type'], 0, 21)))
             && ($oembed_data = File_oembed::_getOembed($given_url))) {
 
-            $fo = File_oembed::staticGet('file_id', $file_id);
+            $fo = File_oembed::staticGet('file_id', $this->id);
 
             if (empty($fo)) {
-                File_oembed::saveNew($oembed_data, $file_id);
+                File_oembed::saveNew($oembed_data, $this->id);
+                return true;
             } else {
                 common_log(LOG_WARNING, "Strangely, a File_oembed object exists for new file $file_id", __FILE__);
             }
         }
-        return $x;
+        return false;
     }
 
     function processNew($given_url, $notice_id=null) {
@@ -105,6 +129,7 @@ class File extends Memcached_DataObject
                     $redir_url = $redir_data['url'];
                 } elseif (is_string($redir_data)) {
                     $redir_url = $redir_data;
+                    $redir_data = array();
                 } else {
                     throw new ServerException("Can't process url '$given_url'");
                 }

classes/File_oembed.php

@@ -81,6 +81,12 @@ class File_oembed extends Memcached_DataObject
         }
     }
 
+    /**
+     * Save embedding info for a new file.
+     *
+     * @param object $data Services_oEmbed_Object_*
+     * @param int $file_id
+     */
     function saveNew($data, $file_id) {
         $file_oembed = new File_oembed;
         $file_oembed->file_id = $file_id;

classes/File_redirection.php

@@ -58,24 +58,30 @@ class File_redirection extends Memcached_DataObject
         return $request;
     }
 
-    function _redirectWhere_imp($short_url, $redirs = 10, $protected = false) {
+    /**
+     * Check if this URL is a redirect and return redir info.
+     *
+     * Most code should call File_redirection::where instead, to check if we
+     * already know that redirection and avoid extra hits to the web.
+     *
+     * The URL is hit and any redirects are followed, up to 10 levels or until
+     * a protected URL is reached.
+     *
+     * @param string $in_url
+     * @return mixed one of:
+     *         string - target URL, if this is a direct link or can't be followed
+     *         array - redirect info if this is an *unknown* redirect:
+     *              associative array with the following elements:
+     *                code: HTTP status code
+     *                redirects: count of redirects followed
+     *                url: URL string of final target
+     *                type (optional): MIME type from Content-Type header
+     *                size (optional): byte size from Content-Length header
+     *                time (optional): timestamp from Last-Modified header
+     */
+    public function lookupWhere($short_url, $redirs = 10, $protected = false) {
         if ($redirs < 0) return false;
 
-        // let's see if we know this...
-        $a = File::staticGet('url', $short_url);
-
-        if (!empty($a)) {
-            // this is a direct link to $a->url
-            return $a->url;
-        } else {
-            $b = File_redirection::staticGet('url', $short_url);
-            if (!empty($b)) {
-                // this is a redirect to $b->file_id
-                $a = File::staticGet('id', $b->file_id);
-                return $a->url;
-            }
-        }
-
         if(strpos($short_url,'://') === false){
             return $short_url;
         }
@@ -93,12 +99,13 @@ class File_redirection extends Memcached_DataObject
             }
         } catch (Exception $e) {
             // Invalid URL or failure to reach server
+            common_log(LOG_ERR, "Error while following redirects for $short_url: " . $e->getMessage());
             return $short_url;
         }
 
         if ($response->getRedirectCount() && File::isProtected($response->getUrl())) {
             // Bump back up the redirect chain until we find a non-protected URL
-            return self::_redirectWhere_imp($short_url, $response->getRedirectCount() - 1, true);
+            return self::lookupWhere($short_url, $response->getRedirectCount() - 1, true);
         }
 
         $ret = array('code' => $response->getStatus()
@@ -115,11 +122,60 @@ class File_redirection extends Memcached_DataObject
         return $ret;
     }
 
-    function where($in_url) {
+    /**
-        $ret = File_redirection::_redirectWhere_imp($in_url);
+     * Check if this URL is a redirect and return redir info.
+     * If a File record is present for this URL, it is not considered a redirect.
+     * If a File_redirection record is present for this URL, the recorded target is returned.
+     *
+     * If no File or File_redirect record is present, the URL is hit and any
+     * redirects are followed, up to 10 levels or until a protected URL is
+     * reached.
+     *
+     * @param string $in_url
+     * @return mixed one of:
+     *         string - target URL, if this is a direct link or a known redirect
+     *         array - redirect info if this is an *unknown* redirect:
+     *              associative array with the following elements:
+     *                code: HTTP status code
+     *                redirects: count of redirects followed
+     *                url: URL string of final target
+     *                type (optional): MIME type from Content-Type header
+     *                size (optional): byte size from Content-Length header
+     *                time (optional): timestamp from Last-Modified header
+     */
+    public function where($in_url) {
+        // let's see if we know this...
+        $a = File::staticGet('url', $in_url);
+
+        if (!empty($a)) {
+            // this is a direct link to $a->url
+            return $a->url;
+        } else {
+            $b = File_redirection::staticGet('url', $in_url);
+            if (!empty($b)) {
+                // this is a redirect to $b->file_id
+                $a = File::staticGet('id', $b->file_id);
+                return $a->url;
+            }
+        }
+
+        $ret = File_redirection::lookupWhere($in_url);
         return $ret;
     }
 
+    /**
+     * Shorten a URL with the current user's configured shortening
+     * options, if applicable.
+     *
+     * If it cannot be shortened or the "short" URL is longer than the
+     * original, the original is returned.
+     *
+     * If the referenced item has not been seen before, embedding data
+     * may be saved.
+     *
+     * @param string $long_url
+     * @return string
+     */
     function makeShort($long_url) {
 
         $canon = File_redirection::_canonUrl($long_url);
@@ -141,11 +197,20 @@ class File_redirection extends Memcached_DataObject
             // store it
             $file = File::staticGet('url', $long_url);
             if (empty($file)) {
+                // Check if the target URL is itself a redirect...
                 $redir_data = File_redirection::where($long_url);
-                $file = File::saveNew($redir_data, $long_url);
+                if (is_array($redir_data)) {
-                $file_id = $file->id;
+                    // We haven't seen the target URL before.
-                if (!empty($redir_data['oembed']['json'])) {
+                    // Save file and embedding data about it!
-                    File_oembed::saveNew($redir_data['oembed']['json'], $file_id);
+                    $file = File::saveNew($redir_data, $long_url);
+                    $file_id = $file->id;
+                    if (!empty($redir_data['oembed']['json'])) {
+                        File_oembed::saveNew($redir_data['oembed']['json'], $file_id);
+                    }
+                } else if (is_string($redir_data)) {
+                    // The file is a known redirect target.
+                    $file = File::staticGet('url', $redir_data);
+                    $file_id = $file->id;
                 }
             } else {
                 $file_id = $file->id;

lib/httpclient.php

@@ -120,6 +120,16 @@ class HTTPClient extends HTTP_Request2
     {
         $this->config['max_redirs'] = 10;
         $this->config['follow_redirects'] = true;
+        
+        // We've had some issues with keepalive breaking with
+        // HEAD requests, such as to youtube which seems to be
+        // emitting chunked encoding info for an empty body
+        // instead of not emitting anything. This may be a
+        // bug on YouTube's end, but the upstream libray
+        // ought to be investigated to see if we can handle
+        // it gracefully in that case as well.
+        $this->config['protocol_version'] = '1.0';
+        
         parent::__construct($url, $method, $config);
         $this->setHeader('User-Agent', $this->userAgent());
     }

plugins/OStatus/OStatusPlugin.php

@@ -929,4 +929,41 @@ class OStatusPlugin extends Plugin
 
         return true;
     }
+
+    /**
+     * Utility function to check if the given URL is a canonical group profile
+     * page, and if so return the ID number.
+     *
+     * @param string $url
+     * @return mixed int or false
+     */
+    public static function localGroupFromUrl($url)
+    {
+        $template = common_local_url('groupbyid', array('id' => '31337'));
+        $template = preg_quote($template, '/');
+        $template = str_replace('31337', '(\d+)', $template);
+        if (preg_match("/$template/", $url, $matches)) {
+            return intval($matches[1]);
+        }
+        return false;
+    }
+
+    /**
+     * Utility function to check if the given URL is a canonical user profile
+     * page, and if so return the ID number.
+     *
+     * @param string $url
+     * @return mixed int or false
+     */
+    public static function localProfileFromUrl($url)
+    {
+        $template = common_local_url('userbyid', array('id' => '31337'));
+        $template = preg_quote($template, '/');
+        $template = str_replace('31337', '(\d+)', $template);
+        if (preg_match("/$template/", $url, $matches)) {
+            return intval($matches[1]);
+        }
+        return false;
+    }
+
 }

plugins/OStatus/classes/Ostatus_profile.php

@@ -675,13 +675,10 @@ class Ostatus_profile extends Memcached_DataObject
             }
 
             // Is the recipient a local group?
-            // @fixme we need a uri on user_group
+            // @fixme uri on user_group isn't reliable yet
             // $group = User_group::staticGet('uri', $recipient);
-            $template = common_local_url('groupbyid', array('id' => '31337'));
+            $id = OStatusPlugin::localGroupFromUrl($recipient);
-            $template = preg_quote($template, '/');
+            if ($id) {
-            $template = str_replace('31337', '(\d+)', $template);
-            if (preg_match("/$template/", $recipient, $matches)) {
-                $id = $matches[1];
                 $group = User_group::staticGet('id', $id);
                 if ($group) {
                     // Deliver to all members of this local group if allowed.
@@ -992,7 +989,15 @@ class Ostatus_profile extends Memcached_DataObject
 
         if (!$homeuri) {
             common_log(LOG_DEBUG, __METHOD__ . " empty actor profile URI: " . var_export($activity, true));
-            throw new ServerException("No profile URI");
+            throw new Exception("No profile URI");
+        }
+
+        if (OStatusPlugin::localProfileFromUrl($homeuri)) {
+            throw new Exception("Local user can't be referenced as remote.");
+        }
+
+        if (OStatusPlugin::localGroupFromUrl($homeuri)) {
+            throw new Exception("Local group can't be referenced as remote.");
         }
 
         if (array_key_exists('feedurl', $hints)) {

plugins/OStatus/scripts/fixup-shadow.php

@@ -0,0 +1,69 @@
+#!/usr/bin/env php
+<?php
+/*
+ * StatusNet - a distributed open-source microblogging tool
+ * Copyright (C) 2010 StatusNet, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+define('INSTALLDIR', realpath(dirname(__FILE__) . '/../../..'));
+
+$longoptions = array('dry-run');
+
+$helptext = <<<END_OF_USERROLE_HELP
+fixup_shadow.php [options]
+Patches up stray ostatus_profile entries with corrupted shadow entries
+for local users and groups.
+
+     --dry-run  look but don't touch
+
+END_OF_USERROLE_HELP;
+
+require_once INSTALLDIR.'/scripts/commandline.inc';
+
+$dry = have_option('dry-run');
+
+$oprofile = new Ostatus_profile();
+
+$marker = mt_rand(31337, 31337000);
+
+$profileTemplate = common_local_url('userbyid', array('id' => $marker));
+$encProfile = $oprofile->escape($profileTemplate, true);
+$encProfile = str_replace($marker, '%', $encProfile);
+
+$groupTemplate = common_local_url('groupbyid', array('id' => $marker));
+$encGroup = $oprofile->escape($groupTemplate, true);
+$encGroup = str_replace($marker, '%', $encGroup);
+
+$sql = "SELECT * FROM ostatus_profile WHERE uri LIKE '%s' OR uri LIKE '%s'";
+$oprofile->query(sprintf($sql, $encProfile, $encGroup));
+
+echo "Found $oprofile->N bogus ostatus_profile entries:\n";
+
+while ($oprofile->fetch()) {
+    echo "$oprofile->uri";
+
+    if ($dry) {
+        echo " (unchanged)\n";
+    } else {
+        echo " deleting...";
+        $evil = clone($oprofile);
+        $evil->delete();
+        echo "  ok\n";
+    }
+}
+
+echo "done.\n";
+