Squashed commit of the following:

commit fb1dfa9e98ded23fb5bdebae6465424a8cb8acd6
Author: Evan Prodromou <evan@status.net>
Date:   Thu Oct 20 10:40:07 2011 -0400

    Use popular notice stream for favorited page

commit e1d409ff738e39061ad35589d546ce9bed456975
Author: Evan Prodromou <evan@status.net>
Date:   Thu Oct 20 10:32:23 2011 -0400

    Use a caching stream for popular notice section

    Instead of a big cached query, we now use a caching notice stream for
    the popular notice section. It uses a single-table query at the
    bottom, then scopes the notices and filters for silenced users. This
    should be much nicer to our database servers.

    Also clears the popular cache when someone favors or disfavors
    something. A nice optimization would be to save the last weights and
    re-calculate them at invalidation time, adding the new notice (or not)
    depending on its own score. That will have to wait for another day,
    though.

commit e9b7ab4c26c95e755adaff53c3957dcfca31c16b
Author: Evan Prodromou <evan@status.net>
Date:   Thu Oct 20 10:31:14 2011 -0400

    Let CachingNoticeStream users skip the ';last' optimization

actions/favorited.php

@@ -172,11 +172,8 @@ class FavoritedAction extends Action
      */
     function showContent()
     {
-        $pop = new Popularity();
+        $stream = new PopularNoticeStream(Profile::current());
-        $pop->offset = ($this->page - 1) * NOTICES_PER_PAGE;
+        $notice = $stream->getNotices(($this->page-1)*NOTICES_PER_PAGE, NOTICES_PER_PAGE+1);
-        $pop->limit  = NOTICES_PER_PAGE;
-        $pop->expiry = 600;
-        $notice = $pop->getNotices();
 
         $nl = new NoticeList($notice, $this);
 

actions/public.php

@@ -255,7 +255,9 @@ class PublicAction extends Action
             $ibs->show();
         }
 
-        $pop = new PopularNoticeSection($this);
+        $p = Profile::current();
+
+        $pop = new PopularNoticeSection($this, $p);
         $pop->show();
         if (!common_config('performance', 'high')) {
             $cloud = new PublicTagCloudSection($this);

classes/Fave.php

@@ -74,6 +74,7 @@ class Fave extends Managed_DataObject
                 return false;
             }
             self::blow('fave:list-ids:notice_id:%d', $fave->notice_id);
+            self::blow('popular');
 
             Event::handle('EndFavorNotice', array($profile, $notice));
         }
@@ -92,6 +93,7 @@ class Fave extends Managed_DataObject
 
             $result = parent::delete();
             self::blow('fave:list-ids:notice_id:%d', $this->notice_id);
+            self::blow('popular');
 
             if ($result) {
                 Event::handle('EndDisfavorNotice', array($profile, $notice));

lib/cachingnoticestream.php

@@ -51,11 +51,13 @@ class CachingNoticeStream extends NoticeStream
 
     public $stream   = null;
     public $cachekey = null;
+    public $useLast  = true;
 
-    function __construct($stream, $cachekey)
+    function __construct($stream, $cachekey, $useLast = true)
     {
         $this->stream   = $stream;
         $this->cachekey = $cachekey;
+        $this->useLast  = $useLast;
     }
 
     function getNoticeIds($offset, $limit, $sinceId, $maxId)
@@ -85,29 +87,31 @@ class CachingNoticeStream extends NoticeStream
             return $ids;
         }
 
-        // Check the cache to see if we have a "last-known-good" version.
+        if ($this->useLast) {
-        // The actual cache gets blown away when new notices are added, but
+            // Check the cache to see if we have a "last-known-good" version.
-        // the "last" value holds a lot of info. We might need to only generate
+            // The actual cache gets blown away when new notices are added, but
-        // a few at the "tip", which can bound our queries and save lots
+            // the "last" value holds a lot of info. We might need to only generate
-        // of time.
+            // a few at the "tip", which can bound our queries and save lots
+            // of time.
 
-        $laststr = $cache->get($idkey.';last');
+            $laststr = $cache->get($idkey.';last');
 
-        if ($laststr !== false) {
+            if ($laststr !== false) {
-            $window = explode(',', $laststr);
+                $window = explode(',', $laststr);
-            $last_id = $window[0];
+                $last_id = $window[0];
-            $new_ids = $this->stream->getNoticeIds(0, self::CACHE_WINDOW, $last_id, 0);
+                $new_ids = $this->stream->getNoticeIds(0, self::CACHE_WINDOW, $last_id, 0);
 
-            $new_window = array_merge($new_ids, $window);
+                $new_window = array_merge($new_ids, $window);
 
-            $new_windowstr = implode(',', $new_window);
+                $new_windowstr = implode(',', $new_window);
 
-            $result = $cache->set($idkey, $new_windowstr);
+                $result = $cache->set($idkey, $new_windowstr);
-            $result = $cache->set($idkey . ';last', $new_windowstr);
+                $result = $cache->set($idkey . ';last', $new_windowstr);
 
-            $ids = array_slice($new_window, $offset, $limit);
+                $ids = array_slice($new_window, $offset, $limit);
 
-            return $ids;
+                return $ids;
+            }
         }
 
         // No cache hits :( Generate directly and stick the results
@@ -118,7 +122,10 @@ class CachingNoticeStream extends NoticeStream
         $windowstr = implode(',', $window);
 
         $result = $cache->set($idkey, $windowstr);
-        $result = $cache->set($idkey . ';last', $windowstr);
+
+        if ($this->useLast) {
+            $result = $cache->set($idkey . ';last', $windowstr);
+        }
 
         // Return just the slice that was requested
 

lib/popularnoticesection.php

@@ -22,7 +22,7 @@
  * @category  Widget
  * @package   StatusNet
  * @author    Evan Prodromou <evan@status.net>
- * @copyright 2009 StatusNet, Inc.
+ * @copyright 2009,2011 StatusNet, Inc.
  * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
  * @link      http://status.net/
  */
@@ -45,15 +45,18 @@ if (!defined('STATUSNET') && !defined('LACONICA')) {
  */
 class PopularNoticeSection extends NoticeSection
 {
+    protected $viewer;
+
+    function __construct($out, $viewer)
+    {
+        parent::__construct($out);
+        $this->viewer = $viewer;
+    }
+
     function getNotices()
     {
-        $pop = new Popularity();
+        $stream = new PopularNoticeStream($this->viewer);
-        if (!empty($this->out->tag)) {
+        return $stream->getNotices(0, NOTICES_PER_SECTION + 1);
-            $pop->tag = $this->out->tag;
-        }
-        $pop->limit = NOTICES_PER_SECTION;
-        $pop->expiry = 1200;
-        return $pop->getNotices();
     }
 
     function title()

lib/popularnoticestream.php

@@ -0,0 +1,109 @@
+<?php
+/**
+ * StatusNet - the distributed open-source microblogging tool
+ * Copyright (C) 2011, StatusNet, Inc.
+ *
+ * Stream of notices sorted by popularity
+ * 
+ * PHP version 5
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * @category  Popular
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2011 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+
+if (!defined('STATUSNET')) {
+    // This check helps protect against security problems;
+    // your code file can't be executed directly from the web.
+    exit(1);
+}
+
+/**
+ * Stream of notices sorted by popularity
+ *
+ * @category  Popular
+ * @package   StatusNet
+ * @author    Evan Prodromou <evan@status.net>
+ * @copyright 2011 StatusNet, Inc.
+ * @license   http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPL 3.0
+ * @link      http://status.net/
+ */
+
+class PopularNoticeStream extends ScopingNoticeStream
+{
+    function __construct($profile=null)
+    {
+        parent::__construct(new HideSilencedStream(new CachingNoticeStream(new RawPopularNoticeStream(),
+                                                                           'popular',
+                                                                           false)),
+                            $profile);
+    }
+}
+
+class HideSilencedStream extends FilteringNoticeStream
+{
+    /**
+     * Only return notices where the profile is in scope
+     *
+     * @param Notice $notice The notice to check
+     *
+     * @return boolean whether to include the notice
+     */
+
+    function filter($notice)
+    {
+        $author = $notice->getProfile();
+        return !$author->isSilenced();
+    }
+}
+
+class RawPopularNoticeStream extends NoticeStream
+{
+    function getNoticeIds($offset, $limit, $since_id, $max_id)
+    {
+        $weightexpr = common_sql_weight('modified', common_config('popular', 'dropoff'));
+        $cutoff = sprintf("modified > '%s'",
+                          common_sql_date(time() - common_config('popular', 'cutoff')));
+
+        $fave = new Fave();
+        $fave->selectAdd();
+        $fave->selectAdd('notice_id');
+        $fave->selectAdd("$weightexpr as weight");
+        $fave->whereAdd($cutoff);
+        $fave->orderBy('weight DESC');
+        $fave->groupBy('notice_id');
+
+        if (!is_null($offset)) {
+            $fave->limit($offset, $limit);
+        }
+
+        // FIXME: $since_id, $max_id are ignored
+
+        $ids = array();
+
+        if ($fave->find()) {
+            while ($fave->fetch()) {
+                $ids[] = $fave->notice_id;
+            }
+        }
+
+        return $ids;
+    }
+}
+