Merge branch 'sorting' into 0.9.x

actions/showstream.php

@@ -317,7 +317,7 @@ class ProfileNoticeListItem extends DoFollowListItem
                            'class' => 'url');
 
             if (!empty($this->profile->fullname)) {
-                $attrs['title'] = $this->getFancyName();
+                $attrs['title'] = $this->profile->getFancyName();
             }
 
             $this->out->elementStart('span', 'repeat');

classes/Fave.php

@@ -85,6 +85,19 @@ class Fave extends Memcached_DataObject
         return $ids;
     }
 
+    /**
+     * Note that the sorting for this is by order of *fave* not order of *notice*.
+     *
+     * @fixme add since_id, max_id support?
+     *
+     * @param <type> $user_id
+     * @param <type> $own
+     * @param <type> $offset
+     * @param <type> $limit
+     * @param <type> $since_id
+     * @param <type> $max_id
+     * @return <type>
+     */
     function _streamDirect($user_id, $own, $offset, $limit, $since_id, $max_id)
     {
         $fav = new Fave();

classes/Notice.php

@@ -654,7 +654,7 @@ class Notice extends Memcached_DataObject
         $notice->selectAdd(); // clears it
         $notice->selectAdd('id');
 
-        $notice->orderBy('id DESC');
+        $notice->orderBy('created DESC, id DESC');
 
         if (!is_null($offset)) {
             $notice->limit($offset, $limit);
@@ -668,13 +668,8 @@ class Notice extends Memcached_DataObject
             $notice->whereAdd('is_local !='. Notice::GATEWAY);
         }
 
-        if ($since_id != 0) {
+        Notice::addWhereSinceId($notice, $since_id);
-            $notice->whereAdd('id > ' . $since_id);
+        Notice::addWhereMaxId($notice, $max_id);
-        }
-
-        if ($max_id != 0) {
-            $notice->whereAdd('id <= ' . $max_id);
-        }
 
         $ids = array();
 
@@ -709,19 +704,14 @@ class Notice extends Memcached_DataObject
 
         $notice->conversation = $id;
 
-        $notice->orderBy('id DESC');
+        $notice->orderBy('created DESC, id DESC');
 
         if (!is_null($offset)) {
             $notice->limit($offset, $limit);
         }
 
-        if ($since_id != 0) {
+        Notice::addWhereSinceId($notice, $since_id);
-            $notice->whereAdd('id > ' . $since_id);
+        Notice::addWhereMaxId($notice, $max_id);
-        }
-
-        if ($max_id != 0) {
-            $notice->whereAdd('id <= ' . $max_id);
-        }
 
         $ids = array();
 
@@ -1695,10 +1685,10 @@ class Notice extends Memcached_DataObject
 
         $notice->repeat_of = $this->id;
 
-        $notice->orderBy('created'); // NB: asc!
+        $notice->orderBy('created, id'); // NB: asc!
 
-        if (!is_null($offset)) {
+        if (!is_null($limit)) {
-            $notice->limit($offset, $limit);
+            $notice->limit(0, $limit);
         }
 
         $ids = array();
@@ -1978,4 +1968,108 @@ class Notice extends Memcached_DataObject
         $d = new DateTime($dateStr, new DateTimeZone('UTC'));
         return $d->format(DATE_W3C);
     }
+
+    /**
+     * Look up the creation timestamp for a given notice ID, even
+     * if it's been deleted.
+     *
+     * @param int $id
+     * @return mixed string recorded creation timestamp, or false if can't be found
+     */
+    public static function getAsTimestamp($id)
+    {
+        if (!$id) {
+            return false;
+        }
+
+        $notice = Notice::staticGet('id', $id);
+        if ($notice) {
+            return $notice->created;
+        }
+
+        $deleted = Deleted_notice::staticGet('id', $id);
+        if ($deleted) {
+            return $deleted->created;
+        }
+
+        return false;
+    }
+
+    /**
+     * Build an SQL 'where' fragment for timestamp-based sorting from a since_id
+     * parameter, matching notices posted after the given one (exclusive).
+     *
+     * If the referenced notice can't be found, will return false.
+     *
+     * @param int $id
+     * @param string $idField
+     * @param string $createdField
+     * @return mixed string or false if no match
+     */
+    public static function whereSinceId($id, $idField='id', $createdField='created')
+    {
+        $since = Notice::getAsTimestamp($id);
+        if ($since) {
+            return sprintf("($createdField = '%s' and $idField > %d) or ($createdField > '%s')", $since, $id, $since);
+        }
+        return false;
+    }
+
+    /**
+     * Build an SQL 'where' fragment for timestamp-based sorting from a since_id
+     * parameter, matching notices posted after the given one (exclusive), and
+     * if necessary add it to the data object's query.
+     *
+     * @param DB_DataObject $obj
+     * @param int $id
+     * @param string $idField
+     * @param string $createdField
+     * @return mixed string or false if no match
+     */
+    public static function addWhereSinceId(DB_DataObject $obj, $id, $idField='id', $createdField='created')
+    {
+        $since = self::whereSinceId($id);
+        if ($since) {
+            $obj->whereAdd($since);
+        }
+    }
+
+    /**
+     * Build an SQL 'where' fragment for timestamp-based sorting from a max_id
+     * parameter, matching notices posted before the given one (inclusive).
+     *
+     * If the referenced notice can't be found, will return false.
+     *
+     * @param int $id
+     * @param string $idField
+     * @param string $createdField
+     * @return mixed string or false if no match
+     */
+    public static function whereMaxId($id, $idField='id', $createdField='created')
+    {
+        $max = Notice::getAsTimestamp($id);
+        if ($max) {
+            return sprintf("($createdField < '%s') or ($createdField = '%s' and $idField <= %d)", $max, $max, $id);
+        }
+        return false;
+    }
+
+    /**
+     * Build an SQL 'where' fragment for timestamp-based sorting from a max_id
+     * parameter, matching notices posted before the given one (inclusive), and
+     * if necessary add it to the data object's query.
+     *
+     * @param DB_DataObject $obj
+     * @param int $id
+     * @param string $idField
+     * @param string $createdField
+     * @return mixed string or false if no match
+     */
+    public static function addWhereMaxId(DB_DataObject $obj, $id, $idField='id', $createdField='created')
+    {
+        $max = self::whereMaxId($id);
+        if ($max) {
+            $obj->whereAdd($max);
+        }
+    }
 }

classes/Notice_tag.php

@@ -55,15 +55,10 @@ class Notice_tag extends Memcached_DataObject
         $nt->selectAdd();
         $nt->selectAdd('notice_id');
 
-        if ($since_id != 0) {
+        Notice::addWhereSinceId($nt, $since_id, 'notice_id');
-            $nt->whereAdd('notice_id > ' . $since_id);
+        Notice::addWhereMaxId($nt, $max_id, 'notice_id');
-        }
 
-        if ($max_id != 0) {
+        $nt->orderBy('created DESC, notice_id DESC');
-            $nt->whereAdd('notice_id <= ' . $max_id);
-        }
-
-        $nt->orderBy('notice_id DESC');
 
         if (!is_null($offset)) {
             $nt->limit($offset, $limit);

classes/Profile.php

@@ -215,26 +215,29 @@ class Profile extends Memcached_DataObject
     function _streamTaggedDirect($tag, $offset, $limit, $since_id, $max_id)
     {
         // XXX It would be nice to do this without a join
+        // (necessary to do it efficiently on accounts with long history)
 
         $notice = new Notice();
 
         $query =
           "select id from notice join notice_tag on id=notice_id where tag='".
           $notice->escape($tag) .
-          "' and profile_id=" . $notice->escape($this->id);
+          "' and profile_id=" . intval($this->id);
 
-        if ($since_id != 0) {
+        $since = Notice::whereSinceId($since_id, 'id', 'notice.created');
-            $query .= " and id > $since_id";
+        if ($since) {
+            $query .= " and ($since)";
         }
 
-        if ($max_id != 0) {
+        $max = Notice::whereMaxId($max_id, 'id', 'notice.created');
-            $query .= " and id <= $max_id";
+        if ($max) {
+            $query .= " and ($max)";
         }
 
-        $query .= ' order by id DESC';
+        $query .= ' order by notice.created DESC, id DESC';
 
         if (!is_null($offset)) {
-            $query .= " LIMIT $limit OFFSET $offset";
+            $query .= " LIMIT " . intval($limit) . " OFFSET " . intval($offset);
         }
 
         $notice->query($query);
@@ -252,58 +255,22 @@ class Profile extends Memcached_DataObject
     {
         $notice = new Notice();
 
-        // Temporary hack until notice_profile_id_idx is updated
+        $notice->profile_id = $this->id;
-        // to (profile_id, id) instead of (profile_id, created, id).
-        // It's been falling back to PRIMARY instead, which is really
-        // very inefficient for a profile that hasn't posted in a few
-        // months. Even though forcing the index will cause a filesort,
-        // it's usually going to be better.
-        if (common_config('db', 'type') == 'mysql') {
-            $index = '';
-            $query =
-              "select id from notice force index (notice_profile_id_idx) ".
-              "where profile_id=" . $notice->escape($this->id);
 
-            if ($since_id != 0) {
+        $notice->selectAdd();
-                $query .= " and id > $since_id";
+        $notice->selectAdd('id');
-            }
 
-            if ($max_id != 0) {
+        Notice::addWhereSinceId($notice, $since_id);
-                $query .= " and id <= $max_id";
+        Notice::addWhereMaxId($notice, $max_id);
-            }
 
-            $query .= ' order by id DESC';
+        $notice->orderBy('created DESC, id DESC');
 
-            if (!is_null($offset)) {
+        if (!is_null($offset)) {
-                $query .= " LIMIT $limit OFFSET $offset";
+            $notice->limit($offset, $limit);
-            }
-
-            $notice->query($query);
-        } else {
-            $index = '';
-
-            $notice->profile_id = $this->id;
-
-            $notice->selectAdd();
-            $notice->selectAdd('id');
-
-            if ($since_id != 0) {
-                $notice->whereAdd('id > ' . $since_id);
-            }
-
-            if ($max_id != 0) {
-                $notice->whereAdd('id <= ' . $max_id);
-            }
-
-            $notice->orderBy('id DESC');
-
-            if (!is_null($offset)) {
-                $notice->limit($offset, $limit);
-            }
-
-            $notice->find();
         }
 
+        $notice->find();
+
         $ids = array();
 
         while ($notice->fetch()) {

classes/Reply.php

@@ -50,15 +50,10 @@ class Reply extends Memcached_DataObject
         $reply = new Reply();
         $reply->profile_id = $user_id;
 
-        if ($since_id != 0) {
+        Notice::addWhereSinceId($reply, $since_id, 'notice_id', 'modified');
-            $reply->whereAdd('notice_id > ' . $since_id);
+        Notice::addWhereMaxId($reply, $max_id, 'notice_id', 'modified');
-        }
 
-        if ($max_id != 0) {
+        $reply->orderBy('modified DESC, notice_id DESC');
-            $reply->whereAdd('notice_id <= ' . $max_id);
-        }
-
-        $reply->orderBy('notice_id DESC');
 
         if (!is_null($offset)) {
             $reply->limit($offset, $limit);

classes/User.php

@@ -755,19 +755,14 @@ class User extends Memcached_DataObject
         $notice->profile_id = $this->id;
         $notice->whereAdd('repeat_of IS NOT NULL');
 
-        $notice->orderBy('id DESC');
+        $notice->orderBy('created DESC, id DESC');
 
         if (!is_null($offset)) {
             $notice->limit($offset, $limit);
         }
 
-        if ($since_id != 0) {
+        Notice::addWhereSinceId($notice, $since_id);
-            $notice->whereAdd('id > ' . $since_id);
+        Notice::addWhereMaxId($notice, $max_id);
-        }
-
-        if ($max_id != 0) {
-            $notice->whereAdd('id <= ' . $max_id);
-        }
 
         $ids = array();
 
@@ -800,17 +795,17 @@ class User extends Memcached_DataObject
           'FROM notice original JOIN notice rept ON original.id = rept.repeat_of ' .
           'WHERE original.profile_id = ' . $this->id . ' ';
 
-        if ($since_id != 0) {
+        $since = Notice::whereSinceId($since_id, 'original.id', 'original.created');
-            $qry .= 'AND original.id > ' . $since_id . ' ';
+        if ($since) {
+            $qry .= "AND ($since) ";
         }
 
-        if ($max_id != 0) {
+        $max = Notice::whereMaxId($max_id, 'original.id', 'original.created');
-            $qry .= 'AND original.id <= ' . $max_id . ' ';
+        if ($max) {
+            $qry .= "AND ($max) ";
         }
 
-        // NOTE: we sort by fave time, not by notice time!
+        $qry .= 'ORDER BY original.created, original.id DESC ';
-
-        $qry .= 'ORDER BY original.id DESC ';
 
         if (!is_null($offset)) {
             $qry .= "LIMIT $limit OFFSET $offset";

classes/User_group.php

@@ -100,15 +100,10 @@ class User_group extends Memcached_DataObject
         $inbox->selectAdd();
         $inbox->selectAdd('notice_id');
 
-        if ($since_id != 0) {
+        Notice::addWhereSinceId($inbox, $since_id, 'notice_id');
-            $inbox->whereAdd('notice_id > ' . $since_id);
+        Notice::addWhereMaxId($inbox, $max_id, 'notice_id');
-        }
 
-        if ($max_id != 0) {
+        $inbox->orderBy('created DESC, notice_id DESC');
-            $inbox->whereAdd('notice_id <= ' . $max_id);
-        }
-
-        $inbox->orderBy('notice_id DESC');
 
         if (!is_null($offset)) {
             $inbox->limit($offset, $limit);

db/096to097.sql

@@ -0,0 +1,26 @@
+-- Add indexes for sorting changes in 0.9.7
+
+-- Allows sorting public timeline, api/statuses/repeats, and conversations by timestamp efficiently
+alter table notice
+    add index notice_created_id_is_local_idx (created,id,is_local),
+
+    add index notice_repeat_of_created_id_idx (repeat_of, created, id),
+    drop index notice_repeatof_idx,
+
+    add index notice_conversation_created_id_idx (conversation, created, id),
+    drop index notice_conversation_idx;
+
+-- Allows sorting tag-filtered public timeline by timestamp efficiently
+alter table notice_tag add index notice_tag_tag_created_notice_id_idx (tag, created, notice_id);
+
+-- Needed for sorting reply/mentions timelines
+alter table reply add index reply_profile_id_modified_notice_id_idx (profile_id, modified, notice_id);
+
+-- Needed for sorting group messages by timestamp
+alter table group_inbox add index group_inbox_group_id_created_notice_id_idx (group_id, created, notice_id);
+
+-- Helps make some reverse role lookups more efficient if there's a lot of assigned accounts
+alter table profile_role add index profile_role_role_created_profile_id_idx (role, created, profile_id);
+
+-- Fix for sorting a user's group memberships by order joined
+alter table group_member add index group_member_profile_id_created_idx (profile_id, created);

db/statusnet.sql

@@ -131,11 +131,21 @@ create table notice (
     location_ns integer comment 'namespace for location',
     repeat_of integer comment 'notice this is a repeat of' references notice (id),
 
+    -- For public timeline...
+    index notice_created_id_is_local_idx (created,id,is_local),
+
+    -- For profile timelines...
     index notice_profile_id_idx (profile_id,created,id),
-    index notice_conversation_idx (conversation),
+
-    index notice_created_idx (created),
+    -- For api/statuses/repeats...
+    index notice_repeat_of_created_id_idx (repeat_of, created, id),
+
+    -- For conversation views
+    index notice_conversation_created_id_idx (conversation, created, id),
+
+    -- Are these needed/used?
     index notice_replyto_idx (reply_to),
-    index notice_repeatof_idx (repeat_of),
+
     FULLTEXT(content)
 ) ENGINE=MyISAM CHARACTER SET utf8 COLLATE utf8_general_ci;
 
@@ -156,7 +166,10 @@ create table reply (
     constraint primary key (notice_id, profile_id),
     index reply_notice_id_idx (notice_id),
     index reply_profile_id_idx (profile_id),
-    index reply_replied_id_idx (replied_id)
+    index reply_replied_id_idx (replied_id),
+
+    -- Needed for sorting reply/mentions timelines
+    index reply_profile_id_modified_notice_id_idx (profile_id, modified, notice_id)
 
 ) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_bin;
 
@@ -301,7 +314,10 @@ create table notice_tag (
 
     constraint primary key (tag, notice_id),
     index notice_tag_created_idx (created),
-    index notice_tag_notice_id_idx (notice_id)
+    index notice_tag_notice_id_idx (notice_id),
+
+    -- For sorting tag-filtered public timeline
+    index notice_tag_tag_created_notice_id_idx (tag, created, notice_id)
 ) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_bin;
 
 /* Synching with foreign services */
@@ -447,7 +463,10 @@ create table group_member (
 
     constraint primary key (group_id, profile_id),
     index group_member_profile_id_idx (profile_id),
-    index group_member_created_idx (created)
+    index group_member_created_idx (created),
+
+    -- To pull up a list of someone's groups in order joined
+    index group_member_profile_id_created_idx (profile_id, created)
 
 ) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_bin;
 
@@ -468,7 +487,10 @@ create table group_inbox (
 
     constraint primary key (group_id, notice_id),
     index group_inbox_created_idx (created),
-    index group_inbox_notice_id_idx (notice_id)
+    index group_inbox_notice_id_idx (notice_id),
+
+    -- Needed for sorting group messages by timestamp
+    index group_inbox_group_id_created_notice_id_idx (group_id, created, notice_id)
 
 ) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_bin;
 
@@ -608,7 +630,8 @@ create table profile_role (
     role    varchar(32) not null comment 'string representing the role',
     created datetime not null comment 'date the role was granted',
 
-    constraint primary key (profile_id, role)
+    constraint primary key (profile_id, role),
+    index profile_role_role_created_profile_id_idx (role, created, profile_id)
 
 ) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_bin;