From 2ecdeb3dddc1c47ba6fd25536cacca88ce659b34 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 29 Dec 2009 21:54:08 -0800 Subject: [PATCH 1/6] add an inbox blob table --- db/statusnet.sql | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/db/statusnet.sql b/db/statusnet.sql index 94b03df639..cb33ccf33e 100644 --- a/db/statusnet.sql +++ b/db/statusnet.sql @@ -596,3 +596,11 @@ create table user_location_prefs ( constraint primary key (user_id) ) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_bin; +create table inbox ( + + user_id integer not null comment 'user receiving the notice' references user (id), + notice_ids blob comment 'packed list of notice ids', + + constraint primary key (user_id) + +) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_bin; From 088ef9b427aae98ee937d9052b10ee9cd6c9e145 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 29 Dec 2009 22:02:46 -0800 Subject: [PATCH 2/6] add inbox data class --- classes/Inbox.php | 51 +++++++++++++++++++++++++++++++++++++++++++ classes/statusnet.ini | 7 ++++++ 2 files changed, 58 insertions(+) create mode 100755 classes/Inbox.php diff --git a/classes/Inbox.php b/classes/Inbox.php new file mode 100755 index 0000000000..35f532c061 --- /dev/null +++ b/classes/Inbox.php @@ -0,0 +1,51 @@ +. + * + * @category Data + * @package StatusNet + * @author Evan Prodromou + * @copyright 2009 StatusNet Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 + * @link http://status.net/ + */ + +require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; + +class Inbox extends Memcached_DataObject +{ + ###START_AUTOCODE + /* the code below is auto generated do not remove the above tag */ + + public $__table = 'inbox'; // table name + public $user_id; // int(4) primary_key not_null + public $notice_ids; // blob + + /* Static get */ + function staticGet($k,$v=NULL) { return Memcached_DataObject::staticGet('Inbox',$k,$v); } + + /* the code above is auto generated do not remove the tag below */ + ###END_AUTOCODE + + function sequenceKey() + { + return array(false, false, false); + } +} diff --git a/classes/statusnet.ini b/classes/statusnet.ini index 4077746c47..e4c6740abe 100644 --- a/classes/statusnet.ini +++ b/classes/statusnet.ini @@ -240,6 +240,13 @@ address = 130 address_type = 130 created = 142 +[inbox] +user_id = 129 +notice_ids = 66 + +[inbox__keys] +user_id = K + [invitation__keys] code = K From 775292eedeec794b69e2bf59185bdbc8b79ed157 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 29 Dec 2009 22:03:06 -0800 Subject: [PATCH 3/6] flip exe bit --- classes/Inbox.php | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 classes/Inbox.php diff --git a/classes/Inbox.php b/classes/Inbox.php old mode 100755 new mode 100644 From 7640d3f07bad0710d69575efc7ceda115f24a60a Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 30 Dec 2009 09:03:06 -1000 Subject: [PATCH 4/6] Replace Notice_inbox with Inbox --- classes/Notice_inbox.php | 184 --------------------------------------- scripts/inbox_users.php | 107 ----------------------- scripts/triminboxes.php | 56 ------------ 3 files changed, 347 deletions(-) delete mode 100644 classes/Notice_inbox.php delete mode 100755 scripts/inbox_users.php delete mode 100644 scripts/triminboxes.php diff --git a/classes/Notice_inbox.php b/classes/Notice_inbox.php deleted file mode 100644 index b39006542c..0000000000 --- a/classes/Notice_inbox.php +++ /dev/null @@ -1,184 +0,0 @@ -. - */ - -if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } - -require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; - -// We keep 5 pages of inbox notices in memcache, +1 for pagination check - -define('INBOX_CACHE_WINDOW', 101); -define('NOTICE_INBOX_GC_BOXCAR', 128); -define('NOTICE_INBOX_GC_MAX', 12800); -define('NOTICE_INBOX_LIMIT', 1000); -define('NOTICE_INBOX_SOFT_LIMIT', 1000); - -define('NOTICE_INBOX_SOURCE_SUB', 1); -define('NOTICE_INBOX_SOURCE_GROUP', 2); -define('NOTICE_INBOX_SOURCE_REPLY', 3); -define('NOTICE_INBOX_SOURCE_FORWARD', 4); -define('NOTICE_INBOX_SOURCE_GATEWAY', -1); - -class Notice_inbox extends Memcached_DataObject -{ - ###START_AUTOCODE - /* the code below is auto generated do not remove the above tag */ - - public $__table = 'notice_inbox'; // table name - public $user_id; // int(4) primary_key not_null - public $notice_id; // int(4) primary_key not_null - public $created; // datetime() not_null - public $source; // tinyint(1) default_1 - - /* Static get */ - function staticGet($k,$v=null) - { return Memcached_DataObject::staticGet('Notice_inbox',$k,$v); } - - /* the code above is auto generated do not remove the tag below */ - ###END_AUTOCODE - - function stream($user_id, $offset, $limit, $since_id, $max_id, $since, $own=false) - { - return Notice::stream(array('Notice_inbox', '_streamDirect'), - array($user_id, $own), - ($own) ? 'notice_inbox:by_user:'.$user_id : - 'notice_inbox:by_user_own:'.$user_id, - $offset, $limit, $since_id, $max_id, $since); - } - - function _streamDirect($user_id, $own, $offset, $limit, $since_id, $max_id, $since) - { - $inbox = new Notice_inbox(); - - $inbox->user_id = $user_id; - - if (!$own) { - $inbox->whereAdd('source != ' . NOTICE_INBOX_SOURCE_GATEWAY); - } - - if ($since_id != 0) { - $inbox->whereAdd('notice_id > ' . $since_id); - } - - if ($max_id != 0) { - $inbox->whereAdd('notice_id <= ' . $max_id); - } - - if (!is_null($since)) { - $inbox->whereAdd('created > \'' . date('Y-m-d H:i:s', $since) . '\''); - } - - $inbox->orderBy('created DESC'); - - if (!is_null($offset)) { - $inbox->limit($offset, $limit); - } - - $ids = array(); - - if ($inbox->find()) { - while ($inbox->fetch()) { - $ids[] = $inbox->notice_id; - } - } - - return $ids; - } - - function &pkeyGet($kv) - { - return Memcached_DataObject::pkeyGet('Notice_inbox', $kv); - } - - static function gc($user_id) - { - $entry = new Notice_inbox(); - $entry->user_id = $user_id; - $entry->orderBy('created DESC'); - $entry->limit(NOTICE_INBOX_LIMIT - 1, NOTICE_INBOX_GC_MAX); - - $total = $entry->find(); - - if ($total > 0) { - $notices = array(); - $cnt = 0; - while ($entry->fetch()) { - $notices[] = $entry->notice_id; - $cnt++; - if ($cnt >= NOTICE_INBOX_GC_BOXCAR) { - self::deleteMatching($user_id, $notices); - $notices = array(); - $cnt = 0; - } - } - - if ($cnt > 0) { - self::deleteMatching($user_id, $notices); - $notices = array(); - } - } - } - - static function deleteMatching($user_id, $notices) - { - $entry = new Notice_inbox(); - return $entry->query('DELETE FROM notice_inbox '. - 'WHERE user_id = ' . $user_id . ' ' . - 'AND notice_id in ('.implode(',', $notices).')'); - } - - static function bulkInsert($notice_id, $created, $ni) - { - $cnt = 0; - - $qryhdr = 'INSERT INTO notice_inbox (user_id, notice_id, source, created) VALUES '; - $qry = $qryhdr; - - foreach ($ni as $id => $source) { - if ($cnt > 0) { - $qry .= ', '; - } - $qry .= '('.$id.', '.$notice_id.', '.$source.", '".$created. "') "; - $cnt++; - if (rand() % NOTICE_INBOX_SOFT_LIMIT == 0) { - // FIXME: Causes lag in replicated servers - // Notice_inbox::gc($id); - } - if ($cnt >= MAX_BOXCARS) { - $inbox = new Notice_inbox(); - $result = $inbox->query($qry); - if (PEAR::isError($result)) { - common_log_db_error($inbox, $qry); - } - $qry = $qryhdr; - $cnt = 0; - } - } - - if ($cnt > 0) { - $inbox = new Notice_inbox(); - $result = $inbox->query($qry); - if (PEAR::isError($result)) { - common_log_db_error($inbox, $qry); - } - } - - return; - } -} diff --git a/scripts/inbox_users.php b/scripts/inbox_users.php deleted file mode 100755 index 32adcea213..0000000000 --- a/scripts/inbox_users.php +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env php -. - */ - -# Abort if called from a web server - -define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); - -$helptext = << - -Update users to use inbox table. Listed in an ID file, default 'ids.txt'. - -ENDOFHELP; - -require_once INSTALLDIR.'/scripts/commandline.inc'; - -$id_file = (count($args) > 1) ? $args[0] : 'ids.txt'; - -common_log(LOG_INFO, 'Updating user inboxes.'); - -$ids = file($id_file); - -foreach ($ids as $id) { - - $user = User::staticGet('id', $id); - - if (!$user) { - common_log(LOG_WARNING, 'No such user: ' . $id); - continue; - } - - if ($user->inboxed) { - common_log(LOG_WARNING, 'Already inboxed: ' . $id); - continue; - } - - common_log(LOG_INFO, 'Updating inbox for user ' . $user->id); - - $user->query('BEGIN'); - - $old_inbox = new Notice_inbox(); - $old_inbox->user_id = $user->id; - - $result = $old_inbox->delete(); - - if (is_null($result) || $result === false) { - common_log_db_error($old_inbox, 'DELETE', __FILE__); - continue; - } - - $old_inbox->free(); - - $inbox = new Notice_inbox(); - - $result = $inbox->query('INSERT INTO notice_inbox (user_id, notice_id, created) ' . - 'SELECT ' . $user->id . ', notice.id, notice.created ' . - 'FROM subscription JOIN notice ON subscription.subscribed = notice.profile_id ' . - 'WHERE subscription.subscriber = ' . $user->id . ' ' . - 'AND notice.created >= subscription.created ' . - 'AND NOT EXISTS (SELECT user_id, notice_id ' . - 'FROM notice_inbox ' . - 'WHERE user_id = ' . $user->id . ' ' . - 'AND notice_id = notice.id) ' . - 'ORDER BY notice.created DESC ' . - 'LIMIT 0, 1000'); - - if (is_null($result) || $result === false) { - common_log_db_error($inbox, 'INSERT', __FILE__); - continue; - } - - $orig = clone($user); - $user->inboxed = 1; - $result = $user->update($orig); - - if (!$result) { - common_log_db_error($user, 'UPDATE', __FILE__); - continue; - } - - $user->query('COMMIT'); - - $inbox->free(); - unset($inbox); - - if ($cache) { - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id)); - $cache->delete(common_cache_key('user:notices_with_friends:' . $user->id . ';last')); - } -} diff --git a/scripts/triminboxes.php b/scripts/triminboxes.php deleted file mode 100644 index da09817e5b..0000000000 --- a/scripts/triminboxes.php +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env php -. - */ - -define('INSTALLDIR', realpath(dirname(__FILE__) . '/..')); - -$shortoptions = 'u::'; -$longoptions = array('start-user-id::'); - -$helptext = << - --start-user-id= User ID to start after. Default is all. - -END_OF_TRIM_HELP; - -require_once INSTALLDIR.'/scripts/commandline.inc'; - -$id = null; - -if (have_option('u')) { - $id = get_option_value('u'); -} else if (have_option('--start-user-id')) { - $id = get_option_value('--start-user-id'); -} else { - $id = null; -} - -$user = new User(); - -if (!empty($id)) { - $user->whereAdd('id > ' . $id); -} - -$cnt = $user->find(); - -while ($user->fetch()) { - Notice_inbox::gc($user->id); -} From ed5e91d60da8c7e2796b8dc42243b39ded009516 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Wed, 30 Dec 2009 09:06:07 -1000 Subject: [PATCH 5/6] Use inbox instead of notice_inbox --- classes/Inbox.php | 53 ++++++++++++++++++++++++++++++ classes/Notice.php | 20 ++---------- classes/User.php | 81 ++++++++++------------------------------------ 3 files changed, 72 insertions(+), 82 deletions(-) diff --git a/classes/Inbox.php b/classes/Inbox.php index 35f532c061..de48d73818 100644 --- a/classes/Inbox.php +++ b/classes/Inbox.php @@ -31,6 +31,8 @@ require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; class Inbox extends Memcached_DataObject { + const BOXCAR = 128; + ###START_AUTOCODE /* the code below is auto generated do not remove the above tag */ @@ -48,4 +50,55 @@ class Inbox extends Memcached_DataObject { return array(false, false, false); } + + static function insertNotice($user_id, $notice_id) + { + $inbox = new Inbox(); + + $inbox->query(sprintf('UPDATE inbox '. + 'set notice_ids = concat(cast(%08x as binary(4)), '. + 'substr(notice_ids, 1, 4092)) '. + 'WHERE user_id = %d', + $notice_id, $user_id)); + } + + static function bulkInsert($notice_id, $user_ids) + { + $cnt = count($user_ids); + + for ($off = 0; $off < $cnt; $off += self::BOXCAR) { + + $boxcar = array_slice($user_ids, $off, self::BOXCAR); + + if (empty($boxcar)) { // jump in, hobo! + break; + } + + $inbox = new Inbox(); + + $inbox->query(sprintf('UPDATE inbox '. + 'set notice_ids = concat(cast(%08x as binary(4)), '. + 'substr(notice_ids, 1, 4092)) '. + 'WHERE user_id in (%s)', + $notice_id, implode(',', $boxcar))); + + $inbox->free(); + } + } + + function stream($user_id, $offset, $limit, $since_id, $max_id, $since, $own=false) + { + $inbox = Inbox::staticGet('user_id', $user_id); + + if (empty($inbox)) { + return array(); + } + + $ids = unpack('L*', $inbox->notice_ids); + + // XXX: handle since_id + // XXX: handle max_id + + $ids = array_slice($ids, $offset, $limit); + } } diff --git a/classes/Notice.php b/classes/Notice.php index fe3f3c0170..4c6d256d3e 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -125,8 +125,7 @@ class Notice extends Memcached_DataObject 'Fave', 'Notice_tag', 'Group_inbox', - 'Queue_item', - 'Notice_inbox'); + 'Queue_item'); foreach ($related as $cls) { $inst = new $cls(); @@ -504,17 +503,6 @@ class Notice extends Memcached_DataObject unset($original); } - $ni = new Notice_inbox(); - - $ni->notice_id = $this->id; - - if ($ni->find()) { - while ($ni->fetch()) { - $tmk = common_cache_key('user:repeated_to_me:'.$ni->user_id); - $cache->delete($tmk); - } - } - $ni->free(); unset($ni); } @@ -844,10 +832,6 @@ class Notice extends Memcached_DataObject function addToInboxes() { - // XXX: loads constants - - $inbox = new Notice_inbox(); - $users = $this->getSubscribedUsers(); // FIXME: kind of ignoring 'transitional'... @@ -887,7 +871,7 @@ class Notice extends Memcached_DataObject } } - Notice_inbox::bulkInsert($this->id, $this->created, $ni); + Inbox::bulkInsert($this->id, array_keys($ni)); return; } diff --git a/classes/User.php b/classes/User.php index 34151778c5..773723da40 100644 --- a/classes/User.php +++ b/classes/User.php @@ -291,6 +291,19 @@ class User extends Memcached_DataObject return false; } + // Everyone gets an inbox + + $inbox = new Inbox(); + + $inbox->user_id = $user->id; + + $result = $inbox->insert(); + + if (!$result) { + common_log_db_error($inbox, 'INSERT', __FILE__); + return false; + } + // Everyone is subscribed to themself $subscription = new Subscription(); @@ -482,89 +495,30 @@ class User extends Memcached_DataObject function noticesWithFriends($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Notice_inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, false); - + $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, false); return Notice::getStreamByIds($ids); } function noticeInbox($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Notice_inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, true); - + $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, true); return Notice::getStreamByIds($ids); } function friendsTimeline($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Notice::stream(array($this, '_friendsTimelineDirect'), - array(false), - 'user:friends_timeline:'.$this->id, - $offset, $limit, $since_id, $before_id, $since); + $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, false); return Notice::getStreamByIds($ids); } function ownFriendsTimeline($offset=0, $limit=NOTICES_PER_PAGE, $since_id=0, $before_id=0, $since=null) { - $ids = Notice::stream(array($this, '_friendsTimelineDirect'), - array(true), - 'user:friends_timeline_own:'.$this->id, - $offset, $limit, $since_id, $before_id, $since); + $ids = Inbox::stream($this->id, $offset, $limit, $since_id, $before_id, $since, true); return Notice::getStreamByIds($ids); } - function _friendsTimelineDirect($own, $offset, $limit, $since_id, $max_id, $since) - { - $qry = - 'SELECT notice.id AS id ' . - 'FROM notice JOIN notice_inbox ON notice.id = notice_inbox.notice_id ' . - 'WHERE notice_inbox.user_id = ' . $this->id . ' ' . - 'AND notice.repeat_of IS NULL '; - - if (!$own) { - // XXX: autoload notice inbox for constant - $inbox = new Notice_inbox(); - - $qry .= 'AND notice_inbox.source != ' . NOTICE_INBOX_SOURCE_GATEWAY . ' '; - } - - if ($since_id != 0) { - $qry .= 'AND notice.id > ' . $since_id . ' '; - } - - if ($max_id != 0) { - $qry .= 'AND notice.id <= ' . $max_id . ' '; - } - - if (!is_null($since)) { - $qry .= 'AND notice.modified > \'' . date('Y-m-d H:i:s', $since) . '\' '; - } - - // NOTE: we sort by fave time, not by notice time! - - $qry .= 'ORDER BY notice_id DESC '; - - if (!is_null($offset)) { - $qry .= "LIMIT $limit OFFSET $offset"; - } - - $ids = array(); - - $notice = new Notice(); - - $notice->query($qry); - - while ($notice->fetch()) { - $ids[] = $notice->id; - } - - $notice->free(); - $notice = NULL; - - return $ids; - } - function blowFavesCache() { $cache = common_memcache(); @@ -777,7 +731,6 @@ class User extends Memcached_DataObject 'Remember_me', 'Foreign_link', 'Invitation', - 'Notice_inbox', ); Event::handle('UserDeleteRelated', array($this, &$related)); From ac7a1387ba35afd725cadac436c26824c9c9c39c Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Thu, 31 Dec 2009 09:09:07 -1000 Subject: [PATCH 6/6] some formatting changes to make inblobs work --- classes/Inbox.php | 8 +++++--- classes/User.php | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/classes/Inbox.php b/classes/Inbox.php index de48d73818..610b5fceb2 100644 --- a/classes/Inbox.php +++ b/classes/Inbox.php @@ -56,7 +56,7 @@ class Inbox extends Memcached_DataObject $inbox = new Inbox(); $inbox->query(sprintf('UPDATE inbox '. - 'set notice_ids = concat(cast(%08x as binary(4)), '. + 'set notice_ids = concat(cast(0x%08x as binary(4)), '. 'substr(notice_ids, 1, 4092)) '. 'WHERE user_id = %d', $notice_id, $user_id)); @@ -77,7 +77,7 @@ class Inbox extends Memcached_DataObject $inbox = new Inbox(); $inbox->query(sprintf('UPDATE inbox '. - 'set notice_ids = concat(cast(%08x as binary(4)), '. + 'set notice_ids = concat(cast(0x%08x as binary(4)), '. 'substr(notice_ids, 1, 4092)) '. 'WHERE user_id in (%s)', $notice_id, implode(',', $boxcar))); @@ -94,11 +94,13 @@ class Inbox extends Memcached_DataObject return array(); } - $ids = unpack('L*', $inbox->notice_ids); + $ids = unpack('N*', $inbox->notice_ids); // XXX: handle since_id // XXX: handle max_id $ids = array_slice($ids, $offset, $limit); + + return $ids; } } diff --git a/classes/User.php b/classes/User.php index 773723da40..bde3f71b92 100644 --- a/classes/User.php +++ b/classes/User.php @@ -296,6 +296,7 @@ class User extends Memcached_DataObject $inbox = new Inbox(); $inbox->user_id = $user->id; + $inbox->notice_ids = ''; $result = $inbox->insert();