From f246b8f2d7a7b26c0d5b5ae8847a537e5ebbb125 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 15 Sep 2008 02:56:16 -0400 Subject: [PATCH] cache noticesWithFriends in memcached noticesWithFriends is turning out to be one of our most expensive queries. The join is costly, and this method is hit over and over and over by desktop clients and other API users. So, I've added a first pass at caching the results. I store a "window" of notices -- equal to the first 3 pages of notices, plus one for pagination -- in the memcached cache. If with-friends notices are requests, I fetch the whole window out of the cache and grab the slice requested. If the requested notices are outside the window, we just do the query. If there's nothing in the cache, we request the window and store it, then return a slice. I had to add a NoticeWrapper class that works like DB_DataObject (well, just the fetch() part...) but just holds an array of notices instead of a DB cursor. Finally, saving a new notice blows away the caches for subscribed users. darcs-hash:20080915065616-84dde-1b1e814c2294498a10b763b779cbb62c3f96aa84.gz --- classes/Notice.php | 21 ++++++++++++++++ classes/User.php | 57 +++++++++++++++++++++++++++++++++++++++++++ lib/common.php | 6 ++++- lib/noticewrapper.php | 57 +++++++++++++++++++++++++++++++++++++++++++ lib/util.php | 10 ++++++++ 5 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 lib/noticewrapper.php diff --git a/classes/Notice.php b/classes/Notice.php index 6cfd0d7865..181a3bd1b3 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -111,6 +111,27 @@ class Notice extends DB_DataObject common_save_replies($notice); $notice->saveTags(); + + # Clear the cache for subscribed users, so they'll update at next request + # XXX: someone clever could prepend instead of clearing the cache + + if (common_config('memcached', 'enabled')) { + $cache = new Memcache(); + if ($cache->connect(common_config('memcached', 'server'), common_config('memcached', 'port'))) { + $user = new User(); + + $user->query('SELECT id ' . + 'FROM user JOIN subscription ON user.id = subscription.subscriber' . + 'WHERE subscription.subscribed = ' . $notice->profile_id); + + while ($user->fetch()) { + $cache->delete(common_cache_key('user:notices_with_friends:' . $this->id)); + } + + $user->free(); + unset($user); + } + } return $notice; } diff --git a/classes/User.php b/classes/User.php index 7eb21eaeb3..17b1c9fea7 100644 --- a/classes/User.php +++ b/classes/User.php @@ -18,11 +18,18 @@ */ if (!defined('LACONICA')) { exit(1); } + +/* We keep the first three 20-notice pages, plus one for pagination check, + * in the memcached cache. */ + +define('WITHFRIENDS_CACHE_WINDOW', 61); + /** * Table Definition for user */ require_once 'DB/DataObject.php'; require_once 'Validate.php'; +require_once($INSTALLDIR.'/lib/noticewrapper.php'); class User extends DB_DataObject { @@ -134,6 +141,19 @@ class User extends DB_DataObject } function noticesWithFriends($offset=0, $limit=20) { + + # We clearly need a more elegant way to make this work. + + if (common_config('memcached', 'enabled')) { + if ($offset + $limit < WITHFRIENDS_CACHE_WINDOW) { + $cached = $this->noticesWithFriendsCachedWindow(); + if (!$cached) { + $cached = $this->noticesWithFriendsWindow(); + } + $wrapper = new NoticeWrapper(array_slice($cached, $offset, $limit)); + return $wrapper; + } + } $notice = new Notice(); @@ -146,6 +166,43 @@ class User extends DB_DataObject return $notice; } + function noticesWithFriendsCachedWindow() { + $cache = new Memcache(); + $res = $cache->connect(common_config('memcached', 'server'), common_config('memcached', 'port')); + if (!$res) { + return NULL; + } + $notices = $cache->get(common_cache_key('user:notices_with_friends:' . $this->id)); + return $notices; + } + + function noticesWithFriendsWindow() { + + $cache = new Memcache(); + $res = $cache->connect(common_config('memcached', 'server'), common_config('memcached', 'port')); + + if (!$res) { + return NULL; + } + + $notice = new Notice(); + + $notice->query('SELECT notice.* ' . + 'FROM notice JOIN subscription on notice.profile_id = subscription.subscribed ' . + 'WHERE subscription.subscriber = ' . $this->id . ' ' . + 'ORDER BY created DESC, notice.id DESC ' . + 'LIMIT 0, ' . WITHFRIENDS_CACHE_WINDOW); + + $notices = array(); + + while ($notice->fetch()) { + $notices[] = clone($notice); + } + + $cache->set(common_cache_key('user:notices_with_friends:' . $this->id), $notices); + return $notices; + } + static function register($fields) { # MAGICALLY put fields into current scope diff --git a/lib/common.php b/lib/common.php index 80aab806f3..1aa60fc42d 100644 --- a/lib/common.php +++ b/lib/common.php @@ -97,7 +97,11 @@ $config = 'daemon' => array('piddir' => '/var/run', 'user' => false, - 'group' => false) + 'group' => false), + 'memcached' => + array('enabled' => false, + 'server' => 'localhost', + 'port' => 11211) ); $config['db'] = &PEAR::getStaticProperty('DB_DataObject','options'); diff --git a/lib/noticewrapper.php b/lib/noticewrapper.php new file mode 100644 index 0000000000..fbf7b59f4f --- /dev/null +++ b/lib/noticewrapper.php @@ -0,0 +1,57 @@ +. + */ + +if (!defined('LACONICA')) { exit(1); } + +class NoticeWrapper { + + public $id; // int(4) primary_key not_null + public $profile_id; // int(4) not_null + public $uri; // varchar(255) unique_key + public $content; // varchar(140) + public $rendered; // text() + public $url; // varchar(255) + public $created; // datetime() not_null + public $modified; // timestamp() not_null default_CURRENT_TIMESTAMP + public $reply_to; // int(4) + public $is_local; // tinyint(1) + public $source; // varchar(32) + + var $notices = NULL; + var $i = -1; + + function __construct($arr) { + $this->notices = $arr; + } + + function fetch() { + static $fields = array('id', 'profile_id', 'uri', 'content', 'rendered', + 'url', 'created', 'modified', 'reply_to', 'is_local', 'source'); + $this->i++; + if ($this->i >= array_count($notices)) { + return false; + } else { + $n = $notices[$this->i]; + foreach ($fields as $f) { + $this->$f = $n->$f; + } + return true; + } + } +} \ No newline at end of file diff --git a/lib/util.php b/lib/util.php index ff22ac644d..349cff7222 100644 --- a/lib/util.php +++ b/lib/util.php @@ -1528,3 +1528,13 @@ function common_session_token() { } return $_SESSION['token']; } + +function common_cache_key($extra) { + return 'laconica:' . common_keyize(common_config('site', 'name')) . ':' . $extra; +} + +function common_keyize($str) { + $str = strtolower($str); + $str = preg_replace('/\s/', '_', $str); + return $str; +}