From 7a2d72fe28e29c67f2a9e8fdb59c6c5c9b38d5e6 Mon Sep 17 00:00:00 2001 From: Craig Andrews Date: Mon, 4 Jan 2010 12:49:25 -0500 Subject: [PATCH 01/18] Enable memcache automatic compression, starting at 20k and only if compression gain is greater than 20%. Allows storage of larger objects (over 1mb in size uncompressed), such as huge LDAP schemas. Should also improve cache efficiency (allows more stuff to be stored in same memory) and reduce network latency (less data transfer) (redo commit 1e9c03e1993b5d2978ac4c5213a8a64e0150b4a2 which was apparently lost during pluginization) --- plugins/MemcachePlugin.php | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/MemcachePlugin.php b/plugins/MemcachePlugin.php index acbec135e4..78e2b24067 100644 --- a/plugins/MemcachePlugin.php +++ b/plugins/MemcachePlugin.php @@ -156,6 +156,11 @@ class MemcachePlugin extends Plugin } $this->_conn->addServer($host, $port); } + //Compress items stored in the cache if they're over 2k in size + //and the compression would save more than 20%. + //Allows the cache to store objects larger than 1MB (if they + //compress to less than 1MB), and improves cache memory efficiency. + $this->_conn->setCompressThreshold(20000, 0.2); } } } From 783a2e249bac2ccd02b6ae3351dfe83630b71cc0 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 4 Jan 2010 10:30:19 -0800 Subject: [PATCH 02/18] Fix for auto_increment parameter in auto-created tables via checkschema. Update FeedSub plugin for non-Plugin_DataObject setup and working checkschema updates. --- lib/columndef.php | 1 + lib/schema.php | 4 ++ plugins/FeedSub/FeedSubPlugin.php | 7 +- plugins/FeedSub/feedinfo.php | 108 +++++++++++++++++++++++------- 4 files changed, 90 insertions(+), 30 deletions(-) diff --git a/lib/columndef.php b/lib/columndef.php index 1bae6b33bb..ac2fcd23ea 100644 --- a/lib/columndef.php +++ b/lib/columndef.php @@ -74,6 +74,7 @@ class ColumnDef * @param string $key type of key * @param value $default default value * @param value $extra unused + * @param boolean $auto_increment */ function __construct($name=null, $type=null, $size=null, diff --git a/lib/schema.php b/lib/schema.php index a8ba91b87f..6fe442d56b 100644 --- a/lib/schema.php +++ b/lib/schema.php @@ -523,6 +523,10 @@ class Schema } else { $sql .= ($cd->nullable) ? "null " : "not null "; } + + if (!empty($cd->auto_increment)) { + $sql .= " auto_increment "; + } return $sql; } diff --git a/plugins/FeedSub/FeedSubPlugin.php b/plugins/FeedSub/FeedSubPlugin.php index 857a9794d5..e49e2a648a 100644 --- a/plugins/FeedSub/FeedSubPlugin.php +++ b/plugins/FeedSub/FeedSubPlugin.php @@ -105,12 +105,11 @@ class FeedSubPlugin extends Plugin return true; } - /* - // auto increment seems to be broken function onCheckSchema() { + // warning: the autoincrement doesn't seem to set. + // alter table feedinfo change column id id int(11) not null auto_increment; $schema = Schema::get(); - $schema->ensureDataObject('Feedinfo'); + $schema->ensureTable('feedinfo', Feedinfo::schemaDef()); return true; } - */ } diff --git a/plugins/FeedSub/feedinfo.php b/plugins/FeedSub/feedinfo.php index fff66afe97..b166bd6e12 100644 --- a/plugins/FeedSub/feedinfo.php +++ b/plugins/FeedSub/feedinfo.php @@ -31,7 +31,7 @@ class FeedDBException extends FeedSubException } } -class Feedinfo extends Plugin_DataObject +class Feedinfo extends Memcached_DataObject { public $__table = 'feedinfo'; @@ -56,34 +56,90 @@ class Feedinfo extends Plugin_DataObject return parent::staticGet(__CLASS__, $k, $v); } - function tableDef() + /** + * return table definition for DB_DataObject + * + * DB_DataObject needs to know something about the table to manipulate + * instances. This method provides all the DB_DataObject needs to know. + * + * @return array array of column definitions + */ + + function table() { - class_exists('Schema'); // autoload hack - // warning: the autoincrement doesn't seem to set. - // alter table feedinfo change column id id int(11) not null auto_increment; - return new TableDef($this->__table, - array(new ColumnDef('id', 'integer', - null, false, 'PRI', '0', null, true), - new ColumnDef('profile_id', 'integer', - null, false), - new ColumnDef('feeduri', 'varchar', - 255, false, 'UNI'), - new ColumnDef('homeuri', 'varchar', - 255, false), - new ColumnDef('huburi', 'varchar', - 255, false), - new ColumnDef('verify_token', 'varchar', - 32, true), - new ColumnDef('sub_start', 'datetime', - null, true), - new ColumnDef('sub_end', 'datetime', - null, true), - new ColumnDef('created', 'datetime', - null, false), - new ColumnDef('lastupdate', 'datetime', - null, false))); + return array('id' => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL, + 'profile_id' => DB_DATAOBJECT_INT + DB_DATAOBJECT_NOTNULL, + 'feeduri' => DB_DATAOBJECT_STR + DB_DATAOBJECT_NOTNULL, + 'homeuri' => DB_DATAOBJECT_STR + DB_DATAOBJECT_NOTNULL, + 'huburi' => DB_DATAOBJECT_STR + DB_DATAOBJECT_NOTNULL, + 'verify_token' => DB_DATAOBJECT_STR, + 'sub_start' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME, + 'sub_end' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME, + 'created' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL, + 'lastupdate' => DB_DATAOBJECT_STR + DB_DATAOBJECT_DATE + DB_DATAOBJECT_TIME + DB_DATAOBJECT_NOTNULL); + } + + static function schemaDef() + { + return array(new ColumnDef('id', 'integer', + /*size*/ null, + /*nullable*/ false, + /*key*/ 'PRI', + /*default*/ '0', + /*extra*/ null, + /*auto_increment*/ true), + new ColumnDef('profile_id', 'integer', + null, false), + new ColumnDef('feeduri', 'varchar', + 255, false, 'UNI'), + new ColumnDef('homeuri', 'varchar', + 255, false), + new ColumnDef('huburi', 'varchar', + 255, false), + new ColumnDef('verify_token', 'varchar', + 32, true), + new ColumnDef('sub_start', 'datetime', + null, true), + new ColumnDef('sub_end', 'datetime', + null, true), + new ColumnDef('created', 'datetime', + null, false), + new ColumnDef('lastupdate', 'datetime', + null, false)); } + /** + * return key definitions for DB_DataObject + * + * DB_DataObject needs to know about keys that the table has; this function + * defines them. + * + * @return array key definitions + */ + + function keys() + { + return array('id' => 'P'); //? + } + + /** + * return key definitions for Memcached_DataObject + * + * Our caching system uses the same key definitions, but uses a different + * method to get them. + * + * @return array key definitions + */ + + function keyTypes() + { + return $this->keys(); + } + + /** + * Fetch the StatusNet-side profile for this feed + * @return Profile + */ public function getProfile() { return Profile::staticGet('id', $this->profile_id); From e440b69e1a3c73796535298b3b20d6678431f9b6 Mon Sep 17 00:00:00 2001 From: Craig Andrews Date: Mon, 4 Jan 2010 13:33:52 -0500 Subject: [PATCH 03/18] Allow an authentication plugin with the same provider_name other than the one that actually checked the password to autoregister a user Allows for SSO-type plugins that don't have any information about the user other than their username to do autoregistration --- .../Authentication/AuthenticationPlugin.php | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/plugins/Authentication/AuthenticationPlugin.php b/plugins/Authentication/AuthenticationPlugin.php index a76848b04e..75e8d2b766 100644 --- a/plugins/Authentication/AuthenticationPlugin.php +++ b/plugins/Authentication/AuthenticationPlugin.php @@ -99,6 +99,23 @@ abstract class AuthenticationPlugin extends Plugin } } + /** + * Internal AutoRegister event handler + * @param nickname + * @param provider_name + * @param user - the newly registered user + */ + function onAutoRegister($nickname, $provider_name, &$user) + { + if($provider_name == $this->provider_name && $this->autoregistration){ + $user = $this->autoregister($nickname); + if($user){ + User_username::register($user,$nickname,$this->provider_name); + return false; + } + } + } + function onStartCheckPassword($nickname, $password, &$authenticatedUser){ //map the nickname to a username $user_username = new User_username(); @@ -127,13 +144,10 @@ abstract class AuthenticationPlugin extends Plugin } } }else{ - if($this->autoregistration){ - $authenticated = $this->checkPassword($nickname, $password); - if($authenticated){ - $user = $this->autoregister($nickname); - if($user){ - $authenticatedUser = $user; - User_username::register($authenticatedUser,$nickname,$this->provider_name); + $authenticated = $this->checkPassword($nickname, $password); + if($authenticated){ + if(Event::handle('AutoRegister', array($nickname, $this->provider_name, &$authenticatedUser))){ + if($authenticatedUser){ return false; } } From 11d7365a15bb8c8cb856bf0d545a0b550f3dbe26 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 4 Jan 2010 10:39:11 -0800 Subject: [PATCH 04/18] Don't spew notices when building tag cloud if there is no popularity sum to divide by. --- actions/publictagcloud.php | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/actions/publictagcloud.php b/actions/publictagcloud.php index e7f6ee36c7..5c70740290 100644 --- a/actions/publictagcloud.php +++ b/actions/publictagcloud.php @@ -136,7 +136,12 @@ class PublictagcloudAction extends Action $this->elementStart('dd'); $this->elementStart('ul', 'tags xoxo tag-cloud'); foreach ($tw as $tag => $weight) { - $this->showTag($tag, $weight, $weight/$sum); + if ($sum) { + $weightedSum = $weight/$sum; + } else { + $weightedSum = 1; + } + $this->showTag($tag, $weight, $weightedSum); } $this->elementEnd('ul'); $this->elementEnd('dd'); From 38912b34c7f9d0a377feeb76572003a36c61ee7e Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 4 Jan 2010 10:41:52 -0800 Subject: [PATCH 05/18] Drop the overly-prominent link to checklibs display; it's unnecessary and just confuses people. The ability's still there to aid in debugging, but it won't be tempting people to click on it. --- install.php | 1 - 1 file changed, 1 deletion(-) diff --git a/install.php b/install.php index 1c62bb2b21..435f6d63bf 100644 --- a/install.php +++ b/install.php @@ -454,7 +454,6 @@ function showForm()

Enter your database connection information below to initialize the database.

-

StatusNet bundles a number of libraries for ease of installation. You can see what bundled libraries you are using, versus what libraries are installed on your server.

From 8f362e9956ef699511344e0a494669ae78aa93eb Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 08:53:28 -1000 Subject: [PATCH 06/18] user_id is a non-autoincrement pkey for user_location_prefs --- classes/statusnet.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/statusnet.ini b/classes/statusnet.ini index 4077746c47..ac31148daa 100644 --- a/classes/statusnet.ini +++ b/classes/statusnet.ini @@ -572,5 +572,5 @@ created = 142 modified = 384 [user_location_prefs__keys] -user_id = U +user_id = K From 5a1ea0b9b28138bc4e06d96e2910b0cf3be1eab2 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 08:59:19 -1000 Subject: [PATCH 07/18] Stop caching unfindable keys There were some problems with the automated cache/uncache system for data objects that made us cache unfindable keys (with null attributes and sometimes null names). Fixed those problems and refactored the encache() and decache() methods so they use a helper to find the cache keys to use. --- classes/Memcached_DataObject.php | 87 +++++++++++++++++++------------- 1 file changed, 52 insertions(+), 35 deletions(-) diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 1608720d14..020d813b2c 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -152,52 +152,69 @@ class Memcached_DataObject extends DB_DataObject function encache() { $c = $this->memcache(); + if (!$c) { return false; - } else { - $pkey = array(); - $pval = array(); - $types = $this->keyTypes(); - ksort($types); - foreach ($types as $key => $type) { - if ($type == 'K') { - $pkey[] = $key; - $pval[] = $this->$key; - } else { - $c->set($this->cacheKey($this->tableName(), $key, $this->$key), $this); - } - } - # XXX: should work for both compound and scalar pkeys - $pvals = implode(',', $pval); - $pkeys = implode(',', $pkey); - $c->set($this->cacheKey($this->tableName(), $pkeys, $pvals), $this); + } + + $keys = $this->_allCacheKeys(); + + foreach ($keys as $key) { + $c->set($key, $this); } } function decache() { $c = $this->memcache(); + if (!$c) { return false; - } else { - $pkey = array(); - $pval = array(); - $types = $this->keyTypes(); - ksort($types); - foreach ($types as $key => $type) { - if ($type == 'K') { - $pkey[] = $key; - $pval[] = $this->$key; - } else { - $c->delete($this->cacheKey($this->tableName(), $key, $this->$key)); - } - } - # should work for both compound and scalar pkeys - # XXX: comma works for now but may not be safe separator for future keys - $pvals = implode(',', $pval); - $pkeys = implode(',', $pkey); - $c->delete($this->cacheKey($this->tableName(), $pkeys, $pvals)); } + + $keys = $this->_allCacheKeys(); + + foreach ($keys as $key) { + $c->delete($key, $this); + } + } + + function _allCacheKeys() + { + $ckeys = array(); + + $types = $this->keyTypes(); + ksort($types); + + $pkey = array(); + $pval = array(); + + foreach ($types as $key => $type) { + + assert(!empty($key)); + + if ($type == 'U') { + if (empty($this->$key)) { + continue; + } + $ckeys[] = $this->cacheKey($this->tableName(), $key, $this->$key); + } else if ($type == 'K' || $type == 'N') { + $pkey[] = $key; + $pval[] = $this->$key; + } else { + throw new Exception("Unknown key type $key => $type for " . $this->tableName()); + } + } + + assert(count($pkey) > 0); + + // XXX: should work for both compound and scalar pkeys + $pvals = implode(',', $pval); + $pkeys = implode(',', $pkey); + + $ckeys[] = $this->cacheKey($this->tableName(), $pkeys, $pvals); + + return $ckeys; } function multicache($cls, $kv) From bcddcb38cebc5ffbffd3df61fb9acaf6f674b133 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 09:09:59 -1000 Subject: [PATCH 08/18] make compression threshold and min savings config attrs for MemcachePlugin --- plugins/MemcachePlugin.php | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/plugins/MemcachePlugin.php b/plugins/MemcachePlugin.php index 78e2b24067..998766313f 100644 --- a/plugins/MemcachePlugin.php +++ b/plugins/MemcachePlugin.php @@ -54,6 +54,9 @@ class MemcachePlugin extends Plugin private $_conn = null; public $servers = array('127.0.0.1;11211'); + public $compressThreshold = 20480; + public $compressMinSaving = 0.2; + /** * Initialize the plugin * @@ -156,11 +159,16 @@ class MemcachePlugin extends Plugin } $this->_conn->addServer($host, $port); } - //Compress items stored in the cache if they're over 2k in size - //and the compression would save more than 20%. - //Allows the cache to store objects larger than 1MB (if they - //compress to less than 1MB), and improves cache memory efficiency. - $this->_conn->setCompressThreshold(20000, 0.2); + + // Compress items stored in the cache if they're over threshold in size + // (default 2KiB) and the compression would save more than min savings + // ratio (default 0.2). + + // Allows the cache to store objects larger than 1MB (if they + // compress to less than 1MB), and improves cache memory efficiency. + + $this->_conn->setCompressThreshold($this->compressThreshold, + $this->compressMinSaving); } } } From c0e4d7bfa2181090f05d3b42ed9f9b135b43aea1 Mon Sep 17 00:00:00 2001 From: Craig Andrews Date: Mon, 4 Jan 2010 14:43:05 -0500 Subject: [PATCH 09/18] Add 'takeOverLogin' parameter for a real SSO feel --- plugins/CasAuthentication/CasAuthenticationPlugin.php | 9 +++++++++ plugins/CasAuthentication/README | 6 +++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/plugins/CasAuthentication/CasAuthenticationPlugin.php b/plugins/CasAuthentication/CasAuthenticationPlugin.php index 8b6ef54621..8f29c7d2ad 100644 --- a/plugins/CasAuthentication/CasAuthenticationPlugin.php +++ b/plugins/CasAuthentication/CasAuthenticationPlugin.php @@ -40,6 +40,7 @@ class CasAuthenticationPlugin extends AuthenticationPlugin public $server; public $port = 443; public $path = ''; + public $takeOverLogin = false; function checkPassword($username, $password) { @@ -62,6 +63,14 @@ class CasAuthenticationPlugin extends AuthenticationPlugin } } + function onArgsInitialize(&$args) + { + if($this->takeOverLogin && $args['action'] == 'login') + { + $args['action'] = 'caslogin'; + } + } + function onStartInitializeRouter($m) { $m->connect('main/cas', array('action' => 'caslogin')); diff --git a/plugins/CasAuthentication/README b/plugins/CasAuthentication/README index 2ee54dc057..c17a28e54a 100644 --- a/plugins/CasAuthentication/README +++ b/plugins/CasAuthentication/README @@ -21,6 +21,9 @@ password_changeable*: must be set to false. This plugin does not support changin server*: CAS server to authentication against port (443): Port the CAS server listens on. Almost always 443 path (): Path on the server to CAS. Usually blank. +takeOverLogin (false): Take over the main login action. If takeOverLogin is + set, anytime the standard username/password login form would be shown, + a CAS login will be done instead. * required default values are in (parenthesis) @@ -33,6 +36,7 @@ addPlugin('casAuthentication', array( 'autoregistration'=>true, 'server'=>'sso-cas.univ-rennes1.fr', 'port'=>443, - 'path'=>'' + 'path'=>'', + 'takeOverLogin'=>true )); From 6911e1c7972c3adec53d0fe04ebdd7da0fbd8b12 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 4 Jan 2010 11:55:27 -0800 Subject: [PATCH 10/18] Ticket 2141: bugs with weighted popularity lists across year boundary. Consolidated several separate implementations of the same weighting algorithm into common_sql_weight() and fixed some bugs... For MySQL, now using timestampdiff() instead of subtraction for the comparison, so we get sane results when the year doesn't match, and utc_timestamp() rather than now() so we don't get negative ages for recent items with local server timezone. Unknown whether the same problems affect PostgreSQL, but note that it lacks the timestampdiff() SQL function. --- actions/favorited.php | 8 ++------ actions/publictagcloud.php | 11 ++++------- lib/grouptagcloudsection.php | 7 +------ lib/personaltagcloudsection.php | 11 +++-------- lib/popularnoticesection.php | 6 +++--- lib/util.php | 20 ++++++++++++++++++++ 6 files changed, 33 insertions(+), 30 deletions(-) diff --git a/actions/favorited.php b/actions/favorited.php index 150b67b0b0..9ffa5b8445 100644 --- a/actions/favorited.php +++ b/actions/favorited.php @@ -185,11 +185,7 @@ class FavoritedAction extends Action function showContent() { - if (common_config('db', 'type') == 'pgsql') { - $weightexpr='sum(exp(-extract(epoch from (now() - fave.modified)) / %s))'; - } else { - $weightexpr='sum(exp(-(now() - fave.modified) / %s))'; - } + $weightexpr = common_sql_weight('fave.modified', common_config('popular', 'dropoff')); $qry = 'SELECT notice.*, '. $weightexpr . ' as weight ' . @@ -207,7 +203,7 @@ class FavoritedAction extends Action } $notice = Memcached_DataObject::cachedQuery('Notice', - sprintf($qry, common_config('popular', 'dropoff')), + $qry, 600); $nl = new NoticeList($notice, $this); diff --git a/actions/publictagcloud.php b/actions/publictagcloud.php index 5c70740290..b5b474f132 100644 --- a/actions/publictagcloud.php +++ b/actions/publictagcloud.php @@ -105,12 +105,8 @@ class PublictagcloudAction extends Action #Add the aggregated columns... $tags->selectAdd('max(notice_id) as last_notice_id'); - if(common_config('db','type')=='pgsql') { - $calc='sum(exp(-extract(epoch from (now()-created))/%s)) as weight'; - } else { - $calc='sum(exp(-(now() - created)/%s)) as weight'; - } - $tags->selectAdd(sprintf($calc, common_config('tag', 'dropoff'))); + $calc = common_sql_weight('created', common_config('tag', 'dropoff')); + $tags->selectAdd($calc . ' as weight'); $tags->groupBy('tag'); $tags->orderBy('weight DESC'); @@ -136,10 +132,11 @@ class PublictagcloudAction extends Action $this->elementStart('dd'); $this->elementStart('ul', 'tags xoxo tag-cloud'); foreach ($tw as $tag => $weight) { + common_log(LOG_DEBUG, "$weight/$sum"); if ($sum) { $weightedSum = $weight/$sum; } else { - $weightedSum = 1; + $weightedSum = 0.5; } $this->showTag($tag, $weight, $weightedSum); } diff --git a/lib/grouptagcloudsection.php b/lib/grouptagcloudsection.php index 091cf48457..14ceda0850 100644 --- a/lib/grouptagcloudsection.php +++ b/lib/grouptagcloudsection.php @@ -58,11 +58,7 @@ class GroupTagCloudSection extends TagCloudSection function getTags() { - if (common_config('db', 'type') == 'pgsql') { - $weightexpr='sum(exp(-extract(epoch from (now() - notice_tag.created)) / %s))'; - } else { - $weightexpr='sum(exp(-(now() - notice_tag.created) / %s))'; - } + $weightexpr = common_sql_weight('notice_tag.created', common_config('tag', 'dropoff')); $names = $this->group->getAliases(); @@ -99,7 +95,6 @@ class GroupTagCloudSection extends TagCloudSection $tag = Memcached_DataObject::cachedQuery('Notice_tag', sprintf($qry, - common_config('tag', 'dropoff'), $this->group->id, $namestring), 3600); diff --git a/lib/personaltagcloudsection.php b/lib/personaltagcloudsection.php index 0b29d58ca6..091425f926 100644 --- a/lib/personaltagcloudsection.php +++ b/lib/personaltagcloudsection.php @@ -58,13 +58,9 @@ class PersonalTagCloudSection extends TagCloudSection function getTags() { - if (common_config('db', 'type') == 'pgsql') { - $weightexpr='sum(exp(-extract(epoch from (now() - notice_tag.created)) / %s))'; - } else { - $weightexpr='sum(exp(-(now() - notice_tag.created) / %s))'; - } - - $qry = 'SELECT notice_tag.tag, '. + $weightexpr = common_sql_weight('notice_tag.created', common_config('tag', 'dropoff')); + + $qry = 'SELECT notice_tag.tag, '. $weightexpr . ' as weight ' . 'FROM notice_tag JOIN notice ' . 'ON notice_tag.notice_id = notice.id ' . @@ -83,7 +79,6 @@ class PersonalTagCloudSection extends TagCloudSection $tag = Memcached_DataObject::cachedQuery('Notice_tag', sprintf($qry, - common_config('tag', 'dropoff'), $this->user->id), 3600); return $tag; diff --git a/lib/popularnoticesection.php b/lib/popularnoticesection.php index 9fbc9d2ddd..fbf9a60ab8 100644 --- a/lib/popularnoticesection.php +++ b/lib/popularnoticesection.php @@ -48,17 +48,17 @@ class PopularNoticeSection extends NoticeSection { function getNotices() { + // @fixme there should be a common func for this if (common_config('db', 'type') == 'pgsql') { - $weightexpr='sum(exp(-extract(epoch from (now() - fave.modified)) / %s))'; if (!empty($this->out->tag)) { $tag = pg_escape_string($this->out->tag); } } else { - $weightexpr='sum(exp(-(now() - fave.modified) / %s))'; if (!empty($this->out->tag)) { $tag = mysql_escape_string($this->out->tag); } } + $weightexpr = common_sql_weight('fave.modified', common_config('popular', 'dropoff')); $qry = "SELECT notice.*, $weightexpr as weight "; if(isset($tag)) { $qry .= 'FROM notice_tag, notice JOIN fave ON notice.id = fave.notice_id ' . @@ -78,7 +78,7 @@ class PopularNoticeSection extends NoticeSection $qry .= ' LIMIT ' . $limit . ' OFFSET ' . $offset; $notice = Memcached_DataObject::cachedQuery('Notice', - sprintf($qry, common_config('popular', 'dropoff')), + $qry, 1200); return $notice; } diff --git a/lib/util.php b/lib/util.php index 63656b6042..50bd0e2ac9 100644 --- a/lib/util.php +++ b/lib/util.php @@ -908,6 +908,26 @@ function common_sql_date($datetime) return strftime('%Y-%m-%d %H:%M:%S', $datetime); } +/** + * Return an SQL fragment to calculate an age-based weight from a given + * timestamp or datetime column. + * + * @param string $column name of field we're comparing against current time + * @param integer $dropoff divisor for age in seconds before exponentiation + * @return string SQL fragment + */ +function common_sql_weight($column, $dropoff) +{ + if (common_config('db', 'type') == 'pgsql') { + // PostgreSQL doesn't support timestampdiff function. + // @fixme will this use the right time zone? + // @fixme does this handle cross-year subtraction correctly? + return "sum(exp(-extract(epoch from (now() - $column)) / $dropoff))"; + } else { + return "sum(exp(timestampdiff(second, utc_timestamp(), $column) / $dropoff))"; + } +} + function common_redirect($url, $code=307) { static $status = array(301 => "Moved Permanently", From 928b5f8f2be554c37b0a435e76e88e92260e667b Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 09:57:48 -1000 Subject: [PATCH 11/18] Differentiate between empty values and cache misses in CacheLogPlugin --- plugins/CacheLogPlugin.php | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/plugins/CacheLogPlugin.php b/plugins/CacheLogPlugin.php index 9eb04641e6..f1e5dd83aa 100644 --- a/plugins/CacheLogPlugin.php +++ b/plugins/CacheLogPlugin.php @@ -61,7 +61,7 @@ class CacheLogPlugin extends Plugin function onEndCacheGet($key, &$value) { - if (is_null($value)) { + if ($value === false) { $this->log(LOG_INFO, "Cache MISS for key '$key'"); } else { $this->log(LOG_INFO, "Cache HIT for key '$key'"); @@ -71,7 +71,21 @@ class CacheLogPlugin extends Plugin function onStartCacheSet(&$key, &$value, &$flag, &$expiry, &$success) { - $this->log(LOG_INFO, "Setting cache value for key '$key'"); + if (empty($value)) { + if (is_array($value)) { + $this->log(LOG_INFO, "Setting empty array for key '$key'"); + } else if (is_null($value)) { + $this->log(LOG_INFO, "Setting null value for key '$key'"); + } else if (is_string($value)) { + $this->log(LOG_INFO, "Setting empty string for key '$key'"); + } else if (is_integer($value)) { + $this->log(LOG_INFO, "Setting integer 0 for key '$key'"); + } else { + $this->log(LOG_INFO, "Setting empty value '$value' for key '$key'"); + } + } else { + $this->log(LOG_INFO, "Setting non-empty value for key '$key'"); + } return true; } From a1821ec8afd40c3ab88546a8fa2e6fc03a19ad25 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 09:59:47 -1000 Subject: [PATCH 12/18] default value for cache::get() changed from null to false --- lib/cache.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cache.php b/lib/cache.php index 253839fb17..bac3499e5e 100644 --- a/lib/cache.php +++ b/lib/cache.php @@ -116,7 +116,7 @@ class Cache function get($key) { - $value = null; + $value = false; if (Event::handle('StartCacheGet', array(&$key, &$value))) { if (array_key_exists($key, $this->_items)) { From abc9b332412921e7ae7f730b23e186bcba412486 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 10:00:17 -1000 Subject: [PATCH 13/18] Memcached_DataObject stores empty values in the cache There's great value in knowing that something doesn't exist. We now cache this information, and carefully compare the results from cache as $results !== false instead of !empty($results), since some empty values (null, 0, empty array, empty string) are stored in the cache. Caching staticGet() and pkeyGet() now store DB misses in the cache, and cachedQuery() checks for empty results from the cache. --- classes/Memcached_DataObject.php | 39 ++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index 020d813b2c..a77e43d38c 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -19,8 +19,6 @@ if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } -require_once INSTALLDIR.'/classes/Memcached_DataObject.php'; - class Memcached_DataObject extends DB_DataObject { /** @@ -57,7 +55,7 @@ class Memcached_DataObject extends DB_DataObject unset($i); } $i = Memcached_DataObject::getcached($cls, $k, $v); - if ($i) { + if ($i !== false) { // false == cache miss return $i; } else { $i = DB_DataObject::factory($cls); @@ -69,6 +67,12 @@ class Memcached_DataObject extends DB_DataObject $i->encache(); return $i; } else { + // save the fact that no such row exists + $c = self::memcache(); + if (!empty($c)) { + $ck = self::cachekey($cls, $k, $v); + $c->set($ck, null); + } return false; } } @@ -77,10 +81,13 @@ class Memcached_DataObject extends DB_DataObject function &pkeyGet($cls, $kv) { $i = Memcached_DataObject::multicache($cls, $kv); - if ($i) { + if ($i !== false) { // false == cache miss return $i; } else { - $i = new $cls(); + $i = DB_DataObject::factory($cls); + if (empty($i)) { + return false; + } foreach ($kv as $k => $v) { $i->$k = $v; } @@ -88,6 +95,11 @@ class Memcached_DataObject extends DB_DataObject $i->encache(); } else { $i = null; + $c = self::memcache(); + if (!empty($c)) { + $ck = self::multicacheKey($cls, $kv); + $c->set($ck, null); + } } return $i; } @@ -220,16 +232,22 @@ class Memcached_DataObject extends DB_DataObject function multicache($cls, $kv) { ksort($kv); - $c = Memcached_DataObject::memcache(); + $c = self::memcache(); if (!$c) { return false; } else { - $pkeys = implode(',', array_keys($kv)); - $pvals = implode(',', array_values($kv)); - return $c->get(Memcached_DataObject::cacheKey($cls, $pkeys, $pvals)); + return $c->get(self::multicacheKey($cls, $kv)); } } + static function multicacheKey($cls, $kv) + { + ksort($kv); + $pkeys = implode(',', array_keys($kv)); + $pvals = implode(',', array_values($kv)); + return self::cacheKey($cls, $pkeys, $pvals); + } + function getSearchEngine($table) { require_once INSTALLDIR.'/lib/search_engines.php'; @@ -264,7 +282,8 @@ class Memcached_DataObject extends DB_DataObject $key_part = common_keyize($cls).':'.md5($qry); $ckey = common_cache_key($key_part); $stored = $c->get($ckey); - if ($stored) { + + if ($stored !== false) { return new ArrayWrapper($stored); } From 254ea279d8c86ed25ed74bff036b16e6293571f7 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 10:02:59 -1000 Subject: [PATCH 14/18] carefully compare cached settings against false for Config --- classes/Config.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/classes/Config.php b/classes/Config.php index 390d75381a..6d914ca1f6 100644 --- a/classes/Config.php +++ b/classes/Config.php @@ -59,7 +59,7 @@ class Config extends Memcached_DataObject if (!empty($c)) { $settings = $c->get(common_cache_key(self::settingsKey)); - if (!empty($settings)) { + if ($settings !== false) { return $settings; } } From 06b6a27d7d31cb0680dffff70c498e27ece56762 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 10:03:57 -1000 Subject: [PATCH 15/18] cached id streams can be empty, compare against false --- classes/Notice.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/classes/Notice.php b/classes/Notice.php index 3e55bd6fa5..e8bc509a68 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -1207,7 +1207,7 @@ class Notice extends Memcached_DataObject $idstr = $cache->get($idkey); - if (!empty($idstr)) { + if ($idstr !== false) { // Cache hit! Woohoo! $window = explode(',', $idstr); $ids = array_slice($window, $offset, $limit); @@ -1216,7 +1216,7 @@ class Notice extends Memcached_DataObject $laststr = $cache->get($idkey.';last'); - if (!empty($laststr)) { + if ($laststr !== false) { $window = explode(',', $laststr); $last_id = $window[0]; $new_ids = call_user_func_array($fn, array_merge($args, array(0, NOTICE_CACHE_WINDOW, @@ -1376,7 +1376,7 @@ class Notice extends Memcached_DataObject $ids = $this->_repeatStreamDirect($limit); } else { $idstr = $cache->get(common_cache_key('notice:repeats:'.$this->id)); - if (!empty($idstr)) { + if ($idstr !== false) { $ids = explode(',', $idstr); } else { $ids = $this->_repeatStreamDirect(100); From 96480aa6c1abda95db272f7c0a2b0e96f17acc70 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Mon, 4 Jan 2010 10:12:19 -1000 Subject: [PATCH 16/18] XCachePlugin returns false value for cache miss --- plugins/XCachePlugin.php | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/XCachePlugin.php b/plugins/XCachePlugin.php index 8eed12cbc9..03cb0c06ea 100644 --- a/plugins/XCachePlugin.php +++ b/plugins/XCachePlugin.php @@ -63,8 +63,10 @@ class XCachePlugin extends Plugin function onStartCacheGet(&$key, &$value) { - $value = xcache_get($key); - if (!is_null($value)) { + if (!xcache_isset($key)) { + $value = false; + } else { + $value = xcache_get($key); $value = unserialize($value); } Event::handle('EndCacheGet', array($key, &$value)); From e668709ce48f366b4477bc62434db28b3e211a33 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 4 Jan 2010 13:02:41 -0800 Subject: [PATCH 17/18] drop debug statement --- actions/publictagcloud.php | 1 - 1 file changed, 1 deletion(-) diff --git a/actions/publictagcloud.php b/actions/publictagcloud.php index b5b474f132..9e4478dbb1 100644 --- a/actions/publictagcloud.php +++ b/actions/publictagcloud.php @@ -132,7 +132,6 @@ class PublictagcloudAction extends Action $this->elementStart('dd'); $this->elementStart('ul', 'tags xoxo tag-cloud'); foreach ($tw as $tag => $weight) { - common_log(LOG_DEBUG, "$weight/$sum"); if ($sum) { $weightedSum = $weight/$sum; } else { From 440b9957f9ae6fdfdb44c39074803fcdbc64112d Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Mon, 4 Jan 2010 14:24:16 -0800 Subject: [PATCH 18/18] Exclude process-specific link & result cache references from serialized Memcached_Data_Object instances. Should fix seemingly-random bugs due to destructor free()ing local resources by mistake. --- classes/Memcached_DataObject.php | 38 +++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/classes/Memcached_DataObject.php b/classes/Memcached_DataObject.php index a77e43d38c..d830884b67 100644 --- a/classes/Memcached_DataObject.php +++ b/classes/Memcached_DataObject.php @@ -21,7 +21,7 @@ if (!defined('STATUSNET') && !defined('LACONICA')) { exit(1); } class Memcached_DataObject extends DB_DataObject { - /** + /** * Destructor to free global memory resources associated with * this data object when it's unset or goes out of scope. * DB_DataObject doesn't do this yet by itself. @@ -35,6 +35,42 @@ class Memcached_DataObject extends DB_DataObject } } + /** + * Magic function called at serialize() time. + * + * We use this to drop a couple process-specific references + * from DB_DataObject which can cause trouble in future + * processes. + * + * @return array of variable names to include in serialization. + */ + function __sleep() + { + $vars = array_keys(get_object_vars($this)); + $skip = array('_DB_resultid', '_link_loaded'); + return array_diff($vars, $skip); + } + + /** + * Magic function called at unserialize() time. + * + * Clean out some process-specific variables which might + * be floating around from a previous process's cached + * objects. + * + * Old cached objects may still have them. + */ + function __wakeup() + { + // Refers to global state info from a previous process. + // Clear this out so we don't accidentally break global + // state in *this* process. + $this->_DB_resultid = null; + + // We don't have any local DBO refs, so clear these out. + $this->_link_loaded = false; + } + /** * Wrapper for DB_DataObject's static lookup using memcached * as backing instead of an in-process cache array.