From cef302cacdf86d1c82f7937d2901f9254c88bf8a Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Fri, 28 May 2010 06:22:12 +0000 Subject: [PATCH 01/12] Bugfix: api/statuses/destroy.:format was outputting deleted notice twice, causing parsers to fail. --- actions/apistatusesdestroy.php | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/actions/apistatusesdestroy.php b/actions/apistatusesdestroy.php index f7d52f0208..0bfcdd060e 100644 --- a/actions/apistatusesdestroy.php +++ b/actions/apistatusesdestroy.php @@ -57,7 +57,7 @@ require_once INSTALLDIR . '/lib/apiauth.php'; class ApiStatusesDestroyAction extends ApiAuthAction { - var $status = null; + var $status = null; /** * Take arguments for running @@ -120,18 +120,11 @@ class ApiStatusesDestroyAction extends ApiAuthAction $replies->get('notice_id', $this->notice_id); $replies->delete(); $this->notice->delete(); - - if ($this->format == 'xml') { - $this->showSingleXmlStatus($this->notice); - } elseif ($this->format == 'json') { - $this->show_single_json_status($this->notice); - } + $this->showNotice(); } else { $this->clientError(_('You may not delete another user\'s status.'), 403, $this->format); } - - $this->showNotice(); } /** From f4539b52ad2c25a87e906c68d955ef921678e18c Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 28 May 2010 16:03:09 -0700 Subject: [PATCH 02/12] Ticket 2329 followup: my clever 'let it use the default' was foiled by PHP gettext module not quite exposing a compatible interface as the backend gettext library. (Most funcs squash null domain parameter into '' empty string, which isn't interpreted as 'use the current default'.) --- lib/language.php | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/language.php b/lib/language.php index 3846b8f358..cb12cca692 100644 --- a/lib/language.php +++ b/lib/language.php @@ -61,7 +61,7 @@ if (!function_exists('dpgettext')) { * Not currently exposed in PHP's gettext module; implemented to be compat * with gettext.h's macros. * - * @param string $domain domain identifier, or null for default domain + * @param string $domain domain identifier * @param string $context context identifier, should be some key like "menu|file" * @param string $msgid English source text * @return string original or translated message @@ -106,7 +106,7 @@ if (!function_exists('dnpgettext')) { * Not currently exposed in PHP's gettext module; implemented to be compat * with gettext.h's macros. * - * @param string $domain domain identifier, or null for default domain + * @param string $domain domain identifier * @param string $context context identifier, should be some key like "menu|file" * @param string $msg singular English source text * @param string $plural plural English source text @@ -180,7 +180,11 @@ function _m($msg/*, ...*/) } /** - * Looks for which plugin we've been called from to set the gettext domain. + * Looks for which plugin we've been called from to set the gettext domain; + * if not in a plugin subdirectory, we'll use the default 'statusnet'. + * + * Note: we can't return null for default domain since most of the PHP gettext + * wrapper functions turn null into "" before passing to the backend library. * * @param array $backtrace debug_backtrace() output * @return string @@ -207,8 +211,8 @@ function _mdomain($backtrace) } $plug = strpos($path, '/plugins/'); if ($plug === false) { - // We're not in a plugin; return null for the default domain. - return null; + // We're not in a plugin; return default domain. + return 'statusnet'; } else { $cut = $plug + 9; $cut2 = strpos($path, '/', $cut); @@ -217,7 +221,7 @@ function _mdomain($backtrace) } else { // We might be running directly from the plugins dir? // If so, there's no place to store locale info. - return null; + return 'statusnet'; } } } From 58fe1a597c76dd6737abbe44e7cb7111d3ae3375 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 28 May 2010 16:52:17 -0700 Subject: [PATCH 03/12] OpenID: add option to enable asking for a username to append to the trusted provider's base URL. Good for hooking up with sites like WikiHow, where usernames are appended to a base URL to get a profile URL which is used as the provider. $config['openid']['append_username'] = true; or check 'Append a username to base URL' in OpenID admin panel. --- plugins/OpenID/openid.php | 2 ++ plugins/OpenID/openidadminpanel.php | 10 ++++++++++ plugins/OpenID/openidlogin.php | 20 +++++++++++++++++++- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/plugins/OpenID/openid.php b/plugins/OpenID/openid.php index 574ecca72b..8be02e031c 100644 --- a/plugins/OpenID/openid.php +++ b/plugins/OpenID/openid.php @@ -144,8 +144,10 @@ function oid_authenticate($openid_url, $returnto, $immediate=false) // Handle failure status return values. if (!$auth_request) { + common_log(LOG_ERR, __METHOD__ . ": mystery fail contacting $openid_url"); return _m('Not a valid OpenID.'); } else if (Auth_OpenID::isFailure($auth_request)) { + common_log(LOG_ERR, __METHOD__ . ": OpenID fail to $openid_url: $auth_request->message"); return sprintf(_m('OpenID failure: %s'), $auth_request->message); } diff --git a/plugins/OpenID/openidadminpanel.php b/plugins/OpenID/openidadminpanel.php index 0633063662..ce4806cc89 100644 --- a/plugins/OpenID/openidadminpanel.php +++ b/plugins/OpenID/openidadminpanel.php @@ -91,6 +91,7 @@ class OpenidadminpanelAction extends AdminPanelAction ); static $booleans = array( + 'openid' => array('append_username'), 'site' => array('openidonly') ); @@ -222,6 +223,15 @@ class OpenIDAdminPanelForm extends AdminForm ); $this->unli(); + $this->li(); + $this->out->checkbox( + 'append_username', _m('Append a username to base URL'), + (bool) $this->value('append_username', 'openid'), + _m('Login form will show the base URL and prompt for a username to add at the end. Use when OpenID provider URL should be the profile page for individual users.'), + 'true' + ); + $this->unli(); + $this->li(); $this->input( 'required_team', diff --git a/plugins/OpenID/openidlogin.php b/plugins/OpenID/openidlogin.php index 8c559c9346..ffedc64810 100644 --- a/plugins/OpenID/openidlogin.php +++ b/plugins/OpenID/openidlogin.php @@ -32,6 +32,9 @@ class OpenidloginAction extends Action $provider = common_config('openid', 'trusted_provider'); if ($provider) { $openid_url = $provider; + if (common_config('openid', 'append_username')) { + $openid_url .= $this->trimmed('openid_username'); + } } else { $openid_url = $this->trimmed('openid_url'); } @@ -94,7 +97,15 @@ class OpenidloginAction extends Action function showScripts() { parent::showScripts(); - $this->autofocus('openid_url'); + if (common_config('openid', 'trusted_provider')) { + if (common_config('openid', 'append_username')) { + $this->autofocus('openid_username'); + } else { + $this->autofocus('rememberme'); + } + } else { + $this->autofocus('openid_url'); + } } function title() @@ -122,10 +133,17 @@ class OpenidloginAction extends Action $this->elementStart('ul', 'form_data'); $this->elementStart('li'); $provider = common_config('openid', 'trusted_provider'); + $appendUsername = common_config('openid', 'append_username'); if ($provider) { $this->element('label', array(), _m('OpenID provider')); $this->element('span', array(), $provider); + if ($appendUsername) { + $this->element('input', array('id' => 'openid_username', + 'name' => 'openid_username', + 'style' => 'float: none')); + } $this->element('p', 'form_guide', + ($appendUsername ? _m('Enter your username.') . ' ' : '') . _m('You will be sent to the provider\'s site for authentication.')); $this->hidden('openid_url', $provider); } else { From 83b976f7eafb74e2ef675262b427be43039428b9 Mon Sep 17 00:00:00 2001 From: Zach Copley Date: Mon, 31 May 2010 15:48:24 -0700 Subject: [PATCH 04/12] Added DarterosStatus to notice sources --- db/notice_source.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/db/notice_source.sql b/db/notice_source.sql index fbcdd6568e..5d86646315 100644 --- a/db/notice_source.sql +++ b/db/notice_source.sql @@ -9,6 +9,7 @@ VALUES ('bti','bti','http://gregkh.github.com/bti/', now()), ('choqok', 'Choqok', 'http://choqok.gnufolks.org/', now()), ('cliqset', 'Cliqset', 'http://www.cliqset.com/', now()), + ('DarterosStatus', 'Darteros Status', 'http://www.darteros.com/doc/Darteros_Status', now()), ('deskbar','Deskbar-Applet','http://www.gnome.org/projects/deskbar-applet/', now()), ('Do','Gnome Do','http://do.davebsd.com/wiki/index.php?title=Microblog_Plugin', now()), ('drupal','Drupal','http://drupal.org/', now()), From b0c589de9aa7bfd41bd59e12ff16d0791009fb18 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Tue, 1 Jun 2010 18:29:01 +0000 Subject: [PATCH 05/12] Ticket #2330: fix Google Maps provider for Mapstraction plugin --- plugins/Mapstraction/MapstractionPlugin.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/Mapstraction/MapstractionPlugin.php b/plugins/Mapstraction/MapstractionPlugin.php index 868933fd43..e7240a6449 100644 --- a/plugins/Mapstraction/MapstractionPlugin.php +++ b/plugins/Mapstraction/MapstractionPlugin.php @@ -125,8 +125,8 @@ class MapstractionPlugin extends Plugin $action->script('http://tile.cloudmade.com/wml/0.2/web-maps-lite.js'); break; case 'google': - $action->script(sprintf('http://maps.google.com/maps?file=api&v=2&sensor=false&key=%s', - $this->apikey)); + $action->script(sprintf('http://maps.google.com/maps?file=api&v=2&sensor=false&key=%s', + urlencode($this->apikey))); break; case 'microsoft': $action->script('http://dev.virtualearth.net/mapcontrol/mapcontrol.ashx?v=6'); @@ -137,7 +137,7 @@ class MapstractionPlugin extends Plugin break; case 'yahoo': $action->script(sprintf('http://api.maps.yahoo.com/ajaxymap?v=3.8&appid=%s', - $this->apikey)); + urlencode($this->apikey))); break; case 'geocommons': // don't support this yet default: From 634752f0d262b4fb02456889250378fca084cd2e Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Tue, 1 Jun 2010 18:41:17 +0000 Subject: [PATCH 06/12] Mapstraction plugin fix: set icon dimensions (24x24 px); Google Maps provider otherwise defaults to stretching them to a funny shape instead of showing square avatars. --- plugins/Mapstraction/usermap.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/Mapstraction/usermap.js b/plugins/Mapstraction/usermap.js index 4b7a6c26b4..53cfe6bb0c 100644 --- a/plugins/Mapstraction/usermap.js +++ b/plugins/Mapstraction/usermap.js @@ -104,7 +104,7 @@ function showMapstraction(element, notices) { pt = new mxn.LatLonPoint(lat, lon); mkr = new mxn.Marker(pt); - mkr.setIcon(n['user']['profile_image_url']); + mkr.setIcon(n['user']['profile_image_url'], [24, 24]); mkr.setInfoBubble('' + n['user']['screen_name'] + '' + ' ' + n['html'] + '
'+ n['created_at'] + ''); From 17ab15a3d02c335f2d9d333ac3773c037e796cf5 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Tue, 1 Jun 2010 13:53:44 -0700 Subject: [PATCH 07/12] Fix memory leak in Inbox::addToInbox() (usage of raw DB_DataObject::staticGet, which leaks memory into a process-global cache). On my test setup, this fixes inbox delivery to 10,000 local recipients from background queuedaemon running with a 32mb memory limit, completes the job within a minute from start. --- classes/Inbox.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/classes/Inbox.php b/classes/Inbox.php index 2533210b73..430419ba5e 100644 --- a/classes/Inbox.php +++ b/classes/Inbox.php @@ -115,9 +115,12 @@ class Inbox extends Memcached_DataObject */ static function insertNotice($user_id, $notice_id) { - $inbox = DB_DataObject::staticGet('inbox', 'user_id', $user_id); - - if (empty($inbox)) { + // Going straight to the DB rather than trusting our caching + // during an update. Note: not using DB_DataObject::staticGet, + // which is unsafe to use directly (in-process caching causes + // memory leaks, which accumulate in queue processes). + $inbox = new Inbox(); + if (!$inbox->get('user_id', $user_id)) { $inbox = Inbox::initialize($user_id); } From 791b98046d2c81aecfa468c06d4b7fd1f06ea8fa Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Thu, 3 Jun 2010 16:09:47 -0700 Subject: [PATCH 08/12] Stomp blocking writes fix --- lib/liberalstomp.php | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/liberalstomp.php b/lib/liberalstomp.php index 3d38953fd2..70c22c17e6 100644 --- a/lib/liberalstomp.php +++ b/lib/liberalstomp.php @@ -147,5 +147,30 @@ class LiberalStomp extends Stomp } return $frame; } -} + + /** + * Write frame to server + * + * @param StompFrame $stompFrame + */ + protected function _writeFrame (StompFrame $stompFrame) + { + if (!is_resource($this->_socket)) { + require_once 'Stomp/Exception.php'; + throw new StompException('Socket connection hasn\'t been established'); + } + + $data = $stompFrame->__toString(); + + // Make sure the socket's in a writable state; if not, wait a bit. + stream_set_blocking($this->_socket, 1); + + $r = fwrite($this->_socket, $data, strlen($data)); + stream_set_blocking($this->_socket, 0); + if ($r === false || $r == 0) { + $this->_reconnect(); + $this->_writeFrame($stompFrame); + } + } + } From 5f4c6ec626d3d641f0712b276deb32b218b7a330 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Thu, 3 Jun 2010 16:58:45 -0700 Subject: [PATCH 09/12] Skip enqueueing to outgoing bridges on incoming remote messages. Twitter, Facebook, RSSCloud, and OStatus checks were enqueued on these when they'd never do anything but churn the queue servers. Notice::isLocal() can replace a number of manual checks for $notice->is_local being LOCAL_PUBLIC or LOCAL_NONPUBLIC. --- classes/Notice.php | 12 ++++++++++++ lib/util.php | 5 ++--- plugins/Facebook/FacebookPlugin.php | 2 +- plugins/OStatus/OStatusPlugin.php | 6 ++++-- plugins/RSSCloud/RSSCloudPlugin.php | 18 +++--------------- plugins/TwitterBridge/TwitterBridgePlugin.php | 2 +- 6 files changed, 23 insertions(+), 22 deletions(-) diff --git a/classes/Notice.php b/classes/Notice.php index 3d7d21533b..cda6328853 100644 --- a/classes/Notice.php +++ b/classes/Notice.php @@ -1861,4 +1861,16 @@ class Notice extends Memcached_DataObject return $ns; } + /** + * Determine whether the notice was locally created + * + * @return boolean locality + */ + + public function isLocal() + { + return ($this->is_local == Notice::LOCAL_PUBLIC || + $this->is_local == Notice::LOCAL_NONPUBLIC); + } + } diff --git a/lib/util.php b/lib/util.php index 59d5132ec6..049001abaf 100644 --- a/lib/util.php +++ b/lib/util.php @@ -1235,9 +1235,8 @@ function common_enqueue_notice($notice) $transports[] = 'jabber'; } - // @fixme move these checks into QueueManager and/or individual handlers - if ($notice->is_local == Notice::LOCAL_PUBLIC || - $notice->is_local == Notice::LOCAL_NONPUBLIC) { + // We can skip these for gatewayed notices. + if ($notice->isLocal()) { $transports = array_merge($transports, $localTransports); if ($xmpp) { $transports[] = 'public'; diff --git a/plugins/Facebook/FacebookPlugin.php b/plugins/Facebook/FacebookPlugin.php index 5dba73a5d8..19989a952e 100644 --- a/plugins/Facebook/FacebookPlugin.php +++ b/plugins/Facebook/FacebookPlugin.php @@ -585,7 +585,7 @@ class FacebookPlugin extends Plugin function onStartEnqueueNotice($notice, &$transports) { - if (self::hasKeys()) { + if (self::hasKeys() && $notice->isLocal()) { array_push($transports, 'facebook'); } return true; diff --git a/plugins/OStatus/OStatusPlugin.php b/plugins/OStatus/OStatusPlugin.php index 5b153216ef..5a657c83d0 100644 --- a/plugins/OStatus/OStatusPlugin.php +++ b/plugins/OStatus/OStatusPlugin.php @@ -102,8 +102,10 @@ class OStatusPlugin extends Plugin */ function onStartEnqueueNotice($notice, &$transports) { - // put our transport first, in case there's any conflict (like OMB) - array_unshift($transports, 'ostatus'); + if ($notice->isLocal()) { + // put our transport first, in case there's any conflict (like OMB) + array_unshift($transports, 'ostatus'); + } return true; } diff --git a/plugins/RSSCloud/RSSCloudPlugin.php b/plugins/RSSCloud/RSSCloudPlugin.php index 661c32141f..c1951cdbf8 100644 --- a/plugins/RSSCloud/RSSCloudPlugin.php +++ b/plugins/RSSCloud/RSSCloudPlugin.php @@ -192,24 +192,12 @@ class RSSCloudPlugin extends Plugin function onStartEnqueueNotice($notice, &$transports) { - array_push($transports, 'rsscloud'); + if ($notice->isLocal()) { + array_push($transports, 'rsscloud'); + } return true; } - /** - * Determine whether the notice was locally created - * - * @param Notice $notice the notice in question - * - * @return boolean locality - */ - - function _isLocal($notice) - { - return ($notice->is_local == Notice::LOCAL_PUBLIC || - $notice->is_local == Notice::LOCAL_NONPUBLIC); - } - /** * Create the rsscloud_subscription table if it's not * already in the DB diff --git a/plugins/TwitterBridge/TwitterBridgePlugin.php b/plugins/TwitterBridge/TwitterBridgePlugin.php index 1a0a69682a..65b3a6b38e 100644 --- a/plugins/TwitterBridge/TwitterBridgePlugin.php +++ b/plugins/TwitterBridge/TwitterBridgePlugin.php @@ -221,7 +221,7 @@ class TwitterBridgePlugin extends Plugin */ function onStartEnqueueNotice($notice, &$transports) { - if (self::hasKeys()) { + if (self::hasKeys() && $notice->isLocal()) { // Avoid a possible loop if ($notice->source != 'twitter') { array_push($transports, 'twitter'); From a75095fa1a3926d1fcc18c3d7285141fa3bef344 Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Thu, 3 Jun 2010 17:41:26 -0700 Subject: [PATCH 10/12] Meteor realtime plugin: use persistent connections by default when pushing updates from our queue threads --- plugins/Meteor/MeteorPlugin.php | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/plugins/Meteor/MeteorPlugin.php b/plugins/Meteor/MeteorPlugin.php index 5600d5fcc0..ec8c9e217c 100644 --- a/plugins/Meteor/MeteorPlugin.php +++ b/plugins/Meteor/MeteorPlugin.php @@ -50,6 +50,7 @@ class MeteorPlugin extends RealtimePlugin public $controlport = null; public $controlserver = null; public $channelbase = null; + public $persistent = true; protected $_socket = null; function __construct($webserver=null, $webport=4670, $controlport=4671, $controlserver=null, $channelbase='') @@ -102,8 +103,14 @@ class MeteorPlugin extends RealtimePlugin function _connect() { $controlserver = (empty($this->controlserver)) ? $this->webserver : $this->controlserver; + + $errno = $errstr = null; + $timeout = 5; + $flags = STREAM_CLIENT_CONNECT; + if ($this->persistent) $flags |= STREAM_CLIENT_PERSISTENT; + // May throw an exception. - $this->_socket = stream_socket_client("tcp://{$controlserver}:{$this->controlport}"); + $this->_socket = stream_socket_client("tcp://{$controlserver}:{$this->controlport}", $errno, $errstr, $timeout, $flags); if (!$this->_socket) { throw new Exception("Couldn't connect to {$controlserver} on {$this->controlport}"); } @@ -124,8 +131,10 @@ class MeteorPlugin extends RealtimePlugin function _disconnect() { - $cnt = fwrite($this->_socket, "QUIT\n"); - @fclose($this->_socket); + if (!$this->persistent) { + $cnt = fwrite($this->_socket, "QUIT\n"); + @fclose($this->_socket); + } } // Meteord flips out with default '/' separator From 8b9436e8ae1ebcc7ef10752bb9666939200e26aa Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Thu, 3 Jun 2010 17:49:20 -0700 Subject: [PATCH 11/12] Option to divert PuSH items directly to the target site's queue when local --- classes/Status_network.php | 29 +++++++++++++++++++--------- lib/stompqueuemanager.php | 9 +++++---- plugins/OStatus/classes/HubSub.php | 31 ++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/classes/Status_network.php b/classes/Status_network.php index a452c32ce0..4a1f2c3747 100644 --- a/classes/Status_network.php +++ b/classes/Status_network.php @@ -149,21 +149,15 @@ class Status_network extends Safe_DataObject $this->decache(); # while we still have the values! return parent::delete(); } - + /** * @param string $servername hostname - * @param string $pathname URL base path * @param string $wildcard hostname suffix to match wildcard config + * @return mixed Status_network or null */ - static function setupSite($servername, $pathname, $wildcard) + static function getFromHostname($servername, $wildcard) { - global $config; - $sn = null; - - // XXX I18N, probably not crucial for hostnames - // XXX This probably needs a tune up - if (0 == strncasecmp(strrev($wildcard), strrev($servername), strlen($wildcard))) { // special case for exact match if (0 == strcasecmp($servername, $wildcard)) { @@ -182,6 +176,23 @@ class Status_network extends Safe_DataObject } } } + return $sn; + } + + /** + * @param string $servername hostname + * @param string $pathname URL base path + * @param string $wildcard hostname suffix to match wildcard config + */ + static function setupSite($servername, $pathname, $wildcard) + { + global $config; + + $sn = null; + + // XXX I18N, probably not crucial for hostnames + // XXX This probably needs a tune up + $sn = self::getFromHostname($servername, $wildcard); if (!empty($sn)) { diff --git a/lib/stompqueuemanager.php b/lib/stompqueuemanager.php index de4ba7f01f..91faa8c367 100644 --- a/lib/stompqueuemanager.php +++ b/lib/stompqueuemanager.php @@ -115,11 +115,12 @@ class StompQueueManager extends QueueManager * * @param mixed $object * @param string $queue + * @param string $siteNickname optional override to drop into another site's queue * * @return boolean true on success * @throws StompException on connection or send error */ - public function enqueue($object, $queue) + public function enqueue($object, $queue, $siteNickname=null) { $this->_connect(); if (common_config('queue', 'stomp_enqueue_on')) { @@ -134,7 +135,7 @@ class StompQueueManager extends QueueManager } else { $idx = $this->defaultIdx; } - return $this->_doEnqueue($object, $queue, $idx); + return $this->_doEnqueue($object, $queue, $idx, $siteNickname); } /** @@ -144,10 +145,10 @@ class StompQueueManager extends QueueManager * @return boolean true on success * @throws StompException on connection or send error */ - protected function _doEnqueue($object, $queue, $idx) + protected function _doEnqueue($object, $queue, $idx, $siteNickname=null) { $rep = $this->logrep($object); - $envelope = array('site' => common_config('site', 'nickname'), + $envelope = array('site' => $siteNickname ? $siteNickname : common_config('site', 'nickname'), 'handler' => $queue, 'payload' => $this->encode($object)); $msg = serialize($envelope); diff --git a/plugins/OStatus/classes/HubSub.php b/plugins/OStatus/classes/HubSub.php index cdace3c1fc..9748b4a569 100644 --- a/plugins/OStatus/classes/HubSub.php +++ b/plugins/OStatus/classes/HubSub.php @@ -260,6 +260,37 @@ class HubSub extends Memcached_DataObject $retries = intval(common_config('ostatus', 'hub_retries')); } + if (common_config('ostatus', 'local_push_bypass')) { + // If target is a local site, bypass the web server and drop the + // item directly into the target's input queue. + $url = parse_url($this->callback); + $wildcard = common_config('ostatus', 'local_wildcard'); + $site = Status_network::getFromHostname($url['host'], $wildcard); + + if ($site) { + if ($this->secret) { + $hmac = 'sha1=' . hash_hmac('sha1', $atom, $this->secret); + } else { + $hmac = ''; + } + + // Hack: at the moment we stick the subscription ID in the callback + // URL so we don't have to look inside the Atom to route the subscription. + // For now this means we need to extract that from the target URL + // so we can include it in the data. + $parts = explode('/', $url['path']); + $subId = intval(array_pop($parts)); + + $data = array('feedsub_id' => $subId, + 'post' => $atom, + 'hmac' => $hmac); + common_log(LOG_DEBUG, "Cross-site PuSH bypass enqueueing straight to $site->nickname feed $subId"); + $qm = QueueManager::get(); + $qm->enqueue($data, 'pushin', $site->nickname); + return; + } + } + // We dare not clone() as when the clone is discarded it'll // destroy the result data for the parent query. // @fixme use clone() again when it's safe to copy an From 41e9dba7297d43b7de0cb7665901869910d1047a Mon Sep 17 00:00:00 2001 From: Brion Vibber Date: Fri, 4 Jun 2010 11:48:54 -0700 Subject: [PATCH 12/12] OStatus plugin: Rolling batch queueing for PuSH output to >50 subscribing sites. Keeps latency down for other things enqueued while we work... --- plugins/OStatus/OStatusPlugin.php | 2 ++ plugins/OStatus/classes/HubSub.php | 20 +++++++++++++ plugins/OStatus/lib/ostatusqueuehandler.php | 31 ++++++++++++++++++++- 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/plugins/OStatus/OStatusPlugin.php b/plugins/OStatus/OStatusPlugin.php index 5a657c83d0..c61e2cc5f3 100644 --- a/plugins/OStatus/OStatusPlugin.php +++ b/plugins/OStatus/OStatusPlugin.php @@ -87,6 +87,8 @@ class OStatusPlugin extends Plugin // Outgoing from our internal PuSH hub $qm->connect('hubconf', 'HubConfQueueHandler'); + $qm->connect('hubprep', 'HubPrepQueueHandler'); + $qm->connect('hubout', 'HubOutQueueHandler'); // Outgoing Salmon replies (when we don't need a return value) diff --git a/plugins/OStatus/classes/HubSub.php b/plugins/OStatus/classes/HubSub.php index 9748b4a569..7db528a4e8 100644 --- a/plugins/OStatus/classes/HubSub.php +++ b/plugins/OStatus/classes/HubSub.php @@ -304,6 +304,26 @@ class HubSub extends Memcached_DataObject $qm->enqueue($data, 'hubout'); } + /** + * Queue up a large batch of pushes to multiple subscribers + * for this same topic update. + * + * If queues are disabled, this will run immediately. + * + * @param string $atom well-formed Atom feed + * @param array $pushCallbacks list of callback URLs + */ + function bulkDistribute($atom, $pushCallbacks) + { + $data = array('atom' => $atom, + 'topic' => $this->topic, + 'pushCallbacks' => $pushCallbacks); + common_log(LOG_INFO, "Queuing PuSH batch: $this->topic to " . + count($pushCallbacks) . " sites"); + $qm = QueueManager::get(); + $qm->enqueue($data, 'hubprep'); + } + /** * Send a 'fat ping' to the subscriber's callback endpoint * containing the given Atom feed chunk. diff --git a/plugins/OStatus/lib/ostatusqueuehandler.php b/plugins/OStatus/lib/ostatusqueuehandler.php index d1e58f1d68..8905d2e210 100644 --- a/plugins/OStatus/lib/ostatusqueuehandler.php +++ b/plugins/OStatus/lib/ostatusqueuehandler.php @@ -25,6 +25,18 @@ */ class OStatusQueueHandler extends QueueHandler { + // If we have more than this many subscribing sites on a single feed, + // break up the PuSH distribution into smaller batches which will be + // rolled into the queue progressively. This reduces disruption to + // other, shorter activities being enqueued while we work. + const MAX_UNBATCHED = 50; + + // Each batch (a 'hubprep' entry) will have this many items. + // Selected to provide a balance between queue packet size + // and number of batches that will end up getting processed. + // For 20,000 target sites, 1000 should work acceptably. + const BATCH_SIZE = 1000; + function transport() { return 'ostatus'; @@ -147,14 +159,31 @@ class OStatusQueueHandler extends QueueHandler /** * Queue up direct feed update pushes to subscribers on our internal hub. + * If there are a large number of subscriber sites, intermediate bulk + * distribution triggers may be queued. + * * @param string $atom update feed, containing only new/changed items * @param HubSub $sub open query of subscribers */ function pushFeedInternal($atom, $sub) { common_log(LOG_INFO, "Preparing $sub->N PuSH distribution(s) for $sub->topic"); + $n = 0; + $batch = array(); while ($sub->fetch()) { - $sub->distribute($atom); + $n++; + if ($n < self::MAX_UNBATCHED) { + $sub->distribute($atom); + } else { + $batch[] = $sub->callback; + if (count($batch) >= self::BATCH_SIZE) { + $sub->bulkDistribute($atom, $batch); + $batch = array(); + } + } + } + if (count($batch) >= 0) { + $sub->bulkDistribute($atom, $batch); } }