From 2ac684f53df3f76aacb4d1be79d7db64567f1964 Mon Sep 17 00:00:00 2001 From: Federico Marani Date: Fri, 20 Feb 2009 18:57:20 +0000 Subject: [PATCH 1/8] * implemented STOMP support for external message queueing systems * (ticket-724) * included PHP STOMP libraries (http://code.google.com/p/stompcli/) --- config.php.sample | 8 + extlib/Stomp.php | 594 +++++++++++++++++++++++++++++++++ extlib/Stomp/Exception.php | 57 ++++ extlib/Stomp/Frame.php | 80 +++++ extlib/Stomp/Message.php | 37 ++ extlib/Stomp/Message/Bytes.php | 47 +++ extlib/Stomp/Message/Map.php | 55 +++ lib/queuehandler.php | 74 +++- lib/util.php | 48 ++- 9 files changed, 980 insertions(+), 20 deletions(-) create mode 100644 extlib/Stomp.php create mode 100644 extlib/Stomp/Exception.php create mode 100644 extlib/Stomp/Frame.php create mode 100644 extlib/Stomp/Message.php create mode 100644 extlib/Stomp/Message/Bytes.php create mode 100644 extlib/Stomp/Message/Map.php diff --git a/config.php.sample b/config.php.sample index a2c5801f45..cc2e4c3c14 100644 --- a/config.php.sample +++ b/config.php.sample @@ -112,6 +112,14 @@ $config['sphinx']['port'] = 3312; #background. See the README for details. #$config['queue']['enabled'] = true; +#Queue subsystem +#subsystems: internal (default) or stomp +#using stomp requires an external message queue server +#$config['queue']['subsystem'] = 'stomp'; +#$config['queue']['stomp_server'] = 'tcp://localhost:61613'; +#use different queue_basename for each laconica instance managed by the server +#$config['queue']['queue_basename'] = 'laconica'; + #The following customise the behaviour of the various daemons: #$config['daemon']['piddir'] = '/var/run'; #$config['daemon']['user'] = false; diff --git a/extlib/Stomp.php b/extlib/Stomp.php new file mode 100644 index 0000000000..9e1c97b3b3 --- /dev/null +++ b/extlib/Stomp.php @@ -0,0 +1,594 @@ + + * @author Dejan Bosanac + * @author Michael Caplan + * @version $Revision: 43 $ + */ +class Stomp +{ + /** + * Perform request synchronously + * + * @var boolean + */ + public $sync = false; + + /** + * Default prefetch size + * + * @var int + */ + public $prefetchSize = 1; + + /** + * Client id used for durable subscriptions + * + * @var string + */ + public $clientId = null; + + protected $_brokerUri = null; + protected $_socket = null; + protected $_hosts = array(); + protected $_params = array(); + protected $_subscriptions = array(); + protected $_defaultPort = 61613; + protected $_currentHost = - 1; + protected $_attempts = 10; + protected $_username = ''; + protected $_password = ''; + protected $_sessionId; + protected $_read_timeout_seconds = 60; + protected $_read_timeout_milliseconds = 0; + + /** + * Constructor + * + * @param string $brokerUri Broker URL + * @throws Stomp_Exception + */ + public function __construct ($brokerUri) + { + $this->_brokerUri = $brokerUri; + $this->_init(); + } + /** + * Initialize connection + * + * @throws Stomp_Exception + */ + protected function _init () + { + $pattern = "|^(([a-zA-Z]+)://)+\(*([a-zA-Z0-9\.:/i,-]+)\)*\??([a-zA-Z0-9=]*)$|i"; + if (preg_match($pattern, $this->_brokerUri, $regs)) { + $scheme = $regs[2]; + $hosts = $regs[3]; + $params = $regs[4]; + if ($scheme != "failover") { + $this->_processUrl($this->_brokerUri); + } else { + $urls = explode(",", $hosts); + foreach ($urls as $url) { + $this->_processUrl($url); + } + } + if ($params != null) { + parse_str($params, $this->_params); + } + } else { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception("Bad Broker URL {$this->_brokerUri}"); + } + } + /** + * Process broker URL + * + * @param string $url Broker URL + * @throws Stomp_Exception + * @return boolean + */ + protected function _processUrl ($url) + { + $parsed = parse_url($url); + if ($parsed) { + array_push($this->_hosts, array($parsed['host'] , $parsed['port'] , $parsed['scheme'])); + } else { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception("Bad Broker URL $url"); + } + } + /** + * Make socket connection to the server + * + * @throws Stomp_Exception + */ + protected function _makeConnection () + { + if (count($this->_hosts) == 0) { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception("No broker defined"); + } + + // force disconnect, if previous established connection exists + $this->disconnect(); + + $i = $this->_currentHost; + $att = 0; + $connected = false; + while (! $connected && $att ++ < $this->_attempts) { + if (isset($this->_params['randomize']) && $this->_params['randomize'] == 'true') { + $i = rand(0, count($this->_hosts) - 1); + } else { + $i = ($i + 1) % count($this->_hosts); + } + $broker = $this->_hosts[$i]; + $host = $broker[0]; + $port = $broker[1]; + $scheme = $broker[2]; + if ($port == null) { + $port = $this->_defaultPort; + } + if ($this->_socket != null) { + fclose($this->_socket); + $this->_socket = null; + } + $this->_socket = @fsockopen($scheme . '://' . $host, $port); + if (!is_resource($this->_socket) && $att >= $this->_attempts && !array_key_exists($i + 1, $this->_hosts)) { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception("Could not connect to $host:$port ($att/{$this->_attempts})"); + } else if (is_resource($this->_socket)) { + $connected = true; + $this->_currentHost = $i; + break; + } + } + if (! $connected) { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception("Could not connect to a broker"); + } + } + /** + * Connect to server + * + * @param string $username + * @param string $password + * @return boolean + * @throws Stomp_Exception + */ + public function connect ($username = '', $password = '') + { + $this->_makeConnection(); + if ($username != '') { + $this->_username = $username; + } + if ($password != '') { + $this->_password = $password; + } + $headers = array('login' => $this->_username , 'passcode' => $this->_password); + if ($this->clientId != null) { + $headers["client-id"] = $this->clientId; + } + $frame = new Stomp_Frame("CONNECT", $headers); + $this->_writeFrame($frame); + $frame = $this->readFrame(); + if ($frame instanceof Stomp_Frame && $frame->command == 'CONNECTED') { + $this->_sessionId = $frame->headers["session"]; + return true; + } else { + require_once 'Stomp/Exception.php'; + if ($frame instanceof Stomp_Frame) { + throw new Stomp_Exception("Unexpected command: {$frame->command}", 0, $frame->body); + } else { + throw new Stomp_Exception("Connection not acknowledged"); + } + } + } + + /** + * Check if client session has ben established + * + * @return boolean + */ + public function isConnected () + { + return !empty($this->_sessionId) && is_resource($this->_socket); + } + /** + * Current stomp session ID + * + * @return string + */ + public function getSessionId() + { + return $this->_sessionId; + } + /** + * Send a message to a destination in the messaging system + * + * @param string $destination Destination queue + * @param string|Stomp_Frame $msg Message + * @param array $properties + * @param boolean $sync Perform request synchronously + * @return boolean + */ + public function send ($destination, $msg, $properties = null, $sync = null) + { + if ($msg instanceof Stomp_Frame) { + $msg->headers['destination'] = $destination; + $msg->headers = array_merge($msg->headers, $properties); + $frame = $msg; + } else { + $headers = $properties; + $headers['destination'] = $destination; + $frame = new Stomp_Frame('SEND', $headers, $msg); + } + $this->_prepareReceipt($frame, $sync); + $this->_writeFrame($frame); + return $this->_waitForReceipt($frame, $sync); + } + /** + * Prepair frame receipt + * + * @param Stomp_Frame $frame + * @param boolean $sync + */ + protected function _prepareReceipt (Stomp_Frame $frame, $sync) + { + $receive = $this->sync; + if ($sync !== null) { + $receive = $sync; + } + if ($receive == true) { + $frame->headers['receipt'] = md5(microtime()); + } + } + /** + * Wait for receipt + * + * @param Stomp_Frame $frame + * @param boolean $sync + * @return boolean + * @throws Stomp_Exception + */ + protected function _waitForReceipt (Stomp_Frame $frame, $sync) + { + + $receive = $this->sync; + if ($sync !== null) { + $receive = $sync; + } + if ($receive == true) { + $id = (isset($frame->headers['receipt'])) ? $frame->headers['receipt'] : null; + if ($id == null) { + return true; + } + $frame = $this->readFrame(); + if ($frame instanceof Stomp_Frame && $frame->command == 'RECEIPT') { + if ($frame->headers['receipt-id'] == $id) { + return true; + } else { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception("Unexpected receipt id {$frame->headers['receipt-id']}", 0, $frame->body); + } + } else { + require_once 'Stomp/Exception.php'; + if ($frame instanceof Stomp_Frame) { + throw new Stomp_Exception("Unexpected command {$frame->command}", 0, $frame->body); + } else { + throw new Stomp_Exception("Receipt not received"); + } + } + } + return true; + } + /** + * Register to listen to a given destination + * + * @param string $destination Destination queue + * @param array $properties + * @param boolean $sync Perform request synchronously + * @return boolean + * @throws Stomp_Exception + */ + public function subscribe ($destination, $properties = null, $sync = null) + { + $headers = array('ack' => 'client'); + $headers['activemq.prefetchSize'] = $this->prefetchSize; + if ($this->clientId != null) { + $headers["activemq.subcriptionName"] = $this->clientId; + } + if (isset($properties)) { + foreach ($properties as $name => $value) { + $headers[$name] = $value; + } + } + $headers['destination'] = $destination; + $frame = new Stomp_Frame('SUBSCRIBE', $headers); + $this->_prepareReceipt($frame, $sync); + $this->_writeFrame($frame); + if ($this->_waitForReceipt($frame, $sync) == true) { + $this->_subscriptions[$destination] = $properties; + return true; + } else { + return false; + } + } + /** + * Remove an existing subscription + * + * @param string $destination + * @param array $properties + * @param boolean $sync Perform request synchronously + * @return boolean + * @throws Stomp_Exception + */ + public function unsubscribe ($destination, $properties = null, $sync = null) + { + $headers = array(); + if (isset($properties)) { + foreach ($properties as $name => $value) { + $headers[$name] = $value; + } + } + $headers['destination'] = $destination; + $frame = new Stomp_Frame('UNSUBSCRIBE', $headers); + $this->_prepareReceipt($frame, $sync); + $this->_writeFrame($frame); + if ($this->_waitForReceipt($frame, $sync) == true) { + unset($this->_subscriptions[$destination]); + return true; + } else { + return false; + } + } + /** + * Start a transaction + * + * @param string $transactionId + * @param boolean $sync Perform request synchronously + * @return boolean + * @throws Stomp_Exception + */ + public function begin ($transactionId = null, $sync = null) + { + $headers = array(); + if (isset($transactionId)) { + $headers['transaction'] = $transactionId; + } + $frame = new Stomp_Frame('BEGIN', $headers); + $this->_prepareReceipt($frame, $sync); + $this->_writeFrame($frame); + return $this->_waitForReceipt($frame, $sync); + } + /** + * Commit a transaction in progress + * + * @param string $transactionId + * @param boolean $sync Perform request synchronously + * @return boolean + * @throws Stomp_Exception + */ + public function commit ($transactionId = null, $sync = null) + { + $headers = array(); + if (isset($transactionId)) { + $headers['transaction'] = $transactionId; + } + $frame = new Stomp_Frame('COMMIT', $headers); + $this->_prepareReceipt($frame, $sync); + $this->_writeFrame($frame); + return $this->_waitForReceipt($frame, $sync); + } + /** + * Roll back a transaction in progress + * + * @param string $transactionId + * @param boolean $sync Perform request synchronously + */ + public function abort ($transactionId = null, $sync = null) + { + $headers = array(); + if (isset($transactionId)) { + $headers['transaction'] = $transactionId; + } + $frame = new Stomp_Frame('ABORT', $headers); + $this->_prepareReceipt($frame, $sync); + $this->_writeFrame($frame); + return $this->_waitForReceipt($frame, $sync); + } + /** + * Acknowledge consumption of a message from a subscription + * Note: This operation is always asynchronous + * + * @param string|Stomp_Frame $messageMessage ID + * @param string $transactionId + * @return boolean + * @throws Stomp_Exception + */ + public function ack ($message, $transactionId = null) + { + if ($message instanceof Stomp_Frame) { + $frame = new Stomp_Frame('ACK', $message->headers); + $this->_writeFrame($frame); + return true; + } else { + $headers = array(); + if (isset($transactionId)) { + $headers['transaction'] = $transactionId; + } + $headers['message-id'] = $message; + $frame = new Stomp_Frame('ACK', $headers); + $this->_writeFrame($frame); + return true; + } + } + /** + * Graceful disconnect from the server + * + */ + public function disconnect () + { + $header = array(); + + if ($this->clientId != null) { + $headers["client-id"] = $this->clientId; + } + + if (is_resource($this->_socket)) { + $this->_writeFrame(new Stomp_Frame('DISCONNECT', $headers)); + fclose($this->_socket); + } + $this->_socket = null; + $this->_sessionId = null; + $this->_currentHost = -1; + $this->_subscriptions = array(); + $this->_username = ''; + $this->_password = ''; + } + /** + * Write frame to server + * + * @param Stomp_Frame $stompFrame + */ + protected function _writeFrame (Stomp_Frame $stompFrame) + { + if (!is_resource($this->_socket)) { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception('Socket connection hasn\'t been established'); + } + + $data = $stompFrame->__toString(); + $r = fwrite($this->_socket, $data, strlen($data)); + if ($r === false || $r == 0) { + $this->_reconnect(); + $this->_writeFrame($stompFrame); + } + } + + /** + * Set timeout to wait for content to read + * + * @param int $seconds_to_wait Seconds to wait for a frame + * @param int $milliseconds Milliseconds to wait for a frame + */ + public function setReadTimeout($seconds, $milliseconds = 0) + { + $this->_read_timeout_seconds = $seconds; + $this->_read_timeout_milliseconds = $milliseconds; + } + + /** + * Read responce frame from server + * + * @return Stomp_Frame|Stomp_Message_Map|boolean False when no frame to read + */ + public function readFrame () + { + if (!$this->hasFrameToRead()) { + return false; + } + + $rb = 1024; + $data = ''; + do { + $read = fgets($this->_socket, $rb); + if ($read === false) { + $this->_reconnect(); + return $this->readFrame(); + } + $data .= $read; + $len = strlen($data); + } while (($len < 2 || ! ($data[$len - 2] == "\x00" && $data[$len - 1] == "\n"))); + + list ($header, $body) = explode("\n\n", $data, 2); + $header = explode("\n", $header); + $headers = array(); + $command = null; + foreach ($header as $v) { + if (isset($command)) { + list ($name, $value) = explode(':', $v, 2); + $headers[$name] = $value; + } else { + $command = $v; + } + } + $frame = new Stomp_Frame($command, $headers, trim($body)); + if (isset($frame->headers['amq-msg-type']) && $frame->headers['amq-msg-type'] == 'MapMessage') { + require_once 'Stomp/Message/Map.php'; + return new Stomp_Message_Map($frame); + } else { + return $frame; + } + } + + /** + * Check if there is a frame to read + * + * @return boolean + */ + public function hasFrameToRead() + { + $read = array($this->_socket); + $write = null; + $except = null; + + $has_frame_to_read = stream_select($read, $write, $except, $this->_read_timeout_seconds, $this->_read_timeout_milliseconds); + + if ($has_frame_to_read === false) { + throw new Stomp_Exception('Check failed to determin if the socket is readable'); + } else if ($has_frame_to_read > 0) { + return true; + } else { + return false; + } + } + + /** + * Reconnects and renews subscriptions (if there were any) + * Call this method when you detect connection problems + */ + protected function _reconnect () + { + $subscriptions = $this->_subscriptions; + + $this->connect($this->_username, $this->_password); + foreach ($subscriptions as $dest => $properties) { + $this->subscribe($dest, $properties); + } + } + /** + * Graceful object desruction + * + */ + public function __destruct() + { + $this->disconnect(); + } +} +?> diff --git a/extlib/Stomp/Exception.php b/extlib/Stomp/Exception.php new file mode 100644 index 0000000000..e6870bc15d --- /dev/null +++ b/extlib/Stomp/Exception.php @@ -0,0 +1,57 @@ + + * @version $Revision: 23 $ + */ +class Stomp_Exception extends Exception +{ + protected $_details; + + /** + * Constructor + * + * @param string $message Error message + * @param int $code Error code + * @param string $details Stomp server error details + */ + public function __construct($message = null, $code = 0, $details = '') + { + $this->_details = $details; + + parent::__construct($message, $code); + } + + /** + * Stomp server error details + * + * @return string + */ + public function getDetails() + { + return $this->_details; + } +} +?> \ No newline at end of file diff --git a/extlib/Stomp/Frame.php b/extlib/Stomp/Frame.php new file mode 100644 index 0000000000..dc59c1cb7f --- /dev/null +++ b/extlib/Stomp/Frame.php @@ -0,0 +1,80 @@ + + * @author Dejan Bosanac + * @author Michael Caplan + * @version $Revision: 36 $ + */ +class Stomp_Frame +{ + public $command; + public $headers = array(); + public $body; + + /** + * Constructor + * + * @param string $command + * @param array $headers + * @param string $body + */ + public function __construct ($command = null, $headers = null, $body = null) + { + $this->_init($command, $headers, $body); + } + + protected function _init ($command = null, $headers = null, $body = null) + { + $this->command = $command; + if ($headers != null) { + $this->headers = $headers; + } + $this->body = $body; + + if ($this->command == 'ERROR') { + require_once 'Stomp/Exception.php'; + throw new Stomp_Exception($this->headers['message'], 0, $this->body); + } + } + + /** + * Convert frame to transportable string + * + * @return string + */ + public function __toString() + { + $data = $this->command . "\n"; + + foreach ($this->headers as $name => $value) { + $data .= $name . ": " . $value . "\n"; + } + + $data .= "\n"; + $data .= $this->body; + return $data .= "\x00\n"; + } +} +?> \ No newline at end of file diff --git a/extlib/Stomp/Message.php b/extlib/Stomp/Message.php new file mode 100644 index 0000000000..6bcad3efd9 --- /dev/null +++ b/extlib/Stomp/Message.php @@ -0,0 +1,37 @@ + + * @version $Revision: 23 $ + */ +class Stomp_Message extends Stomp_Frame +{ + public function __construct ($body, $headers = null) + { + $this->_init("SEND", $headers, $body); + } +} +?> \ No newline at end of file diff --git a/extlib/Stomp/Message/Bytes.php b/extlib/Stomp/Message/Bytes.php new file mode 100644 index 0000000000..c75f23e43a --- /dev/null +++ b/extlib/Stomp/Message/Bytes.php @@ -0,0 +1,47 @@ + + * @version $Revision: 23 $ + */ +class Stomp_Message_Bytes extends Stomp_Message +{ + /** + * Constructor + * + * @param string $body + * @param array $headers + */ + function __construct ($body, $headers = null) + { + $this->_init("SEND", $headers, $body); + if ($this->headers == null) { + $this->headers = array(); + } + $this->headers['content-length'] = count($body); + } +} +?> \ No newline at end of file diff --git a/extlib/Stomp/Message/Map.php b/extlib/Stomp/Message/Map.php new file mode 100644 index 0000000000..288456a849 --- /dev/null +++ b/extlib/Stomp/Message/Map.php @@ -0,0 +1,55 @@ + + * @version $Revision: 23 $ + */ +class Stomp_Message_Map extends Stomp_Message +{ + public $map; + + /** + * Constructor + * + * @param Stomp_Frame|string $msg + * @param array $headers + */ + function __construct ($msg, $headers = null) + { + if ($msg instanceof Stomp_Frame) { + $this->_init($msg->command, $msg->headers, $msg->body); + $this->map = json_decode($msg->body); + } else { + $this->_init("SEND", $headers, $msg); + if ($this->headers == null) { + $this->headers = array(); + } + $this->headers['amq-msg-type'] = 'MapMessage'; + $this->body = json_encode($msg); + } + } +} +?> \ No newline at end of file diff --git a/lib/queuehandler.php b/lib/queuehandler.php index 9ce9e32b3b..752bbc6d26 100644 --- a/lib/queuehandler.php +++ b/lib/queuehandler.php @@ -75,15 +75,9 @@ class QueueHandler extends Daemon return true; } - function run() - { - if (!$this->start()) { - return false; - } - $this->log(LOG_INFO, 'checking for queued notices'); - $transport = $this->transport(); + function db_dispatch() { do { - $qi = Queue_item::top($transport); + $qi = Queue_item::top($this->transport()); if ($qi) { $this->log(LOG_INFO, 'Got item enqueued '.common_exact_date($qi->created)); $notice = Notice::staticGet($qi->notice_id); @@ -115,6 +109,68 @@ class QueueHandler extends Daemon $this->idle(5); } } while (true); + } + + function stomp_dispatch() { + require("Stomp.php"); + $con = new Stomp(common_config('queue','stomp_server')); + if (!$con->connect()) { + $this->log(LOG_ERR, 'Failed to connect to queue server'); + return false; + } + $queue_basename = common_config('queue','queue_basename'); + // subscribe to the relevant queue (format: basename-transport) + $con->subscribe('/queue/'.$queue_basename.'-'.$this->transport()); + + do { + $frame = $con->readFrame(); + if ($frame) { + $this->log(LOG_INFO, 'Got item enqueued '.common_exact_date($frame->headers['created'])); + + // XXX: Now the queue handler receives only the ID of the + // notice, and it has to get it from the DB + // A massive improvement would be avoid DB query by transmitting + // all the notice details via queue server... + $notice = Notice::staticGet($frame->body); + + if ($notice) { + $this->log(LOG_INFO, 'broadcasting notice ID = ' . $notice->id); + $result = $this->handle_notice($notice); + if ($result) { + // if the msg has been handled positively, ack it + // and the queue server will remove it from the queue + $con->ack($frame); + $this->log(LOG_INFO, 'finished broadcasting notice ID = ' . $notice->id); + } + else { + // no ack + $this->log(LOG_WARNING, 'Failed broadcast for notice ID = ' . $notice->id); + } + $notice->free(); + unset($notice); + $notice = null; + } else { + $this->log(LOG_WARNING, 'queue item for notice that does not exist'); + } + } + } while (true); + + $con->disconnect(); + } + + + function run() + { + if (!$this->start()) { + return false; + } + $this->log(LOG_INFO, 'checking for queued notices'); + if (common_config('queue','subsystem') == 'stomp') { + $this->stomp_dispatch(); + } + else { + $this->db_dispatch(); + } if (!$this->finish()) { return false; } @@ -143,4 +199,4 @@ class QueueHandler extends Daemon common_log($level, $this->class_name() . ' ('. $this->get_id() .'): '.$msg); } } - \ No newline at end of file + diff --git a/lib/util.php b/lib/util.php index c5a092f630..8ef19f5796 100644 --- a/lib/util.php +++ b/lib/util.php @@ -1138,18 +1138,44 @@ function common_twitter_broadcast($notice, $flink) function common_enqueue_notice($notice) { - foreach (array('jabber', 'omb', 'sms', 'public') as $transport) { - $qi = new Queue_item(); - $qi->notice_id = $notice->id; - $qi->transport = $transport; - $qi->created = $notice->created; - $result = $qi->insert(); - if (!$result) { - $last_error = &PEAR::getStaticProperty('DB_DataObject','lastError'); - common_log(LOG_ERR, 'DB error inserting queue item: ' . $last_error->message); - return false; + if (common_config('queue','subsystem') == 'stomp') { + // use an external message queue system via STOMP + require_once("Stomp.php"); + $con = new Stomp(common_config('queue','stomp_server')); + if (!$con->connect()) { + common_log(LOG_ERR, 'Failed to connect to queue server'); + return false; + } + $queue_basename = common_config('queue','queue_basename'); + foreach (array('jabber', 'omb', 'sms', 'public') as $transport) { + if (!$con->send( + '/queue/'.$queue_basename.'-'.$transport, // QUEUE + $notice->id, // BODY of the message + array ( // HEADERS of the msg + 'created' => $notice->created + ))) { + common_log(LOG_ERR, 'Error sending to '.$transport.' queue'); + return false; + } + common_log(LOG_DEBUG, 'complete remote queueing notice ID = ' . $notice->id . ' for ' . $transport); + } + $result = true; + } + else { + // in any other case, 'internal' + foreach (array('jabber', 'omb', 'sms', 'public') as $transport) { + $qi = new Queue_item(); + $qi->notice_id = $notice->id; + $qi->transport = $transport; + $qi->created = $notice->created; + $result = $qi->insert(); + if (!$result) { + $last_error = &PEAR::getStaticProperty('DB_DataObject','lastError'); + common_log(LOG_ERR, 'DB error inserting queue item: ' . $last_error->message); + return false; + } + common_log(LOG_DEBUG, 'complete queueing notice ID = ' . $notice->id . ' for ' . $transport); } - common_log(LOG_DEBUG, 'complete queueing notice ID = ' . $notice->id . ' for ' . $transport); } return $result; } From 80d0423026b0bc14c8da820fff7905ba9c0a5d0e Mon Sep 17 00:00:00 2001 From: Federico Marani Date: Sat, 7 Mar 2009 13:47:46 +0000 Subject: [PATCH 2/8] html escape of atom attributes (ticket 1266) --- lib/jabber.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/jabber.php b/lib/jabber.php index f41d984d62..4a96fb54e6 100644 --- a/lib/jabber.php +++ b/lib/jabber.php @@ -176,14 +176,14 @@ function jabber_format_entry($profile, $notice) $entry .= "\n"; $entry .= "" . $profile->nickname . " - " . common_config('site', 'name') . "\n"; $entry .= "\n"; - $entry .= "\n"; + $entry .= "\n"; $entry .= "" . $profile->nickname . "\n"; $entry .= "" . $profile->avatarUrl(AVATAR_PROFILE_SIZE) . "\n"; $entry .= "\n"; $entry .= "" . htmlspecialchars($msg) . "\n"; $entry .= "" . htmlspecialchars($msg) . "\n"; - $entry .= "\n"; - $entry .= "". $notice->uri . "\n"; + $entry .= "\n"; + $entry .= "". htmlspecialchars($notice->uri) . "\n"; $entry .= "".common_date_w3dtf($notice->created)."\n"; $entry .= "".common_date_w3dtf($notice->modified)."\n"; $entry .= "\n"; From a389f157baddafa91347c27194805580b1373e30 Mon Sep 17 00:00:00 2001 From: Federico Marani Date: Sat, 7 Mar 2009 13:48:39 +0000 Subject: [PATCH 3/8] tests with Apache ActiveMQ topics (pubsub) --- lib/util.php | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/util.php b/lib/util.php index 8ef19f5796..8f1c419f20 100644 --- a/lib/util.php +++ b/lib/util.php @@ -1159,6 +1159,20 @@ function common_enqueue_notice($notice) } common_log(LOG_DEBUG, 'complete remote queueing notice ID = ' . $notice->id . ' for ' . $transport); } + $con->send('/topic/laconica.'.$notice->profile_id, + $notice->content, + array( + 'profile_id' => $notice->profile_id, + 'created' => $notice->created + ) + ); + $con->send('/topic/laconica.allusers', + $notice->content, + array( + 'profile_id' => $notice->profile_id, + 'created' => $notice->created + ) + ); $result = true; } else { From 5e9ff6ae973423c06ac42c679f4ad67a46a77bd0 Mon Sep 17 00:00:00 2001 From: Federico Marani Date: Sat, 28 Mar 2009 17:55:20 +0000 Subject: [PATCH 4/8] Testing JMS message selectors for notice tags-based routing --- lib/util.php | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/lib/util.php b/lib/util.php index 4e09bffcec..79d5cc9430 100644 --- a/lib/util.php +++ b/lib/util.php @@ -868,20 +868,38 @@ function common_enqueue_notice($notice) } common_log(LOG_DEBUG, 'complete remote queueing notice ID = ' . $notice->id . ' for ' . $transport); } + + //send tags as headers, so they can be used as JMS selectors + common_log(LOG_DEBUG, 'searching for tags ' . $notice->id); + $tags = array(); + $tag = new Notice_tag(); + $tag->notice_id = $notice->id; + if ($tag->find()) { + while ($tag->fetch()) { + common_log(LOG_DEBUG, 'tag found = ' . $tag->tag); + array_push($tags,$tag->tag); + } + } + $tag->free(); + $con->send('/topic/laconica.'.$notice->profile_id, $notice->content, array( 'profile_id' => $notice->profile_id, - 'created' => $notice->created + 'created' => $notice->created, + 'tags' => implode($tags,' - ') ) ); + common_log(LOG_DEBUG, 'sent to personal topic ' . $notice->id); $con->send('/topic/laconica.allusers', $notice->content, array( 'profile_id' => $notice->profile_id, - 'created' => $notice->created + 'created' => $notice->created, + 'tags' => implode($tags,' - ') ) ); + common_log(LOG_DEBUG, 'sent to catch-all topic ' . $notice->id); $result = true; } else { From bd120bc3153e3502790ec976769febd41f790568 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 3 Apr 2009 12:45:54 -0400 Subject: [PATCH 5/8] reformat queuehandler.php --- lib/queuehandler.php | 79 ++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/lib/queuehandler.php b/lib/queuehandler.php index 752bbc6d26..f76f16e073 100644 --- a/lib/queuehandler.php +++ b/lib/queuehandler.php @@ -36,7 +36,7 @@ class QueueHandler extends Daemon $this->set_id($id); } } - + function class_name() { return ucfirst($this->transport()) . 'Handler'; @@ -46,7 +46,7 @@ class QueueHandler extends Daemon { return strtolower($this->class_name().'.'.$this->get_id()); } - + function get_id() { return $this->_id; @@ -56,16 +56,16 @@ class QueueHandler extends Daemon { $this->_id = $id; } - + function transport() { return null; } - + function start() { } - + function finish() { } @@ -74,7 +74,7 @@ class QueueHandler extends Daemon { return true; } - + function db_dispatch() { do { $qi = Queue_item::top($this->transport()); @@ -107,43 +107,43 @@ class QueueHandler extends Daemon } else { $this->clear_old_claims(); $this->idle(5); - } + } } while (true); } function stomp_dispatch() { - require("Stomp.php"); - $con = new Stomp(common_config('queue','stomp_server')); - if (!$con->connect()) { - $this->log(LOG_ERR, 'Failed to connect to queue server'); - return false; - } - $queue_basename = common_config('queue','queue_basename'); - // subscribe to the relevant queue (format: basename-transport) - $con->subscribe('/queue/'.$queue_basename.'-'.$this->transport()); + require("Stomp.php"); + $con = new Stomp(common_config('queue','stomp_server')); + if (!$con->connect()) { + $this->log(LOG_ERR, 'Failed to connect to queue server'); + return false; + } + $queue_basename = common_config('queue','queue_basename'); + // subscribe to the relevant queue (format: basename-transport) + $con->subscribe('/queue/'.$queue_basename.'-'.$this->transport()); do { $frame = $con->readFrame(); if ($frame) { $this->log(LOG_INFO, 'Got item enqueued '.common_exact_date($frame->headers['created'])); - // XXX: Now the queue handler receives only the ID of the - // notice, and it has to get it from the DB - // A massive improvement would be avoid DB query by transmitting - // all the notice details via queue server... + // XXX: Now the queue handler receives only the ID of the + // notice, and it has to get it from the DB + // A massive improvement would be avoid DB query by transmitting + // all the notice details via queue server... $notice = Notice::staticGet($frame->body); if ($notice) { $this->log(LOG_INFO, 'broadcasting notice ID = ' . $notice->id); $result = $this->handle_notice($notice); if ($result) { - // if the msg has been handled positively, ack it - // and the queue server will remove it from the queue - $con->ack($frame); - $this->log(LOG_INFO, 'finished broadcasting notice ID = ' . $notice->id); - } - else { - // no ack + // if the msg has been handled positively, ack it + // and the queue server will remove it from the queue + $con->ack($frame); + $this->log(LOG_INFO, 'finished broadcasting notice ID = ' . $notice->id); + } + else { + // no ack $this->log(LOG_WARNING, 'Failed broadcast for notice ID = ' . $notice->id); } $notice->free(); @@ -152,12 +152,11 @@ class QueueHandler extends Daemon } else { $this->log(LOG_WARNING, 'queue item for notice that does not exist'); } - } + } } while (true); - - $con->disconnect(); + + $con->disconnect(); } - function run() { @@ -165,12 +164,12 @@ class QueueHandler extends Daemon return false; } $this->log(LOG_INFO, 'checking for queued notices'); - if (common_config('queue','subsystem') == 'stomp') { - $this->stomp_dispatch(); - } - else { - $this->db_dispatch(); - } + if (common_config('queue','subsystem') == 'stomp') { + $this->stomp_dispatch(); + } + else { + $this->db_dispatch(); + } if (!$this->finish()) { return false; } @@ -183,7 +182,7 @@ class QueueHandler extends Daemon sleep($timeout); } } - + function clear_old_claims() { $qi = new Queue_item(); @@ -193,10 +192,10 @@ class QueueHandler extends Daemon $qi->free(); unset($qi); } - + function log($level, $msg) { common_log($level, $this->class_name() . ' ('. $this->get_id() .'): '.$msg); } } - + From fa3ca0548f12a7a8f7774cfa918cba426be45f96 Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 3 Apr 2009 16:12:33 -0400 Subject: [PATCH 6/8] fix output of conversation --- actions/conversation.php | 48 ++++++++++++++-------------------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/actions/conversation.php b/actions/conversation.php index 0c22fe1238..05cfb76e3c 100644 --- a/actions/conversation.php +++ b/actions/conversation.php @@ -31,6 +31,8 @@ if (!defined('LACONICA')) { exit(1); } +require_once(INSTALLDIR.'/lib/noticelist.php'); + /** * Conversation tree in the browser * @@ -43,7 +45,6 @@ if (!defined('LACONICA')) { class ConversationAction extends Action { var $id = null; - var $notices = null; var $page = null; /** @@ -58,10 +59,9 @@ class ConversationAction extends Action { parent::prepare($args); $this->id = $this->trimmed('id'); - if (!$this->id) { + if (empty($this->id)) { return false; } - $this->notices = $this->getNotices(); $this->page = $this->trimmed('page'); if (empty($this->page)) { $this->page = 1; @@ -69,36 +69,9 @@ class ConversationAction extends Action return true; } - /** - * Get notices - * - * @param integer $limit max number of notices to return - * - * @return array notices - */ - - function getNotices($limit=0) - { - $qry = 'SELECT notice.*, '. - 'FROM notice WHERE conversation = %d '. - 'ORDER BY created '; - - $offset = 0; - $limit = NOTICES_PER_PAGE + 1; - - if (common_config('db', 'type') == 'pgsql') { - $qry .= ' LIMIT ' . $limit . ' OFFSET ' . $offset; - } else { - $qry .= ' LIMIT ' . $offset . ', ' . $limit; - } - - return Notice::getStream(sprintf($qry, $this->id), - 'notice:conversation:'.$this->id, - $offset, $limit); - } - function handle($args) { + parent::handle($args); $this->showPage(); } @@ -111,7 +84,18 @@ class ConversationAction extends Action { // FIXME this needs to be a tree, not a list - $nl = new NoticeList($this->notices, $this); + $qry = 'SELECT * FROM notice WHERE conversation = %s '; + + $offset = ($this->page-1)*NOTICES_PER_PAGE; + $limit = NOTICES_PER_PAGE + 1; + + $txt = sprintf($qry, $this->id); + + $notices = Notice::getStream($txt, + 'notice:conversation:'.$this->id, + $offset, $limit); + + $nl = new NoticeList($notices, $this); $cnt = $nl->show(); From 95bb1c6a99766d065ffcde9523de05d8a1162c3d Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Fri, 3 Apr 2009 16:16:39 -0400 Subject: [PATCH 7/8] in context link for conversations --- lib/noticelist.php | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lib/noticelist.php b/lib/noticelist.php index 4182d8808f..8fccba73e1 100644 --- a/lib/noticelist.php +++ b/lib/noticelist.php @@ -197,7 +197,7 @@ class NoticeListItem extends Widget $this->out->elementStart('div', 'entry-content'); $this->showNoticeLink(); $this->showNoticeSource(); - $this->showReplyTo(); + $this->showContext(); $this->out->elementEnd('div'); } @@ -421,17 +421,18 @@ class NoticeListItem extends Widget * @return void */ - function showReplyTo() + function showContext() { - if ($this->notice->reply_to) { - $replyurl = common_local_url('shownotice', - array('notice' => $this->notice->reply_to)); + // XXX: also show context if there are replies to this notice + if (!empty($this->notice->conversation) + && $this->notice->conversation != $this->notice->id) { + $convurl = common_local_url('conversation', + array('id' => $this->notice->conversation)); $this->out->elementStart('dl', 'response'); $this->out->element('dt', null, _('To')); $this->out->elementStart('dd'); - $this->out->element('a', array('href' => $replyurl, - 'rel' => 'in-reply-to'), - _('in reply to')); + $this->out->element('a', array('href' => $convurl), + _('in context')); $this->out->elementEnd('dd'); $this->out->elementEnd('dl'); } From 60861760fc3bdc1cdea407e8ee21eb8c54e77b8f Mon Sep 17 00:00:00 2001 From: Evan Prodromou Date: Tue, 7 Apr 2009 17:10:54 -0400 Subject: [PATCH 8/8] Host multiple sites with the same codebase This is the beginning of the code for status.net and related status farms. It will read basic information about a site from a shared, central database and use the data stored there to switch on the hostname. --- classes/Status_network.php | 61 ++++++++++++++++++++++++++++++++++++++ classes/statusnet.ini | 17 +++++++++++ config.php.sample | 8 +++++ db/site.sql | 17 +++++++++++ 4 files changed, 103 insertions(+) create mode 100755 classes/Status_network.php create mode 100755 classes/statusnet.ini create mode 100644 db/site.sql diff --git a/classes/Status_network.php b/classes/Status_network.php new file mode 100755 index 0000000000..f7747f71d7 --- /dev/null +++ b/classes/Status_network.php @@ -0,0 +1,61 @@ +dbhost)) ? 'localhost' : $sn->dbhost; + $dbuser = (empty($sn->dbuser)) ? $sn->nickname : $sn->dbuser; + $dbpass = $sn->dbpass; + $dbname = (empty($sn->dbname)) ? $sn->nickname : $sn->dbname; + + $config['db']['database'] = "mysqli://$dbuser:$dbpass@$dbhost/$dbname"; + $config['site']['name'] = $sn->sitename; + return true; + } else { + return false; + } + } +} diff --git a/classes/statusnet.ini b/classes/statusnet.ini new file mode 100755 index 0000000000..a70cd41228 --- /dev/null +++ b/classes/statusnet.ini @@ -0,0 +1,17 @@ + +[status_network] +nickname = 130 +hostname = 2 +pathname = 2 +sitename = 2 +dbhost = 2 +dbuser = 2 +dbpass = 2 +dbname = 2 +created = 142 +modified = 384 + +[status_network__keys] +nickname = K +hostname = U +pathname = U diff --git a/config.php.sample b/config.php.sample index 59e0cc7126..eec2bb2c5d 100644 --- a/config.php.sample +++ b/config.php.sample @@ -192,3 +192,11 @@ $config['sphinx']['port'] = 3312; #Use a different hostname for SSL-encrypted pages #$config['site']['sslserver'] = 'secure.example.org'; + +#If you have a lot of status networks on the same server, you can +#store the site data in a database and switch as follows +#Status_network::setupDB('localhost', 'statusnet', 'statuspass', 'statusnet'); +#if (!Status_network::setupSite($_server, $_path)) { +# print "Error\n"; +# exit(1); +#} diff --git a/db/site.sql b/db/site.sql new file mode 100644 index 0000000000..660ba475bb --- /dev/null +++ b/db/site.sql @@ -0,0 +1,17 @@ +/* For managing multiple sites */ + +create table status_network ( + + nickname varchar(64) primary key comment 'nickname', + hostname varchar(255) unique key comment 'alternate hostname if any', + pathname varchar(255) unique key comment 'alternate pathname if any', + sitename varchar(255) comment 'display name', + dbhost varchar(255) comment 'database host', + dbuser varchar(255) comment 'database username', + dbpass varchar(255) comment 'database password', + dbname varchar(255) comment 'database name', + + created datetime not null comment 'date this record was created', + modified timestamp comment 'date this record was modified' + +) ENGINE=InnoDB CHARACTER SET utf8 COLLATE utf8_general_ci;