Add an IMAP daemon so StatusNet can process incoming user posts via catch-all mailbox (in addition to the pre-existing script alias method)

This commit is contained in:
Craig Andrews 2010-01-08 18:52:09 -05:00
parent e22af049a8
commit 055f3fdddb
5 changed files with 542 additions and 260 deletions

275
lib/mailhandler.php Normal file
View File

@ -0,0 +1,275 @@
<?php
/*
* StatusNet - the distributed open-source microblogging tool
* Copyright (C) 2008, 2009, StatusNet, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
require_once(INSTALLDIR . '/lib/mail.php');
require_once(INSTALLDIR . '/lib/mediafile.php');
require_once('Mail/mimeDecode.php');
# FIXME: we use both Mail_mimeDecode and mailparse
# Need to move everything to mailparse
class MailHandler
{
function __construct()
{
}
function handle_message($rawmessage)
{
list($from, $to, $msg, $attachments) = $this->parse_message($rawmessage);
if (!$from || !$to || !$msg) {
$this->error(null, _('Could not parse message.'));
}
common_log(LOG_INFO, "Mail from $from to $to with ".count($attachments) .' attachment(s): ' .substr($msg, 0, 20));
$user = $this->user_from_header($from);
if (!$user) {
$this->error($from, _('Not a registered user.'));
return false;
}
if (!$this->user_match_to($user, $to)) {
$this->error($from, _('Sorry, that is not your incoming email address.'));
return false;
}
if (!$user->emailpost) {
$this->error($from, _('Sorry, no incoming email allowed.'));
return false;
}
$response = $this->handle_command($user, $from, $msg);
if ($response) {
return true;
}
$msg = $this->cleanup_msg($msg);
$msg = common_shorten_links($msg);
if (Notice::contentTooLong($msg)) {
$this->error($from, sprintf(_('That\'s too long. '.
'Max notice size is %d chars.'),
Notice::maxContent()));
}
$mediafiles = array();
foreach($attachments as $attachment){
$mf = null;
try {
$mf = MediaFile::fromFileHandle($attachment, $user);
} catch(ClientException $ce) {
$this->error($from, $ce->getMessage());
}
$msg .= ' ' . $mf->shortUrl();
array_push($mediafiles, $mf);
fclose($attachment);
}
$err = $this->add_notice($user, $msg, $mediafiles);
if (is_string($err)) {
$this->error($from, $err);
return false;
} else {
return true;
}
}
function error($from, $msg)
{
file_put_contents("php://stderr", $msg . "\n");
exit(1);
}
function user_from_header($from_hdr)
{
$froms = mailparse_rfc822_parse_addresses($from_hdr);
if (!$froms) {
return null;
}
$from = $froms[0];
$addr = common_canonical_email($from['address']);
$user = User::staticGet('email', $addr);
if (!$user) {
$user = User::staticGet('smsemail', $addr);
}
return $user;
}
function user_match_to($user, $to_hdr)
{
$incoming = $user->incomingemail;
$tos = mailparse_rfc822_parse_addresses($to_hdr);
foreach ($tos as $to) {
if (strcasecmp($incoming, $to['address']) == 0) {
return true;
}
}
return false;
}
function handle_command($user, $from, $msg)
{
$inter = new CommandInterpreter();
$cmd = $inter->handle_command($user, $msg);
if ($cmd) {
$cmd->execute(new MailChannel($from));
return true;
}
return false;
}
function respond($from, $to, $response)
{
$headers['From'] = $to;
$headers['To'] = $from;
$headers['Subject'] = "Command complete";
return mail_send(array($from), $headers, $response);
}
function log($level, $msg)
{
common_log($level, 'MailDaemon: '.$msg);
}
function add_notice($user, $msg, $mediafiles)
{
try {
$notice = Notice::saveNew($user->id, $msg, 'mail');
} catch (Exception $e) {
$this->log(LOG_ERR, $e->getMessage());
return $e->getMessage();
}
foreach($mediafiles as $mf){
$mf->attachToNotice($notice);
}
common_broadcast_notice($notice);
$this->log(LOG_INFO,
'Added notice ' . $notice->id . ' from user ' . $user->nickname);
return true;
}
function parse_message($contents)
{
$parsed = Mail_mimeDecode::decode(array('input' => $contents,
'include_bodies' => true,
'decode_headers' => true,
'decode_bodies' => true));
if (!$parsed) {
return null;
}
$from = $parsed->headers['from'];
$to = $parsed->headers['to'];
$type = $parsed->ctype_primary . '/' . $parsed->ctype_secondary;
$attachments = array();
$this->extract_part($parsed,$msg,$attachments);
return array($from, $to, $msg, $attachments);
}
function extract_part($parsed,&$msg,&$attachments){
if ($parsed->ctype_primary == 'multipart') {
if($parsed->ctype_secondary == 'alternative'){
$altmsg = $this->extract_msg_from_multipart_alternative_part($parsed);
if(!empty($altmsg)) $msg = $altmsg;
}else{
foreach($parsed->parts as $part){
$this->extract_part($part,$msg,$attachments);
}
}
} else if ($parsed->ctype_primary == 'text'
&& $parsed->ctype_secondary=='plain') {
$msg = $parsed->body;
if(strtolower($parsed->ctype_parameters['charset']) != "utf-8"){
$msg = utf8_encode($msg);
}
}else if(!empty($parsed->body)){
if(common_config('attachments', 'uploads')){
//only save attachments if uploads are enabled
$attachment = tmpfile();
fwrite($attachment, $parsed->body);
$attachments[] = $attachment;
}
}
}
function extract_msg_from_multipart_alternative_part($parsed){
foreach ($parsed->parts as $part) {
$this->extract_part($part,$msg,$attachments);
}
//we don't want any attachments that are a result of this parsing
return $msg;
}
function unsupported_type($type)
{
$this->error(null, "Unsupported message type: " . $type);
}
function cleanup_msg($msg)
{
$lines = explode("\n", $msg);
$output = '';
foreach ($lines as $line) {
// skip quotes
if (preg_match('/^\s*>.*$/', $line)) {
continue;
}
// skip start of quote
if (preg_match('/^\s*On.*wrote:\s*$/', $line)) {
continue;
}
// probably interesting to someone, not us
if (preg_match('/^\s*Sent via/', $line)) {
continue;
}
if (preg_match('/^\s*Sent from my/', $line)) {
continue;
}
// skip everything after a sig
if (preg_match('/^\s*--+\s*$/', $line) ||
preg_match('/^\s*__+\s*$/', $line))
{
break;
}
// skip everything after Outlook quote
if (preg_match('/^\s*-+\s*Original Message\s*-+\s*$/', $line)) {
break;
}
// skip everything after weird forward
if (preg_match('/^\s*Begin\s+forward/', $line)) {
break;
}
$output .= ' ' . $line;
}
preg_replace('/\s+/', ' ', $output);
return trim($output);
}
}

View File

@ -0,0 +1,85 @@
<?php
/**
* StatusNet, the distributed open-source microblogging tool
*
* Plugin to add a StatusNet Facebook application
*
* PHP version 5
*
* LICENCE: This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* @category Plugin
* @package StatusNet
* @author Zach Copley <zach@status.net>
* @copyright 2009 StatusNet, Inc.
* @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
* @link http://status.net/
*/
if (!defined('STATUSNET')) {
exit(1);
}
/**
* IMAP plugin to allow StatusNet to grab incoming emails and handle them as new user posts
*
* @category Plugin
* @package StatusNet
* @author Craig Andrews <candrews@integralblue.com
* @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0
* @link http://status.net/
*/
class ImapPlugin extends Plugin
{
public $mailbox;
public $user;
public $password;
public $poll_frequency = 60;
public static $instances = array();
public static $daemon_added = array();
function initialize(){
if(!isset($this->mailbox)){
throw new Exception("must specify a mailbox");
}
if(!isset($this->user)){
throw new Exception("must specify a user");
}
if(!isset($this->password)){
throw new Exception("must specify a password");
}
if(!isset($this->poll_frequency)){
throw new Exception("must specify a poll_frequency");
}
self::$instances[] = $this;
return true;
}
function cleanup(){
$index = array_search($this, self::$instances);
unset(self::$instances[$index]);
return true;
}
function onGetValidDaemons($daemons)
{
if(! self::$daemon_added){
array_push($daemons, INSTALLDIR .
'/plugins/Imap/imapdaemon.php');
self::$daemon_added = true;
}
return true;
}
}

32
plugins/Imap/README Normal file
View File

@ -0,0 +1,32 @@
The IMAP plugin allows for StatusNet to check a POP or IMAP mailbox for
incoming mail containing user posts.
Installation
============
addPlugin('imap', array(
'mailbox' => '...',
'user' => '...',
'password' => '...'
));
to the bottom of your config.php
Also, make sure:
$config['mail']['domain'] = 'yourdomain.example.net';
is set in your config.php
Create a catch-all account for your domain, and use this account with this
plugin. Whenever a user sends a message to their personal notice posting
address, the message should end up in this mailbox, and then the plugin daemon
will pick it up and post the notice on the user's behalf.
The daemon included with this plugin must be running. It will be started by
the plugin along with their other daemons when you run scripts/startdaemons.sh.
See the StatusNet README for more about queuing and daemons.
Settings
========
mailbox*: the mailbox specifier.
See http://www.php.net/manual/en/function.imap-open.php for details
user*: username to use when authenticating to the mailbox
password*: password to use when authenticating to the mailbox
poll_frequency: how often (in seconds) to check for new messages

147
plugins/Imap/imapdaemon.php Executable file
View File

@ -0,0 +1,147 @@
#!/usr/bin/env php
<?php
/*
* StatusNet - the distributed open-source microblogging tool
* Copyright (C) 2008, 2009, StatusNet, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
define('INSTALLDIR', realpath(dirname(__FILE__) . '/../..'));
$shortoptions = 'fi::';
$longoptions = array('id::', 'foreground');
$helptext = <<<END_OF_IMAP_HELP
Daemon script for receiving new notices from users via a mail box (IMAP, POP3, etc)
-i --id Identity (default none)
-f --foreground Stay in the foreground (default background)
END_OF_IMAP_HELP;
require_once INSTALLDIR.'/scripts/commandline.inc';
require_once INSTALLDIR . '/lib/common.php';
require_once INSTALLDIR . '/lib/daemon.php';
require_once INSTALLDIR.'/lib/mailhandler.php';
class IMAPDaemon extends Daemon
{
function __construct($resource=null, $daemonize=true, $attrs)
{
parent::__construct($daemonize);
foreach ($attrs as $attr=>$value)
{
$this->$attr = $value;
}
$this->log(LOG_INFO, "INITIALIZE IMAPDaemon {" . $this->name() . "}");
}
function name()
{
return strtolower('imapdaemon.'.$this->user.'.'.crc32($this->mailbox));
}
function run()
{
$this->connect();
while(true)
{
if(imap_ping($this->conn) || $this->connect())
{
$this->check_mailbox();
}
sleep($this->poll_frequency);
}
}
function check_mailbox()
{
$count = imap_num_msg($this->conn);
$this->log(LOG_INFO, "Found $count messages");
if($count > 0){
$handler = new IMAPMailHandler();
for($i=1; $i <= $count; $i++)
{
$rawmessage = imap_fetchheader($this->conn, $count, FT_PREFETCHTEXT) . imap_body($this->conn, $i);
$handler->handle_message($rawmessage);
imap_delete($this->conn, $i);
}
imap_expunge($this->conn);
$this->log(LOG_INFO, "Finished processing messages");
}
}
function log($level, $msg)
{
$text = $this->name() . ': '.$msg;
common_log($level, $text);
if (!$this->daemonize)
{
$line = common_log_line($level, $text);
echo $line;
echo "\n";
}
}
function connect()
{
$this->conn = imap_open($this->mailbox, $this->user, $this->password);
if($this->conn){
$this->log(LOG_INFO, "Connected");
return true;
}else{
$this->log(LOG_INFO, "Failed to connect: " . imap_last_error());
return false;
}
}
}
class IMAPMailHandler extends MailHandler
{
function error($from, $msg)
{
$this->log(LOG_INFO, "Error: $from $msg");
$headers['To'] = $from;
$headers['Subject'] = "Error";
return mail_send(array($from), $headers, $msg);
}
}
if (have_option('i', 'id')) {
$id = get_option_value('i', 'id');
} else if (count($args) > 0) {
$id = $args[0];
} else {
$id = null;
}
$foreground = have_option('f', 'foreground');
foreach(ImapPlugin::$instances as $pluginInstance){
$daemon = new IMAPDaemon($id, !$foreground, array(
'mailbox' => $pluginInstance->mailbox,
'user' => $pluginInstance->user,
'password' => $pluginInstance->password,
'poll_frequency' => $pluginInstance->poll_frequency
));
$daemon->runOnce();
}

View File

@ -27,266 +27,9 @@ as STDIN.
END_OF_HELP;
require_once INSTALLDIR.'/scripts/commandline.inc';
require_once(INSTALLDIR . '/lib/mail.php');
require_once(INSTALLDIR . '/lib/mediafile.php');
require_once('Mail/mimeDecode.php');
# FIXME: we use both Mail_mimeDecode and mailparse
# Need to move everything to mailparse
class MailerDaemon
{
function __construct()
{
}
function handle_message($fname='php://stdin')
{
list($from, $to, $msg, $attachments) = $this->parse_message($fname);
if (!$from || !$to || !$msg) {
$this->error(null, _('Could not parse message.'));
}
common_log(LOG_INFO, "Mail from $from to $to with ".count($attachments) .' attachment(s): ' .substr($msg, 0, 20));
$user = $this->user_from($from);
if (!$user) {
$this->error($from, _('Not a registered user.'));
return false;
}
if (!$this->user_match_to($user, $to)) {
$this->error($from, _('Sorry, that is not your incoming email address.'));
return false;
}
if (!$user->emailpost) {
$this->error($from, _('Sorry, no incoming email allowed.'));
return false;
}
$response = $this->handle_command($user, $from, $msg);
if ($response) {
return true;
}
$msg = $this->cleanup_msg($msg);
$msg = common_shorten_links($msg);
if (Notice::contentTooLong($msg)) {
$this->error($from, sprintf(_('That\'s too long. '.
'Max notice size is %d chars.'),
Notice::maxContent()));
}
$mediafiles = array();
foreach($attachments as $attachment){
$mf = null;
try {
$mf = MediaFile::fromFileHandle($attachment, $user);
} catch(ClientException $ce) {
$this->error($from, $ce->getMessage());
}
$msg .= ' ' . $mf->shortUrl();
array_push($mediafiles, $mf);
fclose($attachment);
}
$err = $this->add_notice($user, $msg, $mediafiles);
if (is_string($err)) {
$this->error($from, $err);
return false;
} else {
return true;
}
}
function error($from, $msg)
{
file_put_contents("php://stderr", $msg . "\n");
exit(1);
}
function user_from($from_hdr)
{
$froms = mailparse_rfc822_parse_addresses($from_hdr);
if (!$froms) {
return null;
}
$from = $froms[0];
$addr = common_canonical_email($from['address']);
$user = User::staticGet('email', $addr);
if (!$user) {
$user = User::staticGet('smsemail', $addr);
}
return $user;
}
function user_match_to($user, $to_hdr)
{
$incoming = $user->incomingemail;
$tos = mailparse_rfc822_parse_addresses($to_hdr);
foreach ($tos as $to) {
if (strcasecmp($incoming, $to['address']) == 0) {
return true;
}
}
return false;
}
function handle_command($user, $from, $msg)
{
$inter = new CommandInterpreter();
$cmd = $inter->handle_command($user, $msg);
if ($cmd) {
$cmd->execute(new MailChannel($from));
return true;
}
return false;
}
function respond($from, $to, $response)
{
$headers['From'] = $to;
$headers['To'] = $from;
$headers['Subject'] = "Command complete";
return mail_send(array($from), $headers, $response);
}
function log($level, $msg)
{
common_log($level, 'MailDaemon: '.$msg);
}
function add_notice($user, $msg, $mediafiles)
{
try {
$notice = Notice::saveNew($user->id, $msg, 'mail');
} catch (Exception $e) {
$this->log(LOG_ERR, $e->getMessage());
return $e->getMessage();
}
foreach($mediafiles as $mf){
$mf->attachToNotice($notice);
}
common_broadcast_notice($notice);
$this->log(LOG_INFO,
'Added notice ' . $notice->id . ' from user ' . $user->nickname);
return true;
}
function parse_message($fname)
{
$contents = file_get_contents($fname);
$parsed = Mail_mimeDecode::decode(array('input' => $contents,
'include_bodies' => true,
'decode_headers' => true,
'decode_bodies' => true));
if (!$parsed) {
return null;
}
$from = $parsed->headers['from'];
$to = $parsed->headers['to'];
$type = $parsed->ctype_primary . '/' . $parsed->ctype_secondary;
$attachments = array();
$this->extract_part($parsed,$msg,$attachments);
return array($from, $to, $msg, $attachments);
}
function extract_part($parsed,&$msg,&$attachments){
if ($parsed->ctype_primary == 'multipart') {
if($parsed->ctype_secondary == 'alternative'){
$altmsg = $this->extract_msg_from_multipart_alternative_part($parsed);
if(!empty($altmsg)) $msg = $altmsg;
}else{
foreach($parsed->parts as $part){
$this->extract_part($part,$msg,$attachments);
}
}
} else if ($parsed->ctype_primary == 'text'
&& $parsed->ctype_secondary=='plain') {
$msg = $parsed->body;
if(strtolower($parsed->ctype_parameters['charset']) != "utf-8"){
$msg = utf8_encode($msg);
}
}else if(!empty($parsed->body)){
if(common_config('attachments', 'uploads')){
//only save attachments if uploads are enabled
$attachment = tmpfile();
fwrite($attachment, $parsed->body);
$attachments[] = $attachment;
}
}
}
function extract_msg_from_multipart_alternative_part($parsed){
foreach ($parsed->parts as $part) {
$this->extract_part($part,$msg,$attachments);
}
//we don't want any attachments that are a result of this parsing
return $msg;
}
function unsupported_type($type)
{
$this->error(null, "Unsupported message type: " . $type);
}
function cleanup_msg($msg)
{
$lines = explode("\n", $msg);
$output = '';
foreach ($lines as $line) {
// skip quotes
if (preg_match('/^\s*>.*$/', $line)) {
continue;
}
// skip start of quote
if (preg_match('/^\s*On.*wrote:\s*$/', $line)) {
continue;
}
// probably interesting to someone, not us
if (preg_match('/^\s*Sent via/', $line)) {
continue;
}
if (preg_match('/^\s*Sent from my/', $line)) {
continue;
}
// skip everything after a sig
if (preg_match('/^\s*--+\s*$/', $line) ||
preg_match('/^\s*__+\s*$/', $line))
{
break;
}
// skip everything after Outlook quote
if (preg_match('/^\s*-+\s*Original Message\s*-+\s*$/', $line)) {
break;
}
// skip everything after weird forward
if (preg_match('/^\s*Begin\s+forward/', $line)) {
break;
}
$output .= ' ' . $line;
}
preg_replace('/\s+/', ' ', $output);
return trim($output);
}
}
require_once INSTALLDIR.'/lib/mailhandler.php';
if (common_config('emailpost', 'enabled')) {
$md = new MailerDaemon();
$md->handle_message('php://stdin');
$mh = new MailHandler();
$mh->handle_message(file_get_contents('php://stdin'));
}