Merge branch 'master' into nightly

lib/noticelistitem.php

@@ -537,6 +537,19 @@ class NoticeListItem extends Widget
         if (!$this->notice->isLocal()) {
             $class .= ' external';
         }
+
+        try {
+            if($this->repeat) {
+                $this->out->element('a',
+                            array('href' => $this->repeat->getUrl(),
+                                  'class' => 'u-url'),
+                            '');
+                $class = str_replace('u-url', 'u-repost-of', $class);
+            }
+        } catch (InvalidUrlException $e) {
+            // no permalink available
+        }
+
         try {
             $this->out->element('a',
                         array('href' => $this->notice->getUrl(true),

plugins/Linkback/LinkbackPlugin.php

@@ -32,6 +32,7 @@ if (!defined('STATUSNET')) {
 }
 
 require_once('Auth/Yadis/Yadis.php');
+require_once(__DIR__ . '/lib/util.php');
 
 define('LINKBACKPLUGIN_VERSION', '0.1');
 
@@ -306,6 +307,19 @@ class LinkbackPlugin extends Plugin
         }
     }
 
+
+    public function onRouterInitialized(URLMapper $m)
+    {
+        $m->connect('main/linkback/webmention', array('action' => 'webmention'));
+        $m->connect('main/linkback/pingback', array('action' => 'pingback'));
+    }
+
+    public function onStartShowHTML($action)
+    {
+        header('Link: <' . common_local_url('webmention') . '>; rel="webmention"', false);
+        header('X-Pingback: ' . common_local_url('pingback'));
+    }
+
     public function version()
     {
         return LINKBACKPLUGIN_VERSION;
@@ -344,4 +358,34 @@ class LinkbackPlugin extends Plugin
                           $action_name === 'linkbacksettings');
         return true;
     }
+
+    function onStartNoticeSourceLink($notice, &$name, &$url, &$title)
+    {
+        // If we don't handle this, keep the event handler going
+        if (!in_array($notice->source, array('linkback'))) {
+            return true;
+        }
+
+        try {
+            $url = $notice->getUrl();
+            // If getUrl() throws exception, $url is never set
+
+            $bits = parse_url($url);
+            $domain = $bits['host'];
+            if (substr($domain, 0, 4) == 'www.') {
+                $name = substr($domain, 4);
+            } else {
+                $name = $domain;
+            }
+
+            // TRANS: Title. %s is a domain name.
+            $title = sprintf(_m('Sent from %s via Linkback'), $domain);
+
+            // Abort event handler, we have a name and URL!
+            return false;
+        } catch (InvalidUrlException $e) {
+            // This just means we don't have the notice source data
+            return true;
+        }
+    }
 }

plugins/Linkback/actions/pingback.php

@@ -0,0 +1,87 @@
+<?php
+/*
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+if (!defined('STATUSNET')) {
+    exit(1);
+}
+
+class PingbackAction extends Action
+{
+    protected function handle()
+    {
+        GNUsocial::setApi(true); // Minimize error messages to aid in debugging
+        parent::handle();
+        if ($this->isPost()) {
+            return $this->handlePost();
+        }
+
+        return false;
+    }
+
+    function handlePost()
+    {
+
+        $server = xmlrpc_server_create();
+        xmlrpc_server_register_method($server, 'pingback.ping', array($this, 'ping'));
+        echo xmlrpc_server_call_method($server, file_get_contents('php://input'), null, array('encoding' => 'utf-8'));
+        xmlrpc_server_destroy($server);
+        return true;
+    }
+
+    function ping($method, $parameters) {
+        list($source, $target) = $parameters;
+
+        if(!$source) {
+            return array(
+                'faultCode' => 0x0010,
+                'faultString' => '"source" is missing'
+            );
+        }
+
+        if(!$target) {
+            return array(
+                'faultCode' => 0x0020,
+                'faultString' => '"target" is missing'
+            );
+        }
+
+        $response = linkback_get_source($source, $target);
+        if(!$response) {
+            return array(
+                'faultCode' => 0x0011,
+                'faultString' => 'Source does not link to target'
+            );
+        }
+
+        $notice = linkback_get_target($target);
+        if(!$notice) {
+            return array(
+                'faultCode' => 0x0021,
+                'faultString' => 'Target not found'
+            );
+        }
+
+        $url = linkback_save($source, $target, $response, $notice);
+        if(!$url) {
+            return array(
+                'faultCode' => 0,
+                'faultString' => 'An error occured while saving.'
+            );
+        }
+
+        return array('Success');
+    }
+}

plugins/Linkback/actions/webmention.php

@@ -0,0 +1,73 @@
+<?php
+/*
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+if (!defined('STATUSNET')) {
+    exit(1);
+}
+
+class WebmentionAction extends Action
+{
+    protected function handle()
+    {
+        GNUsocial::setApi(true); // Minimize error messages to aid in debugging
+        parent::handle();
+        if ($this->isPost()) {
+            return $this->handlePost();
+        }
+
+        return false;
+    }
+
+    function handlePost()
+    {
+        $source = $this->arg('source');
+        $target = $this->arg('target');
+
+        header('Content-Type: text/plain; charset=utf-8');
+
+        if(!$source) {
+            echo _m('"source" is missing')."\n";
+            throw new ServerException(_m('"source" is missing'), 400);
+        }
+
+        if(!$target) {
+            echo _m('"target" is missing')."\n";
+            throw new ServerException(_m('"target" is missing'), 400);
+        }
+
+        $response = linkback_get_source($source, $target);
+        if(!$response) {
+            echo _m('Source does not link to target.')."\n";
+            throw new ServerException(_m('Source does not link to target.'), 400);
+        }
+
+        $notice = linkback_get_target($target);
+        if(!$notice) {
+            echo _m('Target not found')."\n";
+            throw new ServerException(_m('Target not found'), 404);
+        }
+
+        $url = linkback_save($source, $target, $response, $notice);
+        if(!$url) {
+            echo _m('An error occured while saving.')."\n";
+            throw new ServerException(_m('An error occured while saving.'), 500);
+        }
+
+        echo $url."\n";
+
+        return true;
+    }
+}

plugins/Linkback/lib/util.php

@@ -0,0 +1,435 @@
+<?php
+
+function linkback_lenient_target_match($body, $target) {
+    return strpos(''.$body, str_replace(array('http://www.', 'http://', 'https://www.', 'https://'), '', preg_replace('/\/+$/', '', preg_replace( '/#.*/', '', $target))));
+}
+
+function linkback_get_source($source, $target) {
+    // Check if we are pinging ourselves and ignore
+    $localprefix = common_config('site', 'server') . '/' . common_config('site', 'path');
+    if(linkback_lenient_target_match($source, $localprefix)) {
+        common_debug('Ignoring self ping from ' . $source . ' to ' . $target);
+        return NULL;
+    }
+
+    $request = HTTPClient::start();
+
+    try {
+        $response = $request->get($source);
+    } catch(Exception $ex) {
+        return NULL;
+    }
+
+    $body = htmlspecialchars_decode($response->getBody());
+    // We're slightly more lenient in our link detection than the spec requires
+    if(!linkback_lenient_target_match($body, $target)) {
+        return NULL;
+    }
+
+    return $response;
+}
+
+function linkback_get_target($target) {
+    // Resolve target (https://github.com/converspace/webmention/issues/43)
+    $request = HTTPClient::start();
+
+    try {
+        $response = $request->head($target);
+    } catch(Exception $ex) {
+        return NULL;
+    }
+
+    try {
+        $notice = Notice::fromUri($response->getEffectiveUrl());
+    } catch(UnknownUriException $ex) {
+        preg_match('/\/notice\/(\d+)(?:#.*)?$/', $response->getEffectiveUrl(), $match);
+        $notice = Notice::getKV('id', $match[1]);
+    }
+
+    if($notice instanceof Notice && $notice->isLocal()) {
+        return $notice;
+    } else {
+        $user = User::getKV('uri', $response->getEffectiveUrl());
+        if(!$user) {
+            preg_match('/\/user\/(\d+)(?:#.*)?$/', $response->getEffectiveUrl(), $match);
+            $user = User::getKV('id', $match[1]);
+        }
+        if(!$user) {
+            preg_match('/\/([^\/\?#]+)(?:#.*)?$/', $response->getEffectiveUrl(), $match);
+            if(linkback_lenient_target_match(common_profile_url($match[1]), $response->getEffectiveUrl())) {
+                $user = User::getKV('nickname', $match[1]);
+            }
+        }
+        if($user instanceof User) {
+            return $user;
+        }
+    }
+
+    return NULL;
+}
+
+function linkback_is_contained_in($entry, $target) {
+    foreach ((array)$entry['properties'] as $key => $values) {
+        if(count(array_filter($values, function($x) use ($target) { return linkback_lenient_target_match($x, $target); })) > 0) {
+            return $entry['properties'];
+        }
+
+        // check included h-* formats and their links
+        foreach ($values as $obj) {
+            if(isset($obj['type']) && array_intersect(array('h-cite', 'h-entry'), $obj['type']) &&
+               isset($obj['properties']) && isset($obj['properties']['url']) &&
+               count(array_filter($obj['properties']['url'],
+                     function($x) use ($target) { return linkback_lenient_target_match($x, $target); })) > 0
+            ) {
+                return $entry['properties'];
+            }
+        }
+
+        // check content for the link
+        if ($key == "content" && preg_match_all("/<a[^>]+?".preg_quote($target, "/")."[^>]*>([^>]+?)<\/a>/i", htmlspecialchars_decode($values[0]['html']), $context)) {
+            return $entry['properties'];
+        // check summary for the link
+        } elseif ($key == "summary" && preg_match_all("/<a[^>]+?".preg_quote($target, "/")."[^>]*>([^>]+?)<\/a>/i", htmlspecialchars_decode($values[0]), $context)) {
+            return $entry['properties'];
+        }
+    }
+
+    foreach((array)$entry['children'] as $mf2) {
+        if(linkback_is_contained_in($mf2, $target)) {
+            return $entry['properties'];
+        }
+    }
+
+    return null;
+}
+
+// Based on https://github.com/acegiak/Semantic-Linkbacks/blob/master/semantic-linkbacks-microformats-handler.php, GPL-2.0+
+function linkback_find_entry($mf2, $target) {
+    if(isset($mf2['items'][0]['type']) && in_array("h-feed", $mf2['items'][0]["type"]) && isset($mf2['items'][0]['children'])) {
+        $mf2['items'] = $mf2['items'][0]['children'];
+    }
+
+    $entries = array_filter($mf2['items'], function($x) { return isset($x['type']) && in_array('h-entry', $x['type']); });
+
+    foreach ($entries as $entry) {
+        if($prop = linkback_is_contained_in($entry, $target)) {
+            return $prop;
+        }
+    }
+
+    // Default to first one
+    if(count($entries) > 0) {
+        return $entries[0]['properties'];
+    }
+
+    return NULL;
+}
+
+function linkback_entry_type($entry, $mf2, $target) {
+    if(!$entry) { return 'mention'; }
+
+    if($mf2['rels'] && $mf2['rels']['in-reply-to']) {
+        foreach($mf2['rels']['in-reply-to'] as $url) {
+            if(linkback_lenient_target_match($url, $target)) {
+                return 'reply';
+            }
+        }
+    }
+
+    $classes = array(
+        'in-reply-to' => 'reply',
+        'repost-of' => 'repost',
+        'like-of' => 'like',
+        'tag-of' => 'tag'
+    );
+
+    foreach((array)$entry as $key => $values) {
+        if(count(array_filter($values, function($x) use ($target) { return linkback_lenient_target_match($x, $target); })) > 0) {
+            if($classes[$key]) { return $classes[$key]; }
+        }
+
+        foreach ($values as $obj) {
+            if(isset($obj['type']) && array_intersect(array('h-cite', 'h-entry'), $obj['type']) &&
+               isset($obj['properties']) && isset($obj['properties']['url']) &&
+               count(array_filter($obj['properties']['url'],
+                     function($x) use ($target) { return linkback_lenient_target_match($x, $target); })) > 0
+            ) {
+                if($classes[$key]) { return $classes[$key]; }
+            }
+        }
+    }
+
+    return 'mention';
+}
+
+function linkback_is_dupe($key, $url) {
+    $dupe = Notice::getKV($key, $url);
+    if ($dupe instanceof Notice) {
+        return $dupe;
+    }
+
+    return false;
+}
+
+
+function linkback_hcard($mf2, $url) {
+    if(empty($mf2['items'])) {
+        return null;
+    }
+  
+    $hcards = array();
+    foreach($mf2['items'] as $item) {
+        if(!in_array('h-card', $item['type'])) {
+            continue;
+        }
+      
+        // We found a match, return it immediately
+        if(isset($item['properties']['url']) && in_array($url, $item['properties']['url'])) {
+            return $item['properties'];
+        }
+
+        // Let's keep all the hcards for later, to return one of them at least
+        $hcards[] = $item['properties'];
+    }
+  
+    // No match immediately for the url we expected, but there were h-cards found
+    if (count($hcards) > 0) {
+        return $hcards[0];
+    }
+  
+    return null;
+}
+
+function linkback_notice($source, $notice_or_user, $entry, $author, $mf2) {
+    $content = $entry['content'] ? $entry['content'][0]['html'] :
+              ($entry['summary'] ? $entry['sumary'][0] : $entry['name'][0]);
+
+    $rendered = common_purify($content);
+
+    if($notice_or_user instanceof Notice && $entry['type'] == 'mention') {
+        $name = $entry['name'] ? $entry['name'][0] : substr(common_strip_html($content), 0, 20).'…';
+        $rendered = _m('linked to this from <a href="'.htmlspecialchars($source).'">'.htmlspecialchars($name).'</a>');
+    }
+
+    $content = common_strip_html($rendered);
+    $shortened = common_shorten_links($content);
+    if(Notice::contentTooLong($shortened)) {
+        $content = substr($content,
+                          0,
+                          Notice::maxContent() - (mb_strlen($source) + 2));
+        $rendered = $content . '<a href="'.htmlspecialchars($source).'">…</a>';
+        $content .= ' ' . $source;
+    }
+
+    $options = array('is_local' => Notice::REMOTE,
+                    'url' => $entry['url'][0],
+                    'uri' => $entry['url'][0],
+                    'rendered' => $rendered,
+                    'replies' => array(),
+                    'groups' => array(),
+                    'peopletags' => array(),
+                    'tags' => array(),
+                    'urls' => array());
+
+    if($notice_or_user instanceof User) {
+        $options['replies'][] = $notice_or_user->getUri();
+    } else {
+        if($entry['type'] == 'repost') {
+            $options['repeat_of'] = $notice_or_user->id;
+        } else {
+            $options['reply_to'] = $notice_or_user->id;
+        }
+    }
+
+    if($entry['published'] || $entry['updated']) {
+        $options['created'] = $entry['published'] ? common_sql_date($entry['published'][0]) : common_sql_date($entry['updated'][0]);
+    }
+
+    if($entry['photo']) {
+        $options['urls'][] = $entry['photo'][0];
+    }
+
+    foreach((array)$entry['category'] as $tag) {
+        $tag = common_canonical_tag($tag);
+        if($tag) { $options['tags'][] = $tag; }
+    }
+
+
+    if($mf2['rels'] && $mf2['rels']['enclosure']) {
+        foreach($mf2['rels']['enclosure'] as $url) {
+            $options['urls'][] = $url;
+        }
+    }
+
+    if($mf2['rels'] && $mf2['rels']['tag']) {
+        foreach($mf2['rels']['tag'] as $url) {
+            preg_match('/\/([^\/]+)\/*$/', $url, $match);
+            $tag = common_canonical_tag($match[1]);
+            if($tag) { $options['tags'][] = $tag; }
+         }
+    }
+
+    if($entry['type'] != 'reply' && $entry['type'] != 'repost') {
+        $options['urls'] = array();
+    }
+
+    return array($content, $options);
+}
+
+function linkback_profile($entry, $mf2, $response, $target) {
+    if(isset($entry['properties']['author']) && isset($entry['properties']['author'][0]['properties'])) {
+        $author = $entry['properties']['author'][0]['properties'];
+    } else {
+        $author = linkback_hcard($mf2, $response->getEffectiveUrl());
+    }
+
+    if(!$author) {
+        $author = array('name' => array($entry['name']));
+    }
+
+    if(!$author['url']) {
+        $author['url'] = array($response->getEffectiveUrl());
+    }
+
+    $user = User::getKV('uri', $author['url'][0]);
+    if ($user instanceof User) {
+        common_log(LOG_INFO, "Linkback: ignoring linkback from local user: $url");
+        return true;
+    }
+
+    try {
+        $profile = Profile::fromUri($author['url'][0]);
+    } catch(UnknownUriException $ex) {}
+
+    if(!($profile instanceof Profile)) {
+        $profile = Profile::getKV('profileurl', $author['url'][0]);
+    }
+
+    if(!($profile instanceof Profile)) {
+        $profile = new Profile();
+        $profile->profileurl = $author['url'][0];
+        $profile->fullname = $author['name'][0];
+        $profile->nickname = $author['nickname'] ? $author['nickname'][0] : str_replace(' ', '', $author['name'][0]);
+        $profile->created = common_sql_now();
+        $profile->insert();
+    }
+
+    return array($profile, $author);
+}
+
+function linkback_save($source, $target, $response, $notice_or_user) {
+    $dupe = linkback_is_dupe('uri', $response->getEffectiveUrl());
+    if(!$dupe) { $dupe = linkback_is_dupe('url', $response->getEffectiveUrl()); }
+    if(!$dupe) { $dupe = linkback_is_dupe('uri', $source); }
+    if(!$dupe) { $dupe = linkback_is_dupe('url', $source); }
+
+    $mf2 = new Mf2\Parser($response->getBody(), $response->getEffectiveUrl());
+    $mf2 = $mf2->parse();
+
+    $entry = linkback_find_entry($mf2, $target);
+    if(!$entry) {
+        preg_match('/<title>([^<]+)', $response->getBody(), $match);
+        $entry = array(
+            'content' => array('html' => $response->getBody()),
+            'name' => $match[1] ? htmlspecialchars_decode($match[1]) : $source
+        );
+    }
+
+    if(!$entry['url']) {
+        $entry['url'] = array($response->getEffectiveUrl());
+    }
+
+    if(!$dupe) { $dupe = linkback_is_dupe('uri', $entry['url'][0]); }
+    if(!$dupe) { $dupe = linkback_is_dupe('url', $entry['url'][0]); }
+
+    $entry['type'] = linkback_entry_type($entry, $mf2, $target);
+    list($profile, $author) =  linkback_profile($entry, $mf2, $response, $target);
+    list($content, $options) = linkback_notice($source, $notice_or_user, $entry, $author, $mf2);
+
+    if($dupe) {
+        $orig = clone($dupe);
+
+        try {
+            // Ignore duplicate save error
+            try { $dupe->saveKnownReplies($options['replies']); } catch (ServerException $ex) {}
+            try { $dupe->saveKnownTags($options['tags']); } catch (ServerException $ex) {}
+            try { $dupe->saveKnownUrls($options['urls']); } catch (ServerException $ex) {}
+
+            if($options['reply_to']) { $dupe->reply_to = $options['reply_to']; }
+            if($options['repeat_of']) { $dupe->repeat_of = $options['repeat_of']; }
+            if($dupe->reply_to != $orig->reply_to || $dupe->repeat_of != $orig->repeat_of) {
+                $parent = Notice::getKV('id', $dupe->repost_of ? $dupe->repost_of : $dupe->reply_to);
+                if($parent instanceof Notice) {
+                    // If we changed the reply_to or repeat_of we might live in a new conversation now
+                    $dupe->conversation = $parent->conversation;
+                }
+            }
+            if($dupe->update($orig)) { $saved = $dupe; }
+            if($dupe->conversation != $orig->conversation && Conversation::noticeCount($orig->conversation) < 1) {
+                // Delete empty conversation
+                $emptyConversation = Conversation::getKV('id', $orig->conversation);
+                $emptyConversation->delete();
+            }
+        } catch (Exception $e) {
+            common_log(LOG_ERR, "Linkback update of remote message $source failed: " . $e->getMessage());
+            return false;
+        }
+        common_log(LOG_INFO, "Linkback updated remote message $source as notice id $saved->id");
+    } else if($entry['type'] == 'like' || ($entry['type'] == 'reply' && $entry['rsvp'])) {
+        $act = new Activity();
+        $act->type    = ActivityObject::ACTIVITY;
+        $act->time    = $options['created'] ? strtotime($options['created']) : time();
+        $act->title   = $entry["name"] ? $entry["name"][0] : _m("Favor");
+        $act->actor   = $profile->asActivityObject();
+        $act->target  = $notice_or_user->asActivityObject();
+        $act->objects = array(clone($act->target));
+
+        // TRANS: Message that is the "content" of a favorite (%1$s is the actor's nickname, %2$ is the favorited
+        //        notice's nickname and %3$s is the content of the favorited notice.)
+        $act->content = sprintf(_('%1$s favorited something by %2$s: %3$s'),
+                                $profile->getNickname(), $notice_or_user->getProfile()->getNickname(),
+                                $notice_or_user->rendered ?: $notice_or_user->content);
+        if($entry['rsvp']) {
+            $act->content = $options['rendered'];
+        }
+
+        $act->verb    = ActivityVerb::FAVORITE;
+        if(strtolower($entry['rsvp'][0]) == 'yes') {
+            $act->verb = 'http://activitystrea.ms/schema/1.0/rsvp-yes';
+        } else if(strtolower($entry['rsvp'][0]) == 'no') {
+            $act->verb = 'http://activitystrea.ms/schema/1.0/rsvp-no';
+        } else if(strtolower($entry['rsvp'][0]) == 'maybe') {
+            $act->verb = 'http://activitystrea.ms/schema/1.0/rsvp-maybe';
+        }
+
+        $act->id = $source;
+        $act->link = $entry['url'][0];
+
+        $options['source'] = 'linkback';
+        $options['mentions'] = $options['replies'];
+        unset($options['reply_to']);
+        unset($options['repeat_of']);
+
+        try {
+            $saved = Notice::saveActivity($act, $profile, $options);
+        } catch (Exception $e) {
+            common_log(LOG_ERR, "Linkback save of remote message $source failed: " . $e->getMessage());
+            return false;
+        }
+        common_log(LOG_INFO, "Linkback saved remote message $source as notice id $saved->id");
+    } else {
+        // Fallback is to make a notice manually
+        try {
+            $saved = Notice::saveNew($profile->id,
+                                     $content,
+                                     'linkback',
+                                     $options);
+        } catch (Exception $e) {
+            common_log(LOG_ERR, "Linkback save of remote message $source failed: " . $e->getMessage());
+            return false;
+        }
+        common_log(LOG_INFO, "Linkback saved remote message $source as notice id $saved->id");
+    }
+
+    return $saved->getLocalUrl();
+}

plugins/TwitterBridge/lib/twitterimport.php

@@ -102,6 +102,17 @@ class TwitterImport
             return Notice::getKV('id', $n2s->notice_id);
         }
 
+        $dupe = Notice::getKV('uri', $statusUri);
+        if($dupe instanceof Notice) {
+            // Add it to our record
+            Notice_to_status::saveNew($dupe->id, $statusId);
+            common_log(
+                LOG_INFO,
+                __METHOD__ . " - Ignoring duplicate import: {$statusId}"
+            );
+            return $dupe;
+        }
+
         // If it's a retweet, save it as a repeat!
         if (!empty($status->retweeted_status)) {
             common_log(LOG_INFO, "Status {$statusId} is a retweet of " . twitter_id($status->retweeted_status) . ".");