From fb4e9b234d5e28a44369b9fea62d92cfc72d463d Mon Sep 17 00:00:00 2001 From: Mikael Nordfeldth Date: Fri, 4 Oct 2013 13:36:45 +0200 Subject: [PATCH] Twitter Import improvements. Still buggy? Apparently mrvdb has problems with duplicate inserts and missing files when unlinking. It could be due to coding, or it could be due to parallelizing. --- classes/Avatar.php | 15 ++-- lib/filenotfoundexception.php | 41 ++++++++++ plugins/TwitterBridge/TwitterBridgePlugin.php | 2 +- .../daemons/synctwitterfriends.php | 2 - .../daemons/twitterstatusfetcher.php | 1 - plugins/TwitterBridge/lib/twitterimport.php | 77 ++++++------------- 6 files changed, 73 insertions(+), 65 deletions(-) create mode 100644 lib/filenotfoundexception.php diff --git a/classes/Avatar.php b/classes/Avatar.php index 29920a5f62..01d42f7406 100644 --- a/classes/Avatar.php +++ b/classes/Avatar.php @@ -89,16 +89,13 @@ class Avatar extends Managed_DataObject if (!$avatar->find(true)) { throw new NoResultException($avatar); } - return $avatar; - } - - public static function hasUploaded(Profile $profile) { - try { - $avatar = Avatar::getUploaded($profile); - } catch (NoResultException $e) { - return false; + if (!file_exists(Avatar::path($avatar->filename))) { + // The delete call may be odd for, say, unmounted filesystems + // that cause a file to currently not exist, but actually it does... + $avatar->delete(); + throw new FileNotFoundException(Avatar::path($avatar->filename)); } - return file_exists(Avatar::path($avatar->filename)); + return $avatar; } public static function getProfileAvatars(Profile $target) { diff --git a/lib/filenotfoundexception.php b/lib/filenotfoundexception.php new file mode 100644 index 0000000000..e7ea7bc23c --- /dev/null +++ b/lib/filenotfoundexception.php @@ -0,0 +1,41 @@ +. + * + * @category Exception + * @package GNUSocial + * @author Mikael Nordfeldth + * @copyright 2013 Free Software Foundation, Inc. + * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html AGPLv3 + * @link http://www.gnu.org/software/social/ + */ + +if (!defined('GNUSOCIAL')) { exit(1); } + +class FileNotFoundException extends ServerException +{ + public $path = null; + + public function __construct($path) + { + $this->path = $path; + parent::__construct(_('File not found in filesystem.'), 404); + } +} diff --git a/plugins/TwitterBridge/TwitterBridgePlugin.php b/plugins/TwitterBridge/TwitterBridgePlugin.php index f11234b5e1..c513772baa 100644 --- a/plugins/TwitterBridge/TwitterBridgePlugin.php +++ b/plugins/TwitterBridge/TwitterBridgePlugin.php @@ -364,7 +364,7 @@ class TwitterBridgePlugin extends Plugin // For saving the last-synched status of various timelines // home_timeline, messages (in), messages (out), ... - $schema->ensureTable('twitter_synch_status', Twitter_sync_status::schemaDef()); + $schema->ensureTable('twitter_synch_status', Twitter_synch_status::schemaDef()); // For storing user-submitted flags on profiles $schema->ensureTable('notice_to_status', Notice_to_status::schemaDef()); diff --git a/plugins/TwitterBridge/daemons/synctwitterfriends.php b/plugins/TwitterBridge/daemons/synctwitterfriends.php index 3f1a8af6fb..651c4d34da 100755 --- a/plugins/TwitterBridge/daemons/synctwitterfriends.php +++ b/plugins/TwitterBridge/daemons/synctwitterfriends.php @@ -31,9 +31,7 @@ Batch script for synching local friends with Twitter friends. END_OF_TRIM_HELP; require_once INSTALLDIR . '/scripts/commandline.inc'; -require_once INSTALLDIR . '/lib/parallelizingdaemon.php'; require_once INSTALLDIR . '/plugins/TwitterBridge/twitter.php'; -require_once INSTALLDIR . '/plugins/TwitterBridge/twitteroauthclient.php'; /** * Daemon to sync local friends with Twitter friends diff --git a/plugins/TwitterBridge/daemons/twitterstatusfetcher.php b/plugins/TwitterBridge/daemons/twitterstatusfetcher.php index 6599058fd4..771544fcce 100755 --- a/plugins/TwitterBridge/daemons/twitterstatusfetcher.php +++ b/plugins/TwitterBridge/daemons/twitterstatusfetcher.php @@ -40,7 +40,6 @@ require_once INSTALLDIR . '/scripts/commandline.inc'; require_once INSTALLDIR . '/lib/common.php'; require_once INSTALLDIR . '/lib/daemon.php'; require_once INSTALLDIR . '/plugins/TwitterBridge/twitter.php'; -require_once INSTALLDIR . '/plugins/TwitterBridge/lib/twitteroauthclient.php'; /** * Fetch statuses from Twitter diff --git a/plugins/TwitterBridge/lib/twitterimport.php b/plugins/TwitterBridge/lib/twitterimport.php index eb58bc7373..d8b9f7a6bb 100644 --- a/plugins/TwitterBridge/lib/twitterimport.php +++ b/plugins/TwitterBridge/lib/twitterimport.php @@ -230,44 +230,19 @@ class TwitterImport $profile->profileurl = $profileurl; $profile->limit(1); - if (!$profile->find()) { + if (!$profile->find(true)) { throw new NoResultException($profile); } - - $profile->fetch(); return $profile; } - /** - * Check to see if this Twitter status has already been imported - * - * @param Profile $profile Twitter user's local profile - * @param string $statusUri URI of the status on Twitter - * - * @return mixed value a matching Notice or null - */ - function checkDupe($profile, $statusUri) - { - $notice = new Notice(); - $notice->uri = $statusUri; - $notice->profile_id = $profile->id; - $notice->limit(1); - - if ($notice->find()) { - $notice->fetch(); - return $notice; - } - - return null; - } - protected function ensureProfile($twuser) { // check to see if there's already a profile for this user $profileurl = 'http://twitter.com/' . $twuser->screen_name; try { $profile = $this->getProfileByUrl($twuser->screen_name, $profileurl); - $this->checkAvatar($twuser, $profile); + $this->updateAvatar($twuser, $profile); return $profile; } catch (NoResultException $e) { common_debug(__METHOD__ . ' - Adding profile and remote profile ' . @@ -323,29 +298,11 @@ class TwitterImport return $profile; } - protected function checkAvatar($twuser, Profile $profile) - { - $path_parts = pathinfo($twuser->profile_image_url); - $ext = isset($path_parts['extension']) - ? '.'.$path_parts['extension'] - : ''; // some lack extension - $img_root = basename($path_parts['basename'], '_normal'.$ext); // cut off extension - $newname = "Twitter_{$twuser->id}_{$img_root}_{$this->avatarsizename}{$ext}"; - - try { - $avatar = Avatar::getUploaded($profile); - $oldname = $avatar->filename; - $avatar->free(); - } catch (Exception $e) { - $oldname = null; - } - - if ($newname != $oldname || !Avatar::hasUploaded($profile)) { - common_debug(__METHOD__ . " - Avatar for {$profile->nickname} has changed."); - $this->updateAvatar($twuser, $profile); - } - } - + /* + * Checks whether we have to update the profile's avatar + * + * @return true when updated, false on failure, null when no action taken + */ protected function updateAvatar($twuser, Profile $profile) { $path_parts = pathinfo($twuser->profile_image_url); @@ -353,17 +310,33 @@ class TwitterImport ? '.'.$path_parts['extension'] : ''; // some lack extension $img_root = basename($path_parts['basename'], '_normal'.$ext); // cut off extension - $url = "{$path_parts['dirname']}/{$img_root}_{$this->avatarsizename}{$ext}"; $filename = "Twitter_{$twuser->id}_{$img_root}_{$this->avatarsizename}{$ext}"; - $mediatype = $this->getMediatype(substr($ext, 1)); try { + $avatar = Avatar::getUploaded($profile); + if ($avatar->filename === $filename) { + return null; + } + // else we continue with creating a new avatar + } catch (Exception $e) { + // Avatar was not found. We can catch NoResultException or FileNotFoundException + // but generally we just want to continue creating a new avatar. + } + + $url = "{$path_parts['dirname']}/{$img_root}_{$this->avatarsizename}{$ext}"; + $mediatype = $this->getMediatype(mb_substr($ext, 1)); + + try { + common_debug(__METHOD__ . " - Updating profile avatar (profile_id={$profile->id} to {$filename}"); $this->newAvatar($profile, $url, $filename, $mediatype); } catch (Exception $e) { if (file_exists(Avatar::path($filename))) { unlink(Avatar::path($filename)); } + return false; } + + return true; } protected function getMediatype($ext)