Major improvements to Twitter bridge friend syncing

lib/twitter.php

@@ -19,7 +19,7 @@
 
 if (!defined('LACONICA')) { exit(1); }
 
-define("TWITTER_SERVICE", 1); // Twitter is foreign_service ID 1
+define('TWITTER_SERVICE', 1); // Twitter is foreign_service ID 1
 
 function get_twitter_data($uri, $screen_name, $password)
 {
@@ -45,6 +45,10 @@ function get_twitter_data($uri, $screen_name, $password)
     if ($errmsg) {
         common_debug("Twitter bridge - cURL error: $errmsg - trying to load: $uri with user $screen_name.",
             __FILE__);
+
+        if (SCRIPT_DEBUG) {
+            print "cURL error: $errmsg - trying to load: $uri with user $screen_name.\n";
+        }
     }
 
     curl_close($ch);
@@ -52,63 +56,141 @@ function get_twitter_data($uri, $screen_name, $password)
     return $data;
 }
 
-function twitter_user_info($screen_name, $password)
+function twitter_json_data($uri, $screen_name, $password)
 {
+    $json_data = get_twitter_data($uri, $screen_name, $password);
 
-    $uri = "http://twitter.com/users/show/$screen_name.json";
+    if (!$json_data) {
-    $data = get_twitter_data($uri, $screen_name, $password);
+        return false;
+    }
+
+    $data = json_decode($json_data);
 
     if (!$data) {
         return false;
     }
 
-    $twit_user = json_decode($data);
+    return $data;
-
-    if (!$twit_user) {
-        return false;
-    }
-
-    return $twit_user;
 }
 
-function update_twitter_user($fuser, $twitter_id, $screen_name)
+function twitter_user_info($screen_name, $password)
 {
+    $uri = "http://twitter.com/users/show/$screen_name.json";
+    return twitter_json_data($uri, $screen_name, $password);
+}
 
-    $original = clone($fuser);
+function twitter_friends_ids($screen_name, $password)
-    $fuser->nickname = $screen_name;
+{
-    $fuser->uri = 'http://twitter.com/' . $screen_name;
+    $uri = "http://twitter.com/friends/ids/$screen_name.json";
-    $result = $fuser->updateKeys($original);
+    return twitter_json_data($uri, $screen_name, $password);
+}
+
+function update_twitter_user($twitter_id, $screen_name)
+{
+    $uri = 'http://twitter.com/' . $screen_name;
+
+    $fuser = new Foreign_user();
+
+    $fuser->query('BEGIN');
+
+    // Dropping down to SQL because regular db_object udpate stuff doesn't seem
+    // to work so good with tables that have multiple column primary keys
+
+    // Any time we update the uri for a forein user we have to make sure there
+    // are no dupe entries first -- unique constraint on the uri column
+
+    $qry = 'UPDATE foreign_user set uri = \'\' WHERE uri = ';
+    $qry .= '\'' . $uri . '\'' . ' AND service = ' . TWITTER_SERVICE;
+
+    $result = $fuser->query($qry);
+
+    if ($result) {
+        common_debug("Removed uri ($uri) from another foreign_user who was squatting on it.");
+        if (SCRIPT_DEBUG) {
+            print("Removed uri ($uri) from another Twitter user who was squatting on it.\n");
+        }
+    }
+
+    // Update the user
+    $qry = 'UPDATE foreign_user SET nickname = ';
+    $qry .= '\'' . $screen_name . '\'' . ', uri = \'' . $uri . '\' ';
+    $qry .= 'WHERE id = ' . $twitter_id . ' AND service = ' . TWITTER_SERVICE;
+
+    $result = $fuser->query($qry);
 
     if (!$result) {
+        common_log(LOG_WARNING,
+            "Couldn't update foreign_user data for Twitter user: $screen_name");
         common_log_db_error($fuser, 'UPDATE', __FILE__);
+        if (SCRIPT_DEBUG) {
+            print "UPDATE failed: for Twitter user:  $twitter_id - $screen_name. - ";
+            print common_log_objstring($fuser) . "\n";
+            $error = &PEAR::getStaticProperty('DB_DataObject','lastError');
+            print "DB_DataObject Error: " . $error->getMessage() . "\n";
+        }
         return false;
     }
 
+    $fuser->query('COMMIT');
+
+    $fuser->free();
+    unset($fuser);
+
     return true;
 }
 
 function add_twitter_user($twitter_id, $screen_name)
 {
 
+    $new_uri = 'http://twitter.com/' . $screen_name;
+
+    // Clear out any bad old foreign_users with the new user's legit URL
+    // This can happen when users move around or fakester accounts get
+    // repoed, and things like that.
+    $luser = new Foreign_user();
+    $luser->uri = $new_uri;
+    $luser->service = TWITTER_SERVICE;
+    $result = $luser->delete();
+
+    if ($result) {
+        common_log(LOG_WARNING,
+            "Twitter bridge - removed invalid Twitter user squatting on uri: $new_uri");
+        if (SCRIPT_DEBUG) {
+            print "Removed invalid Twitter user squatting on uri: $new_uri\n";
+        }
+    }
+
+    $luser->free();
+    unset($luser);
+
     // Otherwise, create a new Twitter user
-    $fuser = DB_DataObject::factory('foreign_user');
+    $fuser = new Foreign_user();
 
     $fuser->nickname = $screen_name;
     $fuser->uri = 'http://twitter.com/' . $screen_name;
     $fuser->id = $twitter_id;
-    $fuser->service = TWITTER_SERVICE; // Twitter
+    $fuser->service = TWITTER_SERVICE;
     $fuser->created = common_sql_now();
     $result = $fuser->insert();
 
     if (!$result) {
-        common_debug("Twitter bridge - failed to add new Twitter user: $twitter_id - $screen_name.");
+        common_log(LOG_WARNING,
+            "Twitter bridge - failed to add new Twitter user: $twitter_id - $screen_name.");
         common_log_db_error($fuser, 'INSERT', __FILE__);
-        return false;
+        if (SCRIPT_DEBUG) {
+            print "INSERT failed: could not add new Twitter user: $twitter_id - $screen_name. - ";
+            print common_log_objstring($fuser) . "\n";
+            $error = &PEAR::getStaticProperty('DB_DataObject','lastError');
+            print "DB_DataObject Error: " . $error->getMessage() . "\n";
+        }
+    } else {
+        common_debug("Twitter bridge - Added new Twitter user: $screen_name ($twitter_id).");
+        if (SCRIPT_DEBUG) {
+            print "Added new Twitter user: $screen_name ($twitter_id).\n";
+        }
     }
 
-    common_debug("Twitter bridge - Added new Twitter user: $screen_name ($twitter_id).");
+    return $result;
-
-    return true;
 }
 
 // Creates or Updates a Twitter user
@@ -117,53 +199,87 @@ function save_twitter_user($twitter_id, $screen_name)
 
     // Check to see whether the Twitter user is already in the system,
     // and update its screen name and uri if so.
-    $fuser = Foreign_user::getForeignUser($twitter_id, 1);
+    $fuser = Foreign_user::getForeignUser($twitter_id, TWITTER_SERVICE);
 
     if ($fuser) {
 
+        $result = true;
+
         // Only update if Twitter screen name has changed
         if ($fuser->nickname != $screen_name) {
+            $result = update_twitter_user($twitter_id, $screen_name);
 
             common_debug('Twitter bridge - Updated nickname (and URI) for Twitter user ' .
                 "$fuser->id to $screen_name, was $fuser->nickname");
 
-            return update_twitter_user($fuser, $twitter_id, $screen_name);
+            if (SCRIPT_DEBUG) {
+                print 'Updated nickname (and URI) for Twitter user ' .
+                    "$fuser->id to $screen_name, was $fuser->nickname\n";
             }
+        }
+
+        return $result;
 
     } else {
         return add_twitter_user($twitter_id, $screen_name);
     }
 
+    $fuser->free();
+    unset($fuser);
+
     return true;
 }
 
 function retreive_twitter_friends($twitter_id, $screen_name, $password)
 {
+    $friends = array();
 
     $uri = "http://twitter.com/statuses/friends/$twitter_id.json?page=";
-    $twitter_user = twitter_user_info($screen_name, $password);
+    $friends_ids = twitter_friends_ids($screen_name, $password);
 
-    // Calculate how many pages to get...
+    if (!$friends_ids) {
-    $pages = ceil($twitter_user->friends_count / 100);
+        return $friends;
-
-    if ($pages == 0) {
-        common_debug("Twitter bridge - Twitter user $screen_name has no friends! Lame.");
     }
 
-    $friends = array();
+    if (SCRIPT_DEBUG) {
+        print "Twitter 'social graph' ids method says $screen_name has " .
+            count($friends_ids) . " friends.\n";
+    }
+
+    // Calculate how many pages to get...
+    $pages = ceil(count($friends_ids) / 100);
+
+    if ($pages == 0) {
+        common_log(LOG_WARNING,
+            "Twitter bridge - $screen_name seems to have no friends.");
+        if (SCRIPT_DEBUG) {
+            print "$screen_name seems to have no friends.\n";
+        }
+    }
 
     for ($i = 1; $i <= $pages; $i++) {
 
         $data = get_twitter_data($uri . $i, $screen_name, $password);
 
         if (!$data) {
-            return null;
+            common_log(LOG_WARNING,
+                "Twitter bridge - Couldn't retrieve page $i of $screen_name's friends.");
+            if (SCRIPT_DEBUG) {
+                print "Couldn't retrieve page $i of $screen_name's friends.\n";
+            }
+            continue;
         }
 
         $more_friends = json_decode($data);
 
         if (!$more_friends) {
-            return null;
+
+            common_log(LOG_WARNING,
+                "Twitter bridge - No data for page $i of $screen_name's friends.");
+            if (SCRIPT_DEBUG) {
+                print "No data for page $i of $screen_name's friends.\n";
+            }
+            continue;
         }
 
          $friends = array_merge($friends, $more_friends);
@@ -177,19 +293,27 @@ function save_twitter_friends($user, $twitter_id, $screen_name, $password)
 
     $friends = retreive_twitter_friends($twitter_id, $screen_name, $password);
 
-    if (is_null($friends)) {
+    if (empty($friends)) {
-        common_debug("Twitter bridge - Couldn't get friends data from Twitter.");
+        common_debug("Twitter bridge - Couldn't get friends data from Twitter for $screen_name.");
+        if (SCRIPT_DEBUG) {
+            print "Couldn't get friends data from Twitter for $screen_name.\n";
+        }
         return false;
     }
 
     foreach ($friends as $friend) {
 
         $friend_name = $friend->screen_name;
-        $friend_id = $friend->id;
+        $friend_id = (int) $friend->id;
 
         // Update or create the Foreign_user record
         if (!save_twitter_user($friend_id, $friend_name)) {
-            return false;
+            common_log(LOG_WARNING,
+                "Twitter bridge - couldn't save $screen_name's friend, $friend_name.");
+            if (SCRIPT_DEBUG) {
+                print "Couldn't save $screen_name's friend, $friend_name.\n";
+            }
+            continue;
         }
 
         // Check to see if there's a related local user
@@ -199,8 +323,20 @@ function save_twitter_friends($user, $twitter_id, $screen_name, $password)
 
             // Get associated user and subscribe her
             $friend_user = User::staticGet('id', $flink->user_id);
-            subs_subscribe_to($user, $friend_user);
+            if (!empty($friend_user)) {
+                $result = subs_subscribe_to($user, $friend_user);
+
+                if ($result === true) {
                     common_debug("Twitter bridge - subscribed $friend_user->nickname to $user->nickname.");
+                    if (SCRIPT_DEBUG) {
+                        print("Subscribed $friend_user->nickname to $user->nickname.\n");
+                    }
+                } else {
+                    if (SCRIPT_DEBUG) {
+                        print "$result ($friend_user->nickname to $user->nickname)\n";
+                    }
+                }
+            }
         }
     }
 
@@ -295,4 +431,3 @@ function broadcast_twitter($notice)
 
     return $success;
 }
-

scripts/synctwitterfriends.php

@@ -18,7 +18,7 @@
  * along with this program.     If not, see <http://www.gnu.org/licenses/>.
  */
 
-# Abort if called from a web server
+// Abort if called from a web server
 if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) {
     print "This script must be run from the command line\n";
     exit();
@@ -27,11 +27,16 @@ if (isset($_SERVER) && array_key_exists('REQUEST_METHOD', $_SERVER)) {
 define('INSTALLDIR', realpath(dirname(__FILE__) . '/..'));
 define('LACONICA', true);
 
+// Set this to true to get useful console output
+define('SCRIPT_DEBUG', false);
+
 require_once(INSTALLDIR . '/lib/common.php');
 
 $flink = new Foreign_link();
 $flink->service = 1; // Twitter
-$flink->find();
+$cnt = $flink->find();
+
+print "Updating Twitter friends subscriptions for $cnt users.\n";
 
 while ($flink->fetch()) {
 
@@ -39,20 +44,30 @@ while ($flink->fetch()) {
 
         $user = User::staticGet($flink->user_id);
 
-        print "Updating Twitter friends for user $user->nickname ($user->id)\n";
+        if (empty($user)) {
+            common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id);
+            print "Unmatched user for ID $flink->user_id\n";
+            continue;
+        }
+
+        print "Updating Twitter friends for $user->nickname (Laconica ID: $user->id)... ";
 
         $fuser = $flink->getForeignUser();
 
-        $result = save_twitter_friends($user, $fuser->id, $fuser->nickname, $flink->credentials);
+        if (empty($fuser)) {
-
+            common_log(LOG_WARNING, "Unmatched user for ID " . $flink->user_id);
-        if ($result == false) {
+            print "Unmatched user for ID $flink->user_id\n";
-            print "Problems updating Twitter friends! Check the log.\n";
+            continue;
-            exit(1);
-        }
         }
 
+        $result = save_twitter_friends($user, $fuser->id,
+                       $fuser->nickname, $flink->credentials);
+        if (SCRIPT_DEBUG) {
+            print "\nDONE\n";
+        } else {
+            print "DONE\n";
+        }
+    }
 }
 
 exit(0);
-
-