From d01f44ee995f0db28ae1a7bb4ae800bd8f1b55ad Mon Sep 17 00:00:00 2001 From: Alexei Sorokin Date: Wed, 10 Jun 2020 16:52:00 +0300 Subject: [PATCH] [DATABASE] Some query improvements Make common_sql_weight employ standard SQL functions for the timestamp difference in seconds. Also replace UTC_TIMESTAMP in the MariaDB-specific part with CURRENT_TIMESTAMP as it is the only occurence and GNU social sets UTC as a default timezone. In a delete_orphan_files.php script simplify the main query considerably. In clean_profiles.php stop using COUNT as if it is ANY, that is unnecessary punishment for the database. Instead implement the anti-join with a left outer join. In Autocomplete and Activitypub_profile use joins instead of a WHERE OR anti-pattern for the semi-joins. In lib/ui/galleryaction.php replace a CROSS JOIN with an INNER JOIN. In actions/sup.php remove a redundant subquery: WHERE is applied before grouping either way. --- actions/sup.php | 18 +-- lib/ui/galleryaction.php | 15 +- lib/util/util.php | 14 +- .../classes/Activitypub_profile.php | 20 ++- plugins/Autocomplete/actions/autocomplete.php | 128 ++++++++++-------- plugins/Nodeinfo/actions/nodeinfo_2_0.php | 3 +- plugins/OStatus/scripts/update-profile.php | 20 +-- scripts/clean_profiles.php | 62 +++++---- scripts/delete_orphan_files.php | 47 ++++--- 9 files changed, 187 insertions(+), 140 deletions(-) diff --git a/actions/sup.php b/actions/sup.php index cf78a2f7d3..f7afff5a48 100644 --- a/actions/sup.php +++ b/actions/sup.php @@ -65,17 +65,17 @@ class SupAction extends Action $divider = common_sql_date(time() - $seconds); - $notice->query('SELECT profile_id, max(id) AS max_id ' . - 'FROM ( ' . - 'SELECT profile_id, id FROM notice ' . - "WHERE created > TIMESTAMP '" . $divider . "' " . - ') AS latest ' . - 'GROUP BY profile_id'); + $notice->selectAdd(); + $notice->selectAdd('profile_id, MAX(id) AS max_id'); + $notice->whereAdd("created > TIMESTAMP '{$divider}'"); + $notice->groupBy('profile_id'); - $updates = array(); + $updates = []; - while ($notice->fetch()) { - $updates[] = array($notice->profile_id, $notice->max_id); + if ($notice->find()) { + while ($notice->fetch()) { + $updates[] = [$notice->profile_id, $notice->max_id]; + } } return $updates; diff --git a/lib/ui/galleryaction.php b/lib/ui/galleryaction.php index b1cffb9c09..26a27f0af6 100644 --- a/lib/ui/galleryaction.php +++ b/lib/ui/galleryaction.php @@ -107,13 +107,14 @@ class GalleryAction extends ProfileAction public function getTags($lst, $usr) { $profile_tag = new Notice_tag(); - $profile_tag->query('SELECT DISTINCT(tag) ' . - 'FROM profile_tag, subscription ' . - 'WHERE tagger = ' . $this->target->id . ' ' . - 'AND ' . $usr . ' = ' . $this->target->id . ' ' . - 'AND ' . $lst . ' = tagged ' . - 'AND tagger <> tagged'); - $tags = array(); + $profile_tag->query( + <<target->id} AND tagger <> tagged; + END + ); + $tags = []; while ($profile_tag->fetch()) { $tags[] = $profile_tag->tag; } diff --git a/lib/util/util.php b/lib/util/util.php index 657a031e85..4978190fbb 100644 --- a/lib/util/util.php +++ b/lib/util/util.php @@ -1647,14 +1647,16 @@ function common_sql_date($datetime) */ function common_sql_weight($column, $dropoff) { - if (common_config('db', 'type') == 'pgsql') { - // PostgreSQL doesn't support timestampdiff function. - // @fixme will this use the right time zone? - // @fixme does this handle cross-year subtraction correctly? - return "sum(exp(-extract(epoch from (now() - $column)) / $dropoff))"; + if (common_config('db', 'type') !== 'mysql') { + $expr = sprintf( + '(((EXTRACT(DAY %1$s) * 24 + EXTRACT(HOUR %1$s)) * 60 + ' + . 'EXTRACT(MINUTE %1$s)) * 60 + EXTRACT(SECOND %1$s))', + "FROM ({$column} - CURRENT_TIMESTAMP)" + ); } else { - return "sum(exp(timestampdiff(second, utc_timestamp(), $column) / $dropoff))"; + $expr = "timestampdiff(SECOND, CURRENT_TIMESTAMP, {$column})"; } + return "SUM(EXP({$expr} / {$dropoff}))"; } function common_redirect($url, $code=307) diff --git a/plugins/ActivityPub/classes/Activitypub_profile.php b/plugins/ActivityPub/classes/Activitypub_profile.php index b6a63c736e..06dfeb29f6 100644 --- a/plugins/ActivityPub/classes/Activitypub_profile.php +++ b/plugins/ActivityPub/classes/Activitypub_profile.php @@ -557,9 +557,15 @@ class Activitypub_profile extends Managed_DataObject $user_table = common_database_tablename('user'); $sub = new Subscription(); $sub->subscribed = $profile->id; + $sub->_join .= "\n" . <<whereAdd('subscriber <> subscribed'); - $sub->whereAdd("subscriber IN (SELECT id FROM {$user_table} UNION SELECT profile_id AS id FROM activitypub_profile)"); - $cnt = $sub->count('distinct subscriber'); + $cnt = $sub->count('DISTINCT subscriber'); self::cacheSet(sprintf('activitypub_profile:subscriberCount:%d', $profile->id), $cnt); @@ -585,9 +591,15 @@ class Activitypub_profile extends Managed_DataObject $user_table = common_database_tablename('user'); $sub = new Subscription(); $sub->subscriber = $profile->id; + $sub->_join .= "\n" . <<whereAdd('subscriber <> subscribed'); - $sub->whereAdd("subscribed IN (SELECT id FROM {$user_table} UNION SELECT profile_id AS id FROM activitypub_profile)"); - $cnt = $sub->count('distinct subscribed'); + $cnt = $sub->count('DISTINCT subscribed'); self::cacheSet(sprintf('activitypub_profile:subscriptionCount:%d', $profile->id), $cnt); diff --git a/plugins/Autocomplete/actions/autocomplete.php b/plugins/Autocomplete/actions/autocomplete.php index f7d79a4999..37014c99b9 100644 --- a/plugins/Autocomplete/actions/autocomplete.php +++ b/plugins/Autocomplete/actions/autocomplete.php @@ -1,47 +1,43 @@ . + /** - * StatusNet, the distributed open-source microblogging tool - * * List profiles and groups for autocompletion * - * PHP version 5 - * - * LICENCE: This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * * @category Plugin - * @package StatusNet + * @package GNUsocial * @author Craig Andrews * @author Mikael Nordfeldth * @copyright 2008-2009 StatusNet, Inc. * @copyright 2009-2013 Free Software Foundation, Inc http://www.fsf.org - * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 - * @link http://status.net/ + * @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later */ -if (!defined('GNUSOCIAL')) { exit(1); } +defined('GNUSOCIAL') || die(); /** * List users for autocompletion * * This is the form for adding a new g * - * @category Plugin - * @package StatusNet - * @author Craig Andrews - * @author Mikael Nordfeldth - * @license http://www.fsf.org/licensing/licenses/agpl-3.0.html GNU Affero General Public License version 3.0 - * @link http://status.net/ + * @category Plugin + * @package GNUsocial + * @author Craig Andrews + * @author Mikael Nordfeldth + * @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later */ class AutocompleteAction extends Action { @@ -57,14 +53,14 @@ class AutocompleteAction extends Action * * @return int last-modified date as unix timestamp */ - function lastModified() + public function lastModified() { - $max=0; - foreach($this->profiles as $profile){ + $max = 0; + foreach ($this->profiles as $profile) { $max = max($max, strtotime($profile->modified)); } - foreach($this->groups as $group){ - $max = max($max,strtotime($group->modified)); + foreach ($this->groups as $group) { + $max = max($max, strtotime($group->modified)); } // but maybe this file has been modified after that and could // respond differently @@ -80,11 +76,13 @@ class AutocompleteAction extends Action * * @return string etag */ - function etag() + public function etag() { return '"' . implode(':', array($this->arg('action'), common_user_cache_hash(), - crc32($this->arg('term')), //the actual string can have funny characters in we don't want showing up in the etag + // the actual string can have funny characters in we don't want + // showing up in the etag + crc32($this->arg('term')), $this->arg('limit'), $this->lastModified())) . '"'; } @@ -96,39 +94,55 @@ class AutocompleteAction extends Action parent::prepare($args); - $this->groups=array(); - $this->profiles=array(); + $this->groups = []; + $this->profiles = []; $term = $this->arg('term'); $limit = $this->arg('limit'); - if($limit > 200) $limit=200; //prevent DOS attacks - if(substr($term,0,1)=='@'){ + // prevent DOS attacks + if ($limit > 200) { + $limit = 200; + } + if (substr($term, 0, 1) === '@') { //profile search - $term=substr($term,1); + $term = substr($term, 1); $user_table = common_database_tablename('user'); $profile = new Profile(); + $profile->_join .= sprintf( + "\n" . <<<'END' + LEFT JOIN ( + SELECT id FROM %s + UNION ALL + SELECT subscribed AS id FROM subscription WHERE subscriber = %d + ) AS t1 USING (id) + END, + $user_table, + $this->scoped->id + ); + $profile->whereAdd('t1.id IS NOT NULL'); + $profile->whereAdd('nickname LIKE \'' . trim($profile->escape($term), '\'') . '%\''); $profile->limit($limit); - $profile->whereAdd('nickname like \'' . trim($profile->escape($term), '\'') . '%\''); - $profile->whereAdd(sprintf('id in (SELECT id FROM %s) OR ' - . 'id in (SELECT subscribed from subscription' - . ' where subscriber = %d)', $user_table, $this->scoped->id)); + if ($profile->find()) { - while($profile->fetch()) { - $this->profiles[]=clone($profile); + while ($profile->fetch()) { + $this->profiles[] = clone($profile); } } } - if(substr($term,0,1)=='!'){ + if (substr($term, 0, 1) === '!') { //group search - $term=substr($term,1); + $term = substr($term, 1); $group = new User_group(); - $group->limit($limit); - $group->whereAdd('nickname like \'' . trim($group->escape($term), '\'') . '%\''); //Can't post to groups we're not subscribed to...: - $group->whereAdd(sprintf('id in (SELECT group_id FROM group_member' - . ' WHERE profile_id = %d)', $this->scoped->id)); - if($group->find()){ - while($group->fetch()) { - $this->groups[]=clone($group); + $group->whereAdd(sprintf( + 'id IN (SELECT group_id FROM group_member WHERE profile_id = %d)', + $this->scoped->id + )); + $group->whereAdd('nickname LIKE \'' . trim($group->escape($term), '\'') . '%\''); + $group->limit($limit); + + if ($group->find()) { + while ($group->fetch()) { + $this->groups[] = clone($group); } } } @@ -140,7 +154,7 @@ class AutocompleteAction extends Action parent::handle(); $results = array(); - foreach($this->profiles as $profile){ + foreach ($this->profiles as $profile) { $avatarUrl = $profile->avatarUrl(AVATAR_MINI_SIZE); $acct = $profile->getAcctUri(); $identifier = explode(':', $profile->getAcctUri(), 2)[1]; @@ -153,7 +167,7 @@ class AutocompleteAction extends Action 'type' => 'user' ); } - foreach($this->groups as $group){ + foreach ($this->groups as $group) { $profile = $group->getProfile(); // sigh.... encapsulate this upstream! if ($group->mini_logo) { @@ -181,7 +195,7 @@ class AutocompleteAction extends Action * * @return boolean is read only action? */ - function isReadOnly($args) + public function isReadOnly($args) { return true; } diff --git a/plugins/Nodeinfo/actions/nodeinfo_2_0.php b/plugins/Nodeinfo/actions/nodeinfo_2_0.php index 9cc114a03d..dc07dd279e 100644 --- a/plugins/Nodeinfo/actions/nodeinfo_2_0.php +++ b/plugins/Nodeinfo/actions/nodeinfo_2_0.php @@ -338,7 +338,8 @@ class Nodeinfo_2_0Action extends Action UNION ALL SELECT id FROM {$userTable} WHERE {$userTable}.created >= CURRENT_DATE - INTERVAL '{$days}' DAY ) AS source - WHERE profile_id NOT IN (SELECT profile_id FROM profile_role WHERE role = 'silenced') + LEFT JOIN profile_role USING (profile_id) + WHERE profile_role.profile_id IS NULL OR profile_role.role <> 'silenced'; END; $activeUsersCount = new DB_DataObject(); diff --git a/plugins/OStatus/scripts/update-profile.php b/plugins/OStatus/scripts/update-profile.php index c67b5a2871..7e2a58ff56 100755 --- a/plugins/OStatus/scripts/update-profile.php +++ b/plugins/OStatus/scripts/update-profile.php @@ -102,14 +102,18 @@ if ($feedurl != $oprofile->feeduri || $salmonuri != $oprofile->salmonuri) { print "\n"; print "Updating...\n"; // @fixme update keys :P - #$orig = clone($oprofile); - #$oprofile->feeduri = $feedurl; - #$oprofile->salmonuri = $salmonuri; - #$ok = $oprofile->update($orig); - $ok = $oprofile->query('UPDATE ostatus_profile SET ' . - 'feeduri=\'' . $oprofile->escape($feedurl) . '\',' . - 'salmonuri=\'' . $oprofile->escape($salmonuri) . '\' ' . - 'WHERE uri=\'' . $oprofile->escape($uri) . '\''); + //$orig = clone($oprofile); + //$oprofile->feeduri = $feedurl; + //$oprofile->salmonuri = $salmonuri; + //$ok = $oprofile->update($orig); + $ok = $oprofile->query( + <<. - */ +// This file is part of GNU social - https://www.gnu.org/software/social +// +// GNU social is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// GNU social is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with GNU social. If not, see . define('INSTALLDIR', dirname(__DIR__)); define('PUBLICDIR', INSTALLDIR . DIRECTORY_SEPARATOR . 'public'); $shortoptions = 'y'; -$longoptions = array('yes'); +$longoptions = ['yes']; $helptext = <<query('SELECT * FROM profile WHERE ' . - 'NOT (SELECT COUNT(*) FROM notice WHERE profile_id=profile.id) ' . - "AND NOT (SELECT COUNT(*) FROM {$user_table} WHERE user.id=profile.id) " . - 'AND NOT (SELECT COUNT(*) FROM user_group WHERE user_group.profile_id=profile.id) ' . - 'AND NOT (SELECT COUNT(*) FROM subscription WHERE subscriber=profile.id OR subscribed=profile.id) '); +$profile->query( + <<fetch()) { echo ' '.$profile->getID().':'.$profile->getNickname(); $profile->delete(); diff --git a/scripts/delete_orphan_files.php b/scripts/delete_orphan_files.php index 8c1212ef51..ee11fd83d8 100755 --- a/scripts/delete_orphan_files.php +++ b/scripts/delete_orphan_files.php @@ -1,21 +1,23 @@ #!/usr/bin/env php . +// This file is part of GNU social - https://www.gnu.org/software/social +// +// GNU social is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// GNU social is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with GNU social. If not, see . + +/** + * @copyright 2008, 2009 StatusNet, Inc. + * @license https://www.gnu.org/licenses/agpl.html GNU AGPL v3 or later */ define('INSTALLDIR', dirname(__DIR__)); @@ -43,12 +45,13 @@ require_once INSTALLDIR.'/scripts/commandline.inc'; print "Finding File entries that are not related to a Notice (or the notice has been deleted)..."; $file = new File(); -$sql = 'SELECT file.* FROM file'. - ' LEFT JOIN file_to_post ON file_to_post.file_id=file.id'. - ' WHERE'. - ' NOT EXISTS (SELECT file_to_post.file_id FROM file_to_post WHERE file.id=file_to_post.file_id)'. - ' OR NOT EXISTS (SELECT notice.id FROM notice WHERE notice.id=file_to_post.post_id)'. - ' GROUP BY file.id;'; +$sql = <<<'END' + SELECT file.* + FROM file_to_post + INNER JOIN notice ON file_to_post.post_id = notice.id + RIGHT JOIN file ON file_to_post.file_id = file.id + WHERE file_to_post.file_id IS NULL; + END; if ($file->query($sql) !== false) { print " {$file->N} found.\n";