diff --git a/boot.php b/boot.php index 8492701a0c..402273bddf 100644 --- a/boot.php +++ b/boot.php @@ -39,7 +39,6 @@ require_once 'include/features.php'; require_once 'include/identity.php'; require_once 'update.php'; require_once 'include/dbstructure.php'; -require_once 'include/poller.php'; define('FRIENDICA_PLATFORM', 'Friendica'); define('FRIENDICA_CODENAME', 'Asparagus'); @@ -630,7 +629,7 @@ function check_db($via_worker) } if ($build != DB_UPDATE_VERSION) { // When we cannot execute the database update via the worker, we will do it directly - if (!Worker::add(PRIORITY_CRITICAL, 'dbupdate') && $via_worker) { + if (!Worker::add(PRIORITY_CRITICAL, 'DBUpdate') && $via_worker) { update_db(get_app()); } } diff --git a/include/Contact.php b/include/Contact.php index 5d34e45262..636bbc4a7a 100644 --- a/include/Contact.php +++ b/include/Contact.php @@ -37,7 +37,7 @@ function user_remove($uid) { Worker::add(PRIORITY_HIGH, "notifier", "removeme", $uid); // Send an update to the directory - Worker::add(PRIORITY_LOW, "directory", $r['url']); + Worker::add(PRIORITY_LOW, "Directory", $r['url']); if($uid == local_user()) { unset($_SESSION['authenticated']); @@ -68,7 +68,7 @@ function contact_remove($id) { dba::delete('contact', array('id' => $id)); // Delete the rest in the background - Worker::add(PRIORITY_LOW, 'remove_contact', $id); + Worker::add(PRIORITY_LOW, 'RemoveContact', $id); } diff --git a/include/api.php b/include/api.php index 33624dcf6e..69569bdc18 100644 --- a/include/api.php +++ b/include/api.php @@ -3870,7 +3870,7 @@ function api_account_update_profile_image($type) //$user = api_get_user(get_app()); $url = System::baseUrl() . '/profile/' . get_app()->user['nickname']; if ($url && strlen(Config::get('system', 'directory'))) { - Worker::add(PRIORITY_LOW, "directory", $url); + Worker::add(PRIORITY_LOW, "Directory", $url); } Worker::add(PRIORITY_LOW, 'profile_update', api_user()); diff --git a/include/cronjobs.php b/include/cronjobs.php deleted file mode 100644 index 2f6ade9c96..0000000000 --- a/include/cronjobs.php +++ /dev/null @@ -1,281 +0,0 @@ - '%s' - AND `account_expires_on` < UTC_TIMESTAMP()", dbesc(NULL_DATE)); - - // delete user records for recently removed accounts - $r = q("SELECT * FROM `user` WHERE `account_removed` AND `account_expires_on` < UTC_TIMESTAMP() - INTERVAL 3 DAY"); - if (DBM::is_result($r)) { - foreach ($r as $user) { - dba::delete('user', array('uid' => $user['uid'])); - } - } -} - -/** - * @brief Clear cache entries - * - * @param App $a - */ -function cron_clear_cache(App $a) { - - $last = Config::get('system','cache_last_cleared'); - - if ($last) { - $next = $last + (3600); // Once per hour - $clear_cache = ($next <= time()); - } else { - $clear_cache = true; - } - - if (!$clear_cache) { - return; - } - - // clear old cache - Cache::clear(); - - // clear old item cache files - clear_cache(); - - // clear cache for photos - clear_cache($a->get_basepath(), $a->get_basepath()."/photo"); - - // clear smarty cache - clear_cache($a->get_basepath()."/view/smarty3/compiled", $a->get_basepath()."/view/smarty3/compiled"); - - // clear cache for image proxy - if (!Config::get("system", "proxy_disabled")) { - clear_cache($a->get_basepath(), $a->get_basepath()."/proxy"); - - $cachetime = Config::get('system','proxy_cache_time'); - if (!$cachetime) { - $cachetime = PROXY_DEFAULT_TIME; - } - q('DELETE FROM `photo` WHERE `uid` = 0 AND `resource-id` LIKE "pic:%%" AND `created` < NOW() - INTERVAL %d SECOND', $cachetime); - } - - // Delete the cached OEmbed entries that are older than one year - q("DELETE FROM `oembed` WHERE `created` < NOW() - INTERVAL 3 MONTH"); - - // Delete the cached "parse_url" entries that are older than one year - q("DELETE FROM `parsed_url` WHERE `created` < NOW() - INTERVAL 3 MONTH"); - - // Maximum table size in megabyte - $max_tablesize = intval(Config::get('system','optimize_max_tablesize')) * 1000000; - if ($max_tablesize == 0) { - $max_tablesize = 100 * 1000000; // Default are 100 MB - } - if ($max_tablesize > 0) { - // Minimum fragmentation level in percent - $fragmentation_level = intval(Config::get('system','optimize_fragmentation')) / 100; - if ($fragmentation_level == 0) { - $fragmentation_level = 0.3; // Default value is 30% - } - - // Optimize some tables that need to be optimized - $r = q("SHOW TABLE STATUS"); - foreach ($r as $table) { - - // Don't optimize tables that are too large - if ($table["Data_length"] > $max_tablesize) { - continue; - } - - // Don't optimize empty tables - if ($table["Data_length"] == 0) { - continue; - } - - // Calculate fragmentation - $fragmentation = $table["Data_free"] / ($table["Data_length"] + $table["Index_length"]); - - logger("Table ".$table["Name"]." - Fragmentation level: ".round($fragmentation * 100, 2), LOGGER_DEBUG); - - // Don't optimize tables that needn't to be optimized - if ($fragmentation < $fragmentation_level) { - continue; - } - - // So optimize it - logger("Optimize Table ".$table["Name"], LOGGER_DEBUG); - q("OPTIMIZE TABLE `%s`", dbesc($table["Name"])); - } - } - - Config::set('system','cache_last_cleared', time()); -} - -/** - * @brief Repair missing values in Diaspora contacts - * - * @param App $a - */ -function cron_repair_diaspora(App $a) { - - $starttime = time(); - - $r = q("SELECT `id`, `url` FROM `contact` - WHERE `network` = '%s' AND (`batch` = '' OR `notify` = '' OR `poll` = '' OR pubkey = '') - ORDER BY RAND() LIMIT 50", dbesc(NETWORK_DIASPORA)); - if (!DBM::is_result($r)) { - return; - } - - foreach ($r AS $contact) { - // Quit the loop after 3 minutes - if (time() > ($starttime + 180)) { - return; - } - - if (!PortableContact::reachable($contact["url"])) { - continue; - } - - $data = Probe::uri($contact["url"]); - if ($data["network"] != NETWORK_DIASPORA) { - continue; - } - - logger("Repair contact ".$contact["id"]." ".$contact["url"], LOGGER_DEBUG); - q("UPDATE `contact` SET `batch` = '%s', `notify` = '%s', `poll` = '%s', pubkey = '%s' WHERE `id` = %d", - dbesc($data["batch"]), dbesc($data["notify"]), dbesc($data["poll"]), dbesc($data["pubkey"]), - intval($contact["id"])); - } -} - -/** - * @brief Do some repairs in database entries - * - */ -function cron_repair_database() { - - // Sometimes there seem to be issues where the "self" contact vanishes. - // We haven't found the origin of the problem by now. - $r = q("SELECT `uid` FROM `user` WHERE NOT EXISTS (SELECT `uid` FROM `contact` WHERE `contact`.`uid` = `user`.`uid` AND `contact`.`self`)"); - if (DBM::is_result($r)) { - foreach ($r AS $user) { - logger('Create missing self contact for user '.$user['uid']); - user_create_self_contact($user['uid']); - } - } - - // Set the parent if it wasn't set. (Shouldn't happen - but does sometimes) - // This call is very "cheap" so we can do it at any time without a problem - q("UPDATE `item` INNER JOIN `item` AS `parent` ON `parent`.`uri` = `item`.`parent-uri` AND `parent`.`uid` = `item`.`uid` SET `item`.`parent` = `parent`.`id` WHERE `item`.`parent` = 0"); - - // There was an issue where the nick vanishes from the contact table - q("UPDATE `contact` INNER JOIN `user` ON `contact`.`uid` = `user`.`uid` SET `nick` = `nickname` WHERE `self` AND `nick`=''"); - - // Update the global contacts for local users - $r = q("SELECT `uid` FROM `user` WHERE `verified` AND NOT `blocked` AND NOT `account_removed` AND NOT `account_expired`"); - if (DBM::is_result($r)) { - foreach ($r AS $user) { - GlobalContact::updateForUser($user["uid"]); - } - } - - /// @todo - /// - remove thread entries without item - /// - remove sign entries without item - /// - remove children when parent got lost - /// - set contact-id in item when not present -} diff --git a/include/dbclean.php b/include/dbclean.php deleted file mode 100644 index 1e1dd90829..0000000000 --- a/include/dbclean.php +++ /dev/null @@ -1,303 +0,0 @@ - 0))) { - Worker::add(PRIORITY_LOW, 'dbclean', $i); - } - } - } else { - remove_orphans($stage); - } -} - -/** - * @brief Remove orphaned database entries - * @param integer $stage What should be deleted? - * - * Values for $stage: - * ------------------ - * 1: Old global item entries from item table without user copy. - * 2: Items without parents. - * 3: Orphaned data from thread table. - * 4: Orphaned data from notify table. - * 5: Orphaned data from notify-threads table. - * 6: Orphaned data from sign table. - * 7: Orphaned data from term table. - * 8: Expired threads. - * 9: Old global item entries from expired threads - */ -function remove_orphans($stage = 0) { - global $db; - - $count = 0; - - // We split the deletion in many small tasks - $limit = 1000; - - // Get the expire days for step 8 and 9 - $days = Config::get('system', 'dbclean-expire-days', 0); - - if ($stage == 1) { - $last_id = Config::get('system', 'dbclean-last-id-1', 0); - - logger("Deleting old global item entries from item table without user copy. Last ID: ".$last_id); - $r = dba::p("SELECT `id` FROM `item` WHERE `uid` = 0 AND - NOT EXISTS (SELECT `guid` FROM `item` AS `i` WHERE `item`.`guid` = `i`.`guid` AND `i`.`uid` != 0) AND - `received` < UTC_TIMESTAMP() - INTERVAL 90 DAY AND `id` >= ? - ORDER BY `id` LIMIT ".intval($limit), $last_id); - $count = dba::num_rows($r); - if ($count > 0) { - logger("found global item orphans: ".$count); - while ($orphan = dba::fetch($r)) { - $last_id = $orphan["id"]; - dba::delete('item', array('id' => $orphan["id"])); - } - } else { - logger("No global item orphans found"); - } - dba::close($r); - logger("Done deleting ".$count." old global item entries from item table without user copy. Last ID: ".$last_id); - - Config::set('system', 'dbclean-last-id-1', $last_id); - } elseif ($stage == 2) { - $last_id = Config::get('system', 'dbclean-last-id-2', 0); - - logger("Deleting items without parents. Last ID: ".$last_id); - $r = dba::p("SELECT `id` FROM `item` - WHERE NOT EXISTS (SELECT `id` FROM `item` AS `i` WHERE `item`.`parent` = `i`.`id`) - AND `id` >= ? ORDER BY `id` LIMIT ".intval($limit), $last_id); - $count = dba::num_rows($r); - if ($count > 0) { - logger("found item orphans without parents: ".$count); - while ($orphan = dba::fetch($r)) { - $last_id = $orphan["id"]; - dba::delete('item', array('id' => $orphan["id"])); - } - } else { - logger("No item orphans without parents found"); - } - dba::close($r); - logger("Done deleting ".$count." items without parents. Last ID: ".$last_id); - - Config::set('system', 'dbclean-last-id-2', $last_id); - - if ($count < $limit) { - Config::set('system', 'finished-dbclean-2', true); - } - } elseif ($stage == 3) { - $last_id = Config::get('system', 'dbclean-last-id-3', 0); - - logger("Deleting orphaned data from thread table. Last ID: ".$last_id); - $r = dba::p("SELECT `iid` FROM `thread` - WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`) AND `iid` >= ? - ORDER BY `iid` LIMIT ".intval($limit), $last_id); - $count = dba::num_rows($r); - if ($count > 0) { - logger("found thread orphans: ".$count); - while ($orphan = dba::fetch($r)) { - $last_id = $orphan["iid"]; - dba::delete('thread', array('iid' => $orphan["iid"])); - } - } else { - logger("No thread orphans found"); - } - dba::close($r); - logger("Done deleting ".$count." orphaned data from thread table. Last ID: ".$last_id); - - Config::set('system', 'dbclean-last-id-3', $last_id); - - if ($count < $limit) { - Config::set('system', 'finished-dbclean-3', true); - } - } elseif ($stage == 4) { - $last_id = Config::get('system', 'dbclean-last-id-4', 0); - - logger("Deleting orphaned data from notify table. Last ID: ".$last_id); - $r = dba::p("SELECT `iid`, `id` FROM `notify` - WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`) AND `id` >= ? - ORDER BY `id` LIMIT ".intval($limit), $last_id); - $count = dba::num_rows($r); - if ($count > 0) { - logger("found notify orphans: ".$count); - while ($orphan = dba::fetch($r)) { - $last_id = $orphan["id"]; - dba::delete('notify', array('iid' => $orphan["iid"])); - } - } else { - logger("No notify orphans found"); - } - dba::close($r); - logger("Done deleting ".$count." orphaned data from notify table. Last ID: ".$last_id); - - Config::set('system', 'dbclean-last-id-4', $last_id); - - if ($count < $limit) { - Config::set('system', 'finished-dbclean-4', true); - } - } elseif ($stage == 5) { - $last_id = Config::get('system', 'dbclean-last-id-5', 0); - - logger("Deleting orphaned data from notify-threads table. Last ID: ".$last_id); - $r = dba::p("SELECT `id` FROM `notify-threads` - WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `notify-threads`.`master-parent-item`) AND `id` >= ? - ORDER BY `id` LIMIT ".intval($limit), $last_id); - $count = dba::num_rows($r); - if ($count > 0) { - logger("found notify-threads orphans: ".$count); - while ($orphan = dba::fetch($r)) { - $last_id = $orphan["id"]; - dba::delete('notify-threads', array('id' => $orphan["id"])); - } - } else { - logger("No notify-threads orphans found"); - } - dba::close($r); - logger("Done deleting ".$count." orphaned data from notify-threads table. Last ID: ".$last_id); - - Config::set('system', 'dbclean-last-id-5', $last_id); - - if ($count < $limit) { - Config::set('system', 'finished-dbclean-5', true); - } - } elseif ($stage == 6) { - $last_id = Config::get('system', 'dbclean-last-id-6', 0); - - logger("Deleting orphaned data from sign table. Last ID: ".$last_id); - $r = dba::p("SELECT `iid`, `id` FROM `sign` - WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`) AND `id` >= ? - ORDER BY `id` LIMIT ".intval($limit), $last_id); - $count = dba::num_rows($r); - if ($count > 0) { - logger("found sign orphans: ".$count); - while ($orphan = dba::fetch($r)) { - $last_id = $orphan["id"]; - dba::delete('sign', array('iid' => $orphan["iid"])); - } - } else { - logger("No sign orphans found"); - } - dba::close($r); - logger("Done deleting ".$count." orphaned data from sign table. Last ID: ".$last_id); - - Config::set('system', 'dbclean-last-id-6', $last_id); - - if ($count < $limit) { - Config::set('system', 'finished-dbclean-6', true); - } - } elseif ($stage == 7) { - $last_id = Config::get('system', 'dbclean-last-id-7', 0); - - logger("Deleting orphaned data from term table. Last ID: ".$last_id); - $r = dba::p("SELECT `oid`, `tid` FROM `term` - WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`) AND `tid` >= ? - ORDER BY `tid` LIMIT ".intval($limit), $last_id); - $count = dba::num_rows($r); - if ($count > 0) { - logger("found term orphans: ".$count); - while ($orphan = dba::fetch($r)) { - $last_id = $orphan["tid"]; - dba::delete('term', array('oid' => $orphan["oid"])); - } - } else { - logger("No term orphans found"); - } - dba::close($r); - logger("Done deleting ".$count." orphaned data from term table. Last ID: ".$last_id); - - Config::set('system', 'dbclean-last-id-7', $last_id); - - if ($count < $limit) { - Config::set('system', 'finished-dbclean-7', true); - } - } elseif ($stage == 8) { - if ($days <= 0) { - return; - } - - $last_id = Config::get('system', 'dbclean-last-id-8', 0); - - logger("Deleting expired threads. Last ID: ".$last_id); - $r = dba::p("SELECT `thread`.`iid` FROM `thread` - INNER JOIN `contact` ON `thread`.`contact-id` = `contact`.`id` AND NOT `notify_new_posts` - WHERE `thread`.`received` < UTC_TIMESTAMP() - INTERVAL ? DAY - AND NOT `thread`.`mention` AND NOT `thread`.`starred` - AND NOT `thread`.`wall` AND NOT `thread`.`origin` - AND `thread`.`uid` != 0 AND `thread`.`iid` >= ? - AND NOT `thread`.`iid` IN (SELECT `parent` FROM `item` - WHERE (`item`.`starred` OR (`item`.`resource-id` != '') - OR (`item`.`file` != '') OR (`item`.`event-id` != '') - OR (`item`.`attach` != '') OR `item`.`wall` OR `item`.`origin`) - AND `item`.`parent` = `thread`.`iid`) - ORDER BY `thread`.`iid` LIMIT 1000", $days, $last_id); - $count = dba::num_rows($r); - if ($count > 0) { - logger("found expired threads: ".$count); - while ($thread = dba::fetch($r)) { - $last_id = $thread["iid"]; - dba::delete('thread', array('iid' => $thread["iid"])); - } - } else { - logger("No expired threads found"); - } - dba::close($r); - logger("Done deleting ".$count." expired threads. Last ID: ".$last_id); - - Config::set('system', 'dbclean-last-id-8', $last_id); - } elseif ($stage == 9) { - if ($days <= 0) { - return; - } - - $last_id = Config::get('system', 'dbclean-last-id-9', 0); - $till_id = Config::get('system', 'dbclean-last-id-8', 0); - - logger("Deleting old global item entries from expired threads from ID ".$last_id." to ID ".$till_id); - $r = dba::p("SELECT `id` FROM `item` WHERE `uid` = 0 AND - NOT EXISTS (SELECT `guid` FROM `item` AS `i` WHERE `item`.`guid` = `i`.`guid` AND `i`.`uid` != 0) AND - `received` < UTC_TIMESTAMP() - INTERVAL 90 DAY AND `id` >= ? AND `id` <= ? - ORDER BY `id` LIMIT ".intval($limit), $last_id, $till_id); - $count = dba::num_rows($r); - if ($count > 0) { - logger("found global item entries from expired threads: ".$count); - while ($orphan = dba::fetch($r)) { - $last_id = $orphan["id"]; - dba::delete('item', array('id' => $orphan["id"])); - } - } else { - logger("No global item entries from expired threads"); - } - dba::close($r); - logger("Done deleting ".$count." old global item entries from expired threads. Last ID: ".$last_id); - - Config::set('system', 'dbclean-last-id-9', $last_id); - } - - // Call it again if not all entries were purged - if (($stage != 0) && ($count > 0)) { - Worker::add(PRIORITY_MEDIUM, 'dbclean'); - } -} diff --git a/include/dbupdate.php b/include/dbupdate.php deleted file mode 100644 index 799ca262c8..0000000000 --- a/include/dbupdate.php +++ /dev/null @@ -1,13 +0,0 @@ - $argv[1]); - - call_hooks('globaldir_update', $arr); - - logger('Updating directory: ' . $arr['url'], LOGGER_DEBUG); - if (strlen($arr['url'])) { - fetch_url($dir . '?url=' . bin2hex($arr['url'])); - } - - return; -} - -function directory_update_all() { - $r = q("SELECT `url` FROM `contact` - INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid` - INNER JOIN `user` ON `user`.`uid` = `contact`.`uid` - WHERE `contact`.`self` AND `profile`.`net-publish` AND `profile`.`is-default` AND - NOT `user`.`account_expired` AND `user`.`verified`"); - - if (DBM::is_result($r)) { - foreach ($r AS $user) { - Worker::add(PRIORITY_LOW, 'directory', $user['url']); - } - } -} diff --git a/include/discover_poco.php b/include/discover_poco.php deleted file mode 100644 index fd72d69401..0000000000 --- a/include/discover_poco.php +++ /dev/null @@ -1,299 +0,0 @@ -: Searches for "search pattern" in the directory. "search pattern" is url encoded. - - checkcontact: Updates gcontact entries - - suggestions: Discover other servers for their contacts. - - server : Searches for the poco server list. "poco url" is base64 encoded. - - update_server: Frequently check the first 250 servers for vitality. - - update_server_directory: Discover the given server id for their contacts - - PortableContact::load: Load POCO data from a given POCO address - - check_profile: Update remote profile data - */ - - if (($argc > 2) && ($argv[1] == "dirsearch")) { - $search = urldecode($argv[2]); - $mode = 1; - } elseif (($argc == 2) && ($argv[1] == "checkcontact")) { - $mode = 2; - } elseif (($argc == 2) && ($argv[1] == "suggestions")) { - $mode = 3; - } elseif (($argc == 3) && ($argv[1] == "server")) { - $mode = 4; - } elseif (($argc == 2) && ($argv[1] == "update_server")) { - $mode = 5; - } elseif (($argc == 3) && ($argv[1] == "update_server_directory")) { - $mode = 6; - } elseif (($argc > 5) && ($argv[1] == "load")) { - $mode = 7; - } elseif (($argc == 3) && ($argv[1] == "check_profile")) { - $mode = 8; - } elseif ($argc == 1) { - $search = ""; - $mode = 0; - } else { - logger("Unknown or missing parameter ".$argv[1]."\n"); - return; - } - - logger('start '.$search); - - if ($mode == 8) { - if ($argv[2] != "") { - PortableContact::lastUpdated($argv[2], true); - } - } elseif ($mode == 7) { - if ($argc == 6) { - $url = $argv[5]; - } else { - $url = ''; - } - PortableContact::load(intval($argv[2]), intval($argv[3]), intval($argv[4]), $url); - } elseif ($mode == 6) { - PortableContact::discoverSingleServer(intval($argv[2])); - } elseif ($mode == 5) { - update_server(); - } elseif ($mode == 4) { - $server_url = $argv[2]; - if ($server_url == "") { - return; - } - $server_url = filter_var($server_url, FILTER_SANITIZE_URL); - if (substr(normalise_link($server_url), 0, 7) != "http://") { - return; - } - $result = "Checking server ".$server_url." - "; - $ret = PortableContact::checkServer($server_url); - if ($ret) { - $result .= "success"; - } else { - $result .= "failed"; - } - logger($result, LOGGER_DEBUG); - } elseif ($mode == 3) { - GlobalContact::updateSuggestions(); - } elseif (($mode == 2) && Config::get('system', 'poco_completion')) { - discover_users(); - } elseif (($mode == 1) && ($search != "") && Config::get('system', 'poco_local_search')) { - discover_directory($search); - gs_search_user($search); - } elseif (($mode == 0) && ($search == "") && (Config::get('system', 'poco_discovery') > 0)) { - // Query Friendica and Hubzilla servers for their users - PortableContact::discover(); - - // Query GNU Social servers for their users ("statistics" addon has to be enabled on the GS server) - if (!Config::get('system', 'ostatus_disabled')) { - GlobalContact::discoverGsUsers(); - } - } - - logger('end '.$search); - - return; -} - -/** - * @brief Updates the first 250 servers - * - */ -function update_server() { - $r = q("SELECT `url`, `created`, `last_failure`, `last_contact` FROM `gserver` ORDER BY rand()"); - - if (!DBM::is_result($r)) { - return; - } - - $updated = 0; - - foreach ($r AS $server) { - if (!PortableContact::updateNeeded($server["created"], "", $server["last_failure"], $server["last_contact"])) { - continue; - } - logger('Update server status for server '.$server["url"], LOGGER_DEBUG); - - Worker::add(PRIORITY_LOW, "discover_poco", "server", $server["url"]); - - if (++$updated > 250) { - return; - } - } -} - -function discover_users() { - logger("Discover users", LOGGER_DEBUG); - - $starttime = time(); - - $users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url`, `network` FROM `gcontact` - WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND - `last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND - `network` IN ('%s', '%s', '%s', '%s', '') ORDER BY rand()", - dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), - dbesc(NETWORK_OSTATUS), dbesc(NETWORK_FEED)); - - if (!$users) { - return; - } - $checked = 0; - - foreach ($users AS $user) { - - $urlparts = parse_url($user["url"]); - if (!isset($urlparts["scheme"])) { - q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'", - dbesc(NETWORK_PHANTOM), dbesc(normalise_link($user["url"]))); - continue; - } - - if (in_array($urlparts["host"], array("www.facebook.com", "facebook.com", "twitter.com", - "identi.ca", "alpha.app.net"))) { - $networks = array("www.facebook.com" => NETWORK_FACEBOOK, - "facebook.com" => NETWORK_FACEBOOK, - "twitter.com" => NETWORK_TWITTER, - "identi.ca" => NETWORK_PUMPIO, - "alpha.app.net" => NETWORK_APPNET); - - q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'", - dbesc($networks[$urlparts["host"]]), dbesc(normalise_link($user["url"]))); - continue; - } - - $server_url = PortableContact::detectServer($user["url"]); - $force_update = false; - - if ($user["server_url"] != "") { - - $force_update = (normalise_link($user["server_url"]) != normalise_link($server_url)); - - $server_url = $user["server_url"]; - } - - if ((($server_url == "") && ($user["network"] == NETWORK_FEED)) || $force_update || PortableContact::checkServer($server_url, $user["network"])) { - logger('Check profile '.$user["url"]); - Worker::add(PRIORITY_LOW, "discover_poco", "check_profile", $user["url"]); - - if (++$checked > 100) { - return; - } - } else { - q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", - dbesc(datetime_convert()), dbesc(normalise_link($user["url"]))); - } - - // Quit the loop after 3 minutes - if (time() > ($starttime + 180)) { - return; - } - } -} - -function discover_directory($search) { - - $data = Cache::get("dirsearch:".$search); - if (!is_null($data)) { - // Only search for the same item every 24 hours - if (time() < $data + (60 * 60 * 24)) { - logger("Already searched for ".$search." in the last 24 hours", LOGGER_DEBUG); - return; - } - } - - $x = fetch_url(get_server()."/lsearch?p=1&n=500&search=".urlencode($search)); - $j = json_decode($x); - - if (count($j->results)) { - foreach ($j->results as $jj) { - // Check if the contact already exists - $exists = q("SELECT `id`, `last_contact`, `last_failure`, `updated` FROM `gcontact` WHERE `nurl` = '%s'", normalise_link($jj->url)); - if (DBM::is_result($exists)) { - logger("Profile ".$jj->url." already exists (".$search.")", LOGGER_DEBUG); - - if (($exists[0]["last_contact"] < $exists[0]["last_failure"]) && - ($exists[0]["updated"] < $exists[0]["last_failure"])) { - continue; - } - // Update the contact - PortableContact::lastUpdated($jj->url); - continue; - } - - $server_url = PortableContact::detectServer($jj->url); - if ($server_url != '') { - if (!PortableContact::checkServer($server_url)) { - logger("Friendica server ".$server_url." doesn't answer.", LOGGER_DEBUG); - continue; - } - logger("Friendica server ".$server_url." seems to be okay.", LOGGER_DEBUG); - } - - $data = Probe::uri($jj->url); - if ($data["network"] == NETWORK_DFRN) { - logger("Profile ".$jj->url." is reachable (".$search.")", LOGGER_DEBUG); - logger("Add profile ".$jj->url." to local directory (".$search.")", LOGGER_DEBUG); - - if ($jj->tags != "") { - $data["keywords"] = $jj->tags; - } - - $data["server_url"] = $data["baseurl"]; - - GlobalContact::update($data); - } else { - logger("Profile ".$jj->url." is not responding or no Friendica contact - but network ".$data["network"], LOGGER_DEBUG); - } - } - } - Cache::set("dirsearch:".$search, time(), CACHE_DAY); -} - -/** - * @brief Search for GNU Social user with gstools.org - * - * @param str $search User name - */ -function gs_search_user($search) { - - // Currently disabled, since the service isn't available anymore. - // It is not removed since I hope that there will be a successor. - return false; - - $a = get_app(); - - $url = "http://gstools.org/api/users_search/".urlencode($search); - - $result = z_fetch_url($url); - if (!$result["success"]) { - return false; - } - - $contacts = json_decode($result["body"]); - - if ($contacts->status == 'ERROR') { - return false; - } - - /// @TODO AS is considered as a notation for constants (as they usually being written all upper-case) - /// @TODO find all those and convert to all lower-case which is a keyword then - foreach ($contacts->data AS $user) { - $contact = Probe::uri($user->site_address."/".$user->name); - if ($contact["network"] != NETWORK_PHANTOM) { - $contact["about"] = $user->description; - GlobalContact::update($contact); - } - } -} diff --git a/include/poller.php b/include/poller.php index 3f6290a98f..6fcf4c7f41 100644 --- a/include/poller.php +++ b/include/poller.php @@ -3,6 +3,7 @@ use Friendica\App; use Friendica\Core\Worker; use Friendica\Core\Config; +// Ensure that poller.php is executed from the base path of the installation if (!file_exists("boot.php") && (sizeof($_SERVER["argv"]) != 0)) { $directory = dirname($_SERVER["argv"][0]); @@ -14,45 +15,35 @@ if (!file_exists("boot.php") && (sizeof($_SERVER["argv"]) != 0)) { chdir($directory); } -require_once("boot.php"); +require_once "boot.php"; +require_once "include/dba.php"; -function poller_run($argv, $argc) { - global $a; +$a = new App(dirname(__DIR__)); - if (empty($a)) { - $a = new App(dirname(__DIR__)); - } +require_once ".htconfig.php"; +dba::connect($db_host, $db_user, $db_pass, $db_data); +unset($db_host, $db_user, $db_pass, $db_data); - require_once ".htconfig.php"; - require_once "include/dba.php"; - dba::connect($db_host, $db_user, $db_pass, $db_data); - unset($db_host, $db_user, $db_pass, $db_data); +Config::load(); - Config::load(); +// Check the database structure and possibly fixes it +check_db(true); - // Check the database structure and possibly fixes it - check_db(true); - - // Quit when in maintenance - if (Config::get('system', 'maintenance', true)) { - return; - } - - $a->set_baseurl(Config::get('system', 'url')); - - load_hooks(); - - $run_cron = (($argc <= 1) || ($argv[1] != "no_cron")); - Worker::processQueue($run_cron); +// Quit when in maintenance +if (Config::get('system', 'maintenance', true)) { return; } -if (array_search(__file__, get_included_files()) === 0) { - poller_run($_SERVER["argv"], $_SERVER["argc"]); +$a->set_baseurl(Config::get('system', 'url')); - Worker::unclaimProcess(); +load_hooks(); - get_app()->end_process(); +$run_cron = (($_SERVER["argc"] <= 1) || ($_SERVER["argv"][1] != "no_cron")); +Worker::processQueue($run_cron); + +Worker::unclaimProcess(); + +$a->end_process(); + +killme(); - killme(); -} diff --git a/include/remove_contact.php b/include/remove_contact.php deleted file mode 100644 index 9d4b1e4c37..0000000000 --- a/include/remove_contact.php +++ /dev/null @@ -1,24 +0,0 @@ - $id)); - if ($r) { - return; - } - - // Now we delete all the depending table entries - dba::delete('contact', array('id' => $id)); -} diff --git a/include/shadowupdate.php b/include/shadowupdate.php deleted file mode 100644 index c41b231221..0000000000 --- a/include/shadowupdate.php +++ /dev/null @@ -1,29 +0,0 @@ -get_path() != "") { diff --git a/mod/dirfind.php b/mod/dirfind.php index 84fe6f0ff4..89df7c885f 100644 --- a/mod/dirfind.php +++ b/mod/dirfind.php @@ -169,7 +169,7 @@ function dirfind_content(App $a, $prefix = "") { } // Add found profiles from the global directory to the local directory - Worker::add(PRIORITY_LOW, 'discover_poco', "dirsearch", urlencode($search)); + Worker::add(PRIORITY_LOW, 'DiscoverPoCo', "dirsearch", urlencode($search)); } else { $p = (($a->pager['page'] != 1) ? '&p=' . $a->pager['page'] : ''); diff --git a/mod/profile_photo.php b/mod/profile_photo.php index 5f52451414..3ef0118da0 100644 --- a/mod/profile_photo.php +++ b/mod/profile_photo.php @@ -132,7 +132,7 @@ function profile_photo_post(App $a) { // Update global directory in background $url = System::baseUrl() . '/profile/' . $a->user['nickname']; if ($url && strlen(Config::get('system','directory'))) { - Worker::add(PRIORITY_LOW, "directory", $url); + Worker::add(PRIORITY_LOW, "Directory", $url); } Worker::add(PRIORITY_LOW, 'profile_update', local_user()); @@ -232,7 +232,7 @@ function profile_photo_content(App $a) { // Update global directory in background $url = $_SESSION['my_url']; if ($url && strlen(Config::get('system','directory'))) { - Worker::add(PRIORITY_LOW, "directory", $url); + Worker::add(PRIORITY_LOW, "Directory", $url); } goaway(System::baseUrl() . '/profiles'); diff --git a/mod/profiles.php b/mod/profiles.php index 4263897d0b..545fb2fc78 100644 --- a/mod/profiles.php +++ b/mod/profiles.php @@ -503,7 +503,7 @@ function profiles_post(App $a) { // Update global directory in background $url = $_SESSION['my_url']; if ($url && strlen(Config::get('system', 'directory'))) { - Worker::add(PRIORITY_LOW, "directory", $url); + Worker::add(PRIORITY_LOW, "Directory", $url); } Worker::add(PRIORITY_LOW, 'profile_update', local_user()); diff --git a/mod/register.php b/mod/register.php index 4d0ef96485..cd6385144c 100644 --- a/mod/register.php +++ b/mod/register.php @@ -72,7 +72,7 @@ function register_post(App $a) { if($netpublish && $a->config['register_policy'] != REGISTER_APPROVE) { $url = System::baseUrl() . '/profile/' . $user['nickname']; - Worker::add(PRIORITY_LOW, "directory", $url); + Worker::add(PRIORITY_LOW, "Directory", $url); } $using_invites = Config::get('system','invitation_only'); diff --git a/mod/regmod.php b/mod/regmod.php index d6e122a286..6d76e7ea7e 100644 --- a/mod/regmod.php +++ b/mod/regmod.php @@ -45,7 +45,7 @@ function user_allow($hash) { if (DBM::is_result($r) && $r[0]['net-publish']) { $url = System::baseUrl() . '/profile/' . $user[0]['nickname']; if ($url && strlen(Config::get('system','directory'))) { - Worker::add(PRIORITY_LOW, "directory", $url); + Worker::add(PRIORITY_LOW, "Directory", $url); } } diff --git a/mod/settings.php b/mod/settings.php index e693f77595..c0b3d2cc16 100644 --- a/mod/settings.php +++ b/mod/settings.php @@ -645,7 +645,7 @@ function settings_post(App $a) { // Update global directory in background $url = $_SESSION['my_url']; if ($url && strlen(Config::get('system', 'directory'))) { - Worker::add(PRIORITY_LOW, "directory", $url); + Worker::add(PRIORITY_LOW, "Directory", $url); } } diff --git a/mod/worker.php b/mod/worker.php index 930a807e38..f540010bd6 100644 --- a/mod/worker.php +++ b/mod/worker.php @@ -3,7 +3,6 @@ * @file mod/worker.php * @brief Module for running the poller as frontend process */ -require_once("include/poller.php"); use Friendica\Core\Worker; use Friendica\Core\Config; diff --git a/src/Core/Worker.php b/src/Core/Worker.php index b06c4920ee..276179131b 100644 --- a/src/Core/Worker.php +++ b/src/Core/Worker.php @@ -899,7 +899,7 @@ class Worker { logger('Add cron entries', LOGGER_DEBUG); // Check for spooled items - self::add(PRIORITY_HIGH, "spool_post"); + self::add(PRIORITY_HIGH, "SpoolPost"); // Run the cron job that calls all other jobs self::add(PRIORITY_MEDIUM, "Cron"); diff --git a/src/Protocol/PortableContact.php b/src/Protocol/PortableContact.php index 826d921599..1c0ce15a8f 100644 --- a/src/Protocol/PortableContact.php +++ b/src/Protocol/PortableContact.php @@ -51,7 +51,7 @@ class PortableContact public static function loadWorker($cid, $uid = 0, $zcid = 0, $url = null) { // Call the function "load" via the worker - Worker::add(PRIORITY_LOW, "discover_poco", "load", (int)$cid, (int)$uid, (int)$zcid, $url); + Worker::add(PRIORITY_LOW, "DiscoverPoCo", "load", (int)$cid, (int)$uid, (int)$zcid, $url); } /** @@ -1317,7 +1317,7 @@ class PortableContact $r = q("SELECT `nurl` FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url))); if (!DBM::is_result($r)) { logger("Call server check for server ".$server_url, LOGGER_DEBUG); - Worker::add(PRIORITY_LOW, "discover_poco", "server", $server_url); + Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $server_url); } } } @@ -1340,7 +1340,7 @@ class PortableContact $servers = json_decode($serverdata); foreach ($servers->pods as $server) { - Worker::add(PRIORITY_LOW, "discover_poco", "server", "https://".$server->host); + Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", "https://".$server->host); } } @@ -1353,7 +1353,7 @@ class PortableContact foreach ($servers as $server) { $url = (is_null($server->https_score) ? 'http' : 'https').'://'.$server->name; - Worker::add(PRIORITY_LOW, "discover_poco", "server", $url); + Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $url); } } } @@ -1462,7 +1462,7 @@ class PortableContact } logger('Update directory from server '.$server['url'].' with ID '.$server['id'], LOGGER_DEBUG); - Worker::add(PRIORITY_LOW, "discover_poco", "update_server_directory", (int)$server['id']); + Worker::add(PRIORITY_LOW, "DiscoverPoCo", "update_server_directory", (int)$server['id']); if (!$complete && (--$no_of_queries == 0)) { break; diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 00b28b6a52..b3f0f1f691 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -1,9 +1,13 @@ '%s' + AND `account_expires_on` < UTC_TIMESTAMP()", dbesc(NULL_DATE)); + + // delete user records for recently removed accounts + $r = q("SELECT * FROM `user` WHERE `account_removed` AND `account_expires_on` < UTC_TIMESTAMP() - INTERVAL 3 DAY"); + if (DBM::is_result($r)) { + foreach ($r as $user) { + dba::delete('user', array('uid' => $user['uid'])); + } + } + } + + /** + * @brief Clear cache entries + * + * @param App $a + */ + private static function clearCache(App $a) { + + $last = Config::get('system','cache_last_cleared'); + + if ($last) { + $next = $last + (3600); // Once per hour + $clear_cache = ($next <= time()); + } else { + $clear_cache = true; + } + + if (!$clear_cache) { + return; + } + + // clear old cache + Cache::clear(); + + // clear old item cache files + clear_cache(); + + // clear cache for photos + clear_cache($a->get_basepath(), $a->get_basepath()."/photo"); + + // clear smarty cache + clear_cache($a->get_basepath()."/view/smarty3/compiled", $a->get_basepath()."/view/smarty3/compiled"); + + // clear cache for image proxy + if (!Config::get("system", "proxy_disabled")) { + clear_cache($a->get_basepath(), $a->get_basepath()."/proxy"); + + $cachetime = Config::get('system','proxy_cache_time'); + if (!$cachetime) { + $cachetime = PROXY_DEFAULT_TIME; + } + q('DELETE FROM `photo` WHERE `uid` = 0 AND `resource-id` LIKE "pic:%%" AND `created` < NOW() - INTERVAL %d SECOND', $cachetime); + } + + // Delete the cached OEmbed entries that are older than one year + q("DELETE FROM `oembed` WHERE `created` < NOW() - INTERVAL 3 MONTH"); + + // Delete the cached "parse_url" entries that are older than one year + q("DELETE FROM `parsed_url` WHERE `created` < NOW() - INTERVAL 3 MONTH"); + + // Maximum table size in megabyte + $max_tablesize = intval(Config::get('system','optimize_max_tablesize')) * 1000000; + if ($max_tablesize == 0) { + $max_tablesize = 100 * 1000000; // Default are 100 MB + } + if ($max_tablesize > 0) { + // Minimum fragmentation level in percent + $fragmentation_level = intval(Config::get('system','optimize_fragmentation')) / 100; + if ($fragmentation_level == 0) { + $fragmentation_level = 0.3; // Default value is 30% + } + + // Optimize some tables that need to be optimized + $r = q("SHOW TABLE STATUS"); + foreach ($r as $table) { + + // Don't optimize tables that are too large + if ($table["Data_length"] > $max_tablesize) { + continue; + } + + // Don't optimize empty tables + if ($table["Data_length"] == 0) { + continue; + } + + // Calculate fragmentation + $fragmentation = $table["Data_free"] / ($table["Data_length"] + $table["Index_length"]); + + logger("Table ".$table["Name"]." - Fragmentation level: ".round($fragmentation * 100, 2), LOGGER_DEBUG); + + // Don't optimize tables that needn't to be optimized + if ($fragmentation < $fragmentation_level) { + continue; + } + + // So optimize it + logger("Optimize Table ".$table["Name"], LOGGER_DEBUG); + q("OPTIMIZE TABLE `%s`", dbesc($table["Name"])); + } + } + + Config::set('system','cache_last_cleared', time()); + } + + /** + * @brief Repair missing values in Diaspora contacts + * + * @param App $a + */ + private static function repairDiaspora(App $a) { + + $starttime = time(); + + $r = q("SELECT `id`, `url` FROM `contact` + WHERE `network` = '%s' AND (`batch` = '' OR `notify` = '' OR `poll` = '' OR pubkey = '') + ORDER BY RAND() LIMIT 50", dbesc(NETWORK_DIASPORA)); + if (!DBM::is_result($r)) { + return; + } + + foreach ($r AS $contact) { + // Quit the loop after 3 minutes + if (time() > ($starttime + 180)) { + return; + } + + if (!PortableContact::reachable($contact["url"])) { + continue; + } + + $data = Probe::uri($contact["url"]); + if ($data["network"] != NETWORK_DIASPORA) { + continue; + } + + logger("Repair contact ".$contact["id"]." ".$contact["url"], LOGGER_DEBUG); + q("UPDATE `contact` SET `batch` = '%s', `notify` = '%s', `poll` = '%s', pubkey = '%s' WHERE `id` = %d", + dbesc($data["batch"]), dbesc($data["notify"]), dbesc($data["poll"]), dbesc($data["pubkey"]), + intval($contact["id"])); + } + } + + /** + * @brief Do some repairs in database entries + * + */ + private static function repairDatabase() { + + // Sometimes there seem to be issues where the "self" contact vanishes. + // We haven't found the origin of the problem by now. + $r = q("SELECT `uid` FROM `user` WHERE NOT EXISTS (SELECT `uid` FROM `contact` WHERE `contact`.`uid` = `user`.`uid` AND `contact`.`self`)"); + if (DBM::is_result($r)) { + foreach ($r AS $user) { + logger('Create missing self contact for user '.$user['uid']); + user_create_self_contact($user['uid']); + } + } + + // Set the parent if it wasn't set. (Shouldn't happen - but does sometimes) + // This call is very "cheap" so we can do it at any time without a problem + q("UPDATE `item` INNER JOIN `item` AS `parent` ON `parent`.`uri` = `item`.`parent-uri` AND `parent`.`uid` = `item`.`uid` SET `item`.`parent` = `parent`.`id` WHERE `item`.`parent` = 0"); + + // There was an issue where the nick vanishes from the contact table + q("UPDATE `contact` INNER JOIN `user` ON `contact`.`uid` = `user`.`uid` SET `nick` = `nickname` WHERE `self` AND `nick`=''"); + + // Update the global contacts for local users + $r = q("SELECT `uid` FROM `user` WHERE `verified` AND NOT `blocked` AND NOT `account_removed` AND NOT `account_expired`"); + if (DBM::is_result($r)) { + foreach ($r AS $user) { + GlobalContact::updateForUser($user["uid"]); + } + } + + /// @todo + /// - remove thread entries without item + /// - remove sign entries without item + /// - remove children when parent got lost + /// - set contact-id in item when not present + } +} diff --git a/src/Worker/DBClean.php b/src/Worker/DBClean.php new file mode 100644 index 0000000000..7030081dc9 --- /dev/null +++ b/src/Worker/DBClean.php @@ -0,0 +1,303 @@ + 0))) { + Worker::add(PRIORITY_LOW, 'DBClean', $i); + } + } + } else { + self::removeOrphans($stage); + } + } + + /** + * @brief Remove orphaned database entries + * @param integer $stage What should be deleted? + * + * Values for $stage: + * ------------------ + * 1: Old global item entries from item table without user copy. + * 2: Items without parents. + * 3: Orphaned data from thread table. + * 4: Orphaned data from notify table. + * 5: Orphaned data from notify-threads table. + * 6: Orphaned data from sign table. + * 7: Orphaned data from term table. + * 8: Expired threads. + * 9: Old global item entries from expired threads + */ + private static function removeOrphans($stage = 0) { + global $db; + + $count = 0; + + // We split the deletion in many small tasks + $limit = 1000; + + // Get the expire days for step 8 and 9 + $days = Config::get('system', 'dbclean-expire-days', 0); + + if ($stage == 1) { + $last_id = Config::get('system', 'dbclean-last-id-1', 0); + + logger("Deleting old global item entries from item table without user copy. Last ID: ".$last_id); + $r = dba::p("SELECT `id` FROM `item` WHERE `uid` = 0 AND + NOT EXISTS (SELECT `guid` FROM `item` AS `i` WHERE `item`.`guid` = `i`.`guid` AND `i`.`uid` != 0) AND + `received` < UTC_TIMESTAMP() - INTERVAL 90 DAY AND `id` >= ? + ORDER BY `id` LIMIT ".intval($limit), $last_id); + $count = dba::num_rows($r); + if ($count > 0) { + logger("found global item orphans: ".$count); + while ($orphan = dba::fetch($r)) { + $last_id = $orphan["id"]; + dba::delete('item', array('id' => $orphan["id"])); + } + } else { + logger("No global item orphans found"); + } + dba::close($r); + logger("Done deleting ".$count." old global item entries from item table without user copy. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-1', $last_id); + } elseif ($stage == 2) { + $last_id = Config::get('system', 'dbclean-last-id-2', 0); + + logger("Deleting items without parents. Last ID: ".$last_id); + $r = dba::p("SELECT `id` FROM `item` + WHERE NOT EXISTS (SELECT `id` FROM `item` AS `i` WHERE `item`.`parent` = `i`.`id`) + AND `id` >= ? ORDER BY `id` LIMIT ".intval($limit), $last_id); + $count = dba::num_rows($r); + if ($count > 0) { + logger("found item orphans without parents: ".$count); + while ($orphan = dba::fetch($r)) { + $last_id = $orphan["id"]; + dba::delete('item', array('id' => $orphan["id"])); + } + } else { + logger("No item orphans without parents found"); + } + dba::close($r); + logger("Done deleting ".$count." items without parents. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-2', $last_id); + + if ($count < $limit) { + Config::set('system', 'finished-dbclean-2', true); + } + } elseif ($stage == 3) { + $last_id = Config::get('system', 'dbclean-last-id-3', 0); + + logger("Deleting orphaned data from thread table. Last ID: ".$last_id); + $r = dba::p("SELECT `iid` FROM `thread` + WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`) AND `iid` >= ? + ORDER BY `iid` LIMIT ".intval($limit), $last_id); + $count = dba::num_rows($r); + if ($count > 0) { + logger("found thread orphans: ".$count); + while ($orphan = dba::fetch($r)) { + $last_id = $orphan["iid"]; + dba::delete('thread', array('iid' => $orphan["iid"])); + } + } else { + logger("No thread orphans found"); + } + dba::close($r); + logger("Done deleting ".$count." orphaned data from thread table. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-3', $last_id); + + if ($count < $limit) { + Config::set('system', 'finished-dbclean-3', true); + } + } elseif ($stage == 4) { + $last_id = Config::get('system', 'dbclean-last-id-4', 0); + + logger("Deleting orphaned data from notify table. Last ID: ".$last_id); + $r = dba::p("SELECT `iid`, `id` FROM `notify` + WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`) AND `id` >= ? + ORDER BY `id` LIMIT ".intval($limit), $last_id); + $count = dba::num_rows($r); + if ($count > 0) { + logger("found notify orphans: ".$count); + while ($orphan = dba::fetch($r)) { + $last_id = $orphan["id"]; + dba::delete('notify', array('iid' => $orphan["iid"])); + } + } else { + logger("No notify orphans found"); + } + dba::close($r); + logger("Done deleting ".$count." orphaned data from notify table. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-4', $last_id); + + if ($count < $limit) { + Config::set('system', 'finished-dbclean-4', true); + } + } elseif ($stage == 5) { + $last_id = Config::get('system', 'dbclean-last-id-5', 0); + + logger("Deleting orphaned data from notify-threads table. Last ID: ".$last_id); + $r = dba::p("SELECT `id` FROM `notify-threads` + WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `notify-threads`.`master-parent-item`) AND `id` >= ? + ORDER BY `id` LIMIT ".intval($limit), $last_id); + $count = dba::num_rows($r); + if ($count > 0) { + logger("found notify-threads orphans: ".$count); + while ($orphan = dba::fetch($r)) { + $last_id = $orphan["id"]; + dba::delete('notify-threads', array('id' => $orphan["id"])); + } + } else { + logger("No notify-threads orphans found"); + } + dba::close($r); + logger("Done deleting ".$count." orphaned data from notify-threads table. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-5', $last_id); + + if ($count < $limit) { + Config::set('system', 'finished-dbclean-5', true); + } + } elseif ($stage == 6) { + $last_id = Config::get('system', 'dbclean-last-id-6', 0); + + logger("Deleting orphaned data from sign table. Last ID: ".$last_id); + $r = dba::p("SELECT `iid`, `id` FROM `sign` + WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`) AND `id` >= ? + ORDER BY `id` LIMIT ".intval($limit), $last_id); + $count = dba::num_rows($r); + if ($count > 0) { + logger("found sign orphans: ".$count); + while ($orphan = dba::fetch($r)) { + $last_id = $orphan["id"]; + dba::delete('sign', array('iid' => $orphan["iid"])); + } + } else { + logger("No sign orphans found"); + } + dba::close($r); + logger("Done deleting ".$count." orphaned data from sign table. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-6', $last_id); + + if ($count < $limit) { + Config::set('system', 'finished-dbclean-6', true); + } + } elseif ($stage == 7) { + $last_id = Config::get('system', 'dbclean-last-id-7', 0); + + logger("Deleting orphaned data from term table. Last ID: ".$last_id); + $r = dba::p("SELECT `oid`, `tid` FROM `term` + WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`) AND `tid` >= ? + ORDER BY `tid` LIMIT ".intval($limit), $last_id); + $count = dba::num_rows($r); + if ($count > 0) { + logger("found term orphans: ".$count); + while ($orphan = dba::fetch($r)) { + $last_id = $orphan["tid"]; + dba::delete('term', array('oid' => $orphan["oid"])); + } + } else { + logger("No term orphans found"); + } + dba::close($r); + logger("Done deleting ".$count." orphaned data from term table. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-7', $last_id); + + if ($count < $limit) { + Config::set('system', 'finished-dbclean-7', true); + } + } elseif ($stage == 8) { + if ($days <= 0) { + return; + } + + $last_id = Config::get('system', 'dbclean-last-id-8', 0); + + logger("Deleting expired threads. Last ID: ".$last_id); + $r = dba::p("SELECT `thread`.`iid` FROM `thread` + INNER JOIN `contact` ON `thread`.`contact-id` = `contact`.`id` AND NOT `notify_new_posts` + WHERE `thread`.`received` < UTC_TIMESTAMP() - INTERVAL ? DAY + AND NOT `thread`.`mention` AND NOT `thread`.`starred` + AND NOT `thread`.`wall` AND NOT `thread`.`origin` + AND `thread`.`uid` != 0 AND `thread`.`iid` >= ? + AND NOT `thread`.`iid` IN (SELECT `parent` FROM `item` + WHERE (`item`.`starred` OR (`item`.`resource-id` != '') + OR (`item`.`file` != '') OR (`item`.`event-id` != '') + OR (`item`.`attach` != '') OR `item`.`wall` OR `item`.`origin`) + AND `item`.`parent` = `thread`.`iid`) + ORDER BY `thread`.`iid` LIMIT 1000", $days, $last_id); + $count = dba::num_rows($r); + if ($count > 0) { + logger("found expired threads: ".$count); + while ($thread = dba::fetch($r)) { + $last_id = $thread["iid"]; + dba::delete('thread', array('iid' => $thread["iid"])); + } + } else { + logger("No expired threads found"); + } + dba::close($r); + logger("Done deleting ".$count." expired threads. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-8', $last_id); + } elseif ($stage == 9) { + if ($days <= 0) { + return; + } + + $last_id = Config::get('system', 'dbclean-last-id-9', 0); + $till_id = Config::get('system', 'dbclean-last-id-8', 0); + + logger("Deleting old global item entries from expired threads from ID ".$last_id." to ID ".$till_id); + $r = dba::p("SELECT `id` FROM `item` WHERE `uid` = 0 AND + NOT EXISTS (SELECT `guid` FROM `item` AS `i` WHERE `item`.`guid` = `i`.`guid` AND `i`.`uid` != 0) AND + `received` < UTC_TIMESTAMP() - INTERVAL 90 DAY AND `id` >= ? AND `id` <= ? + ORDER BY `id` LIMIT ".intval($limit), $last_id, $till_id); + $count = dba::num_rows($r); + if ($count > 0) { + logger("found global item entries from expired threads: ".$count); + while ($orphan = dba::fetch($r)) { + $last_id = $orphan["id"]; + dba::delete('item', array('id' => $orphan["id"])); + } + } else { + logger("No global item entries from expired threads"); + } + dba::close($r); + logger("Done deleting ".$count." old global item entries from expired threads. Last ID: ".$last_id); + + Config::set('system', 'dbclean-last-id-9', $last_id); + } + + // Call it again if not all entries were purged + if (($stage != 0) && ($count > 0)) { + Worker::add(PRIORITY_MEDIUM, 'dbclean'); + } + } +} diff --git a/src/Worker/DBUpdate.php b/src/Worker/DBUpdate.php new file mode 100644 index 0000000000..5f85c8ebef --- /dev/null +++ b/src/Worker/DBUpdate.php @@ -0,0 +1,20 @@ + $argv[1]); + + call_hooks('globaldir_update', $arr); + + logger('Updating directory: ' . $arr['url'], LOGGER_DEBUG); + if (strlen($arr['url'])) { + fetch_url($dir . '?url=' . bin2hex($arr['url'])); + } + + return; + } + + private static function updateAll() { + $r = q("SELECT `url` FROM `contact` + INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid` + INNER JOIN `user` ON `user`.`uid` = `contact`.`uid` + WHERE `contact`.`self` AND `profile`.`net-publish` AND `profile`.`is-default` AND + NOT `user`.`account_expired` AND `user`.`verified`"); + + if (DBM::is_result($r)) { + foreach ($r AS $user) { + Worker::add(PRIORITY_LOW, 'Directory', $user['url']); + } + } + } +} diff --git a/src/Worker/DiscoverPoCo.php b/src/Worker/DiscoverPoCo.php new file mode 100644 index 0000000000..96cfae00b6 --- /dev/null +++ b/src/Worker/DiscoverPoCo.php @@ -0,0 +1,304 @@ +: Searches for "search pattern" in the directory. "search pattern" is url encoded. + - checkcontact: Updates gcontact entries + - suggestions: Discover other servers for their contacts. + - server : Searches for the poco server list. "poco url" is base64 encoded. + - update_server: Frequently check the first 250 servers for vitality. + - update_server_directory: Discover the given server id for their contacts + - PortableContact::load: Load POCO data from a given POCO address + - check_profile: Update remote profile data + */ + + if ($command == "dirsearch") { + $search = urldecode($param1); + $mode = 1; + } elseif ($command == "checkcontact") { + $mode = 2; + } elseif ($command == "suggestions") { + $mode = 3; + } elseif ($command == "server") { + $mode = 4; + } elseif ($command == "update_server") { + $mode = 5; + } elseif ($command == "update_server_directory") { + $mode = 6; + } elseif ($command == "load") { + $mode = 7; + } elseif ($command == "check_profile") { + $mode = 8; + } elseif ($command == '') { + $search = ""; + $mode = 0; + } else { + logger("Unknown or missing parameter ".$command."\n"); + return; + } + + logger('start '.$search); + + if ($mode == 8) { + if ($param1 != "") { + PortableContact::lastUpdated($param1, true); + } + } elseif ($mode == 7) { + if (!empty($param4)) { + $url = $param4; + } else { + $url = ''; + } + PortableContact::load(intval($param1), intval($param2), intval($param3), $url); + } elseif ($mode == 6) { + PortableContact::discoverSingleServer(intval($param1)); + } elseif ($mode == 5) { + self::updateServer(); + } elseif ($mode == 4) { + $server_url = $param1; + if ($server_url == "") { + return; + } + $server_url = filter_var($server_url, FILTER_SANITIZE_URL); + if (substr(normalise_link($server_url), 0, 7) != "http://") { + return; + } + $result = "Checking server ".$server_url." - "; + $ret = PortableContact::checkServer($server_url); + if ($ret) { + $result .= "success"; + } else { + $result .= "failed"; + } + logger($result, LOGGER_DEBUG); + } elseif ($mode == 3) { + GlobalContact::updateSuggestions(); + } elseif (($mode == 2) && Config::get('system', 'poco_completion')) { + self::discoverUsers(); + } elseif (($mode == 1) && ($search != "") && Config::get('system', 'poco_local_search')) { + self::discoverDirectory($search); + self::gsSearchUser($search); + } elseif (($mode == 0) && ($search == "") && (Config::get('system', 'poco_discovery') > 0)) { + // Query Friendica and Hubzilla servers for their users + PortableContact::discover(); + + // Query GNU Social servers for their users ("statistics" addon has to be enabled on the GS server) + if (!Config::get('system', 'ostatus_disabled')) { + GlobalContact::discoverGsUsers(); + } + } + + logger('end '.$search); + + return; + } + + /** + * @brief Updates the first 250 servers + * + */ + private static function updateServer() { + $r = q("SELECT `url`, `created`, `last_failure`, `last_contact` FROM `gserver` ORDER BY rand()"); + + if (!DBM::is_result($r)) { + return; + } + + $updated = 0; + + foreach ($r AS $server) { + if (!PortableContact::updateNeeded($server["created"], "", $server["last_failure"], $server["last_contact"])) { + continue; + } + logger('Update server status for server '.$server["url"], LOGGER_DEBUG); + + Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $server["url"]); + + if (++$updated > 250) { + return; + } + } + } + + private static function discoverUsers() { + logger("Discover users", LOGGER_DEBUG); + + $starttime = time(); + + $users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url`, `network` FROM `gcontact` + WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND + `last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND + `network` IN ('%s', '%s', '%s', '%s', '') ORDER BY rand()", + dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), + dbesc(NETWORK_OSTATUS), dbesc(NETWORK_FEED)); + + if (!$users) { + return; + } + $checked = 0; + + foreach ($users AS $user) { + + $urlparts = parse_url($user["url"]); + if (!isset($urlparts["scheme"])) { + q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'", + dbesc(NETWORK_PHANTOM), dbesc(normalise_link($user["url"]))); + continue; + } + + if (in_array($urlparts["host"], array("www.facebook.com", "facebook.com", "twitter.com", + "identi.ca", "alpha.app.net"))) { + $networks = array("www.facebook.com" => NETWORK_FACEBOOK, + "facebook.com" => NETWORK_FACEBOOK, + "twitter.com" => NETWORK_TWITTER, + "identi.ca" => NETWORK_PUMPIO, + "alpha.app.net" => NETWORK_APPNET); + + q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'", + dbesc($networks[$urlparts["host"]]), dbesc(normalise_link($user["url"]))); + continue; + } + + $server_url = PortableContact::detectServer($user["url"]); + $force_update = false; + + if ($user["server_url"] != "") { + + $force_update = (normalise_link($user["server_url"]) != normalise_link($server_url)); + + $server_url = $user["server_url"]; + } + + if ((($server_url == "") && ($user["network"] == NETWORK_FEED)) || $force_update || PortableContact::checkServer($server_url, $user["network"])) { + logger('Check profile '.$user["url"]); + Worker::add(PRIORITY_LOW, "DiscoverPoCo", "check_profile", $user["url"]); + + if (++$checked > 100) { + return; + } + } else { + q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($user["url"]))); + } + + // Quit the loop after 3 minutes + if (time() > ($starttime + 180)) { + return; + } + } + } + + private static function discoverDirectory($search) { + + $data = Cache::get("dirsearch:".$search); + if (!is_null($data)) { + // Only search for the same item every 24 hours + if (time() < $data + (60 * 60 * 24)) { + logger("Already searched for ".$search." in the last 24 hours", LOGGER_DEBUG); + return; + } + } + + $x = fetch_url(get_server()."/lsearch?p=1&n=500&search=".urlencode($search)); + $j = json_decode($x); + + if (count($j->results)) { + foreach ($j->results as $jj) { + // Check if the contact already exists + $exists = q("SELECT `id`, `last_contact`, `last_failure`, `updated` FROM `gcontact` WHERE `nurl` = '%s'", normalise_link($jj->url)); + if (DBM::is_result($exists)) { + logger("Profile ".$jj->url." already exists (".$search.")", LOGGER_DEBUG); + + if (($exists[0]["last_contact"] < $exists[0]["last_failure"]) && + ($exists[0]["updated"] < $exists[0]["last_failure"])) { + continue; + } + // Update the contact + PortableContact::lastUpdated($jj->url); + continue; + } + + $server_url = PortableContact::detectServer($jj->url); + if ($server_url != '') { + if (!PortableContact::checkServer($server_url)) { + logger("Friendica server ".$server_url." doesn't answer.", LOGGER_DEBUG); + continue; + } + logger("Friendica server ".$server_url." seems to be okay.", LOGGER_DEBUG); + } + + $data = Probe::uri($jj->url); + if ($data["network"] == NETWORK_DFRN) { + logger("Profile ".$jj->url." is reachable (".$search.")", LOGGER_DEBUG); + logger("Add profile ".$jj->url." to local directory (".$search.")", LOGGER_DEBUG); + + if ($jj->tags != "") { + $data["keywords"] = $jj->tags; + } + + $data["server_url"] = $data["baseurl"]; + + GlobalContact::update($data); + } else { + logger("Profile ".$jj->url." is not responding or no Friendica contact - but network ".$data["network"], LOGGER_DEBUG); + } + } + } + Cache::set("dirsearch:".$search, time(), CACHE_DAY); + } + + /** + * @brief Search for GNU Social user with gstools.org + * + * @param str $search User name + */ + private static function gsSearchUser($search) { + + // Currently disabled, since the service isn't available anymore. + // It is not removed since I hope that there will be a successor. + return false; + + $a = get_app(); + + $url = "http://gstools.org/api/users_search/".urlencode($search); + + $result = z_fetch_url($url); + if (!$result["success"]) { + return false; + } + + $contacts = json_decode($result["body"]); + + if ($contacts->status == 'ERROR') { + return false; + } + + /// @TODO AS is considered as a notation for constants (as they usually being written all upper-case) + /// @TODO find all those and convert to all lower-case which is a keyword then + foreach ($contacts->data AS $user) { + $contact = Probe::uri($user->site_address."/".$user->name); + if ($contact["network"] != NETWORK_PHANTOM) { + $contact["about"] = $user->description; + GlobalContact::update($contact); + } + } + } +} diff --git a/src/Worker/RemoveContact.php b/src/Worker/RemoveContact.php new file mode 100644 index 0000000000..811b0295fb --- /dev/null +++ b/src/Worker/RemoveContact.php @@ -0,0 +1,23 @@ + $id)); + if ($r) { + return; + } + + // Now we delete all the depending table entries + dba::delete('contact', array('id' => $id)); + } +} diff --git a/src/Worker/SpoolPost.php b/src/Worker/SpoolPost.php new file mode 100644 index 0000000000..a394ee0fe1 --- /dev/null +++ b/src/Worker/SpoolPost.php @@ -0,0 +1,58 @@ +