From 8d999f54d025cebcaeef9386ae9910def7af52c3 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 3 Dec 2020 22:32:51 +0000 Subject: [PATCH] Check and add a server --- src/Model/GServer.php | 95 +++++++------------------------- src/Worker/UpdateGServer.php | 18 +++++- src/Worker/UpdateGServers.php | 26 +++++++-- src/Worker/UpdateServerPeers.php | 4 +- static/defaults.config.php | 4 ++ 5 files changed, 63 insertions(+), 84 deletions(-) diff --git a/src/Model/GServer.php b/src/Model/GServer.php index c035abc73f..0b79b3f8fd 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -70,6 +70,22 @@ class GServer const DETECT_NODEINFO_1 = 101; const DETECT_NODEINFO_2 = 102; + /** + * Check for the existance of a server and adds it in the background if not existant + * + * @param string $url + * @param boolean $only_nodeinfo + * @return void + */ + public static function add(string $url, bool $only_nodeinfo = false) + { + if (self::getID($url, false)) { + return; + } + + Worker::add(PRIORITY_LOW, 'UpdateGServer', $url, $only_nodeinfo); + } + /** * Get the ID for the given server URL * @@ -173,61 +189,6 @@ class GServer return DateTimeFormat::utc('now +3 month'); } - /** - * Decides if a server needs to be updated, based upon several date fields - * - * @param date $created Creation date of that server entry - * @param date $updated When had the server entry be updated - * @param date $last_failure Last failure when contacting that server - * @param date $last_contact Last time the server had been contacted - * - * @return boolean Does the server record needs an update? - */ - public static function updateNeeded($created, $updated, $last_failure, $last_contact) - { - $now = strtotime(DateTimeFormat::utcNow()); - - if ($updated > $last_contact) { - $contact_time = strtotime($updated); - } else { - $contact_time = strtotime($last_contact); - } - - $failure_time = strtotime($last_failure); - $created_time = strtotime($created); - - // If there is no "created" time then use the current time - if ($created_time <= 0) { - $created_time = $now; - } - - // If the last contact was less than 24 hours then don't update - if (($now - $contact_time) < (60 * 60 * 24)) { - return false; - } - - // If the last failure was less than 24 hours then don't update - if (($now - $failure_time) < (60 * 60 * 24)) { - return false; - } - - // If the last contact was less than a week ago and the last failure is older than a week then don't update - //if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time)) - // return false; - - // If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week - if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) { - return false; - } - - // If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month - if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) { - return false; - } - - return true; - } - /** * Checks the state of the given server. * @@ -241,7 +202,6 @@ class GServer public static function check(string $server_url, string $network = '', bool $force = false, bool $only_nodeinfo = false) { $server_url = self::cleanURL($server_url); - if ($server_url == '') { return false; } @@ -254,24 +214,11 @@ class GServer DBA::update('gserver', $fields, $condition); } - $last_contact = $gserver['last_contact']; - $last_failure = $gserver['last_failure']; - - // See discussion under https://forum.friendi.ca/display/0b6b25a8135aabc37a5a0f5684081633 - // It can happen that a zero date is in the database, but storing it again is forbidden. - if ($last_contact < DBA::NULL_DATETIME) { - $last_contact = DBA::NULL_DATETIME; - } - - if ($last_failure < DBA::NULL_DATETIME) { - $last_failure = DBA::NULL_DATETIME; - } - - if (!$force && !self::updateNeeded($gserver['created'], '', $last_failure, $last_contact)) { + if (!$force && (strtotime($gserver['next_contact']) > time())) { Logger::info('No update needed', ['server' => $server_url]); - return ($last_contact >= $last_failure); + return (!$gserver['failed']); } - Logger::info('Server is outdated. Start discovery.', ['Server' => $server_url, 'Force' => $force, 'Created' => $gserver['created'], 'Failure' => $last_failure, 'Contact' => $last_contact]); + Logger::info('Server is outdated. Start discovery.', ['Server' => $server_url, 'Force' => $force]); } else { Logger::info('Server is unknown. Start discovery.', ['Server' => $server_url]); } @@ -1690,7 +1637,7 @@ class GServer if (!empty($data['data']['nodes'])) { foreach ($data['data']['nodes'] as $server) { // Using "only_nodeinfo" since servers that are listed on that page should always have it. - Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $server['host'], true); + self::add('https://' . $server['host'], true); } } } @@ -1709,7 +1656,7 @@ class GServer foreach ($servers['instances'] as $server) { $url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name']; - Worker::add(PRIORITY_LOW, 'UpdateGServer', $url); + self::add($url); } } } diff --git a/src/Worker/UpdateGServer.php b/src/Worker/UpdateGServer.php index 696ec125d8..0c42a67ec3 100644 --- a/src/Worker/UpdateGServer.php +++ b/src/Worker/UpdateGServer.php @@ -22,6 +22,7 @@ namespace Friendica\Worker; use Friendica\Core\Logger; +use Friendica\Database\DBA; use Friendica\Model\GServer; use Friendica\Util\Strings; @@ -32,7 +33,7 @@ class UpdateGServer * @param string $server_url Server URL * @param boolean $only_nodeinfo Only use nodeinfo for server detection */ - public static function execute(string $server_url, bool $only_nodeinfo = false, bool $force = false) + public static function execute(string $server_url, bool $only_nodeinfo = false) { if (empty($server_url)) { return; @@ -40,11 +41,22 @@ class UpdateGServer $filtered = filter_var($server_url, FILTER_SANITIZE_URL); if (substr(Strings::normaliseLink($filtered), 0, 7) != 'http://') { - GServer::setFailure($filtered); + GServer::setFailure($server_url); return; } - $ret = GServer::check($filtered, '', $force, $only_nodeinfo); + if (($filtered != $server_url) && DBA::exists('gserver', ['nurl' => Strings::normaliseLink($server_url)])) { + GServer::setFailure($server_url); + return; + } + + $cleaned = GServer::cleanURL($server_url); + if (($cleaned != $server_url) && DBA::exists('gserver', ['nurl' => Strings::normaliseLink($server_url)])) { + GServer::setFailure($server_url); + return; + } + + $ret = GServer::check($filtered, '', true, $only_nodeinfo); Logger::info('Updated gserver', ['url' => $filtered, 'result' => $ret]); } } diff --git a/src/Worker/UpdateGServers.php b/src/Worker/UpdateGServers.php index 12022a3468..486dbc93c2 100644 --- a/src/Worker/UpdateGServers.php +++ b/src/Worker/UpdateGServers.php @@ -24,33 +24,49 @@ namespace Friendica\Worker; use Friendica\Core\Logger; use Friendica\Core\Worker; use Friendica\Database\DBA; +use Friendica\DI; +use Friendica\Util\Strings; class UpdateGServers { /** - * Updates up to 100 servers + * Updates a defined number of servers */ public static function execute() { + $update_limit = DI::config()->get('system', 'gserver_update_limit'); + if (empty($update_limit)) { + return; + } + $updating = Worker::countWorkersByCommand('UpdateGServer'); - $limit = 100 - $updating; + $limit = $update_limit - $updating; if ($limit <= 0) { Logger::info('The number of currently running jobs exceed the limit'); return; } - $outdated = DBA::count('gserver', ["`next_contact` < UTC_TIMESTAMP()"]); $total = DBA::count('gserver'); + $condition = ["`next_contact` < UTC_TIMESTAMP() AND (`nurl` != ? OR `url` != ?)", '', '']; + $outdated = DBA::count('gserver', $condition); Logger::info('Server status', ['total' => $total, 'outdated' => $outdated, 'updating' => $limit]); - $gservers = DBA::select('gserver', ['url'], ["`next_contact` < UTC_TIMESTAMP()"], ['limit' => $limit]); + $gservers = DBA::select('gserver', ['url', 'nurl'], $condition, ['limit' => $limit]); if (!DBA::isResult($gservers)) { return; } $count = 0; while ($gserver = DBA::fetch($gservers)) { - Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['url'], false, true); + // Sometimes the "nurl" and "url" doesn't seem to fit, see https://forum.friendi.ca/display/ec054ce7-155f-c94d-6159-f50372664245 + // There are duplicated "url" but not "nurl". So we check both addresses instead of just overwriting them, + // since that would mean loosing data. + if (!empty($gserver['url'])) { + Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['url']); + } + if (!empty($gserver['nurl']) && ($gserver['nurl'] != Strings::normaliseLink($gserver['url']))) { + Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['nurl']); + } $count++; } DBA::close($gservers); diff --git a/src/Worker/UpdateServerPeers.php b/src/Worker/UpdateServerPeers.php index ff0cdfa730..d7dc7562cf 100644 --- a/src/Worker/UpdateServerPeers.php +++ b/src/Worker/UpdateServerPeers.php @@ -22,9 +22,9 @@ namespace Friendica\Worker; use Friendica\Core\Logger; -use Friendica\Core\Worker; use Friendica\Database\DBA; use Friendica\DI; +use Friendica\Model\GServer; use Friendica\Util\Strings; class UpdateServerPeers @@ -58,7 +58,7 @@ class UpdateServerPeers continue; } // This endpoint doesn't offer the schema. So we assume that it is HTTPS. - Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $peer); + GServer::add('https://' . $peer); ++$added; } Logger::info('Server peer update ended', ['total' => $total, 'added' => $added, 'url' => $url]); diff --git a/static/defaults.config.php b/static/defaults.config.php index e7d7985aa4..053811b7f1 100644 --- a/static/defaults.config.php +++ b/static/defaults.config.php @@ -254,6 +254,10 @@ return [ // This can alternatively be set on a per account basis in the pconfig table. 'groupedit_image_limit' => 400, + // gserver_update_limit (Integer) + // How much servers should be checked at a time? + 'gserver_update_limit' => 100, + // hsts (Boolean) // Enables the sending of HTTP Strict Transport Security headers. 'hsts' => false,