Check and add a server

This commit is contained in:
Michael 2020-12-03 22:32:51 +00:00
parent 8cbe2f025b
commit 8d999f54d0
5 changed files with 63 additions and 84 deletions

View file

@ -70,6 +70,22 @@ class GServer
const DETECT_NODEINFO_1 = 101; const DETECT_NODEINFO_1 = 101;
const DETECT_NODEINFO_2 = 102; const DETECT_NODEINFO_2 = 102;
/**
* Check for the existance of a server and adds it in the background if not existant
*
* @param string $url
* @param boolean $only_nodeinfo
* @return void
*/
public static function add(string $url, bool $only_nodeinfo = false)
{
if (self::getID($url, false)) {
return;
}
Worker::add(PRIORITY_LOW, 'UpdateGServer', $url, $only_nodeinfo);
}
/** /**
* Get the ID for the given server URL * Get the ID for the given server URL
* *
@ -173,61 +189,6 @@ class GServer
return DateTimeFormat::utc('now +3 month'); return DateTimeFormat::utc('now +3 month');
} }
/**
* Decides if a server needs to be updated, based upon several date fields
*
* @param date $created Creation date of that server entry
* @param date $updated When had the server entry be updated
* @param date $last_failure Last failure when contacting that server
* @param date $last_contact Last time the server had been contacted
*
* @return boolean Does the server record needs an update?
*/
public static function updateNeeded($created, $updated, $last_failure, $last_contact)
{
$now = strtotime(DateTimeFormat::utcNow());
if ($updated > $last_contact) {
$contact_time = strtotime($updated);
} else {
$contact_time = strtotime($last_contact);
}
$failure_time = strtotime($last_failure);
$created_time = strtotime($created);
// If there is no "created" time then use the current time
if ($created_time <= 0) {
$created_time = $now;
}
// If the last contact was less than 24 hours then don't update
if (($now - $contact_time) < (60 * 60 * 24)) {
return false;
}
// If the last failure was less than 24 hours then don't update
if (($now - $failure_time) < (60 * 60 * 24)) {
return false;
}
// If the last contact was less than a week ago and the last failure is older than a week then don't update
//if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time))
// return false;
// If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week
if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) {
return false;
}
// If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month
if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) {
return false;
}
return true;
}
/** /**
* Checks the state of the given server. * Checks the state of the given server.
* *
@ -241,7 +202,6 @@ class GServer
public static function check(string $server_url, string $network = '', bool $force = false, bool $only_nodeinfo = false) public static function check(string $server_url, string $network = '', bool $force = false, bool $only_nodeinfo = false)
{ {
$server_url = self::cleanURL($server_url); $server_url = self::cleanURL($server_url);
if ($server_url == '') { if ($server_url == '') {
return false; return false;
} }
@ -254,24 +214,11 @@ class GServer
DBA::update('gserver', $fields, $condition); DBA::update('gserver', $fields, $condition);
} }
$last_contact = $gserver['last_contact']; if (!$force && (strtotime($gserver['next_contact']) > time())) {
$last_failure = $gserver['last_failure'];
// See discussion under https://forum.friendi.ca/display/0b6b25a8135aabc37a5a0f5684081633
// It can happen that a zero date is in the database, but storing it again is forbidden.
if ($last_contact < DBA::NULL_DATETIME) {
$last_contact = DBA::NULL_DATETIME;
}
if ($last_failure < DBA::NULL_DATETIME) {
$last_failure = DBA::NULL_DATETIME;
}
if (!$force && !self::updateNeeded($gserver['created'], '', $last_failure, $last_contact)) {
Logger::info('No update needed', ['server' => $server_url]); Logger::info('No update needed', ['server' => $server_url]);
return ($last_contact >= $last_failure); return (!$gserver['failed']);
} }
Logger::info('Server is outdated. Start discovery.', ['Server' => $server_url, 'Force' => $force, 'Created' => $gserver['created'], 'Failure' => $last_failure, 'Contact' => $last_contact]); Logger::info('Server is outdated. Start discovery.', ['Server' => $server_url, 'Force' => $force]);
} else { } else {
Logger::info('Server is unknown. Start discovery.', ['Server' => $server_url]); Logger::info('Server is unknown. Start discovery.', ['Server' => $server_url]);
} }
@ -1690,7 +1637,7 @@ class GServer
if (!empty($data['data']['nodes'])) { if (!empty($data['data']['nodes'])) {
foreach ($data['data']['nodes'] as $server) { foreach ($data['data']['nodes'] as $server) {
// Using "only_nodeinfo" since servers that are listed on that page should always have it. // Using "only_nodeinfo" since servers that are listed on that page should always have it.
Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $server['host'], true); self::add('https://' . $server['host'], true);
} }
} }
} }
@ -1709,7 +1656,7 @@ class GServer
foreach ($servers['instances'] as $server) { foreach ($servers['instances'] as $server) {
$url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name']; $url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name'];
Worker::add(PRIORITY_LOW, 'UpdateGServer', $url); self::add($url);
} }
} }
} }

View file

@ -22,6 +22,7 @@
namespace Friendica\Worker; namespace Friendica\Worker;
use Friendica\Core\Logger; use Friendica\Core\Logger;
use Friendica\Database\DBA;
use Friendica\Model\GServer; use Friendica\Model\GServer;
use Friendica\Util\Strings; use Friendica\Util\Strings;
@ -32,7 +33,7 @@ class UpdateGServer
* @param string $server_url Server URL * @param string $server_url Server URL
* @param boolean $only_nodeinfo Only use nodeinfo for server detection * @param boolean $only_nodeinfo Only use nodeinfo for server detection
*/ */
public static function execute(string $server_url, bool $only_nodeinfo = false, bool $force = false) public static function execute(string $server_url, bool $only_nodeinfo = false)
{ {
if (empty($server_url)) { if (empty($server_url)) {
return; return;
@ -40,11 +41,22 @@ class UpdateGServer
$filtered = filter_var($server_url, FILTER_SANITIZE_URL); $filtered = filter_var($server_url, FILTER_SANITIZE_URL);
if (substr(Strings::normaliseLink($filtered), 0, 7) != 'http://') { if (substr(Strings::normaliseLink($filtered), 0, 7) != 'http://') {
GServer::setFailure($filtered); GServer::setFailure($server_url);
return; return;
} }
$ret = GServer::check($filtered, '', $force, $only_nodeinfo); if (($filtered != $server_url) && DBA::exists('gserver', ['nurl' => Strings::normaliseLink($server_url)])) {
GServer::setFailure($server_url);
return;
}
$cleaned = GServer::cleanURL($server_url);
if (($cleaned != $server_url) && DBA::exists('gserver', ['nurl' => Strings::normaliseLink($server_url)])) {
GServer::setFailure($server_url);
return;
}
$ret = GServer::check($filtered, '', true, $only_nodeinfo);
Logger::info('Updated gserver', ['url' => $filtered, 'result' => $ret]); Logger::info('Updated gserver', ['url' => $filtered, 'result' => $ret]);
} }
} }

View file

@ -24,33 +24,49 @@ namespace Friendica\Worker;
use Friendica\Core\Logger; use Friendica\Core\Logger;
use Friendica\Core\Worker; use Friendica\Core\Worker;
use Friendica\Database\DBA; use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Util\Strings;
class UpdateGServers class UpdateGServers
{ {
/** /**
* Updates up to 100 servers * Updates a defined number of servers
*/ */
public static function execute() public static function execute()
{ {
$update_limit = DI::config()->get('system', 'gserver_update_limit');
if (empty($update_limit)) {
return;
}
$updating = Worker::countWorkersByCommand('UpdateGServer'); $updating = Worker::countWorkersByCommand('UpdateGServer');
$limit = 100 - $updating; $limit = $update_limit - $updating;
if ($limit <= 0) { if ($limit <= 0) {
Logger::info('The number of currently running jobs exceed the limit'); Logger::info('The number of currently running jobs exceed the limit');
return; return;
} }
$outdated = DBA::count('gserver', ["`next_contact` < UTC_TIMESTAMP()"]);
$total = DBA::count('gserver'); $total = DBA::count('gserver');
$condition = ["`next_contact` < UTC_TIMESTAMP() AND (`nurl` != ? OR `url` != ?)", '', ''];
$outdated = DBA::count('gserver', $condition);
Logger::info('Server status', ['total' => $total, 'outdated' => $outdated, 'updating' => $limit]); Logger::info('Server status', ['total' => $total, 'outdated' => $outdated, 'updating' => $limit]);
$gservers = DBA::select('gserver', ['url'], ["`next_contact` < UTC_TIMESTAMP()"], ['limit' => $limit]); $gservers = DBA::select('gserver', ['url', 'nurl'], $condition, ['limit' => $limit]);
if (!DBA::isResult($gservers)) { if (!DBA::isResult($gservers)) {
return; return;
} }
$count = 0; $count = 0;
while ($gserver = DBA::fetch($gservers)) { while ($gserver = DBA::fetch($gservers)) {
Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['url'], false, true); // Sometimes the "nurl" and "url" doesn't seem to fit, see https://forum.friendi.ca/display/ec054ce7-155f-c94d-6159-f50372664245
// There are duplicated "url" but not "nurl". So we check both addresses instead of just overwriting them,
// since that would mean loosing data.
if (!empty($gserver['url'])) {
Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['url']);
}
if (!empty($gserver['nurl']) && ($gserver['nurl'] != Strings::normaliseLink($gserver['url']))) {
Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['nurl']);
}
$count++; $count++;
} }
DBA::close($gservers); DBA::close($gservers);

View file

@ -22,9 +22,9 @@
namespace Friendica\Worker; namespace Friendica\Worker;
use Friendica\Core\Logger; use Friendica\Core\Logger;
use Friendica\Core\Worker;
use Friendica\Database\DBA; use Friendica\Database\DBA;
use Friendica\DI; use Friendica\DI;
use Friendica\Model\GServer;
use Friendica\Util\Strings; use Friendica\Util\Strings;
class UpdateServerPeers class UpdateServerPeers
@ -58,7 +58,7 @@ class UpdateServerPeers
continue; continue;
} }
// This endpoint doesn't offer the schema. So we assume that it is HTTPS. // This endpoint doesn't offer the schema. So we assume that it is HTTPS.
Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $peer); GServer::add('https://' . $peer);
++$added; ++$added;
} }
Logger::info('Server peer update ended', ['total' => $total, 'added' => $added, 'url' => $url]); Logger::info('Server peer update ended', ['total' => $total, 'added' => $added, 'url' => $url]);

View file

@ -254,6 +254,10 @@ return [
// This can alternatively be set on a per account basis in the pconfig table. // This can alternatively be set on a per account basis in the pconfig table.
'groupedit_image_limit' => 400, 'groupedit_image_limit' => 400,
// gserver_update_limit (Integer)
// How much servers should be checked at a time?
'gserver_update_limit' => 100,
// hsts (Boolean) // hsts (Boolean)
// Enables the sending of HTTP Strict Transport Security headers. // Enables the sending of HTTP Strict Transport Security headers.
'hsts' => false, 'hsts' => false,