From 585af09b7dfbeb3425ab21f6488e7eee9b7b486c Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 6 Feb 2022 05:58:37 +0000 Subject: [PATCH 1/6] Only display federated servers --- src/Module/Admin/Federation.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Module/Admin/Federation.php b/src/Module/Admin/Federation.php index 0e315c37c..119196189 100644 --- a/src/Module/Admin/Federation.php +++ b/src/Module/Admin/Federation.php @@ -21,6 +21,7 @@ namespace Friendica\Module\Admin; +use Friendica\Core\Protocol; use Friendica\Core\Renderer; use Friendica\Database\DBA; use Friendica\DI; @@ -72,15 +73,15 @@ class Federation extends BaseAdmin $gservers = DBA::p("SELECT COUNT(*) AS `total`, SUM(`registered-users`) AS `users`, `platform`, ANY_VALUE(`network`) AS `network`, MAX(`version`) AS `version` - FROM `gserver` WHERE NOT `failed` AND `detection-method` != ? GROUP BY `platform`", GServer::DETECT_MANUAL); + FROM `gserver` WHERE NOT `failed` AND `detection-method` != ? AND NOT `network` IN (?, ?) GROUP BY `platform`", GServer::DETECT_MANUAL, Protocol::PHANTOM, Protocol::FEED); while ($gserver = DBA::fetch($gservers)) { $total += $gserver['total']; $users += $gserver['users']; $versionCounts = []; $versions = DBA::p("SELECT COUNT(*) AS `total`, `version` FROM `gserver` - WHERE NOT `failed` AND `platform` = ? AND `detection-method` != ? - GROUP BY `version` ORDER BY `version`", $gserver['platform'], GServer::DETECT_MANUAL); + WHERE NOT `failed` AND `platform` = ? AND `detection-method` != ? AND NOT `network` IN (?, ?) + GROUP BY `version` ORDER BY `version`", $gserver['platform'], GServer::DETECT_MANUAL, Protocol::PHANTOM, Protocol::FEED); while ($version = DBA::fetch($versions)) { $version['version'] = str_replace(["\n", "\r", "\t"], " ", $version['version']); From ea2faa28fe899735a9673dfadd4db7a0356509d7 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 6 Feb 2022 05:59:47 +0000 Subject: [PATCH 2/6] Use first word as platform / improvements for registered users --- src/Model/GServer.php | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 26c36d6a9..4db851571 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -496,17 +496,17 @@ class GServer $serverdata['url'] = $url; $serverdata['nurl'] = Strings::normaliseLink($url); - // We take the highest number that we do find - $registeredUsers = $serverdata['registered-users'] ?? 0; - - // On an active server there has to be at least a single user - if (($serverdata['network'] != Protocol::PHANTOM) && ($registeredUsers == 0)) { - $registeredUsers = 1; + if ($serverdata['network'] == Protocol::PHANTOM) { + $serverdata = self::detectNetworkViaContacts($url, $serverdata); } - if ($serverdata['network'] == Protocol::PHANTOM) { - $serverdata['registered-users'] = max($registeredUsers, 1); - $serverdata = self::detectNetworkViaContacts($url, $serverdata); + $serverdata['registered-users'] = $serverdata['registered-users'] ?? 0; + + // On an active server there has to be at least a single user + if (!in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED]) && ($serverdata['registered-users'] == 0)) { + $serverdata['registered-users'] = 1; + } elseif (in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED])) { + $serverdata['registered-users'] = 0; } $serverdata['next_contact'] = self::getNextUpdateDate(true); @@ -520,11 +520,6 @@ class GServer $ret = DBA::insert('gserver', $serverdata); $id = DBA::lastInsertId(); } else { - // Don't override the network with 'unknown' when there had been a valid entry before - if (($serverdata['network'] == Protocol::PHANTOM) && !empty($gserver['network'])) { - unset($serverdata['network']); - } - $ret = DBA::update('gserver', $serverdata, ['nurl' => $serverdata['nurl']]); $gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => $serverdata['nurl']]); if (DBA::isResult($gserver)) { @@ -532,11 +527,12 @@ class GServer } } - if (!empty($serverdata['network']) && !empty($id) && ($serverdata['network'] != Protocol::PHANTOM)) { + // Count the number of known contacts from this server + if (!empty($id) && !in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED])) { $apcontacts = DBA::count('apcontact', ['gsid' => $id]); - $contacts = DBA::count('contact', ['uid' => 0, 'gsid' => $id]); - $max_users = max($apcontacts, $contacts, $registeredUsers, 1); - if ($max_users > $registeredUsers) { + $contacts = DBA::count('contact', ['uid' => 0, 'gsid' => $id, 'failed' => false]); + $max_users = max($apcontacts, $contacts); + if ($max_users > $serverdata['registered-users']) { Logger::info('Update registered users', ['id' => $id, 'url' => $serverdata['nurl'], 'registered-users' => $max_users]); DBA::update('gserver', ['registered-users' => $max_users], ['id' => $id]); } @@ -1056,14 +1052,12 @@ class GServer foreach ($contacts as $contact) { $probed = Contact::getByURL($contact); - if (!empty($probed) && in_array($probed['network'], Protocol::FEDERATED)) { + if (!empty($probed) && !$probed['failed'] && in_array($probed['network'], Protocol::FEDERATED)) { $serverdata['network'] = $probed['network']; break; } } - $serverdata['registered-users'] = max($serverdata['registered-users'], count($contacts), 1); - return $serverdata; } @@ -1598,7 +1592,8 @@ class GServer } } - if (!in_array(strtolower($attr['content']), $valid_platforms)) { + $platform = explode(' ', strtolower($attr['content'])); + if (!in_array($platform[0], $valid_platforms)) { continue; } @@ -1661,7 +1656,8 @@ class GServer } } - if (!in_array(strtolower($attr['content']), $valid_platforms)) { + $platform = explode(' ', strtolower($attr['content'])); + if (!in_array($platform[0], $valid_platforms)) { continue; } From f1cac5170b3e9fb7d07a25a7ba3e7f06325079e7 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 6 Feb 2022 08:02:19 +0000 Subject: [PATCH 3/6] mark sites as "phantom" that are up but couldn't be detected --- src/Model/GServer.php | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 4db851571..6a4b14a7b 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -1562,7 +1562,7 @@ class GServer // Using only body information we cannot safely detect a lot of systems. // So we define a list of platforms that we can detect safely. - $valid_platforms = ['friendica', 'friendika', 'hubzilla', 'misskey', 'peertube', 'wordpress', 'write.as']; + $valid_platforms = ['friendica', 'friendika', 'diaspora', 'mastodon', 'hubzilla', 'misskey', 'peertube', 'wordpress', 'write.as']; $doc = new DOMDocument(); @$doc->loadHTML($curlResult->getBody()); @@ -1592,11 +1592,6 @@ class GServer } } - $platform = explode(' ', strtolower($attr['content'])); - if (!in_array($platform[0], $valid_platforms)) { - continue; - } - if ($attr['name'] == 'description') { $serverdata['info'] = $attr['content']; } @@ -1656,11 +1651,6 @@ class GServer } } - $platform = explode(' ', strtolower($attr['content'])); - if (!in_array($platform[0], $valid_platforms)) { - continue; - } - if ($attr['property'] == 'og:site_name') { $serverdata['site_name'] = $attr['content']; } @@ -1687,7 +1677,11 @@ class GServer } } - if (!empty($serverdata['network']) && ($serverdata['detection-method'] == self::DETECT_MANUAL)) { + if (!empty($serverdata['platform']) && in_array($serverdata['detection-method'], [self::DETECT_MANUAL, self::DETECT_BODY]) && !in_array($serverdata['platform'], $valid_platforms)) { + $serverdata['network'] = Protocol::PHANTOM; + $serverdata['version'] = ''; + $serverdata['detection-method'] = self::DETECT_MANUAL; + } elseif (!empty($serverdata['network']) && ($serverdata['detection-method'] == self::DETECT_MANUAL)) { $serverdata['detection-method'] = self::DETECT_BODY; } From 85f029a7244b4ecea37c94a8fb99eb7878224286 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 6 Feb 2022 09:03:48 +0000 Subject: [PATCH 4/6] Enforce probing --- src/Model/GServer.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 6a4b14a7b..d38078ea5 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -1050,11 +1050,15 @@ class GServer return $serverdata; } + $retrial = 0; foreach ($contacts as $contact) { - $probed = Contact::getByURL($contact); + $probed = Contact::getByURL($contact, true); if (!empty($probed) && !$probed['failed'] && in_array($probed['network'], Protocol::FEDERATED)) { $serverdata['network'] = $probed['network']; break; + } elseif (++$retrial > 10) { + // To reduce the stress on remote systems we probe a maximum of 10 contacts + break; } } From b3da88ceaa597b73e223671eaefe097ab19822fa Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 6 Feb 2022 09:21:14 +0000 Subject: [PATCH 5/6] Don't assume AP with Wordpress --- src/Model/GServer.php | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/Model/GServer.php b/src/Model/GServer.php index d38078ea5..3b4866cbf 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -496,7 +496,7 @@ class GServer $serverdata['url'] = $url; $serverdata['nurl'] = Strings::normaliseLink($url); - if ($serverdata['network'] == Protocol::PHANTOM) { + if (in_array($serverdata['network'], [Protocol::PHANTOM, Protocol::FEED])) { $serverdata = self::detectNetworkViaContacts($url, $serverdata); } @@ -1617,11 +1617,8 @@ class GServer $serverdata['version'] = $version_part[1]; // We still do need a reliable test if some AP plugin is activated - if (DBA::exists('apcontact', ['baseurl' => $url])) { - $serverdata['network'] = Protocol::ACTIVITYPUB; - } else { - $serverdata['network'] = Protocol::FEED; - } + // By now we just check in a later process for some known contacts + $serverdata['network'] = Protocol::FEED; if ($serverdata['detection-method'] == self::DETECT_MANUAL) { $serverdata['detection-method'] = self::DETECT_BODY; From 8b585c6cccd84eecef44778a67d9183be4c3bf0f Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 6 Feb 2022 10:07:18 +0000 Subject: [PATCH 6/6] Set the timeout to 10 seconds --- src/Model/GServer.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 3b4866cbf..90b023f6f 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -1050,14 +1050,14 @@ class GServer return $serverdata; } - $retrial = 0; + $time = time(); foreach ($contacts as $contact) { $probed = Contact::getByURL($contact, true); if (!empty($probed) && !$probed['failed'] && in_array($probed['network'], Protocol::FEDERATED)) { $serverdata['network'] = $probed['network']; break; - } elseif (++$retrial > 10) { - // To reduce the stress on remote systems we probe a maximum of 10 contacts + } elseif ((time() - $time) > 10) { + // To reduce the stress on remote systems we probe a maximum of 10 seconds break; } }