diff --git a/database.sql b/database.sql index a64094732f..bcd8c9843d 100644 --- a/database.sql +++ b/database.sql @@ -1,9 +1,37 @@ -- ------------------------------------------ -- Friendica 2020.06-dev (Red Hot Poker) --- DB_UPDATE_VERSION 1348 +-- DB_UPDATE_VERSION 1350 -- ------------------------------------------ +-- +-- TABLE gserver +-- +CREATE TABLE IF NOT EXISTS `gserver` ( + `id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID', + `url` varchar(255) NOT NULL DEFAULT '' COMMENT '', + `nurl` varchar(255) NOT NULL DEFAULT '' COMMENT '', + `version` varchar(255) NOT NULL DEFAULT '' COMMENT '', + `site_name` varchar(255) NOT NULL DEFAULT '' COMMENT '', + `info` text COMMENT '', + `register_policy` tinyint NOT NULL DEFAULT 0 COMMENT '', + `registered-users` int unsigned NOT NULL DEFAULT 0 COMMENT 'Number of registered users', + `directory-type` tinyint DEFAULT 0 COMMENT 'Type of directory service (Poco, Mastodon)', + `poco` varchar(255) NOT NULL DEFAULT '' COMMENT '', + `noscrape` varchar(255) NOT NULL DEFAULT '' COMMENT '', + `network` char(4) NOT NULL DEFAULT '' COMMENT '', + `platform` varchar(255) NOT NULL DEFAULT '' COMMENT '', + `relay-subscribe` boolean NOT NULL DEFAULT '0' COMMENT 'Has the server subscribed to the relay system', + `relay-scope` varchar(10) NOT NULL DEFAULT '' COMMENT 'The scope of messages that the server wants to get', + `detection-method` tinyint unsigned COMMENT 'Method that had been used to detect that server', + `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', + `last_poco_query` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', + `last_contact` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', + `last_failure` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', + PRIMARY KEY(`id`), + UNIQUE INDEX `nurl` (`nurl`(190)) +) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Global servers'; + -- -- TABLE clients -- @@ -87,6 +115,7 @@ CREATE TABLE IF NOT EXISTS `contact` ( `unsearchable` boolean NOT NULL DEFAULT '0' COMMENT 'Contact prefers to not be searchable', `sensitive` boolean NOT NULL DEFAULT '0' COMMENT 'Contact posts sensitive content', `baseurl` varchar(255) DEFAULT '' COMMENT 'baseurl of the contact', + `gsid` int unsigned COMMENT 'Global Server ID', `reason` text COMMENT '', `closeness` tinyint unsigned NOT NULL DEFAULT 99 COMMENT '', `info` mediumtext COMMENT '', @@ -108,7 +137,9 @@ CREATE TABLE IF NOT EXISTS `contact` ( INDEX `nurl_uid` (`nurl`(32),`uid`), INDEX `nick_uid` (`nick`(32),`uid`), INDEX `dfrn-id` (`dfrn-id`(64)), - INDEX `issued-id` (`issued-id`(64)) + INDEX `issued-id` (`issued-id`(64)), + INDEX `gsid` (`gsid`), + FOREIGN KEY (`gsid`) REFERENCES `gserver` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='contact table'; -- @@ -210,6 +241,7 @@ CREATE TABLE IF NOT EXISTS `apcontact` ( `alias` varchar(255) COMMENT '', `pubkey` text COMMENT '', `baseurl` varchar(255) COMMENT 'baseurl of the ap contact', + `gsid` int unsigned COMMENT 'Global Server ID', `generator` varchar(255) COMMENT 'Name of the contact\'s system', `following_count` int unsigned DEFAULT 0 COMMENT 'Number of following contacts', `followers_count` int unsigned DEFAULT 0 COMMENT 'Number of followers', @@ -218,7 +250,9 @@ CREATE TABLE IF NOT EXISTS `apcontact` ( PRIMARY KEY(`url`), INDEX `addr` (`addr`(32)), INDEX `alias` (`alias`(190)), - INDEX `url` (`followers`(190)) + INDEX `followers` (`followers`(190)), + INDEX `gsid` (`gsid`), + FOREIGN KEY (`gsid`) REFERENCES `gserver` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='ActivityPub compatible contacts - used in the ActivityPub implementation'; -- @@ -463,13 +497,16 @@ CREATE TABLE IF NOT EXISTS `gcontact` ( `alias` varchar(255) NOT NULL DEFAULT '' COMMENT '', `generation` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '', `server_url` varchar(255) NOT NULL DEFAULT '' COMMENT 'baseurl of the contacts server', + `gsid` int unsigned COMMENT 'Global Server ID', PRIMARY KEY(`id`), UNIQUE INDEX `nurl` (`nurl`(190)), INDEX `name` (`name`(64)), INDEX `nick` (`nick`(32)), INDEX `addr` (`addr`(64)), INDEX `hide_network_updated` (`hide`,`network`,`updated`), - INDEX `updated` (`updated`) + INDEX `updated` (`updated`), + INDEX `gsid` (`gsid`), + FOREIGN KEY (`gsid`) REFERENCES `gserver` (`id`) ON UPDATE RESTRICT ON DELETE RESTRICT ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='global contacts'; -- @@ -523,33 +560,6 @@ CREATE TABLE IF NOT EXISTS `group_member` ( UNIQUE INDEX `gid_contactid` (`gid`,`contact-id`) ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='privacy groups, member info'; --- --- TABLE gserver --- -CREATE TABLE IF NOT EXISTS `gserver` ( - `id` int unsigned NOT NULL auto_increment COMMENT 'sequential ID', - `url` varchar(255) NOT NULL DEFAULT '' COMMENT '', - `nurl` varchar(255) NOT NULL DEFAULT '' COMMENT '', - `version` varchar(255) NOT NULL DEFAULT '' COMMENT '', - `site_name` varchar(255) NOT NULL DEFAULT '' COMMENT '', - `info` text COMMENT '', - `register_policy` tinyint NOT NULL DEFAULT 0 COMMENT '', - `registered-users` int unsigned NOT NULL DEFAULT 0 COMMENT 'Number of registered users', - `directory-type` tinyint DEFAULT 0 COMMENT 'Type of directory service (Poco, Mastodon)', - `poco` varchar(255) NOT NULL DEFAULT '' COMMENT '', - `noscrape` varchar(255) NOT NULL DEFAULT '' COMMENT '', - `network` char(4) NOT NULL DEFAULT '' COMMENT '', - `platform` varchar(255) NOT NULL DEFAULT '' COMMENT '', - `relay-subscribe` boolean NOT NULL DEFAULT '0' COMMENT 'Has the server subscribed to the relay system', - `relay-scope` varchar(10) NOT NULL DEFAULT '' COMMENT 'The scope of messages that the server wants to get', - `created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '', - `last_poco_query` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', - `last_contact` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', - `last_failure` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '', - PRIMARY KEY(`id`), - UNIQUE INDEX `nurl` (`nurl`(190)) -) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Global servers'; - -- -- TABLE gserver-tag -- diff --git a/src/Database/PostUpdate.php b/src/Database/PostUpdate.php index 73ac374acc..a4fadce498 100644 --- a/src/Database/PostUpdate.php +++ b/src/Database/PostUpdate.php @@ -25,6 +25,7 @@ use Friendica\Core\Logger; use Friendica\Core\Protocol; use Friendica\DI; use Friendica\Model\Contact; +use Friendica\Model\GServer; use Friendica\Model\Item; use Friendica\Model\ItemURI; use Friendica\Model\PermissionSet; @@ -86,6 +87,15 @@ class PostUpdate if (!self::update1347()) { return false; } + if (!self::update1348()) { + return false; + } + if (!self::update1349()) { + return false; + } + if (!self::update1350()) { + return false; + } return true; } @@ -870,4 +880,163 @@ class PostUpdate return false; } + + /** + * update the "gsid" (global server id) field in the contact table + * + * @return bool "true" when the job is done + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + * @throws \ImagickException + */ + private static function update1348() + { + // Was the script completed? + if (DI::config()->get("system", "post_update_version") >= 1348) { + return true; + } + + $id = DI::config()->get("system", "post_update_version_1348_id", 0); + + Logger::info('Start', ['contact' => $id]); + + $start_id = $id; + $rows = 0; + $condition = ["`id` > ? AND `gsid` IS NULL AND `baseurl` != '' AND NOT `baseurl` IS NULL", $id]; + $params = ['order' => ['id'], 'limit' => 10000]; + $contacts = DBA::select('contact', ['id', 'baseurl'], $condition, $params); + + if (DBA::errorNo() != 0) { + Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]); + return false; + } + + while ($contact = DBA::fetch($contacts)) { + $id = $contact['id']; + + DBA::update('contact', + ['gsid' => GServer::getID($contact['baseurl']), 'baseurl' => GServer::cleanURL($contact['baseurl'])], + ['id' => $contact['id']]); + + ++$rows; + } + DBA::close($contacts); + + DI::config()->set("system", "post_update_version_1348_id", $id); + + Logger::info('Processed', ['rows' => $rows, 'last' => $id]); + + if ($start_id == $id) { + DI::config()->set("system", "post_update_version", 1348); + Logger::info('Done'); + return true; + } + + return false; + } + + /** + * update the "gsid" (global server id) field in the apcontact table + * + * @return bool "true" when the job is done + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + * @throws \ImagickException + */ + private static function update1349() + { + // Was the script completed? + if (DI::config()->get("system", "post_update_version") >= 1349) { + return true; + } + + $id = DI::config()->get("system", "post_update_version_1349_id", 0); + + Logger::info('Start', ['apcontact' => $id]); + + $start_id = $id; + $rows = 0; + $condition = ["`url` > ? AND `gsid` IS NULL AND `baseurl` != '' AND NOT `baseurl` IS NULL", $id]; + $params = ['order' => ['url'], 'limit' => 10000]; + $apcontacts = DBA::select('apcontact', ['url', 'baseurl'], $condition, $params); + + if (DBA::errorNo() != 0) { + Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]); + return false; + } + + while ($apcontact = DBA::fetch($apcontacts)) { + $id = $apcontact['url']; + + DBA::update('apcontact', + ['gsid' => GServer::getID($apcontact['baseurl']), 'baseurl' => GServer::cleanURL($apcontact['baseurl'])], + ['url' => $apcontact['url']]); + + ++$rows; + } + DBA::close($apcontacts); + + DI::config()->set("system", "post_update_version_1349_id", $id); + + Logger::info('Processed', ['rows' => $rows, 'last' => $id]); + + if ($start_id == $id) { + DI::config()->set("system", "post_update_version", 1349); + Logger::info('Done'); + return true; + } + + return false; + } + + /** + * update the "gsid" (global server id) field in the gcontact table + * + * @return bool "true" when the job is done + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + * @throws \ImagickException + */ + private static function update1350() + { + // Was the script completed? + if (DI::config()->get("system", "post_update_version") >= 1350) { + return true; + } + + $id = DI::config()->get("system", "post_update_version_1350_id", 0); + + Logger::info('Start', ['gcontact' => $id]); + + $start_id = $id; + $rows = 0; + $condition = ["`id` > ? AND `gsid` IS NULL AND `server_url` != '' AND NOT `server_url` IS NULL", $id]; + $params = ['order' => ['id'], 'limit' => 10000]; + $gcontacts = DBA::select('gcontact', ['id', 'server_url'], $condition, $params); + + if (DBA::errorNo() != 0) { + Logger::error('Database error', ['no' => DBA::errorNo(), 'message' => DBA::errorMessage()]); + return false; + } + + while ($gcontact = DBA::fetch($gcontacts)) { + $id = $gcontact['id']; + + DBA::update('gcontact', + ['gsid' => GServer::getID($gcontact['server_url']), 'server_url' => GServer::cleanURL($gcontact['server_url'])], + ['id' => $gcontact['id']]); + + ++$rows; + } + DBA::close($gcontacts); + + DI::config()->set("system", "post_update_version_1350_id", $id); + + Logger::info('Processed', ['rows' => $rows, 'last' => $id]); + + if ($start_id == $id) { + DI::config()->set("system", "post_update_version", 1350); + Logger::info('Done'); + return true; + } + + return false; + } } diff --git a/src/Model/APContact.php b/src/Model/APContact.php index ec33864f4d..73b6143b48 100644 --- a/src/Model/APContact.php +++ b/src/Model/APContact.php @@ -291,6 +291,12 @@ class APContact $apcontact['baseurl'] = null; } + if (!empty($apcontact['baseurl']) && empty($fetched_contact['gsid'])) { + $apcontact['gsid'] = GServer::getID($apcontact['baseurl']); + } elseif (!empty($fetched_contact['gsid'])) { + $apcontact['gsid'] = $fetched_contact['gsid']; + } + if ($apcontact['url'] == $apcontact['alias']) { $apcontact['alias'] = null; } diff --git a/src/Model/Contact.php b/src/Model/Contact.php index 0d321189ff..ee49d9b75c 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -1530,10 +1530,6 @@ class Contact if (empty($data)) { $data = Probe::uri($url, "", $uid); - // Ensure that there is a gserver entry - if (!empty($data['baseurl']) && ($data['network'] != Protocol::PHANTOM)) { - GServer::check($data['baseurl']); - } } // Take the default values when probing failed @@ -1575,6 +1571,7 @@ class Contact 'confirm' => $data['confirm'] ?? '', 'poco' => $data['poco'] ?? '', 'baseurl' => $data['baseurl'] ?? '', + 'gsid' => $data['gsid'] ?? null, 'name-date' => DateTimeFormat::utcNow(), 'uri-date' => DateTimeFormat::utcNow(), 'avatar-date' => DateTimeFormat::utcNow(), @@ -2082,7 +2079,7 @@ class Contact $fields = ['uid', 'avatar', 'name', 'nick', 'location', 'keywords', 'about', 'unsearchable', 'url', 'addr', 'batch', 'notify', 'poll', 'request', 'confirm', 'poco', - 'network', 'alias', 'baseurl', 'forum', 'prv', 'contact-type', 'pubkey']; + 'network', 'alias', 'baseurl', 'gsid', 'forum', 'prv', 'contact-type', 'pubkey']; $contact = DBA::selectFirst('contact', $fields, ['id' => $id]); if (!DBA::isResult($contact)) { return false; @@ -2415,6 +2412,7 @@ class Contact 'nick' => $ret['nick'], 'network' => $ret['network'], 'baseurl' => $ret['baseurl'], + 'gsid' => $ret['gsid'] ?? null, 'protocol' => $protocol, 'pubkey' => $ret['pubkey'], 'rel' => $new_relation, diff --git a/src/Model/GContact.php b/src/Model/GContact.php index becfd61b08..3da5513b50 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -690,7 +690,7 @@ class GContact } $public_contact = DBA::selectFirst('gcontact', [ - 'name', 'nick', 'photo', 'location', 'about', 'addr', 'generation', 'birthday', 'keywords', + 'name', 'nick', 'photo', 'location', 'about', 'addr', 'generation', 'birthday', 'keywords', 'gsid', 'contact-type', 'hide', 'nsfw', 'network', 'alias', 'notify', 'server_url', 'connect', 'updated', 'url' ], ['id' => $gcontact_id]); @@ -752,6 +752,10 @@ class GContact $contact['server_url'] = Strings::normaliseLink($contact['server_url']); } + if (!empty($contact['server_url']) && empty($contact['gsid'])) { + $contact['gsid'] = GServer::getID($contact['server_url']); + } + if (empty($contact['addr']) && !empty($contact['server_url']) && !empty($contact['nick'])) { $hostname = str_replace('http://', '', $contact['server_url']); $contact['addr'] = $contact['nick'] . '@' . $hostname; @@ -791,7 +795,8 @@ class GContact 'notify' => $contact['notify'], 'url' => $contact['url'], 'location' => $contact['location'], 'about' => $contact['about'], 'generation' => $contact['generation'], 'updated' => $contact['updated'], - 'server_url' => $contact['server_url'], 'connect' => $contact['connect'] + 'server_url' => $contact['server_url'], 'connect' => $contact['connect'], + 'gsid' => $contact['gsid'] ]; DBA::update('gcontact', $updated, $condition, $fields); @@ -1016,7 +1021,7 @@ class GContact $fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords', 'bd', 'contact-type', 'network', 'addr', 'notify', 'alias', 'archive', 'term-date', 'created', 'updated', 'avatar', 'success_update', 'failure_update', 'forum', 'prv', - 'baseurl', 'sensitive', 'unsearchable']; + 'baseurl', 'gsid', 'sensitive', 'unsearchable']; $contact = DBA::selectFirst('contact', $fields, array_merge($condition, ['uid' => 0, 'network' => Protocol::FEDERATED])); if (!DBA::isResult($contact)) { @@ -1026,7 +1031,7 @@ class GContact $fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords', 'generation', 'birthday', 'contact-type', 'network', 'addr', 'notify', 'alias', 'archived', 'archive_date', 'created', 'updated', 'photo', 'last_contact', 'last_failure', 'community', 'connect', - 'server_url', 'nsfw', 'hide', 'id']; + 'server_url', 'gsid', 'nsfw', 'hide', 'id']; $old_gcontact = DBA::selectFirst('gcontact', $fields, ['nurl' => $contact['nurl']]); $do_insert = !DBA::isResult($old_gcontact); @@ -1037,7 +1042,7 @@ class GContact $gcontact = []; // These fields are identical in both contact and gcontact - $fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords', + $fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords', 'gsid', 'contact-type', 'network', 'addr', 'notify', 'alias', 'created', 'updated']; foreach ($fields as $field) { diff --git a/src/Model/GServer.php b/src/Model/GServer.php index ba60d89659..560c0831d7 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -34,6 +34,7 @@ use Friendica\Util\DateTimeFormat; use Friendica\Util\Strings; use Friendica\Util\XML; use Friendica\Core\Logger; +use Friendica\Core\System; use Friendica\Protocol\PortableContact; use Friendica\Protocol\Diaspora; use Friendica\Network\Probe; @@ -47,6 +48,57 @@ class GServer const DT_NONE = 0; const DT_POCO = 1; const DT_MASTODON = 2; + + // Methods to detect server types + + // Non endpoint specific methods + const DETECT_MANUAL = 0; + const DETECT_HEADER = 1; + const DETECT_BODY = 2; + + // Implementation specific endpoints + const DETECT_FRIENDIKA = 10; + const DETECT_FRIENDICA = 11; + const DETECT_STATUSNET = 12; + const DETECT_GNUSOCIAL = 13; + const DETECT_CONFIG_JSON = 14; // Statusnet, GNU Social, Older Hubzilla/Redmatrix + const DETECT_SITEINFO_JSON = 15; // Newer Hubzilla + const DETECT_MASTODON_API = 16; + const DETECT_STATUS_PHP = 17; // Nextcloud + + // Standardized endpoints + const DETECT_STATISTICS_JSON = 100; + const DETECT_NODEINFO_1 = 101; + const DETECT_NODEINFO_2 = 102; + + /** + * Get the ID for the given server URL + * + * @param string $url + * @param boolean $no_check Don't check if the server hadn't been found + * @return int gserver id + */ + public static function getID(string $url, bool $no_check = false) + { + if (empty($url)) { + return null; + } + + $url = self::cleanURL($url); + + $gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => Strings::normaliseLink($url)]); + if (DBA::isResult($gserver)) { + Logger::info('Got ID for URL', ['id' => $gserver['id'], 'url' => $url, 'callstack' => System::callstack(20)]); + return $gserver['id']; + } + + if ($no_check || !self::check($url)) { + return null; + } + + return self::getID($url, true); + } + /** * Checks if the given server is reachable * @@ -137,9 +189,7 @@ class GServer */ public static function check(string $server_url, string $network = '', bool $force = false, bool $only_nodeinfo = false) { - // Unify the server address - $server_url = trim($server_url, '/'); - $server_url = str_replace('/index.php', '', $server_url); + $server_url = self::cleanURL($server_url); if ($server_url == '') { return false; @@ -186,7 +236,8 @@ class GServer private static function setFailure(string $url) { if (DBA::exists('gserver', ['nurl' => Strings::normaliseLink($url)])) { - DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($url)]); + DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow(), 'detection-method' => null], + ['nurl' => Strings::normaliseLink($url)]); Logger::info('Set failed status for existing server', ['url' => $url]); return; } @@ -196,6 +247,38 @@ class GServer Logger::info('Set failed status for new server', ['url' => $url]); } + /** + * Remove unwanted content from the given URL + * + * @param string $url + * @return string cleaned URL + */ + public static function cleanURL(string $url) + { + $url = trim($url, '/'); + $url = str_replace('/index.php', '', $url); + + $urlparts = parse_url($url); + unset($urlparts['user']); + unset($urlparts['pass']); + unset($urlparts['query']); + unset($urlparts['fragment']); + return Network::unparseURL($urlparts); + } + + /** + * Return the base URL + * + * @param string $url + * @return string base URL + */ + private static function getBaseURL(string $url) + { + $urlparts = parse_url(self::cleanURL($url)); + unset($urlparts['path']); + return Network::unparseURL($urlparts); + } + /** * Detect server data (type, protocol, version number, ...) * The detected data is then updated or inserted in the gserver table. @@ -209,17 +292,15 @@ class GServer public static function detect(string $url, string $network = '', bool $only_nodeinfo = false) { Logger::info('Detect server type', ['server' => $url]); - $serverdata = []; + $serverdata = ['detection-method' => self::DETECT_MANUAL]; $original_url = $url; // Remove URL content that is not supposed to exist for a server url - $urlparts = parse_url($url); - unset($urlparts['user']); - unset($urlparts['pass']); - unset($urlparts['query']); - unset($urlparts['fragment']); - $url = Network::unparseURL($urlparts); + $url = self::cleanURL($url); + + // Get base URL + $baseurl = self::getBaseURL($url); // If the URL missmatches, then we mark the old entry as failure if ($url != $original_url) { @@ -249,18 +330,53 @@ class GServer // If that didn't work out well, we use some protocol specific endpoints // For Friendica and Zot based networks we have to dive deeper to reveal more details if (empty($nodeinfo['network']) || in_array($nodeinfo['network'], [Protocol::DFRN, Protocol::ZOT])) { + if (!empty($nodeinfo['detection-method'])) { + $serverdata['detection-method'] = $nodeinfo['detection-method']; + } + // Fetch the landing page, possibly it reveals some data if (empty($nodeinfo['network'])) { - $curlResult = Network::curl($url, false, ['timeout' => $xrd_timeout]); + if ($baseurl == $url) { + $basedata = $serverdata; + } else { + $basedata = ['detection-method' => self::DETECT_MANUAL]; + } + + $curlResult = Network::curl($baseurl, false, ['timeout' => $xrd_timeout]); if ($curlResult->isSuccess()) { - $serverdata = self::analyseRootHeader($curlResult, $serverdata); - $serverdata = self::analyseRootBody($curlResult, $serverdata, $url); + $basedata = self::analyseRootHeader($curlResult, $basedata); + $basedata = self::analyseRootBody($curlResult, $basedata, $baseurl); } if (!$curlResult->isSuccess() || empty($curlResult->getBody()) || self::invalidBody($curlResult->getBody())) { self::setFailure($url); return false; } + + if ($baseurl == $url) { + $serverdata = $basedata; + } else { + // When the base path doesn't seem to contain a social network we try the complete path. + // Most detectable system have to be installed in the root directory. + // We checked the base to avoid false positives. + $curlResult = Network::curl($url, false, ['timeout' => $xrd_timeout]); + if ($curlResult->isSuccess()) { + $urldata = self::analyseRootHeader($curlResult, $serverdata); + $urldata = self::analyseRootBody($curlResult, $urldata, $url); + + $comparebase = $basedata; + unset($comparebase['info']); + unset($comparebase['site_name']); + $compareurl = $urldata; + unset($compareurl['info']); + unset($compareurl['site_name']); + + // We assume that no one will install the identical system in the root and a subfolder + if (!empty(array_diff($comparebase, $compareurl))) { + $serverdata = $urldata; + } + } + } } if (empty($serverdata['network']) || ($serverdata['network'] == Protocol::ACTIVITYPUB)) { @@ -296,6 +412,8 @@ class GServer if (empty($serverdata['network'])) { $serverdata = self::detectGNUSocial($url, $serverdata); } + + $serverdata = array_merge($nodeinfo, $serverdata); } else { $serverdata = $nodeinfo; } @@ -326,12 +444,7 @@ class GServer $registeredUsers = 1; } - if ($serverdata['network'] != Protocol::PHANTOM) { - $gcontacts = DBA::count('gcontact', ['server_url' => [$url, $serverdata['nurl']]]); - $apcontacts = DBA::count('apcontact', ['baseurl' => [$url, $serverdata['nurl']]]); - $contacts = DBA::count('contact', ['uid' => 0, 'baseurl' => [$url, $serverdata['nurl']]]); - $serverdata['registered-users'] = max($gcontacts, $apcontacts, $contacts, $registeredUsers); - } else { + if ($serverdata['network'] == Protocol::PHANTOM) { $serverdata['registered-users'] = $registeredUsers; $serverdata = self::detectNetworkViaContacts($url, $serverdata); } @@ -342,6 +455,7 @@ class GServer if (!DBA::isResult($gserver)) { $serverdata['created'] = DateTimeFormat::utcNow(); $ret = DBA::insert('gserver', $serverdata); + $id = DBA::lastInsertId(); } else { // Don't override the network with 'unknown' when there had been a valid entry before if (($serverdata['network'] == Protocol::PHANTOM) && !empty($gserver['network'])) { @@ -349,11 +463,26 @@ class GServer } $ret = DBA::update('gserver', $serverdata, ['nurl' => $serverdata['nurl']]); + $gserver = DBA::selectFirst('gserver', ['id'], ['nurl' => $serverdata['nurl']]); + if (DBA::isResult($gserver)) { + $id = $gserver['id']; + } + } + + if (!empty($id) && ($serverdata['network'] != Protocol::PHANTOM)) { + $gcontacts = DBA::count('gcontact', ['gsid' => $id]); + $apcontacts = DBA::count('apcontact', ['gsid' => $id]); + $contacts = DBA::count('contact', ['uid' => 0, 'gsid' => $id]); + $max_users = max($gcontacts, $apcontacts, $contacts, $registeredUsers); + if ($max_users > $registeredUsers) { + Logger::info('Update registered users', ['id' => $id, 'url' => $serverdata['nurl'], 'registered-users' => $max_users]); + DBA::update('gserver', ['registered-users' => $max_users], ['id' => $id]); + } } if (!empty($serverdata['network']) && in_array($serverdata['network'], [Protocol::DFRN, Protocol::DIASPORA])) { - self::discoverRelay($url); - } + self::discoverRelay($url); + } return $ret; } @@ -459,7 +588,7 @@ class GServer return []; } - $serverdata = []; + $serverdata = ['detection-method' => self::DETECT_STATISTICS_JSON]; if (!empty($data['version'])) { $serverdata['version'] = $data['version']; @@ -560,7 +689,6 @@ class GServer private static function parseNodeinfo1(string $nodeinfo_url) { $curlResult = Network::curl($nodeinfo_url); - if (!$curlResult->isSuccess()) { return []; } @@ -571,9 +699,8 @@ class GServer return []; } - $server = []; - - $server['register_policy'] = Register::CLOSED; + $server = ['detection-method' => self::DETECT_NODEINFO_1, + 'register_policy' => Register::CLOSED]; if (!empty($nodeinfo['openRegistrations'])) { $server['register_policy'] = Register::OPEN; @@ -648,9 +775,8 @@ class GServer return []; } - $server = []; - - $server['register_policy'] = Register::CLOSED; + $server = ['detection-method' => self::DETECT_NODEINFO_2, + 'register_policy' => Register::CLOSED]; if (!empty($nodeinfo['openRegistrations'])) { $server['register_policy'] = Register::OPEN; @@ -725,6 +851,10 @@ class GServer return $serverdata; } + if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + $serverdata['detection-method'] = self::DETECT_SITEINFO_JSON; + } + if (!empty($data['url'])) { $serverdata['platform'] = strtolower($data['platform']); $serverdata['version'] = $data['version']; @@ -934,7 +1064,6 @@ class GServer private static function detectNextcloud(string $url, array $serverdata) { $curlResult = Network::curl($url . '/status.php'); - if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; } @@ -948,6 +1077,10 @@ class GServer $serverdata['platform'] = 'nextcloud'; $serverdata['version'] = $data['version']; $serverdata['network'] = Protocol::ACTIVITYPUB; + + if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + $serverdata['detection-method'] = self::DETECT_STATUS_PHP; + } } return $serverdata; @@ -964,7 +1097,6 @@ class GServer private static function detectMastodonAlikes(string $url, array $serverdata) { $curlResult = Network::curl($url . '/api/v1/instance'); - if (!$curlResult->isSuccess() || ($curlResult->getBody() == '')) { return $serverdata; } @@ -974,6 +1106,10 @@ class GServer return $serverdata; } + if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + $serverdata['detection-method'] = self::DETECT_MASTODON_API; + } + if (!empty($data['version'])) { $serverdata['platform'] = 'mastodon'; $serverdata['version'] = $data['version'] ?? ''; @@ -1031,7 +1167,7 @@ class GServer } $data = json_decode($curlResult->getBody(), true); - if (empty($data)) { + if (empty($data) || empty($data['site'])) { return $serverdata; } @@ -1079,11 +1215,16 @@ class GServer } if (!$closed && !$private and $inviteonly) { - $register_policy = Register::APPROVE; + $serverdata['register_policy'] = Register::APPROVE; } elseif (!$closed && !$private) { - $register_policy = Register::OPEN; + $serverdata['register_policy'] = Register::OPEN; } else { - $register_policy = Register::CLOSED; + $serverdata['register_policy'] = Register::CLOSED; + } + + if (!empty($serverdata['network']) && in_array($serverdata['detection-method'], + [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + $serverdata['detection-method'] = self::DETECT_CONFIG_JSON; } return $serverdata; @@ -1127,6 +1268,11 @@ class GServer $serverdata['version'] = str_replace(["\r", "\n", "\t"], '', $serverdata['version']); $serverdata['version'] = trim($serverdata['version'], '"'); $serverdata['network'] = Protocol::OSTATUS; + + if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + $serverdata['detection-method'] = self::DETECT_GNUSOCIAL; + } + return $serverdata; } @@ -1148,6 +1294,10 @@ class GServer $serverdata['platform'] = 'statusnet'; $serverdata['network'] = Protocol::OSTATUS; } + + if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + $serverdata['detection-method'] = self::DETECT_STATUSNET; + } } return $serverdata; @@ -1166,6 +1316,11 @@ class GServer $curlResult = Network::curl($url . '/friendica/json'); if (!$curlResult->isSuccess()) { $curlResult = Network::curl($url . '/friendika/json'); + $friendika = true; + $platform = 'Friendika'; + } else { + $friendika = false; + $platform = 'Friendica'; } if (!$curlResult->isSuccess()) { @@ -1177,6 +1332,10 @@ class GServer return $serverdata; } + if (in_array($serverdata['detection-method'], [self::DETECT_HEADER, self::DETECT_BODY, self::DETECT_MANUAL])) { + $serverdata['detection-method'] = $friendika ? self::DETECT_FRIENDIKA : self::DETECT_FRIENDICA; + } + $serverdata['network'] = Protocol::DFRN; $serverdata['version'] = $data['version']; @@ -1212,7 +1371,7 @@ class GServer break; } - $serverdata['platform'] = strtolower($data['platform'] ?? ''); + $serverdata['platform'] = strtolower($data['platform'] ?? $platform); return $serverdata; } @@ -1260,7 +1419,8 @@ class GServer $serverdata['info'] = $attr['content']; } - if ($attr['name'] == 'application-name') { + if (in_array($attr['name'], ['application-name', 'al:android:app_name', 'al:ios:app_name', + 'twitter:app:name:googleplay', 'twitter:app:name:iphone', 'twitter:app:name:ipad'])) { $serverdata['platform'] = strtolower($attr['content']); if (in_array($attr['content'], ['Misskey', 'Write.as'])) { $serverdata['network'] = Protocol::ACTIVITYPUB; @@ -1281,6 +1441,10 @@ class GServer } else { $serverdata['network'] = Protocol::FEED; } + + if ($serverdata['detection-method'] == self::DETECT_MANUAL) { + $serverdata['detection-method'] = self::DETECT_BODY; + } } if (in_array($version_part[0], ['Friendika', 'Friendica'])) { $serverdata['platform'] = strtolower($version_part[0]); @@ -1336,6 +1500,10 @@ class GServer } } + if (!empty($serverdata['network']) && ($serverdata['detection-method'] == self::DETECT_MANUAL)) { + $serverdata['detection-method'] = self::DETECT_BODY; + } + return $serverdata; } @@ -1351,16 +1519,23 @@ class GServer { if ($curlResult->getHeader('server') == 'Mastodon') { $serverdata['platform'] = 'mastodon'; - $serverdata['network'] = $network = Protocol::ACTIVITYPUB; + $serverdata['network'] = Protocol::ACTIVITYPUB; } elseif ($curlResult->inHeader('x-diaspora-version')) { $serverdata['platform'] = 'diaspora'; - $serverdata['network'] = $network = Protocol::DIASPORA; + $serverdata['network'] = Protocol::DIASPORA; $serverdata['version'] = $curlResult->getHeader('x-diaspora-version'); } elseif ($curlResult->inHeader('x-friendica-version')) { $serverdata['platform'] = 'friendica'; - $serverdata['network'] = $network = Protocol::DFRN; + $serverdata['network'] = Protocol::DFRN; $serverdata['version'] = $curlResult->getHeader('x-friendica-version'); + } else { + return $serverdata; } + + if ($serverdata['detection-method'] == self::DETECT_MANUAL) { + $serverdata['detection-method'] = self::DETECT_HEADER; + } + return $serverdata; } @@ -1472,7 +1647,6 @@ class GServer $api = 'https://instances.social/api/1.0/instances/list?count=0'; $header = ['Authorization: Bearer '.$accesstoken]; $curlResult = Network::curl($api, false, ['headers' => $header]); - if ($curlResult->isSuccess()) { $servers = json_decode($curlResult->getBody(), true); diff --git a/src/Network/Probe.php b/src/Network/Probe.php index d5b3e0be8a..4333196547 100644 --- a/src/Network/Probe.php +++ b/src/Network/Probe.php @@ -30,6 +30,7 @@ use Friendica\Core\System; use Friendica\Database\DBA; use Friendica\DI; use Friendica\Model\Contact; +use Friendica\Model\GServer; use Friendica\Model\Profile; use Friendica\Protocol\ActivityNamespace; use Friendica\Protocol\ActivityPub; @@ -86,14 +87,16 @@ class Probe "community", "keywords", "location", "about", "hide", "batch", "notify", "poll", "request", "confirm", "poco", "following", "followers", "inbox", "outbox", "sharedinbox", - "priority", "network", "pubkey", "baseurl"]; + "priority", "network", "pubkey", "baseurl", "gsid"]; $newdata = []; foreach ($fields as $field) { if (isset($data[$field])) { $newdata[$field] = $data[$field]; - } else { + } elseif ($field != "gsid") { $newdata[$field] = ""; + } else { + $newdata[$field] = null; } } @@ -461,6 +464,10 @@ class Probe $data['baseurl'] = self::$baseurl; } + if (!empty($data['baseurl']) && empty($data['gsid'])) { + $data['gsid'] = GServer::getID($data['baseurl']); + } + if (empty($data['network'])) { $data['network'] = Protocol::PHANTOM; } diff --git a/src/Protocol/ActivityPub.php b/src/Protocol/ActivityPub.php index c3168f5509..9d6223a748 100644 --- a/src/Protocol/ActivityPub.php +++ b/src/Protocol/ActivityPub.php @@ -171,6 +171,7 @@ class ActivityPub $profile['poll'] = $apcontact['outbox']; $profile['pubkey'] = $apcontact['pubkey']; $profile['baseurl'] = $apcontact['baseurl']; + $profile['gsid'] = $apcontact['gsid']; // Remove all "null" fields foreach ($profile as $field => $content) { diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 512002f25c..fc7abded50 100755 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -54,11 +54,40 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1349); + define('DB_UPDATE_VERSION', 1350); } return [ // Side tables + "gserver" => [ + "comment" => "Global servers", + "fields" => [ + "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => "sequential ID"], + "url" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], + "nurl" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], + "version" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], + "site_name" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], + "info" => ["type" => "text", "comment" => ""], + "register_policy" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => ""], + "registered-users" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "comment" => "Number of registered users"], + "directory-type" => ["type" => "tinyint", "default" => "0", "comment" => "Type of directory service (Poco, Mastodon)"], + "poco" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], + "noscrape" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], + "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""], + "platform" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], + "relay-subscribe" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Has the server subscribed to the relay system"], + "relay-scope" => ["type" => "varchar(10)", "not null" => "1", "default" => "", "comment" => "The scope of messages that the server wants to get"], + "detection-method" => ["type" => "tinyint unsigned", "comment" => "Method that had been used to detect that server"], + "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""], + "last_poco_query" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], + "last_contact" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], + "last_failure" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], + ], + "indexes" => [ + "PRIMARY" => ["id"], + "nurl" => ["UNIQUE", "nurl(190)"], + ] + ], "clients" => [ "comment" => "OAuth usage", "fields" => [ @@ -142,6 +171,7 @@ return [ "unsearchable" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Contact prefers to not be searchable"], "sensitive" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Contact posts sensitive content"], "baseurl" => ["type" => "varchar(255)", "default" => "", "comment" => "baseurl of the contact"], + "gsid" => ["type" => "int unsigned", "foreign" => ["gserver" => "id", "on delete" => "restrict"], "comment" => "Global Server ID"], "reason" => ["type" => "text", "comment" => ""], "closeness" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "99", "comment" => ""], "info" => ["type" => "mediumtext", "comment" => ""], @@ -166,6 +196,7 @@ return [ "nick_uid" => ["nick(32)", "uid"], "dfrn-id" => ["dfrn-id(64)"], "issued-id" => ["issued-id(64)"], + "gsid" => ["gsid"] ] ], "item-uri" => [ @@ -273,6 +304,7 @@ return [ "alias" => ["type" => "varchar(255)", "comment" => ""], "pubkey" => ["type" => "text", "comment" => ""], "baseurl" => ["type" => "varchar(255)", "comment" => "baseurl of the ap contact"], + "gsid" => ["type" => "int unsigned", "foreign" => ["gserver" => "id", "on delete" => "restrict"], "comment" => "Global Server ID"], "generator" => ["type" => "varchar(255)", "comment" => "Name of the contact's system"], "following_count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of following contacts"], "followers_count" => ["type" => "int unsigned", "default" => 0, "comment" => "Number of followers"], @@ -283,7 +315,8 @@ return [ "PRIMARY" => ["url"], "addr" => ["addr(32)"], "alias" => ["alias(190)"], - "url" => ["followers(190)"] + "followers" => ["followers(190)"], + "gsid" => ["gsid"] ] ], "attach" => [ @@ -539,6 +572,7 @@ return [ "alias" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], "generation" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""], "server_url" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => "baseurl of the contacts server"], + "gsid" => ["type" => "int unsigned", "foreign" => ["gserver" => "id", "on delete" => "restrict"], "comment" => "Global Server ID"], ], "indexes" => [ "PRIMARY" => ["id"], @@ -548,6 +582,7 @@ return [ "addr" => ["addr(64)"], "hide_network_updated" => ["hide", "network", "updated"], "updated" => ["updated"], + "gsid" => ["gsid"] ] ], "gfollower" => [ @@ -605,34 +640,6 @@ return [ "gid_contactid" => ["UNIQUE", "gid", "contact-id"], ] ], - "gserver" => [ - "comment" => "Global servers", - "fields" => [ - "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => "sequential ID"], - "url" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], - "nurl" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], - "version" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], - "site_name" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], - "info" => ["type" => "text", "comment" => ""], - "register_policy" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => ""], - "registered-users" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "comment" => "Number of registered users"], - "directory-type" => ["type" => "tinyint", "default" => "0", "comment" => "Type of directory service (Poco, Mastodon)"], - "poco" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], - "noscrape" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], - "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""], - "platform" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], - "relay-subscribe" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Has the server subscribed to the relay system"], - "relay-scope" => ["type" => "varchar(10)", "not null" => "1", "default" => "", "comment" => "The scope of messages that the server wants to get"], - "created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => ""], - "last_poco_query" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], - "last_contact" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], - "last_failure" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""], - ], - "indexes" => [ - "PRIMARY" => ["id"], - "nurl" => ["UNIQUE", "nurl(190)"], - ] - ], "gserver-tag" => [ "comment" => "Tags that the server has subscribed", "fields" => [