diff --git a/database.sql b/database.sql index cbb724d18..6a55db526 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2019.12-rc (Dalmatian Bellflower) --- DB_UPDATE_VERSION 1326 +-- DB_UPDATE_VERSION 1328 -- ------------------------------------------ @@ -470,6 +470,7 @@ CREATE TABLE IF NOT EXISTS `gserver` ( `info` text COMMENT '', `register_policy` tinyint NOT NULL DEFAULT 0 COMMENT '', `registered-users` int unsigned NOT NULL DEFAULT 0 COMMENT 'Number of registered users', + `directory-type` tinyint DEFAULT 0 COMMENT 'Type of directory service (Poco, Mastodon)', `poco` varchar(255) NOT NULL DEFAULT '' COMMENT '', `noscrape` varchar(255) NOT NULL DEFAULT '' COMMENT '', `network` char(4) NOT NULL DEFAULT '' COMMENT '', diff --git a/src/Core/Search.php b/src/Core/Search.php index 9700c6472..e43e621d2 100644 --- a/src/Core/Search.php +++ b/src/Core/Search.php @@ -242,7 +242,7 @@ class Search extends BaseObject DBA::close($data); // Add found profiles from the global directory to the local directory - Worker::add(PRIORITY_LOW, 'DiscoverPoCo', "dirsearch", urlencode($search)); + Worker::add(PRIORITY_LOW, 'SearchDirectory', $search); return $resultList; } diff --git a/src/Model/GContact.php b/src/Model/GContact.php index 2caad9459..61e44b410 100644 --- a/src/Model/GContact.php +++ b/src/Model/GContact.php @@ -580,9 +580,6 @@ class GContact public static function getId($contact) { $gcontact_id = 0; - $doprobing = false; - $last_failure_str = ''; - $last_contact_str = ''; if (empty($contact['network'])) { Logger::notice('Empty network', ['url' => $contact['url'], 'callstack' => System::callstack()]); @@ -613,15 +610,6 @@ class GContact $gcnt = DBA::selectFirst('gcontact', $fields, ['nurl' => Strings::normaliseLink($contact['url'])]); if (DBA::isResult($gcnt)) { $gcontact_id = $gcnt['id']; - - // Update every 90 days - if (empty($gcnt['network']) || in_array($gcnt['network'], Protocol::FEDERATED)) { - $last_failure_str = $gcnt['last_failure']; - $last_failure = strtotime($gcnt['last_failure']); - $last_contact_str = $gcnt['last_contact']; - $last_contact = strtotime($gcnt['last_contact']); - $doprobing = (((time() - $last_contact) > (90 * 86400)) && ((time() - $last_failure) > (90 * 86400))); - } } else { $contact['location'] = $contact['location'] ?? ''; $contact['about'] = $contact['about'] ?? ''; @@ -638,16 +626,10 @@ class GContact $cnt = DBA::selectFirst('gcontact', ['id', 'network'], $condition, ['order' => ['id']]); if (DBA::isResult($cnt)) { $gcontact_id = $cnt['id']; - $doprobing = (empty($cnt['network']) || in_array($cnt['network'], Protocol::FEDERATED)); } } DBA::unlock(); - if ($doprobing) { - Logger::notice('Probing', ['contact' => $last_contact_str, "failure" => $last_failure_str, "checking" => $contact['url']]); - Worker::add(PRIORITY_LOW, 'GProbe', $contact['url']); - } - return $gcontact_id; } @@ -801,7 +783,7 @@ class GContact return; } - if (!$force && !PortableContact::updateNeeded($gcontact['created'], $gcontact['updated'], $gcontact['last_failure'], $gcontact['last_contact'])) { + if (!$force && !GServer::updateNeeded($gcontact['created'], $gcontact['updated'], $gcontact['last_failure'], $gcontact['last_contact'])) { Logger::info("Don't update profile", ['url' => $data['url'], 'updated' => $gcontact['updated']]); return; } diff --git a/src/Model/GServer.php b/src/Model/GServer.php index 41c8fa44b..3fffd4ed4 100644 --- a/src/Model/GServer.php +++ b/src/Model/GServer.php @@ -26,6 +26,10 @@ use Friendica\Network\Probe; */ class GServer { + // Directory types + const DT_NONE = 0; + const DT_POCO = 1; + const DT_MASTODON = 2; /** * Checks if the given server is reachable * @@ -49,6 +53,61 @@ class GServer return self::check($server, $network, $force); } + /** + * Decides if a server needs to be updated, based upon several date fields + * + * @param date $created Creation date of that server entry + * @param date $updated When had the server entry be updated + * @param date $last_failure Last failure when contacting that server + * @param date $last_contact Last time the server had been contacted + * + * @return boolean Does the server record needs an update? + */ + public static function updateNeeded($created, $updated, $last_failure, $last_contact) + { + $now = strtotime(DateTimeFormat::utcNow()); + + if ($updated > $last_contact) { + $contact_time = strtotime($updated); + } else { + $contact_time = strtotime($last_contact); + } + + $failure_time = strtotime($last_failure); + $created_time = strtotime($created); + + // If there is no "created" time then use the current time + if ($created_time <= 0) { + $created_time = $now; + } + + // If the last contact was less than 24 hours then don't update + if (($now - $contact_time) < (60 * 60 * 24)) { + return false; + } + + // If the last failure was less than 24 hours then don't update + if (($now - $failure_time) < (60 * 60 * 24)) { + return false; + } + + // If the last contact was less than a week ago and the last failure is older than a week then don't update + //if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time)) + // return false; + + // If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week + if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) { + return false; + } + + // If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month + if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) { + return false; + } + + return true; + } + /** * Checks the state of the given server. * @@ -89,7 +148,7 @@ class GServer $last_failure = DBA::NULL_DATETIME; } - if (!$force && !PortableContact::updateNeeded($gserver['created'], '', $last_failure, $last_contact)) { + if (!$force && !self::updateNeeded($gserver['created'], '', $last_failure, $last_contact)) { Logger::info('No update needed', ['server' => $server_url]); return ($last_contact >= $last_failure); } @@ -112,8 +171,24 @@ class GServer */ public static function detect(string $url, string $network = '') { + Logger::info('Detect server type', ['server' => $url]); $serverdata = []; + $original_url = $url; + + // Remove URL content that is not supposed to exist for a server url + $urlparts = parse_url($url); + unset($urlparts['user']); + unset($urlparts['pass']); + unset($urlparts['query']); + unset($urlparts['fragment']); + $url = Network::unparseURL($urlparts); + + // If the URL missmatches, then we mark the old entry as failure + if ($url != $original_url) { + DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($original_url)]); + } + // When a nodeinfo is present, we don't need to dig further $xrd_timeout = Config::get('system', 'xrd_timeout'); $curlResult = Network::curl($url . '/.well-known/nodeinfo', false, ['timeout' => $xrd_timeout]); @@ -183,7 +258,10 @@ class GServer $serverdata = $nodeinfo; } + // Detect the directory type + $serverdata['directory-type'] = self::DT_NONE; $serverdata = self::checkPoCo($url, $serverdata); + $serverdata = self::checkMastodonDirectory($url, $serverdata); // We can't detect the network type. Possibly it is some system that we don't know yet if (empty($serverdata['network'])) { @@ -740,6 +818,8 @@ class GServer */ private static function checkPoCo(string $url, array $serverdata) { + $serverdata['poco'] = ''; + $curlResult = Network::curl($url. '/poco'); if (!$curlResult->isSuccess()) { return $serverdata; @@ -753,9 +833,35 @@ class GServer if (!empty($data['totalResults'])) { $registeredUsers = $serverdata['registered-users'] ?? 0; $serverdata['registered-users'] = max($data['totalResults'], $registeredUsers); + $serverdata['directory-type'] = self::DT_POCO; $serverdata['poco'] = $url . '/poco'; - } else { - $serverdata['poco'] = ''; + } + + return $serverdata; + } + + /** + * Checks if the given server does have a Mastodon style directory endpoint. + * + * @param string $url URL of the given server + * @param array $serverdata array with server data + * + * @return array server data + */ + public static function checkMastodonDirectory(string $url, array $serverdata) + { + $curlResult = Network::curl($url . '/api/v1/directory?limit=1'); + if (!$curlResult->isSuccess()) { + return $serverdata; + } + + $data = json_decode($curlResult->getBody(), true); + if (empty($data)) { + return $serverdata; + } + + if (count($data) == 1) { + $serverdata['directory-type'] = self::DT_MASTODON; } return $serverdata; @@ -1184,4 +1290,18 @@ class GServer } return $serverdata; } + + /** + * Update the user directory of a given gserver record + * + * @param array $gserver gserver record + */ + public static function updateDirectory(array $gserver) + { + /// @todo Add Mastodon API directory + + if (!empty($gserver['poco'])) { + PortableContact::discoverSingleServer($gserver['id']); + } + } } diff --git a/src/Model/Profile.php b/src/Model/Profile.php index eb274a640..8659506cb 100644 --- a/src/Model/Profile.php +++ b/src/Model/Profile.php @@ -1074,8 +1074,6 @@ class Profile Logger::log('Not authenticated. Invoking reverse magic-auth for ' . $my_url, Logger::DEBUG); - Worker::add(PRIORITY_LOW, 'GProbe', $my_url); - // Remove the "addr" parameter from the destination. It is later added as separate parameter again. $addr_request = 'addr=' . urlencode($addr); $query = rtrim(str_replace($addr_request, '', $a->query_string), '?&'); diff --git a/src/Protocol/PortableContact.php b/src/Protocol/PortableContact.php index 228a96762..49da85e61 100644 --- a/src/Protocol/PortableContact.php +++ b/src/Protocol/PortableContact.php @@ -57,7 +57,7 @@ class PortableContact public static function loadWorker($cid, $uid = 0, $zcid = 0, $url = null) { // Call the function "load" via the worker - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "load", (int)$cid, (int)$uid, (int)$zcid, $url); + Worker::add(PRIORITY_LOW, 'FetchPoCo', (int)$cid, (int)$uid, (int)$zcid, $url); } /** @@ -219,51 +219,6 @@ class PortableContact return(preg_match("=https?://.+/user/\d+=ism", $url, $matches)); } - public static function updateNeeded($created, $updated, $last_failure, $last_contact) - { - $now = strtotime(DateTimeFormat::utcNow()); - - if ($updated > $last_contact) { - $contact_time = strtotime($updated); - } else { - $contact_time = strtotime($last_contact); - } - - $failure_time = strtotime($last_failure); - $created_time = strtotime($created); - - // If there is no "created" time then use the current time - if ($created_time <= 0) { - $created_time = $now; - } - - // If the last contact was less than 24 hours then don't update - if (($now - $contact_time) < (60 * 60 * 24)) { - return false; - } - - // If the last failure was less than 24 hours then don't update - if (($now - $failure_time) < (60 * 60 * 24)) { - return false; - } - - // If the last contact was less than a week ago and the last failure is older than a week then don't update - //if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time)) - // return false; - - // If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week - if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) { - return false; - } - - // If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month - if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) { - return false; - } - - return true; - } - /** * @brief Returns a list of all known servers * @return array List of server urls @@ -315,7 +270,7 @@ class PortableContact if (!DBA::isResult($r)) { Logger::log("Call server check for server ".$server_url, Logger::DEBUG); - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $server_url); + Worker::add(PRIORITY_LOW, 'UpdateGServer', $server_url); } } } @@ -340,7 +295,7 @@ class PortableContact if (!empty($servers['pods'])) { foreach ($servers['pods'] as $server) { - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", "https://" . $server['host']); + Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $server['host']); } } } @@ -359,7 +314,7 @@ class PortableContact foreach ($servers['instances'] as $server) { $url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name']; - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $url); + Worker::add(PRIORITY_LOW, 'UpdateGServer', $url); } } } @@ -470,7 +425,7 @@ class PortableContact $last_update = date('c', time() - (60 * 60 * 24 * $requery_days)); - $gservers = q("SELECT `id`, `url`, `nurl`, `network` + $gservers = q("SELECT `id`, `url`, `nurl`, `network`, `poco` FROM `gserver` WHERE `last_contact` >= `last_failure` AND `poco` != '' @@ -488,7 +443,7 @@ class PortableContact } Logger::log('Update directory from server ' . $gserver['url'] . ' with ID ' . $gserver['id'], Logger::DEBUG); - Worker::add(PRIORITY_LOW, 'DiscoverPoCo', 'update_server_directory', (int) $gserver['id']); + Worker::add(PRIORITY_LOW, 'UpdateServerDirectory', $gserver); if (!$complete && ( --$no_of_queries == 0)) { break; diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index e43fde28e..6928cb8d7 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -38,11 +38,11 @@ class Cron // Fork the cron jobs in separate parts to avoid problems when one of them is crashing Hook::fork($a->queue['priority'], "cron"); - // run the process to discover global contacts in the background - Worker::add(PRIORITY_LOW, "DiscoverPoCo"); + // run the process to update server directories in the background + Worker::add(PRIORITY_LOW, 'UpdateServerDirectories'); // run the process to update locally stored global contacts in the background - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "checkcontact"); + Worker::add(PRIORITY_LOW, 'UpdateGContacts'); // Expire and remove user entries Worker::add(PRIORITY_MEDIUM, "CronJobs", "expire_and_remove_users"); @@ -73,9 +73,9 @@ class Cron // update nodeinfo data Worker::add(PRIORITY_LOW, "CronJobs", "nodeinfo"); - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "update_server"); + Worker::add(PRIORITY_LOW, 'UpdateGServers'); - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "suggestions"); + Worker::add(PRIORITY_LOW, 'UpdateSuggestions'); Worker::add(PRIORITY_LOW, 'Expire'); diff --git a/src/Worker/DiscoverPoCo.php b/src/Worker/DiscoverPoCo.php deleted file mode 100644 index d495b4c2a..000000000 --- a/src/Worker/DiscoverPoCo.php +++ /dev/null @@ -1,305 +0,0 @@ -: Searches for "search pattern" in the directory. "search pattern" is url encoded. - - checkcontact: Updates gcontact entries - - suggestions: Discover other servers for their contacts. - - server : Searches for the poco server list. "poco url" is base64 encoded. - - update_server: Frequently check the first 250 servers for vitality. - - update_server_directory: Discover the given server id for their contacts - - PortableContact::load: Load POCO data from a given POCO address - - check_profile: Update remote profile data - */ - - $search = ""; - $mode = 0; - if ($command == "dirsearch") { - $search = urldecode($param1); - $mode = 1; - } elseif ($command == "checkcontact") { - $mode = 2; - } elseif ($command == "suggestions") { - $mode = 3; - } elseif ($command == "server") { - $mode = 4; - } elseif ($command == "update_server") { - $mode = 5; - } elseif ($command == "update_server_directory") { - $mode = 6; - } elseif ($command == "load") { - $mode = 7; - } elseif ($command == "check_profile") { - $mode = 8; - } elseif ($command !== "") { - Logger::log("Unknown or missing parameter ".$command."\n"); - return; - } - - Logger::log('start '.$search); - - if ($mode == 8) { - if ($param1 != "") { - GContact::updateFromProbe($param1, true); - } - } elseif ($mode == 7) { - if (!empty($param4)) { - $url = $param4; - } else { - $url = ''; - } - PortableContact::load(intval($param1), intval($param2), intval($param3), $url); - } elseif ($mode == 6) { - PortableContact::discoverSingleServer(intval($param1)); - } elseif ($mode == 5) { - self::updateServer(); - } elseif ($mode == 4) { - $server_url = $param1; - if ($server_url == "") { - return; - } - $server_url = filter_var($server_url, FILTER_SANITIZE_URL); - if (substr(Strings::normaliseLink($server_url), 0, 7) != "http://") { - return; - } - $result = "Checking server ".$server_url." - "; - $ret = GServer::check($server_url); - if ($ret) { - $result .= "success"; - } else { - $result .= "failed"; - } - Logger::log($result, Logger::DEBUG); - } elseif ($mode == 3) { - GContact::updateSuggestions(); - } elseif (($mode == 2) && Config::get('system', 'poco_completion')) { - self::discoverUsers(); - } elseif (($mode == 1) && ($search != "") && Config::get('system', 'poco_local_search')) { - self::discoverDirectory($search); - self::gsSearchUser($search); - } elseif (($mode == 0) && ($search == "") && (Config::get('system', 'poco_discovery') != PortableContact::DISABLED)) { - // Query Friendica and Hubzilla servers for their users - PortableContact::discover(); - - // Query GNU Social servers for their users ("statistics" addon has to be enabled on the GS server) - if (!Config::get('system', 'ostatus_disabled')) { - GContact::discoverGsUsers(); - } - } - - Logger::log('end '.$search); - - return; - } - - /** - * @brief Updates the first 250 servers - * - */ - private static function updateServer() { - $r = q("SELECT `url`, `created`, `last_failure`, `last_contact` FROM `gserver` ORDER BY rand()"); - - if (!DBA::isResult($r)) { - return; - } - - $updated = 0; - - foreach ($r AS $server) { - if (!PortableContact::updateNeeded($server["created"], "", $server["last_failure"], $server["last_contact"])) { - continue; - } - Logger::log('Update server status for server '.$server["url"], Logger::DEBUG); - - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $server["url"]); - - if (++$updated > 250) { - return; - } - } - } - - private static function discoverUsers() { - Logger::log("Discover users", Logger::DEBUG); - - $starttime = time(); - - $users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url`, `network` FROM `gcontact` - WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND - `last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND - `network` IN ('%s', '%s', '%s', '%s', '') ORDER BY rand()", - DBA::escape(Protocol::DFRN), DBA::escape(Protocol::DIASPORA), - DBA::escape(Protocol::OSTATUS), DBA::escape(Protocol::FEED)); - - if (!$users) { - return; - } - $checked = 0; - - foreach ($users AS $user) { - - $urlparts = parse_url($user["url"]); - if (!isset($urlparts["scheme"])) { - DBA::update('gcontact', ['network' => Protocol::PHANTOM], - ['nurl' => Strings::normaliseLink($user["url"])]); - continue; - } - - if (in_array($urlparts["host"], ["twitter.com", "identi.ca"])) { - $networks = ["twitter.com" => Protocol::TWITTER, "identi.ca" => Protocol::PUMPIO]; - - DBA::update('gcontact', ['network' => $networks[$urlparts["host"]]], - ['nurl' => Strings::normaliseLink($user["url"])]); - continue; - } - - $server_url = Contact::getBasepath($user["url"]); - $force_update = false; - - if ($user["server_url"] != "") { - - $force_update = (Strings::normaliseLink($user["server_url"]) != Strings::normaliseLink($server_url)); - - $server_url = $user["server_url"]; - } - - if ((($server_url == "") && ($user["network"] == Protocol::FEED)) || $force_update || GServer::check($server_url, $user["network"])) { - Logger::log('Check profile '.$user["url"]); - Worker::add(PRIORITY_LOW, "DiscoverPoCo", "check_profile", $user["url"]); - - if (++$checked > 100) { - return; - } - } else { - DBA::update('gcontact', ['last_failure' => DateTimeFormat::utcNow()], - ['nurl' => Strings::normaliseLink($user["url"])]); - } - - // Quit the loop after 3 minutes - if (time() > ($starttime + 180)) { - return; - } - } - } - - private static function discoverDirectory($search) { - - $data = Cache::get("dirsearch:".$search); - if (!is_null($data)) { - // Only search for the same item every 24 hours - if (time() < $data + (60 * 60 * 24)) { - Logger::log("Already searched for ".$search." in the last 24 hours", Logger::DEBUG); - return; - } - } - - $x = Network::fetchUrl(get_server()."/lsearch?p=1&n=500&search=".urlencode($search)); - $j = json_decode($x); - - if (!empty($j->results)) { - foreach ($j->results as $jj) { - // Check if the contact already exists - $exists = q("SELECT `id`, `last_contact`, `last_failure`, `updated` FROM `gcontact` WHERE `nurl` = '%s'", Strings::normaliseLink($jj->url)); - if (DBA::isResult($exists)) { - Logger::log("Profile ".$jj->url." already exists (".$search.")", Logger::DEBUG); - - if (($exists[0]["last_contact"] < $exists[0]["last_failure"]) && - ($exists[0]["updated"] < $exists[0]["last_failure"])) { - continue; - } - // Update the contact - GContact::updateFromProbe($jj->url); - continue; - } - - $server_url = Contact::getBasepath($jj->url); - if ($server_url != '') { - if (!GServer::check($server_url)) { - Logger::log("Friendica server ".$server_url." doesn't answer.", Logger::DEBUG); - continue; - } - Logger::log("Friendica server ".$server_url." seems to be okay.", Logger::DEBUG); - } - - $data = Probe::uri($jj->url); - if ($data["network"] == Protocol::DFRN) { - Logger::log("Profile ".$jj->url." is reachable (".$search.")", Logger::DEBUG); - Logger::log("Add profile ".$jj->url." to local directory (".$search.")", Logger::DEBUG); - - if ($jj->tags != "") { - $data["keywords"] = $jj->tags; - } - - $data["server_url"] = $data["baseurl"]; - - GContact::update($data); - } else { - Logger::log("Profile ".$jj->url." is not responding or no Friendica contact - but network ".$data["network"], Logger::DEBUG); - } - } - } - Cache::set("dirsearch:".$search, time(), Cache::DAY); - } - - /** - * @brief Search for GNU Social user with gstools.org - * - * @param string $search User name - * @return bool - * @throws \Friendica\Network\HTTPException\InternalServerErrorException - * @throws \ImagickException - */ - private static function gsSearchUser($search) { - - // Currently disabled, since the service isn't available anymore. - // It is not removed since I hope that there will be a successor. - return false; - - $url = "http://gstools.org/api/users_search/".urlencode($search); - - $curlResult = Network::curl($url); - if (!$curlResult->isSuccess()) { - return false; - } - - $contacts = json_decode($curlResult->getBody()); - - if ($contacts->status == 'ERROR') { - return false; - } - - /// @TODO AS is considered as a notation for constants (as they usually being written all upper-case) - /// @TODO find all those and convert to all lower-case which is a keyword then - foreach ($contacts->data AS $user) { - $contact = Probe::uri($user->site_address."/".$user->name); - if ($contact["network"] != Protocol::PHANTOM) { - $contact["about"] = $user->description; - GContact::update($contact); - } - } - } -} diff --git a/src/Worker/FetchPoCo.php b/src/Worker/FetchPoCo.php new file mode 100644 index 000000000..67f2e548b --- /dev/null +++ b/src/Worker/FetchPoCo.php @@ -0,0 +1,24 @@ + $_SERVER]); - return; - } - } - - $arr = Probe::uri($url); - - if (is_null($result)) { - Cache::set("gprobe:".$urlparts["host"], $arr); - } - - if (!in_array($arr["network"], [Protocol::FEED, Protocol::PHANTOM])) { - GContact::update($arr); - } - - $r = q( - "SELECT `id`, `url`, `network` FROM `gcontact` WHERE `nurl` = '%s' ORDER BY `id` LIMIT 1", - DBA::escape(Strings::normaliseLink($url)) - ); - } - if (DBA::isResult($r)) { - // Check for accessibility and do a poco discovery - if (GContact::updateFromProbe($r[0]['url'], true) && ($r[0]["network"] == Protocol::DFRN)) { - PortableContact::loadWorker(0, 0, $r[0]['id'], str_replace('/profile/', '/poco/', $r[0]['url'])); - } - } - - Logger::log("gprobe end for ".Strings::normaliseLink($url), Logger::DEBUG); - return; - } -} diff --git a/src/Worker/SearchDirectory.php b/src/Worker/SearchDirectory.php new file mode 100644 index 000000000..4fd6d44d5 --- /dev/null +++ b/src/Worker/SearchDirectory.php @@ -0,0 +1,85 @@ +: Searches for "search pattern" in the directory. + public static function execute($search) + { + if (!Config::get('system', 'poco_local_search')) { + Logger::info('Local search is not enabled'); + return; + } + + $data = Cache::get('SearchDirectory:' . $search); + if (!is_null($data)) { + // Only search for the same item every 24 hours + if (time() < $data + (60 * 60 * 24)) { + Logger::info('Already searched this in the last 24 hours', ['search' => $search]); + return; + } + } + + $x = Network::fetchUrl(get_server() . '/lsearch?p=1&n=500&search=' . urlencode($search)); + $j = json_decode($x); + + if (!empty($j->results)) { + foreach ($j->results as $jj) { + // Check if the contact already exists + $gcontact = DBA::selectFirst('gcontact', ['id', 'last_contact', 'last_failure', 'updated'], ['nurl' => Strings::normaliseLink($jj->url)]); + if (DBA::isResult($gcontact)) { + Logger::info('Profile already exists', ['profile' => $jj->url, 'search' => $search]); + + if (($gcontact['last_contact'] < $gcontact['last_failure']) && + ($gcontact['updated'] < $gcontact['last_failure'])) { + continue; + } + + // Update the contact + GContact::updateFromProbe($jj->url); + continue; + } + + $server_url = Contact::getBasepath($jj->url); + if ($server_url != '') { + if (!GServer::check($server_url)) { + Logger::info("Friendica server doesn't answer.", ['server' => $server_url]); + continue; + } + Logger::info('Friendica server seems to be okay.', ['server' => $server_url]); + } + + $data = Probe::uri($jj->url); + if ($data['network'] == Protocol::DFRN) { + Logger::info('Add profile to local directory', ['profile' => $jj->url]); + + if ($jj->tags != '') { + $data['keywords'] = $jj->tags; + } + + $data['server_url'] = $data['baseurl']; + + GContact::update($data); + } else { + Logger::info('Profile is not responding or no Friendica contact', ['profile' => $jj->url, 'network' => $data['network']]); + } + } + } + Cache::set('SearchDirectory:' . $search, time(), Cache::DAY); + } +} diff --git a/src/Worker/UpdateContact.php b/src/Worker/UpdateContact.php index f23c5c0a0..0e5fd1412 100644 --- a/src/Worker/UpdateContact.php +++ b/src/Worker/UpdateContact.php @@ -13,6 +13,11 @@ use Friendica\Database\DBA; class UpdateContact { + /** + * Update contact data via probe + * @param int $contact_id Contact ID + * @param string $command + */ public static function execute($contact_id, $command = '') { $force = ($command == "force"); diff --git a/src/Worker/UpdateGContact.php b/src/Worker/UpdateGContact.php index aacebcb80..1057d0a27 100644 --- a/src/Worker/UpdateGContact.php +++ b/src/Worker/UpdateGContact.php @@ -12,6 +12,11 @@ use Friendica\Database\DBA; class UpdateGContact { + /** + * Update global contact via probe + * @param string $url Global contact url + * @param string $command + */ public static function execute($url, $command = '') { $force = ($command == "force"); diff --git a/src/Worker/UpdateGContacts.php b/src/Worker/UpdateGContacts.php new file mode 100644 index 000000000..1d9d86bcf --- /dev/null +++ b/src/Worker/UpdateGContacts.php @@ -0,0 +1,83 @@ + Protocol::PHANTOM], + ['nurl' => Strings::normaliseLink($contact['url'])]); + continue; + } + + if (in_array($urlparts['host'], ['twitter.com', 'identi.ca'])) { + $networks = ['twitter.com' => Protocol::TWITTER, 'identi.ca' => Protocol::PUMPIO]; + + DBA::update('gcontact', ['network' => $networks[$urlparts['host']]], + ['nurl' => Strings::normaliseLink($contact['url'])]); + continue; + } + + $server_url = Contact::getBasepath($contact['url']); + $force_update = false; + + if (!empty($contact['server_url'])) { + $force_update = (Strings::normaliseLink($contact['server_url']) != Strings::normaliseLink($server_url)); + + $server_url = $contact['server_url']; + } + + if ((empty($server_url) && ($contact['network'] == Protocol::FEED)) || $force_update || GServer::check($server_url, $contact['network'])) { + Logger::info('Check profile', ['profile' => $contact['url']]); + Worker::add(PRIORITY_LOW, 'UpdateGContact', $contact['url'], 'force'); + + if (++$checked > 100) { + return; + } + } else { + DBA::update('gcontact', ['last_failure' => DateTimeFormat::utcNow()], + ['nurl' => Strings::normaliseLink($contact['url'])]); + } + + // Quit the loop after 3 minutes + if (time() > ($starttime + 180)) { + return; + } + } + } +} diff --git a/src/Worker/UpdateGServer.php b/src/Worker/UpdateGServer.php new file mode 100644 index 000000000..b94c5343a --- /dev/null +++ b/src/Worker/UpdateGServer.php @@ -0,0 +1,31 @@ + $server_url, 'result' => $ret]); + } +} diff --git a/src/Worker/UpdateGServers.php b/src/Worker/UpdateGServers.php new file mode 100644 index 000000000..d55d8df01 --- /dev/null +++ b/src/Worker/UpdateGServers.php @@ -0,0 +1,39 @@ + $gserver['url']]); + + Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['url']); + + if (++$updated > 250) { + return; + } + } + } +} diff --git a/src/Worker/UpdateServerDirectories.php b/src/Worker/UpdateServerDirectories.php new file mode 100644 index 000000000..433685cb2 --- /dev/null +++ b/src/Worker/UpdateServerDirectories.php @@ -0,0 +1,31 @@ + ["type" => "text", "comment" => ""], "register_policy" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => ""], "registered-users" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "comment" => "Number of registered users"], + "directory-type" => ["type" => "tinyint", "default" => "0", "comment" => "Type of directory service (Poco, Mastodon)"], "poco" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], "noscrape" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""], "network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""],