Tupambae_Org/friendica_2021-01
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #7996 from annando/poco-cleanup

Browse source 
The PoCo-Workers had been split
This commit is contained in:
Hypolite Petovan 2019-12-24 05:13:36 -05:00 committed by GitHub
parent e7ea5dac24 8608e8ff63
commit 94954c810b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 483 additions and 456 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
database.sql
View file

@ -1,6 +1,6 @@
-- ------------------------------------------
-- Friendica 2019.12-rc (Dalmatian Bellflower)
-- DB_UPDATE_VERSION 1326
-- DB_UPDATE_VERSION 1328
-- ------------------------------------------
@ -470,6 +470,7 @@ CREATE TABLE IF NOT EXISTS `gserver` (
`info` text COMMENT '',
`register_policy` tinyint NOT NULL DEFAULT 0 COMMENT '',
`registered-users` int unsigned NOT NULL DEFAULT 0 COMMENT 'Number of registered users',
`directory-type` tinyint DEFAULT 0 COMMENT 'Type of directory service (Poco, Mastodon)',
`poco` varchar(255) NOT NULL DEFAULT '' COMMENT '',
`noscrape` varchar(255) NOT NULL DEFAULT '' COMMENT '',
`network` char(4) NOT NULL DEFAULT '' COMMENT '',

2
src/Core/Search.php
View file

@ -242,7 +242,7 @@ class Search extends BaseObject
DBA::close($data);
// Add found profiles from the global directory to the local directory
Worker::add(PRIORITY_LOW, 'DiscoverPoCo', "dirsearch", urlencode($search));
Worker::add(PRIORITY_LOW, 'SearchDirectory', $search);
return $resultList;
}

20
src/Model/GContact.php
View file

@ -580,9 +580,6 @@ class GContact
public static function getId($contact)
{
$gcontact_id = 0;
$doprobing = false;
$last_failure_str = '';
$last_contact_str = '';
if (empty($contact['network'])) {
Logger::notice('Empty network', ['url' => $contact['url'], 'callstack' => System::callstack()]);
@ -613,15 +610,6 @@ class GContact
$gcnt = DBA::selectFirst('gcontact', $fields, ['nurl' => Strings::normaliseLink($contact['url'])]);
if (DBA::isResult($gcnt)) {
$gcontact_id = $gcnt['id'];
// Update every 90 days
if (empty($gcnt['network']) || in_array($gcnt['network'], Protocol::FEDERATED)) {
$last_failure_str = $gcnt['last_failure'];
$last_failure = strtotime($gcnt['last_failure']);
$last_contact_str = $gcnt['last_contact'];
$last_contact = strtotime($gcnt['last_contact']);
$doprobing = (((time() - $last_contact) > (90 * 86400)) && ((time() - $last_failure) > (90 * 86400)));
}
} else {
$contact['location'] = $contact['location'] ?? '';
$contact['about'] = $contact['about'] ?? '';
@ -638,16 +626,10 @@ class GContact
$cnt = DBA::selectFirst('gcontact', ['id', 'network'], $condition, ['order' => ['id']]);
if (DBA::isResult($cnt)) {
$gcontact_id = $cnt['id'];
$doprobing = (empty($cnt['network']) || in_array($cnt['network'], Protocol::FEDERATED));
}
}
DBA::unlock();
if ($doprobing) {
Logger::notice('Probing', ['contact' => $last_contact_str, "failure" => $last_failure_str, "checking" => $contact['url']]);
Worker::add(PRIORITY_LOW, 'GProbe', $contact['url']);
}
return $gcontact_id;
}
@ -801,7 +783,7 @@ class GContact
return;
}
if (!$force && !PortableContact::updateNeeded($gcontact['created'], $gcontact['updated'], $gcontact['last_failure'], $gcontact['last_contact'])) {
if (!$force && !GServer::updateNeeded($gcontact['created'], $gcontact['updated'], $gcontact['last_failure'], $gcontact['last_contact'])) {
Logger::info("Don't update profile", ['url' => $data['url'], 'updated' => $gcontact['updated']]);
return;
}

126
src/Model/GServer.php
View file

@ -26,6 +26,10 @@ use Friendica\Network\Probe;
*/
class GServer
{
// Directory types
const DT_NONE = 0;
const DT_POCO = 1;
const DT_MASTODON = 2;
/**
* Checks if the given server is reachable
*
@ -49,6 +53,61 @@ class GServer
return self::check($server, $network, $force);
}
/**
* Decides if a server needs to be updated, based upon several date fields
*
* @param date $created Creation date of that server entry
* @param date $updated When had the server entry be updated
* @param date $last_failure Last failure when contacting that server
* @param date $last_contact Last time the server had been contacted
*
* @return boolean Does the server record needs an update?
*/
public static function updateNeeded($created, $updated, $last_failure, $last_contact)
{
$now = strtotime(DateTimeFormat::utcNow());
if ($updated > $last_contact) {
$contact_time = strtotime($updated);
} else {
$contact_time = strtotime($last_contact);
}
$failure_time = strtotime($last_failure);
$created_time = strtotime($created);
// If there is no "created" time then use the current time
if ($created_time <= 0) {
$created_time = $now;
}
// If the last contact was less than 24 hours then don't update
if (($now - $contact_time) < (60 * 60 * 24)) {
return false;
}
// If the last failure was less than 24 hours then don't update
if (($now - $failure_time) < (60 * 60 * 24)) {
return false;
}
// If the last contact was less than a week ago and the last failure is older than a week then don't update
//if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time))
// return false;
// If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week
if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) {
return false;
}
// If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month
if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) {
return false;
}
return true;
}
/**
* Checks the state of the given server.
*
@ -89,7 +148,7 @@ class GServer
$last_failure = DBA::NULL_DATETIME;
}
if (!$force && !PortableContact::updateNeeded($gserver['created'], '', $last_failure, $last_contact)) {
if (!$force && !self::updateNeeded($gserver['created'], '', $last_failure, $last_contact)) {
Logger::info('No update needed', ['server' => $server_url]);
return ($last_contact >= $last_failure);
}
@ -112,8 +171,24 @@ class GServer
*/
public static function detect(string $url, string $network = '')
{
Logger::info('Detect server type', ['server' => $url]);
$serverdata = [];
$original_url = $url;
// Remove URL content that is not supposed to exist for a server url
$urlparts = parse_url($url);
unset($urlparts['user']);
unset($urlparts['pass']);
unset($urlparts['query']);
unset($urlparts['fragment']);
$url = Network::unparseURL($urlparts);
// If the URL missmatches, then we mark the old entry as failure
if ($url != $original_url) {
DBA::update('gserver', ['last_failure' => DateTimeFormat::utcNow()], ['nurl' => Strings::normaliseLink($original_url)]);
}
// When a nodeinfo is present, we don't need to dig further
$xrd_timeout = Config::get('system', 'xrd_timeout');
$curlResult = Network::curl($url . '/.well-known/nodeinfo', false, ['timeout' => $xrd_timeout]);
@ -183,7 +258,10 @@ class GServer
$serverdata = $nodeinfo;
}
// Detect the directory type
$serverdata['directory-type'] = self::DT_NONE;
$serverdata = self::checkPoCo($url, $serverdata);
$serverdata = self::checkMastodonDirectory($url, $serverdata);
// We can't detect the network type. Possibly it is some system that we don't know yet
if (empty($serverdata['network'])) {
@ -740,6 +818,8 @@ class GServer
*/
private static function checkPoCo(string $url, array $serverdata)
{
$serverdata['poco'] = '';
$curlResult = Network::curl($url. '/poco');
if (!$curlResult->isSuccess()) {
return $serverdata;
@ -753,9 +833,35 @@ class GServer
if (!empty($data['totalResults'])) {
$registeredUsers = $serverdata['registered-users'] ?? 0;
$serverdata['registered-users'] = max($data['totalResults'], $registeredUsers);
$serverdata['directory-type'] = self::DT_POCO;
$serverdata['poco'] = $url . '/poco';
} else {
$serverdata['poco'] = '';
}
return $serverdata;
}
/**
* Checks if the given server does have a Mastodon style directory endpoint.
*
* @param string $url URL of the given server
* @param array $serverdata array with server data
*
* @return array server data
*/
public static function checkMastodonDirectory(string $url, array $serverdata)
{
$curlResult = Network::curl($url . '/api/v1/directory?limit=1');
if (!$curlResult->isSuccess()) {
return $serverdata;
}
$data = json_decode($curlResult->getBody(), true);
if (empty($data)) {
return $serverdata;
}
if (count($data) == 1) {
$serverdata['directory-type'] = self::DT_MASTODON;
}
return $serverdata;
@ -1184,4 +1290,18 @@ class GServer
}
return $serverdata;
}
/**
* Update the user directory of a given gserver record
*
* @param array $gserver gserver record
*/
public static function updateDirectory(array $gserver)
{
/// @todo Add Mastodon API directory
if (!empty($gserver['poco'])) {
PortableContact::discoverSingleServer($gserver['id']);
}
}
}

2
src/Model/Profile.php
View file

@ -1074,8 +1074,6 @@ class Profile
Logger::log('Not authenticated. Invoking reverse magic-auth for ' . $my_url, Logger::DEBUG);
Worker::add(PRIORITY_LOW, 'GProbe', $my_url);
// Remove the "addr" parameter from the destination. It is later added as separate parameter again.
$addr_request = 'addr=' . urlencode($addr);
$query = rtrim(str_replace($addr_request, '', $a->query_string), '?&');

57
src/Protocol/PortableContact.php
View file

@ -57,7 +57,7 @@ class PortableContact
public static function loadWorker($cid, $uid = 0, $zcid = 0, $url = null)
{
// Call the function "load" via the worker
Worker::add(PRIORITY_LOW, "DiscoverPoCo", "load", (int)$cid, (int)$uid, (int)$zcid, $url);
Worker::add(PRIORITY_LOW, 'FetchPoCo', (int)$cid, (int)$uid, (int)$zcid, $url);
}
/**
@ -219,51 +219,6 @@ class PortableContact
return(preg_match("=https?://.+/user/\d+=ism", $url, $matches));
}
public static function updateNeeded($created, $updated, $last_failure, $last_contact)
{
$now = strtotime(DateTimeFormat::utcNow());
if ($updated > $last_contact) {
$contact_time = strtotime($updated);
} else {
$contact_time = strtotime($last_contact);
}
$failure_time = strtotime($last_failure);
$created_time = strtotime($created);
// If there is no "created" time then use the current time
if ($created_time <= 0) {
$created_time = $now;
}
// If the last contact was less than 24 hours then don't update
if (($now - $contact_time) < (60 * 60 * 24)) {
return false;
}
// If the last failure was less than 24 hours then don't update
if (($now - $failure_time) < (60 * 60 * 24)) {
return false;
}
// If the last contact was less than a week ago and the last failure is older than a week then don't update
//if ((($now - $contact_time) < (60 * 60 * 24 * 7)) && ($contact_time > $failure_time))
// return false;
// If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week
if ((($now - $contact_time) > (60 * 60 * 24 * 7)) && (($now - $created_time) > (60 * 60 * 24 * 7)) && (($now - $failure_time) < (60 * 60 * 24 * 7))) {
return false;
}
// If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month
if ((($now - $contact_time) > (60 * 60 * 24 * 30)) && (($now - $created_time) > (60 * 60 * 24 * 30)) && (($now - $failure_time) < (60 * 60 * 24 * 30))) {
return false;
}
return true;
}
/**
* @brief Returns a list of all known servers
* @return array List of server urls
@ -315,7 +270,7 @@ class PortableContact
if (!DBA::isResult($r)) {
Logger::log("Call server check for server ".$server_url, Logger::DEBUG);
Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $server_url);
Worker::add(PRIORITY_LOW, 'UpdateGServer', $server_url);
}
}
}
@ -340,7 +295,7 @@ class PortableContact
if (!empty($servers['pods'])) {
foreach ($servers['pods'] as $server) {
Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", "https://" . $server['host']);
Worker::add(PRIORITY_LOW, 'UpdateGServer', 'https://' . $server['host']);
}
}
}
@ -359,7 +314,7 @@ class PortableContact
foreach ($servers['instances'] as $server) {
$url = (is_null($server['https_score']) ? 'http' : 'https') . '://' . $server['name'];
Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $url);
Worker::add(PRIORITY_LOW, 'UpdateGServer', $url);
}
}
}
@ -470,7 +425,7 @@ class PortableContact
$last_update = date('c', time() - (60 * 60 * 24 * $requery_days));
$gservers = q("SELECT `id`, `url`, `nurl`, `network`
$gservers = q("SELECT `id`, `url`, `nurl`, `network`, `poco`
FROM `gserver`
WHERE `last_contact` >= `last_failure`
AND `poco` != ''
@ -488,7 +443,7 @@ class PortableContact
}
Logger::log('Update directory from server ' . $gserver['url'] . ' with ID ' . $gserver['id'], Logger::DEBUG);
Worker::add(PRIORITY_LOW, 'DiscoverPoCo', 'update_server_directory', (int) $gserver['id']);
Worker::add(PRIORITY_LOW, 'UpdateServerDirectory', $gserver);
if (!$complete && ( --$no_of_queries == 0)) {
break;

10
src/Worker/Cron.php
View file

@ -38,11 +38,11 @@ class Cron
// Fork the cron jobs in separate parts to avoid problems when one of them is crashing
Hook::fork($a->queue['priority'], "cron");
// run the process to discover global contacts in the background
Worker::add(PRIORITY_LOW, "DiscoverPoCo");
// run the process to update server directories in the background
Worker::add(PRIORITY_LOW, 'UpdateServerDirectories');
// run the process to update locally stored global contacts in the background
Worker::add(PRIORITY_LOW, "DiscoverPoCo", "checkcontact");
Worker::add(PRIORITY_LOW, 'UpdateGContacts');
// Expire and remove user entries
Worker::add(PRIORITY_MEDIUM, "CronJobs", "expire_and_remove_users");
@ -73,9 +73,9 @@ class Cron
// update nodeinfo data
Worker::add(PRIORITY_LOW, "CronJobs", "nodeinfo");
Worker::add(PRIORITY_LOW, "DiscoverPoCo", "update_server");
Worker::add(PRIORITY_LOW, 'UpdateGServers');
Worker::add(PRIORITY_LOW, "DiscoverPoCo", "suggestions");
Worker::add(PRIORITY_LOW, 'UpdateSuggestions');
Worker::add(PRIORITY_LOW, 'Expire');

305
src/Worker/DiscoverPoCo.php
View file

@ -1,305 +0,0 @@
<?php
/**
* @file src/Worker/DiscoverPoCo.php
*/
namespace Friendica\Worker;
use Friendica\Core\Cache;
use Friendica\Core\Config;
use Friendica\Core\Logger;
use Friendica\Core\Protocol;
use Friendica\Core\Worker;
use Friendica\Database\DBA;
use Friendica\Model\GContact;
use Friendica\Model\Contact;
use Friendica\Model\GServer;
use Friendica\Network\Probe;
use Friendica\Protocol\PortableContact;
use Friendica\Util\DateTimeFormat;
use Friendica\Util\Network;
use Friendica\Util\Strings;
class DiscoverPoCo
{
/// @todo Clean up this mess of a parameter hell and split it in several classes
public static function execute($command = '', $param1 = '', $param2 = '', $param3 = '', $param4 = '')
{
/*
This function can be called in these ways:
- dirsearch <search pattern>: Searches for "search pattern" in the directory. "search pattern" is url encoded.
- checkcontact: Updates gcontact entries
- suggestions: Discover other servers for their contacts.
- server <poco url>: Searches for the poco server list. "poco url" is base64 encoded.
- update_server: Frequently check the first 250 servers for vitality.
- update_server_directory: Discover the given server id for their contacts
- PortableContact::load: Load POCO data from a given POCO address
- check_profile: Update remote profile data
*/
$search = "";
$mode = 0;
if ($command == "dirsearch") {
$search = urldecode($param1);
$mode = 1;
} elseif ($command == "checkcontact") {
$mode = 2;
} elseif ($command == "suggestions") {
$mode = 3;
} elseif ($command == "server") {
$mode = 4;
} elseif ($command == "update_server") {
$mode = 5;
} elseif ($command == "update_server_directory") {
$mode = 6;
} elseif ($command == "load") {
$mode = 7;
} elseif ($command == "check_profile") {
$mode = 8;
} elseif ($command !== "") {
Logger::log("Unknown or missing parameter ".$command."\n");
return;
}
Logger::log('start '.$search);
if ($mode == 8) {
if ($param1 != "") {
GContact::updateFromProbe($param1, true);
}
} elseif ($mode == 7) {
if (!empty($param4)) {
$url = $param4;
} else {
$url = '';
}
PortableContact::load(intval($param1), intval($param2), intval($param3), $url);
} elseif ($mode == 6) {
PortableContact::discoverSingleServer(intval($param1));
} elseif ($mode == 5) {
self::updateServer();
} elseif ($mode == 4) {
$server_url = $param1;
if ($server_url == "") {
return;
}
$server_url = filter_var($server_url, FILTER_SANITIZE_URL);
if (substr(Strings::normaliseLink($server_url), 0, 7) != "http://") {
return;
}
$result = "Checking server ".$server_url." - ";
$ret = GServer::check($server_url);
if ($ret) {
$result .= "success";
} else {
$result .= "failed";
}
Logger::log($result, Logger::DEBUG);
} elseif ($mode == 3) {
GContact::updateSuggestions();
} elseif (($mode == 2) && Config::get('system', 'poco_completion')) {
self::discoverUsers();
} elseif (($mode == 1) && ($search != "") && Config::get('system', 'poco_local_search')) {
self::discoverDirectory($search);
self::gsSearchUser($search);
} elseif (($mode == 0) && ($search == "") && (Config::get('system', 'poco_discovery') != PortableContact::DISABLED)) {
// Query Friendica and Hubzilla servers for their users
PortableContact::discover();
// Query GNU Social servers for their users ("statistics" addon has to be enabled on the GS server)
if (!Config::get('system', 'ostatus_disabled')) {
GContact::discoverGsUsers();
}
}
Logger::log('end '.$search);
return;
}
/**
* @brief Updates the first 250 servers
*
*/
private static function updateServer() {
$r = q("SELECT `url`, `created`, `last_failure`, `last_contact` FROM `gserver` ORDER BY rand()");
if (!DBA::isResult($r)) {
return;
}
$updated = 0;
foreach ($r AS $server) {
if (!PortableContact::updateNeeded($server["created"], "", $server["last_failure"], $server["last_contact"])) {
continue;
}
Logger::log('Update server status for server '.$server["url"], Logger::DEBUG);
Worker::add(PRIORITY_LOW, "DiscoverPoCo", "server", $server["url"]);
if (++$updated > 250) {
return;
}
}
}
private static function discoverUsers() {
Logger::log("Discover users", Logger::DEBUG);
$starttime = time();
$users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url`, `network` FROM `gcontact`
WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
`last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
`network` IN ('%s', '%s', '%s', '%s', '') ORDER BY rand()",
DBA::escape(Protocol::DFRN), DBA::escape(Protocol::DIASPORA),
DBA::escape(Protocol::OSTATUS), DBA::escape(Protocol::FEED));
if (!$users) {
return;
}
$checked = 0;
foreach ($users AS $user) {
$urlparts = parse_url($user["url"]);
if (!isset($urlparts["scheme"])) {
DBA::update('gcontact', ['network' => Protocol::PHANTOM],
['nurl' => Strings::normaliseLink($user["url"])]);
continue;
}
if (in_array($urlparts["host"], ["twitter.com", "identi.ca"])) {
$networks = ["twitter.com" => Protocol::TWITTER, "identi.ca" => Protocol::PUMPIO];
DBA::update('gcontact', ['network' => $networks[$urlparts["host"]]],
['nurl' => Strings::normaliseLink($user["url"])]);
continue;
}
$server_url = Contact::getBasepath($user["url"]);
$force_update = false;
if ($user["server_url"] != "") {
$force_update = (Strings::normaliseLink($user["server_url"]) != Strings::normaliseLink($server_url));
$server_url = $user["server_url"];
}
if ((($server_url == "") && ($user["network"] == Protocol::FEED)) || $force_update || GServer::check($server_url, $user["network"])) {
Logger::log('Check profile '.$user["url"]);
Worker::add(PRIORITY_LOW, "DiscoverPoCo", "check_profile", $user["url"]);
if (++$checked > 100) {
return;
}
} else {
DBA::update('gcontact', ['last_failure' => DateTimeFormat::utcNow()],
['nurl' => Strings::normaliseLink($user["url"])]);
}
// Quit the loop after 3 minutes
if (time() > ($starttime + 180)) {
return;
}
}
}
private static function discoverDirectory($search) {
$data = Cache::get("dirsearch:".$search);
if (!is_null($data)) {
// Only search for the same item every 24 hours
if (time() < $data + (60 * 60 * 24)) {
Logger::log("Already searched for ".$search." in the last 24 hours", Logger::DEBUG);
return;
}
}
$x = Network::fetchUrl(get_server()."/lsearch?p=1&n=500&search=".urlencode($search));
$j = json_decode($x);
if (!empty($j->results)) {
foreach ($j->results as $jj) {
// Check if the contact already exists
$exists = q("SELECT `id`, `last_contact`, `last_failure`, `updated` FROM `gcontact` WHERE `nurl` = '%s'", Strings::normaliseLink($jj->url));
if (DBA::isResult($exists)) {
Logger::log("Profile ".$jj->url." already exists (".$search.")", Logger::DEBUG);
if (($exists[0]["last_contact"] < $exists[0]["last_failure"]) &&
($exists[0]["updated"] < $exists[0]["last_failure"])) {
continue;
}
// Update the contact
GContact::updateFromProbe($jj->url);
continue;
}
$server_url = Contact::getBasepath($jj->url);
if ($server_url != '') {
if (!GServer::check($server_url)) {
Logger::log("Friendica server ".$server_url." doesn't answer.", Logger::DEBUG);
continue;
}
Logger::log("Friendica server ".$server_url." seems to be okay.", Logger::DEBUG);
}
$data = Probe::uri($jj->url);
if ($data["network"] == Protocol::DFRN) {
Logger::log("Profile ".$jj->url." is reachable (".$search.")", Logger::DEBUG);
Logger::log("Add profile ".$jj->url." to local directory (".$search.")", Logger::DEBUG);
if ($jj->tags != "") {
$data["keywords"] = $jj->tags;
}
$data["server_url"] = $data["baseurl"];
GContact::update($data);
} else {
Logger::log("Profile ".$jj->url." is not responding or no Friendica contact - but network ".$data["network"], Logger::DEBUG);
}
}
}
Cache::set("dirsearch:".$search, time(), Cache::DAY);
}
/**
* @brief Search for GNU Social user with gstools.org
*
* @param string $search User name
* @return bool
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @throws \ImagickException
*/
private static function gsSearchUser($search) {
// Currently disabled, since the service isn't available anymore.
// It is not removed since I hope that there will be a successor.
return false;
$url = "http://gstools.org/api/users_search/".urlencode($search);
$curlResult = Network::curl($url);
if (!$curlResult->isSuccess()) {
return false;
}
$contacts = json_decode($curlResult->getBody());
if ($contacts->status == 'ERROR') {
return false;
}
/// @TODO AS is considered as a notation for constants (as they usually being written all upper-case)
/// @TODO find all those and convert to all lower-case which is a keyword then
foreach ($contacts->data AS $user) {
$contact = Probe::uri($user->site_address."/".$user->name);
if ($contact["network"] != Protocol::PHANTOM) {
$contact["about"] = $user->description;
GContact::update($contact);
}
}
}
}

24
src/Worker/FetchPoCo.php Normal file
View file

@ -0,0 +1,24 @@
<?php
/**
* @file src/Worker/FetchPoCo.php
*/
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Protocol\PortableContact;
class FetchPoCo
{
/**
* Fetch PortableContacts from a given PoCo server address
*
* @param integer $cid Contact ID
* @param integer $uid User ID
* @param integer $zcid Global Contact ID
* @param integer $url PoCo address that should be polled
*/
public static function execute($cid, $uid, $zcid, $url)
{
PortableContact::load($cid, $uid, $zcid, $url);
}
}

68
src/Worker/GProbe.php
View file

@ -1,68 +0,0 @@
<?php
/**
* @file src/Worker/GProbe.php
*/
namespace Friendica\Worker;
use Friendica\Core\Cache;
use Friendica\Core\Logger;
use Friendica\Core\Protocol;
use Friendica\Database\DBA;
use Friendica\Model\GContact;
use Friendica\Network\Probe;
use Friendica\Protocol\PortableContact;
use Friendica\Util\Strings;
class GProbe {
public static function execute($url = '')
{
if (empty($url)) {
return;
}
$r = q(
"SELECT `id`, `url`, `network` FROM `gcontact` WHERE `nurl` = '%s' ORDER BY `id` LIMIT 1",
DBA::escape(Strings::normaliseLink($url))
);
Logger::log("gprobe start for ".Strings::normaliseLink($url), Logger::DEBUG);
if (!DBA::isResult($r)) {
// Is it a DDoS attempt?
$urlparts = parse_url($url);
$result = Cache::get("gprobe:".$urlparts["host"]);
if (!is_null($result)) {
if (in_array($result["network"], [Protocol::FEED, Protocol::PHANTOM])) {
Logger::debug("DDoS attempt detected for " . $urlparts["host"] . " by " . ($_SERVER["REMOTE_ADDR"] ?? ''), ['$_SERVER' => $_SERVER]);
return;
}
}
$arr = Probe::uri($url);
if (is_null($result)) {
Cache::set("gprobe:".$urlparts["host"], $arr);
}
if (!in_array($arr["network"], [Protocol::FEED, Protocol::PHANTOM])) {
GContact::update($arr);
}
$r = q(
"SELECT `id`, `url`, `network` FROM `gcontact` WHERE `nurl` = '%s' ORDER BY `id` LIMIT 1",
DBA::escape(Strings::normaliseLink($url))
);
}
if (DBA::isResult($r)) {
// Check for accessibility and do a poco discovery
if (GContact::updateFromProbe($r[0]['url'], true) && ($r[0]["network"] == Protocol::DFRN)) {
PortableContact::loadWorker(0, 0, $r[0]['id'], str_replace('/profile/', '/poco/', $r[0]['url']));
}
}
Logger::log("gprobe end for ".Strings::normaliseLink($url), Logger::DEBUG);
return;
}
}

85
src/Worker/SearchDirectory.php Normal file
View file

@ -0,0 +1,85 @@
<?php
/**
* @file src/Worker/SearchDirectory.php
*/
namespace Friendica\Worker;
use Friendica\Core\Cache;
use Friendica\Core\Config;
use Friendica\Core\Logger;
use Friendica\Core\Protocol;
use Friendica\Database\DBA;
use Friendica\Model\GContact;
use Friendica\Model\Contact;
use Friendica\Model\GServer;
use Friendica\Network\Probe;
use Friendica\Util\Network;
use Friendica\Util\Strings;
class SearchDirectory
{
// <search pattern>: Searches for "search pattern" in the directory.
public static function execute($search)
{
if (!Config::get('system', 'poco_local_search')) {
Logger::info('Local search is not enabled');
return;
}
$data = Cache::get('SearchDirectory:' . $search);
if (!is_null($data)) {
// Only search for the same item every 24 hours
if (time() < $data + (60 * 60 * 24)) {
Logger::info('Already searched this in the last 24 hours', ['search' => $search]);
return;
}
}
$x = Network::fetchUrl(get_server() . '/lsearch?p=1&n=500&search=' . urlencode($search));
$j = json_decode($x);
if (!empty($j->results)) {
foreach ($j->results as $jj) {
// Check if the contact already exists
$gcontact = DBA::selectFirst('gcontact', ['id', 'last_contact', 'last_failure', 'updated'], ['nurl' => Strings::normaliseLink($jj->url)]);
if (DBA::isResult($gcontact)) {
Logger::info('Profile already exists', ['profile' => $jj->url, 'search' => $search]);
if (($gcontact['last_contact'] < $gcontact['last_failure']) &&
($gcontact['updated'] < $gcontact['last_failure'])) {
continue;
}
// Update the contact
GContact::updateFromProbe($jj->url);
continue;
}
$server_url = Contact::getBasepath($jj->url);
if ($server_url != '') {
if (!GServer::check($server_url)) {
Logger::info("Friendica server doesn't answer.", ['server' => $server_url]);
continue;
}
Logger::info('Friendica server seems to be okay.', ['server' => $server_url]);
}
$data = Probe::uri($jj->url);
if ($data['network'] == Protocol::DFRN) {
Logger::info('Add profile to local directory', ['profile' => $jj->url]);
if ($jj->tags != '') {
$data['keywords'] = $jj->tags;
}
$data['server_url'] = $data['baseurl'];
GContact::update($data);
} else {
Logger::info('Profile is not responding or no Friendica contact', ['profile' => $jj->url, 'network' => $data['network']]);
}
}
}
Cache::set('SearchDirectory:' . $search, time(), Cache::DAY);
}
}

5
src/Worker/UpdateContact.php
View file

@ -13,6 +13,11 @@ use Friendica\Database\DBA;
class UpdateContact
{
/**
* Update contact data via probe
* @param int $contact_id Contact ID
* @param string $command
*/
public static function execute($contact_id, $command = '')
{
$force = ($command == "force");

5
src/Worker/UpdateGContact.php
View file

@ -12,6 +12,11 @@ use Friendica\Database\DBA;
class UpdateGContact
{
/**
* Update global contact via probe
* @param string $url Global contact url
* @param string $command
*/
public static function execute($url, $command = '')
{
$force = ($command == "force");

83
src/Worker/UpdateGContacts.php Normal file
View file

@ -0,0 +1,83 @@
<?php
/**
* @file src/Worker/UpdateGContacts.php
*/
namespace Friendica\Worker;
use Friendica\Core\Config;
use Friendica\Core\Logger;
use Friendica\Core\Protocol;
use Friendica\Core\Worker;
use Friendica\Database\DBA;
use Friendica\Model\Contact;
use Friendica\Model\GServer;
use Friendica\Util\DateTimeFormat;
use Friendica\Util\Strings;
class UpdateGContacts
{
/**
* Updates global contacts
*/
public static function execute()
{
if (!Config::get('system', 'poco_completion')) {
return;
}
Logger::info('Update global contacts');
$starttime = time();
$contacts = DBA::p("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url`, `network` FROM `gcontact`
WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
`last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
`network` IN (?, ?, ?, ?, ?, '') ORDER BY rand()",
Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::FEED);
$checked = 0;
while ($contact = DBA::fetch($contacts)) {
$urlparts = parse_url($contact['url']);
if (empty($urlparts['scheme'])) {
DBA::update('gcontact', ['network' => Protocol::PHANTOM],
['nurl' => Strings::normaliseLink($contact['url'])]);
continue;
}
if (in_array($urlparts['host'], ['twitter.com', 'identi.ca'])) {
$networks = ['twitter.com' => Protocol::TWITTER, 'identi.ca' => Protocol::PUMPIO];
DBA::update('gcontact', ['network' => $networks[$urlparts['host']]],
['nurl' => Strings::normaliseLink($contact['url'])]);
continue;
}
$server_url = Contact::getBasepath($contact['url']);
$force_update = false;
if (!empty($contact['server_url'])) {
$force_update = (Strings::normaliseLink($contact['server_url']) != Strings::normaliseLink($server_url));
$server_url = $contact['server_url'];
}
if ((empty($server_url) && ($contact['network'] == Protocol::FEED)) || $force_update || GServer::check($server_url, $contact['network'])) {
Logger::info('Check profile', ['profile' => $contact['url']]);
Worker::add(PRIORITY_LOW, 'UpdateGContact', $contact['url'], 'force');
if (++$checked > 100) {
return;
}
} else {
DBA::update('gcontact', ['last_failure' => DateTimeFormat::utcNow()],
['nurl' => Strings::normaliseLink($contact['url'])]);
}
// Quit the loop after 3 minutes
if (time() > ($starttime + 180)) {
return;
}
}
}
}

31
src/Worker/UpdateGServer.php Normal file
View file

@ -0,0 +1,31 @@
<?php
/**
* @file src/Worker/UpdateGServer.php
*/
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Model\GServer;
use Friendica\Util\Strings;
class UpdateGServer
{
/**
* Update the given server
* @param string $server_url Server URL
*/
public static function execute($server_url)
{
if (empty($server_url)) {
return;
}
$server_url = filter_var($server_url, FILTER_SANITIZE_URL);
if (substr(Strings::normaliseLink($server_url), 0, 7) != 'http://') {
return;
}
$ret = GServer::check($server_url);
Logger::info('Updated gserver', ['url' => $server_url, 'result' => $ret]);
}
}

39
src/Worker/UpdateGServers.php Normal file
View file

@ -0,0 +1,39 @@
<?php
/**
* @file src/Worker/UpdateGServers.php
*/
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Core\Worker;
use Friendica\Database\DBA;
use Friendica\Model\GServer;
class UpdateGServers
{
/**
* Updates the first 250 servers
*/
public static function execute()
{
$gservers = DBA::p("SELECT `url`, `created`, `last_failure`, `last_contact` FROM `gserver` ORDER BY rand()");
if (!DBA::isResult($gservers)) {
return;
}
$updated = 0;
while ($gserver = DBA::fetch($gservers)) {
if (!GServer::updateNeeded($gserver['created'], '', $gserver['last_failure'], $gserver['last_contact'])) {
continue;
}
Logger::info('Update server status', ['server' => $gserver['url']]);
Worker::add(PRIORITY_LOW, 'UpdateGServer', $gserver['url']);
if (++$updated > 250) {
return;
}
}
}
}

31
src/Worker/UpdateServerDirectories.php Normal file
View file

@ -0,0 +1,31 @@
<?php
/**
* @file src/Worker/UpdateServerDirectories.php
*/
namespace Friendica\Worker;
use Friendica\Core\Config;
use Friendica\Core\Logger;
use Friendica\Model\GContact;
use Friendica\Protocol\PortableContact;
class UpdateServerDirectories
{
/**
* Query global servers for their users
*/
public static function execute()
{
if (Config::get('system', 'poco_discovery') == PortableContact::DISABLED) {
return;
}
// Query Friendica and Hubzilla servers for their users
PortableContact::discover();
// Query GNU Social servers for their users ("statistics" addon has to be enabled on the GS server)
if (!Config::get('system', 'ostatus_disabled')) {
GContact::discoverGsUsers();
}
}
}

21
src/Worker/UpdateServerDirectory.php Normal file
View file

@ -0,0 +1,21 @@
<?php
/**
* @file src/Worker/UpdateServerDirectory.php
*/
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Model\GServer;
class UpdateServerDirectory
{
/**
* Query the given server for their users
* @param string $gserver Server URL
*/
public static function execute($gserver)
{
GServer::updateDirectory($gserver);
return;
}
}

19
src/Worker/UpdateSuggestions.php Normal file
View file

@ -0,0 +1,19 @@
<?php
/**
* @file src/Worker/UpdateSuggestions.php
*/
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Model\GContact;
class UpdateSuggestions
{
/**
* Discover other servers for their contacts.
*/
public static function execute()
{
GContact::updateSuggestions();
}
}

3
static/dbstructure.config.php
View file

@ -34,7 +34,7 @@
use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1327);
define('DB_UPDATE_VERSION', 1328);
}
return [
@ -526,6 +526,7 @@ return [
"info" => ["type" => "text", "comment" => ""],
"register_policy" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => ""],
"registered-users" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "comment" => "Number of registered users"],
"directory-type" => ["type" => "tinyint", "default" => "0", "comment" => "Type of directory service (Poco, Mastodon)"],
"poco" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"noscrape" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
"network" => ["type" => "char(4)", "not null" => "1", "default" => "", "comment" => ""],