bitPickups/friendica_2020-09-1_sharedHosting
1
0
Fork 0
Code Pull requests Activity

Poco and gcontact (mostly) removed

Browse source
This commit is contained in:
Michael 2020-08-01 16:15:18 +00:00
commit 0c73531da1
34 changed files with 370 additions and 2032 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/Worker/Cron.php
View file

@ -57,9 +57,6 @@ class Cron
// run the process to update server directories in the background
Worker::add(PRIORITY_LOW, 'UpdateServerDirectories');
// run the process to update locally stored global contacts in the background
Worker::add(PRIORITY_LOW, 'UpdateGContacts');
// Expire and remove user entries
Worker::add(PRIORITY_MEDIUM, "CronJobs", "expire_and_remove_users");
@ -88,8 +85,6 @@ class Cron
Worker::add(PRIORITY_LOW, 'UpdateGServers');
Worker::add(PRIORITY_LOW, 'UpdateSuggestions');
Worker::add(PRIORITY_LOW, 'Expire');
Worker::add(PRIORITY_MEDIUM, 'DBClean');

9
src/Worker/CronJobs.php
View file

@ -29,7 +29,6 @@ use Friendica\Database\DBA;
use Friendica\Database\PostUpdate;
use Friendica\DI;
use Friendica\Model\Contact;
use Friendica\Model\GContact;
use Friendica\Model\Nodeinfo;
use Friendica\Model\Photo;
use Friendica\Model\User;
@ -258,14 +257,6 @@ class CronJobs
// There was an issue where the nick vanishes from the contact table
q("UPDATE `contact` INNER JOIN `user` ON `contact`.`uid` = `user`.`uid` SET `nick` = `nickname` WHERE `self` AND `nick`=''");
// Update the global contacts for local users
$r = q("SELECT `uid` FROM `user` WHERE `verified` AND NOT `blocked` AND NOT `account_removed` AND NOT `account_expired`");
if (DBA::isResult($r)) {
foreach ($r AS $user) {
GContact::updateForUser($user["uid"]);
}
}
/// @todo
/// - remove thread entries without item
/// - remove sign entries without item

41
src/Worker/FetchPoCo.php
View file

@ -1,41 +0,0 @@
<?php
/**
* @copyright Copyright (C) 2020, Friendica
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Protocol\PortableContact;
class FetchPoCo
{
/**
* Fetch PortableContacts from a given PoCo server address
*
* @param integer $cid Contact ID
* @param integer $uid User ID
* @param integer $zcid Global Contact ID
* @param integer $url PoCo address that should be polled
*/
public static function execute($cid, $uid, $zcid, $url)
{
PortableContact::load($cid, $uid, $zcid, $url);
}
}

8
src/Worker/OnePoll.php
View file

@ -32,7 +32,6 @@ use Friendica\Protocol\Activity;
use Friendica\Protocol\ActivityPub;
use Friendica\Protocol\Email;
use Friendica\Protocol\Feed;
use Friendica\Protocol\PortableContact;
use Friendica\Util\DateTimeFormat;
use Friendica\Util\Strings;
use Friendica\Util\XML;
@ -95,13 +94,6 @@ class OnePoll
$contact = DBA::selectFirst('contact', [], ['id' => $contact_id]);
}
// load current friends if possible.
if (!empty($contact['poco']) && !$contact['failed']) {
if (!DBA::exists('glink', ["`cid` = ? AND updated > UTC_TIMESTAMP() - INTERVAL 1 DAY", $contact['id']])) {
PortableContact::loadWorker($contact['id'], $importer_uid, 0, $contact['poco']);
}
}
// Don't poll if polling is deactivated (But we poll feeds and mails anyway)
if (!in_array($protocol, [Protocol::FEED, Protocol::MAIL]) && DI::config()->get('system', 'disable_polling')) {
Logger::log('Polling is disabled');

17
src/Worker/PullDirectory.php
View file

@ -60,22 +60,11 @@ class PullDirectory
return;
}
$result = Contact::addContactsByArray($contacts['results']);
$now = $contacts['now'] ?? 0;
$count = $contacts['count'] ?? 0;
$added = 0;
$updated = 0;
foreach ($contacts['results'] as $url) {
$contact = Contact::getByURL($url, false, ['id']);
if (empty($contact['id'])) {
Worker::add(PRIORITY_LOW, 'AddContact', 0, $url);
++$added;
} else {
Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id']);
++$updated;
}
}
DI::config()->set('system', 'last-directory-sync', $now);
Logger::info('Synchronization ended.', ['now' => $now, 'count' => $count, 'added' => $added, 'updated' => $updated, 'directory' => $directory]);
Logger::info('Synchronization ended', ['now' => $now, 'count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'directory' => $directory]);
}
}

43
src/Worker/SearchDirectory.php
View file

@ -23,14 +23,9 @@ namespace Friendica\Worker;
use Friendica\Core\Cache\Duration;
use Friendica\Core\Logger;
use Friendica\Core\Protocol;
use Friendica\Core\Search;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\Contact;
use Friendica\Model\GContact;
use Friendica\Model\GServer;
use Friendica\Util\Strings;
class SearchDirectory
{
@ -56,43 +51,7 @@ class SearchDirectory
if (!empty($j->results)) {
foreach ($j->results as $jj) {
// Check if the contact already exists
$gcontact = DBA::selectFirst('gcontact', ['failed'], ['nurl' => Strings::normaliseLink($jj->url)]);
if (DBA::isResult($gcontact)) {
Logger::info('Profile already exists', ['profile' => $jj->url, 'search' => $search]);
if ($gcontact['failed']) {
continue;
}
// Update the contact
GContact::updateFromProbe($jj->url);
continue;
}
$server_url = GContact::getBasepath($jj->url, true);
if ($server_url != '') {
if (!GServer::check($server_url)) {
Logger::info("Friendica server doesn't answer.", ['server' => $server_url]);
continue;
}
Logger::info('Friendica server seems to be okay.', ['server' => $server_url]);
}
$data = Contact::getByURL($jj->url);
if ($data['network'] == Protocol::DFRN) {
Logger::info('Add profile to local directory', ['profile' => $jj->url]);
if ($jj->tags != '') {
$data['keywords'] = $jj->tags;
}
$data['server_url'] = $data['baseurl'];
GContact::update($data);
} else {
Logger::info('Profile is not responding or no Friendica contact', ['profile' => $jj->url, 'network' => $data['network']]);
}
Contact::getByURL($jj->url);
}
}
DI::cache()->set('SearchDirectory:' . $search, time(), Duration::DAY);

44
src/Worker/UpdateGContact.php
View file

@ -1,44 +0,0 @@
<?php
/**
* @copyright Copyright (C) 2020, Friendica
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\DI;
use Friendica\Model\GContact;
class UpdateGContact
{
/**
* Update global contact via probe
* @param string $url Global contact url
* @param string $command
*/
public static function execute(string $url, string $command = '')
{
$force = ($command == "force");
$nodiscover = ($command == "nodiscover");
$success = GContact::updateFromProbe($url, $force);
Logger::info('Updated from probe', ['url' => $url, 'force' => $force, 'success' => $success]);
}
}

101
src/Worker/UpdateGContacts.php
View file

@ -1,101 +0,0 @@
<?php
/**
* @copyright Copyright (C) 2020, Friendica
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Core\Protocol;
use Friendica\Core\Worker;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\GContact;
use Friendica\Model\GServer;
use Friendica\Util\DateTimeFormat;
use Friendica\Util\Strings;
class UpdateGContacts
{
/**
* Updates global contacts
*/
public static function execute()
{
if (!DI::config()->get('system', 'poco_completion')) {
return;
}
Logger::info('Update global contacts');
$starttime = time();
$contacts = DBA::p("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url`, `network` FROM `gcontact`
WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
`last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
`network` IN (?, ?, ?, ?, ?, '') ORDER BY rand()",
Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::FEED);
$checked = 0;
while ($contact = DBA::fetch($contacts)) {
$urlparts = parse_url($contact['url']);
if (empty($urlparts['scheme'])) {
DBA::update('gcontact', ['network' => Protocol::PHANTOM],
['nurl' => Strings::normaliseLink($contact['url'])]);
continue;
}
if (in_array($urlparts['host'], ['twitter.com', 'identi.ca'])) {
$networks = ['twitter.com' => Protocol::TWITTER, 'identi.ca' => Protocol::PUMPIO];
DBA::update('gcontact', ['network' => $networks[$urlparts['host']]],
['nurl' => Strings::normaliseLink($contact['url'])]);
continue;
}
$server_url = GContact::getBasepath($contact['url'], true);
$force_update = false;
if (!empty($contact['server_url'])) {
$force_update = (Strings::normaliseLink($contact['server_url']) != Strings::normaliseLink($server_url));
$server_url = $contact['server_url'];
}
if ((empty($server_url) && ($contact['network'] == Protocol::FEED)) || $force_update || GServer::check($server_url, $contact['network'])) {
Logger::info('Check profile', ['profile' => $contact['url']]);
Worker::add(PRIORITY_LOW, 'UpdateGContact', $contact['url'], 'force');
if (++$checked > 100) {
return;
}
} else {
DBA::update('gcontact', ['last_failure' => DateTimeFormat::utcNow()],
['nurl' => Strings::normaliseLink($contact['url'])]);
}
// Quit the loop after 3 minutes
if (time() > ($starttime + 180)) {
return;
}
}
DBA::close($contacts);
}
}

10
src/Worker/UpdateServerDirectories.php
View file

@ -22,9 +22,7 @@
namespace Friendica\Worker;
use Friendica\DI;
use Friendica\Model\GContact;
use Friendica\Model\GServer;
use Friendica\Protocol\PortableContact;
class UpdateServerDirectories
{
@ -33,16 +31,10 @@ class UpdateServerDirectories
*/
public static function execute()
{
if (DI::config()->get('system', 'poco_discovery') == PortableContact::DISABLED) {
if (!DI::config()->get('system', 'poco_discovery')) {
return;
}
// Query Friendica and Hubzilla servers for their users
GServer::discover();
// Query GNU Social servers for their users ("statistics" addon has to be enabled on the GS server)
if (!DI::config()->get('system', 'ostatus_disabled')) {
GContact::discoverGsUsers();
}
}
}

77
src/Worker/UpdateServerDirectory.php
View file

@ -22,17 +22,86 @@
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Database\DBA;
use Friendica\DI;
use Friendica\Model\Contact;
use Friendica\Model\GServer;
class UpdateServerDirectory
{
/**
* Query the given server for their users
* @param string $gserver Server URL
*
* @param array $gserver Server record
*/
public static function execute($gserver)
public static function execute(array $gserver)
{
GServer::updateDirectory($gserver);
return;
$gserver = DBA::selectFirst('gserver', [], ['url' => $gserver['url']]);
if ($gserver['directory-type'] == GServer::DT_MASTODON) {
self::discoverMastodonDirectory($gserver);
} elseif (!empty($gserver['poco'])) {
self::discoverPoCo($gserver);
}
}
private static function discoverPoCo(array $gserver)
{
$result = DI::httpRequest()->fetch($gserver['poco'] . '?fields=urls');
if (empty($result)) {
Logger::info('Empty result', ['url' => $gserver['url']]);
return;
}
$contacts = json_decode($result, true);
if (empty($contacts['entry'])) {
Logger::info('No contacts', ['url' => $gserver['url']]);
return;
}
Logger::info('PoCo discovery started', ['poco' => $gserver['poco']]);
$urls = [];
foreach ($contacts['entry'] as $entry) {
foreach ($entry['urls'] as $url_entry) {
if (empty($url_entry['type']) || empty($url_entry['value'])) {
continue;
}
if ($url_entry['type'] == 'profile') {
$urls[] = $url_entry['value'];
}
}
}
$result = Contact::addContactsByArray($urls);
Logger::info('PoCo discovery ended', ['count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'poco' => $gserver['poco']]);
}
private static function discoverMastodonDirectory(array $gserver)
{
$result = DI::httpRequest()->fetch($gserver['url'] . '/api/v1/directory?order=new&local=true&limit=200&offset=0');
if (empty($result)) {
Logger::info('Empty result', ['url' => $gserver['url']]);
return;
}
$accounts = json_decode($result, true);
if (empty($accounts)) {
Logger::info('No contacts', ['url' => $gserver['url']]);
return;
}
Logger::info('Account discovery started', ['url' => $gserver['url']]);
$urls = [];
foreach ($accounts as $account) {
if (!empty($account['url'])) {
$urls[] = $account['url'];
}
}
$result = Contact::addContactsByArray($urls);
Logger::info('Account discovery ended', ['count' => $result['count'], 'added' => $result['added'], 'updated' => $result['updated'], 'url' => $gserver['url']]);
}
}

37
src/Worker/UpdateServerPeers.php
View file

@ -63,4 +63,41 @@ class UpdateServerPeers
}
Logger::info('Server peer update ended', ['total' => $total, 'added' => $added, 'url' => $url]);
}
/**
* Fetch server list from remote servers and adds them when they are new.
*
* @param string $poco URL to the POCO endpoint
*/
private static function fetchServerlist($poco)
{
$curlResult = DI::httpRequest()->get($poco . '/@server');
if (!$curlResult->isSuccess()) {
Logger::info('Server is not reachable or does not offer the "poco" endpoint', ['poco' => $poco]);
return;
}
$serverlist = json_decode($curlResult->getBody(), true);
if (!is_array($serverlist)) {
Logger::info('Server does not have any servers listed', ['poco' => $poco]);
return;
}
Logger::info('PoCo Server update start', ['poco' => $poco]);
$total = 0;
$added = 0;
foreach ($serverlist as $server) {
++$total;
if (DBA::exists('gserver', ['nurl' => Strings::normaliseLink($server['url'])])) {
// We already know this server
continue;
}
// This endpoint doesn't offer the schema. So we assume that it is HTTPS.
Worker::add(PRIORITY_LOW, 'UpdateGServer', $server['url']);
++$added;
}
Logger::info('PoCo Server update ended', ['total' => $total, 'added' => $added, 'poco' => $poco]);
}
}

36
src/Worker/UpdateSuggestions.php
View file

@ -1,36 +0,0 @@
<?php
/**
* @copyright Copyright (C) 2020, Friendica
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
namespace Friendica\Worker;
use Friendica\Core\Logger;
use Friendica\Model\GContact;
class UpdateSuggestions
{
/**
* Discover other servers for their contacts.
*/
public static function execute()
{
GContact::updateSuggestions();
}
}