strk/friendica
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #6455 from annando/issue-6282

Browse source 
Issue 6282: Update the contact data regularly (including the network)
This commit is contained in:
Hypolite Petovan 2019-01-16 18:53:27 -05:00 committed by GitHub
parent f13b8f8ec4 fe0339dabe
commit 8dbab8f8ce
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 107 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src/Core/Protocol.php
View file

@ -14,25 +14,27 @@ use Friendica\Util\Network;
class Protocol
{
// Native support
const ACTIVITYPUB = 'apub'; // ActivityPub
const ACTIVITYPUB = 'apub'; // ActivityPub (Pleroma, Mastodon, Osada, ...)
const DFRN = 'dfrn'; // Friendica, Mistpark, other DFRN implementations
const DIASPORA = 'dspr'; // Diaspora
const DIASPORA = 'dspr'; // Diaspora, Hubzilla, Socialhome, Ganggo
const FEED = 'feed'; // RSS/Atom feeds with no known "post/notify" protocol
const MAIL = 'mail'; // IMAP/POP
const OSTATUS = 'stat'; // GNU-social, Pleroma, Mastodon, other OStatus implementations
const OSTATUS = 'stat'; // GNU Social and other OStatus implementations
const NATIVE_SUPPORT = [self::DFRN, self::DIASPORA, self::OSTATUS, self::FEED, self::MAIL, self::ACTIVITYPUB];
// Supported through a connector
const APPNET = 'apdn'; // app.net - Dead protocol
const DIASPORA2 = 'dspc'; // Diaspora connector
const FACEBOOK = 'face'; // Facebook API
const GPLUS = 'goog'; // Google+
const LINKEDIN = 'lnkd'; // LinkedIn
const PUMPIO = 'pump'; // pump.io
const STATUSNET = 'stac'; // Statusnet connector
const TWITTER = 'twit'; // Twitter
// Dead protocols
const APPNET = 'apdn'; // app.net - Dead protocol
const FACEBOOK = 'face'; // Facebook API - Not working anymore, API is closed
const GPLUS = 'goog'; // Google+ - Dead in 2019
// Currently unsupported
const ICALENDAR = 'ical'; // iCalendar
const MYSPACE = 'mysp'; // MySpace

8
src/Model/Contact.php
View file

@ -634,6 +634,8 @@ class Contact extends BaseObject
Logger::log('Empty contact: ' . json_encode($contact) . ' - ' . System::callstack(20), Logger::DEBUG);
}
Logger::log('Contact '.$contact['id'].' is marked for archival', Logger::DEBUG);
// Contact already archived or "self" contact? => nothing to do
if ($contact['archive'] || $contact['self']) {
return;
@ -682,6 +684,8 @@ class Contact extends BaseObject
return;
}
Logger::log('Contact '.$contact['id'].' is marked as vital again', Logger::DEBUG);
if (!isset($contact['url']) && !empty($contact['id'])) {
$fields = ['id', 'url', 'batch'];
$contact = DBA::selectFirst('contact', [], ['id' => $contact['id']]);
@ -1571,8 +1575,8 @@ class Contact extends BaseObject
$ret = Probe::uri($contact["url"], $network);
// If Probe::uri fails the network code will be different
if (($ret["network"] != $contact["network"]) && !in_array($ret["network"], [Protocol::ACTIVITYPUB, $network])) {
// If Probe::uri fails the network code will be different (mostly "feed" or "unkn")
if (($ret["network"] != $contact["network"]) && !in_array($ret["network"], [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, $network])) {
return false;
}

2
src/Model/Item.php
View file

@ -2370,7 +2370,7 @@ class Item extends BaseObject
$update = (!$arr['private'] && ((defaults($arr, 'author-link', '') === defaults($arr, 'owner-link', '')) || ($arr["parent-uri"] === $arr["uri"])));
// Is it a forum? Then we don't care about the rules from above
if (!$update && ($arr["network"] == Protocol::DFRN) && ($arr["parent-uri"] === $arr["uri"])) {
if (!$update && in_array($arr["network"], [Protocol::ACTIVITYPUB, Protocol::DFRN]) && ($arr["parent-uri"] === $arr["uri"])) {
if (DBA::exists('contact', ['id' => $arr['contact-id'], 'forum' => true])) {
$update = true;
}

2
src/Protocol/PortableContact.php
View file

@ -334,7 +334,7 @@ class PortableContact
$server_url = Strings::normaliseLink(self::detectServer($profile));
}
if (!in_array($gcontacts[0]["network"], [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::FEED, Protocol::OSTATUS, ""])) {
if (!in_array($gcontacts[0]["network"], [Protocol::DFRN, Protocol::DIASPORA, Protocol::FEED, Protocol::OSTATUS, ""])) {
Logger::log("Profile ".$profile.": Network type ".$gcontacts[0]["network"]." can't be checked", Logger::DEBUG);
return false;
}

152
src/Worker/OnePoll.php
View file

@ -28,7 +28,7 @@ class OnePoll
{
$a = BaseObject::getApp();
Logger::log('start');
Logger::log('Start for contact ' . $contact_id);
$manual_id = 0;
$generation = 0;
@ -63,29 +63,23 @@ class OnePoll
// These three networks can be able to speak AP, so we are trying to fetch AP profile data here
if (in_array($contact['network'], [Protocol::ACTIVITYPUB, Protocol::DIASPORA, Protocol::DFRN])) {
APContact::getByURL($contact['url']);
}
$apcontact = APContact::getByURL($contact['url'], true);
// We currently don't do anything more with AP here
if ($contact['network'] === Protocol::ACTIVITYPUB) {
$updated = DateTimeFormat::utcNow();
if (($contact['network'] === Protocol::ACTIVITYPUB) && empty($apcontact)) {
self::updateContact($contact, ['last-update' => $updated, 'failure_update' => $updated]);
Contact::markForArchival($contact);
Logger::log('Contact archived');
return;
}
// load current friends if possible.
if (($contact['poco'] != "") && ($contact['success_update'] > $contact['failure_update'])) {
$r = q("SELECT count(*) AS total FROM glink
WHERE `cid` = %d AND updated > UTC_TIMESTAMP() - INTERVAL 1 DAY",
intval($contact['id'])
);
if (DBA::isResult($r)) {
if (!$r[0]['total']) {
PortableContact::loadWorker($contact['id'], $importer_uid, 0, $contact['poco']);
}
} elseif (!empty($apcontact)) {
$fields = ['last-update' => $updated, 'success_update' => $updated];
self::updateContact($contact, $fields);
Contact::unmarkForArchival($contact);
}
}
// Diaspora users, archived users and followers are only checked if they still exist.
if ($contact['archive'] || ($contact["network"] == Protocol::DIASPORA) || ($contact["rel"] == Contact::FOLLOWER)) {
if (($contact['network'] != Protocol::ACTIVITYPUB) && ($contact['archive'] || ($contact["network"] == Protocol::DIASPORA) || ($contact["rel"] == Contact::FOLLOWER))) {
$last_updated = PortableContact::lastUpdated($contact["url"], true);
$updated = DateTimeFormat::utcNow();
@ -103,56 +97,64 @@ class OnePoll
} else {
self::updateContact($contact, ['last-update' => $updated, 'failure_update' => $updated]);
Contact::markForArchival($contact);
Logger::log('Contact '.$contact['id'].' is marked for archival', Logger::DEBUG);
}
Logger::log('Contact archived');
return;
}
$xml = false;
$t = $contact['last-update'];
if ($contact['subhub']) {
$poll_interval = Config::get('system', 'pushpoll_frequency', 3);
$contact['priority'] = intval($poll_interval);
$hub_update = false;
if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 day")) {
$hub_update = true;
}
} else {
$hub_update = false;
}
$last_update = (($contact['last-update'] <= DBA::NULL_DATETIME)
? DateTimeFormat::utc('now - 7 days', DateTimeFormat::ATOM)
: DateTimeFormat::utc($contact['last-update'], DateTimeFormat::ATOM)
);
// Update the contact entry
if (($contact['network'] === Protocol::OSTATUS) || ($contact['network'] === Protocol::DIASPORA) || ($contact['network'] === Protocol::DFRN)) {
if (!PortableContact::reachable($contact['url'])) {
if (in_array($contact['network'], [Protocol::ACTIVITYPUB, Protocol::OSTATUS, Protocol::DIASPORA, Protocol::DFRN])) {
$updated = DateTimeFormat::utcNow();
// Currently we can't check every AP implementation, so we don't do it at all
if (($contact['network' != Protocol::ACTIVITYPUB]) && !PortableContact::reachable($contact['url'])) {
Logger::log("Skipping probably dead contact ".$contact['url']);
// set the last-update so we don't keep polling
DBA::update('contact', ['last-update' => DateTimeFormat::utcNow()], ['id' => $contact['id']]);
self::updateContact($contact, ['last-update' => $updated]);
return;
}
if (!Contact::updateFromProbe($contact["id"])) {
Contact::markForArchival($contact);
Logger::log('Contact is marked dead');
// set the last-update so we don't keep polling
DBA::update('contact', ['last-update' => DateTimeFormat::utcNow()], ['id' => $contact['id']]);
self::updateContact($contact, ['last-update' => $updated]);
Contact::markForArchival($contact);
Logger::log('Contact archived');
return;
} else {
$fields = ['last-update' => $updated, 'success_update' => $updated];
self::updateContact($contact, $fields);
Contact::unmarkForArchival($contact);
}
}
// load current friends if possible.
if (!empty($contact['poco']) && ($contact['success_update'] > $contact['failure_update'])) {
$r = q("SELECT count(*) AS total FROM glink
WHERE `cid` = %d AND updated > UTC_TIMESTAMP() - INTERVAL 1 DAY",
intval($contact['id'])
);
if (DBA::isResult($r)) {
if (!$r[0]['total']) {
PortableContact::loadWorker($contact['id'], $importer_uid, 0, $contact['poco']);
}
}
}
// We don't poll our followers
if ($contact["rel"] == Contact::FOLLOWER) {
Logger::log("Don't poll follower");
return;
}
// Don't poll if polling is deactivated (But we poll feeds and mails anyway)
if (!in_array($contact['network'], [Protocol::FEED, Protocol::MAIL]) && Config::get('system', 'disable_polling')) {
Logger::log('Polling is disabled');
return;
}
// We don't poll AP contacts by now
if ($contact['network'] === Protocol::ACTIVITYPUB) {
Logger::log("Don't poll AP contact");
return;
}
@ -178,6 +180,24 @@ class OnePoll
$importer = $r[0];
$url = '';
$xml = false;
if ($contact['subhub']) {
$poll_interval = Config::get('system', 'pushpoll_frequency', 3);
$contact['priority'] = intval($poll_interval);
$hub_update = false;
if (DateTimeFormat::utcNow() > DateTimeFormat::utc($contact['last-update'] . " + 1 day")) {
$hub_update = true;
}
} else {
$hub_update = false;
}
$last_update = (($contact['last-update'] <= DBA::NULL_DATETIME)
? DateTimeFormat::utc('now - 7 days', DateTimeFormat::ATOM)
: DateTimeFormat::utc($contact['last-update'], DateTimeFormat::ATOM)
);
Logger::log("poll: ({$contact['network']}-{$contact['id']}) IMPORTER: {$importer['name']}, CONTACT: {$contact['name']}");
@ -209,8 +229,9 @@ class OnePoll
if (!$curlResult->isSuccess() && ($curlResult->getErrorNumber() == CURLE_OPERATION_TIMEDOUT)) {
// set the last-update so we don't keep polling
DBA::update('contact', ['last-update' => DateTimeFormat::utcNow()], ['id' => $contact['id']]);
self::updateContact($contact, ['last-update' => DateTimeFormat::utcNow()]);
Contact::markForArchival($contact);
Logger::log('Contact archived');
return;
}
@ -220,27 +241,24 @@ class OnePoll
Logger::log('handshake with url ' . $url . ' returns xml: ' . $handshake_xml, Logger::DATA);
if (!strlen($handshake_xml) || ($html_code >= 400) || !$html_code) {
Logger::log("$url appears to be dead - marking for death ");
// dead connection - might be a transient event, or this might
// mean the software was uninstalled or the domain expired.
// Will keep trying for one month.
Contact::markForArchival($contact);
Logger::log("$url appears to be dead - marking for death ");
// set the last-update so we don't keep polling
$fields = ['last-update' => DateTimeFormat::utcNow(), 'failure_update' => DateTimeFormat::utcNow()];
self::updateContact($contact, $fields);
Contact::markForArchival($contact);
return;
}
if (!strstr($handshake_xml, '<')) {
Logger::log('response from ' . $url . ' did not contain XML.');
Contact::markForArchival($contact);
$fields = ['last-update' => DateTimeFormat::utcNow(), 'failure_update' => DateTimeFormat::utcNow()];
self::updateContact($contact, $fields);
Contact::markForArchival($contact);
return;
}
@ -248,22 +266,21 @@ class OnePoll
$res = XML::parseString($handshake_xml);
if (intval($res->status) == 1) {
// we may not be friends anymore. Will keep trying for one month.
Logger::log("$url replied status 1 - marking for death ");
// we may not be friends anymore. Will keep trying for one month.
// set the last-update so we don't keep polling
$fields = ['last-update' => DateTimeFormat::utcNow(), 'failure_update' => DateTimeFormat::utcNow()];
self::updateContact($contact, $fields);
Contact::markForArchival($contact);
} elseif ($contact['term-date'] > DBA::NULL_DATETIME) {
Logger::log("$url back from the dead - removing mark for death");
Contact::unmarkForArchival($contact);
}
if ((intval($res->status) != 0) || !strlen($res->challenge) || !strlen($res->dfrn_id)) {
// set the last-update so we don't keep polling
DBA::update('contact', ['last-update' => DateTimeFormat::utcNow()], ['id' => $contact['id']]);
Logger::log('Contact status is ' . $res->status);
return;
}
@ -293,6 +310,13 @@ class OnePoll
$final_dfrn_id = substr($final_dfrn_id, 2);
}
// There are issues with the legacy DFRN transport layer.
// Since we mostly don't use it anyway, we won't dig into it deeper, but simply ignore it.
if (empty($final_dfrn_id) || empty($orig_id)) {
Logger::log('Contact has got no ID - quitting');
return;
}
if ($final_dfrn_id != $orig_id) {
// did not decode properly - cannot trust this site
Logger::log('ID did not decode: ' . $contact['id'] . ' orig: ' . $orig_id . ' final: ' . $final_dfrn_id);
@ -330,10 +354,10 @@ class OnePoll
}
// Are we allowed to import from this person?
if ($contact['rel'] == Contact::FOLLOWER || $contact['blocked']) {
// set the last-update so we don't keep polling
DBA::update('contact', ['last-update' => DateTimeFormat::utcNow()], ['id' => $contact['id']]);
Logger::log('Contact is blocked or only a follower');
return;
}
@ -343,8 +367,9 @@ class OnePoll
if ($curlResult->isTimeout()) {
// set the last-update so we don't keep polling
DBA::update('contact', ['last-update' => DateTimeFormat::utcNow()], ['id' => $contact['id']]);
self::updateContact($contact, ['last-update' => DateTimeFormat::utcNow()]);
Contact::markForArchival($contact);
Logger::log('Contact archived');
return;
}
@ -356,8 +381,9 @@ class OnePoll
$mail_disabled = ((function_exists('imap_open') && !Config::get('system', 'imap_disabled')) ? 0 : 1);
if ($mail_disabled) {
// set the last-update so we don't keep polling
DBA::update('contact', ['last-update' => DateTimeFormat::utcNow()], ['id' => $contact['id']]);
self::updateContact($contact, ['last-update' => DateTimeFormat::utcNow()]);
Contact::markForArchival($contact);
Logger::log('Contact archived');
return;
}
@ -652,10 +678,10 @@ class OnePoll
DBA::update('gcontact', ['last_failure' => $updated], ['nurl' => $contact['nurl']]);
Contact::markForArchival($contact);
} else {
$updated = DateTimeFormat::utcNow();
DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]);
self::updateContact($contact, ['last-update' => DateTimeFormat::utcNow()]);
}
Logger::log('End');
return;
}