From c359c162a7c7feb8c3b233b9c6b6418e13d3471a Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 16 Jan 2019 21:39:56 +0000 Subject: [PATCH] Issue 6282: Update the contact data regularly (including the network) --- src/Core/Protocol.php | 14 ++-- src/Model/Contact.php | 8 +- src/Model/Item.php | 2 +- src/Protocol/PortableContact.php | 2 +- src/Worker/OnePoll.php | 135 ++++++++++++++++--------------- 5 files changed, 87 insertions(+), 74 deletions(-) diff --git a/src/Core/Protocol.php b/src/Core/Protocol.php index 949e2350c3..00552bf787 100644 --- a/src/Core/Protocol.php +++ b/src/Core/Protocol.php @@ -14,25 +14,27 @@ use Friendica\Util\Network; class Protocol { // Native support - const ACTIVITYPUB = 'apub'; // ActivityPub + const ACTIVITYPUB = 'apub'; // ActivityPub (Pleroma, Mastodon, Osada, ...) const DFRN = 'dfrn'; // Friendica, Mistpark, other DFRN implementations - const DIASPORA = 'dspr'; // Diaspora + const DIASPORA = 'dspr'; // Diaspora, Hubzilla, Socialhome, Ganggo const FEED = 'feed'; // RSS/Atom feeds with no known "post/notify" protocol const MAIL = 'mail'; // IMAP/POP - const OSTATUS = 'stat'; // GNU-social, Pleroma, Mastodon, other OStatus implementations + const OSTATUS = 'stat'; // GNU Social and other OStatus implementations const NATIVE_SUPPORT = [self::DFRN, self::DIASPORA, self::OSTATUS, self::FEED, self::MAIL, self::ACTIVITYPUB]; // Supported through a connector - const APPNET = 'apdn'; // app.net - Dead protocol const DIASPORA2 = 'dspc'; // Diaspora connector - const FACEBOOK = 'face'; // Facebook API - const GPLUS = 'goog'; // Google+ const LINKEDIN = 'lnkd'; // LinkedIn const PUMPIO = 'pump'; // pump.io const STATUSNET = 'stac'; // Statusnet connector const TWITTER = 'twit'; // Twitter + // Dead protocols + const APPNET = 'apdn'; // app.net - Dead protocol + const FACEBOOK = 'face'; // Facebook API - Not working anymore, API is closed + const GPLUS = 'goog'; // Google+ - Dead in 2019 + // Currently unsupported const ICALENDAR = 'ical'; // iCalendar const MYSPACE = 'mysp'; // MySpace diff --git a/src/Model/Contact.php b/src/Model/Contact.php index c6e11bca67..7f309925f9 100644 --- a/src/Model/Contact.php +++ b/src/Model/Contact.php @@ -634,6 +634,8 @@ class Contact extends BaseObject Logger::log('Empty contact: ' . json_encode($contact) . ' - ' . System::callstack(20), Logger::DEBUG); } + Logger::log('Contact '.$contact['id'].' is marked for archival', Logger::DEBUG); + // Contact already archived or "self" contact? => nothing to do if ($contact['archive'] || $contact['self']) { return; @@ -682,6 +684,8 @@ class Contact extends BaseObject return; } + Logger::log('Contact '.$contact['id'].' is marked as vital again', Logger::DEBUG); + if (!isset($contact['url']) && !empty($contact['id'])) { $fields = ['id', 'url', 'batch']; $contact = DBA::selectFirst('contact', [], ['id' => $contact['id']]); @@ -1571,8 +1575,8 @@ class Contact extends BaseObject $ret = Probe::uri($contact["url"], $network); - // If Probe::uri fails the network code will be different - if (($ret["network"] != $contact["network"]) && !in_array($ret["network"], [Protocol::ACTIVITYPUB, $network])) { + // If Probe::uri fails the network code will be different (mostly "feed" or "unkn") + if (($ret["network"] != $contact["network"]) && !in_array($ret["network"], [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, $network])) { return false; } diff --git a/src/Model/Item.php b/src/Model/Item.php index ca6556d275..95256219a1 100644 --- a/src/Model/Item.php +++ b/src/Model/Item.php @@ -2370,7 +2370,7 @@ class Item extends BaseObject $update = (!$arr['private'] && ((defaults($arr, 'author-link', '') === defaults($arr, 'owner-link', '')) || ($arr["parent-uri"] === $arr["uri"]))); // Is it a forum? Then we don't care about the rules from above - if (!$update && ($arr["network"] == Protocol::DFRN) && ($arr["parent-uri"] === $arr["uri"])) { + if (!$update && in_array($arr["network"], [Protocol::ACTIVITYPUB, Protocol::DFRN]) && ($arr["parent-uri"] === $arr["uri"])) { if (DBA::exists('contact', ['id' => $arr['contact-id'], 'forum' => true])) { $update = true; } diff --git a/src/Protocol/PortableContact.php b/src/Protocol/PortableContact.php index c61fe128a1..dcdc9e2b14 100644 --- a/src/Protocol/PortableContact.php +++ b/src/Protocol/PortableContact.php @@ -334,7 +334,7 @@ class PortableContact $server_url = Strings::normaliseLink(self::detectServer($profile)); } - if (!in_array($gcontacts[0]["network"], [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::FEED, Protocol::OSTATUS, ""])) { + if (!in_array($gcontacts[0]["network"], [Protocol::DFRN, Protocol::DIASPORA, Protocol::FEED, Protocol::OSTATUS, ""])) { Logger::log("Profile ".$profile.": Network type ".$gcontacts[0]["network"]." can't be checked", Logger::DEBUG); return false; } diff --git a/src/Worker/OnePoll.php b/src/Worker/OnePoll.php index 4b4790c279..4e0de487e1 100644 --- a/src/Worker/OnePoll.php +++ b/src/Worker/OnePoll.php @@ -63,29 +63,22 @@ class OnePoll // These three networks can be able to speak AP, so we are trying to fetch AP profile data here if (in_array($contact['network'], [Protocol::ACTIVITYPUB, Protocol::DIASPORA, Protocol::DFRN])) { - APContact::getByURL($contact['url']); - } + $apcontact = APContact::getByURL($contact['url'], true); - // We currently don't do anything more with AP here - if ($contact['network'] === Protocol::ACTIVITYPUB) { - return; - } - - // load current friends if possible. - if (($contact['poco'] != "") && ($contact['success_update'] > $contact['failure_update'])) { - $r = q("SELECT count(*) AS total FROM glink - WHERE `cid` = %d AND updated > UTC_TIMESTAMP() - INTERVAL 1 DAY", - intval($contact['id']) - ); - if (DBA::isResult($r)) { - if (!$r[0]['total']) { - PortableContact::loadWorker($contact['id'], $importer_uid, 0, $contact['poco']); - } + $updated = DateTimeFormat::utcNow(); + if (($contact['network'] === Protocol::ACTIVITYPUB) && empty($apcontact)) { + self::updateContact($contact, ['last-update' => $updated, 'failure_update' => $updated]); + Contact::markForArchival($contact); + return; + } elseif (!empty($apcontact)) { + $fields = ['last-update' => $updated, 'success_update' => $updated]; + self::updateContact($contact, $fields); + Contact::unmarkForArchival($contact); } } // Diaspora users, archived users and followers are only checked if they still exist. - if ($contact['archive'] || ($contact["network"] == Protocol::DIASPORA) || ($contact["rel"] == Contact::FOLLOWER)) { + if (($contact['network'] != Protocol::ACTIVITYPUB) && ($contact['archive'] || ($contact["network"] == Protocol::DIASPORA) || ($contact["rel"] == Contact::FOLLOWER))) { $last_updated = PortableContact::lastUpdated($contact["url"], true); $updated = DateTimeFormat::utcNow(); @@ -103,59 +96,62 @@ class OnePoll } else { self::updateContact($contact, ['last-update' => $updated, 'failure_update' => $updated]); Contact::markForArchival($contact); - Logger::log('Contact '.$contact['id'].' is marked for archival', Logger::DEBUG); + return; } - - return; } - $xml = false; - - $t = $contact['last-update']; - - if ($contact['subhub']) { - $poll_interval = Config::get('system', 'pushpoll_frequency', 3); - $contact['priority'] = intval($poll_interval); - $hub_update = false; - - if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + 1 day")) { - $hub_update = true; - } - } else { - $hub_update = false; - } - - $last_update = (($contact['last-update'] <= DBA::NULL_DATETIME) - ? DateTimeFormat::utc('now - 7 days', DateTimeFormat::ATOM) - : DateTimeFormat::utc($contact['last-update'], DateTimeFormat::ATOM) - ); - // Update the contact entry - if (($contact['network'] === Protocol::OSTATUS) || ($contact['network'] === Protocol::DIASPORA) || ($contact['network'] === Protocol::DFRN)) { - if (!PortableContact::reachable($contact['url'])) { + if (in_array($contact['network'], [Protocol::ACTIVITYPUB, Protocol::OSTATUS, Protocol::DIASPORA, Protocol::DFRN])) { + $updated = DateTimeFormat::utcNow(); + // Currently we can't check every AP implementation, so we don't do it at all + if (($contact['network' != Protocol::ACTIVITYPUB]) && !PortableContact::reachable($contact['url'])) { Logger::log("Skipping probably dead contact ".$contact['url']); // set the last-update so we don't keep polling - DBA::update('contact', ['last-update' => DateTimeFormat::utcNow()], ['id' => $contact['id']]); + self::updateContact($contact, ['last-update' => $updated]); return; } if (!Contact::updateFromProbe($contact["id"])) { - Contact::markForArchival($contact); - Logger::log('Contact is marked dead'); - // set the last-update so we don't keep polling - DBA::update('contact', ['last-update' => DateTimeFormat::utcNow()], ['id' => $contact['id']]); + self::updateContact($contact, ['last-update' => $updated]); + Contact::markForArchival($contact); return; } else { + $fields = ['last-update' => $updated, 'success_update' => $updated]; + self::updateContact($contact, $fields); Contact::unmarkForArchival($contact); } } + // load current friends if possible. + if (!emoty($contact['poco']) && ($contact['success_update'] > $contact['failure_update'])) { + $r = q("SELECT count(*) AS total FROM glink + WHERE `cid` = %d AND updated > UTC_TIMESTAMP() - INTERVAL 1 DAY", + intval($contact['id']) + ); + if (DBA::isResult($r)) { + if (!$r[0]['total']) { + PortableContact::loadWorker($contact['id'], $importer_uid, 0, $contact['poco']); + } + } + } + + // We don't poll our followers + if ($contact["rel"] == Contact::FOLLOWER) { + return; + } + + // Don't poll if polling is deactivated (But we poll feeds and mails anyway) if (!in_array($contact['network'], [Protocol::FEED, Protocol::MAIL]) && Config::get('system', 'disable_polling')) { return; } + // We don't poll AP contacts by now + if ($contact['network'] === Protocol::ACTIVITYPUB) { + return; + } + if ($importer_uid == 0) { Logger::log('Ignore public contacts'); @@ -178,6 +174,24 @@ class OnePoll $importer = $r[0]; $url = ''; + $xml = false; + + if ($contact['subhub']) { + $poll_interval = Config::get('system', 'pushpoll_frequency', 3); + $contact['priority'] = intval($poll_interval); + $hub_update = false; + + if (DateTimeFormat::utcNow() > DateTimeFormat::utc($contact['last-update'] . " + 1 day")) { + $hub_update = true; + } + } else { + $hub_update = false; + } + + $last_update = (($contact['last-update'] <= DBA::NULL_DATETIME) + ? DateTimeFormat::utc('now - 7 days', DateTimeFormat::ATOM) + : DateTimeFormat::utc($contact['last-update'], DateTimeFormat::ATOM) + ); Logger::log("poll: ({$contact['network']}-{$contact['id']}) IMPORTER: {$importer['name']}, CONTACT: {$contact['name']}"); @@ -209,7 +223,7 @@ class OnePoll if (!$curlResult->isSuccess() && ($curlResult->getErrorNumber() == CURLE_OPERATION_TIMEDOUT)) { // set the last-update so we don't keep polling - DBA::update('contact', ['last-update' => DateTimeFormat::utcNow()], ['id' => $contact['id']]); + self::updateContact($contact, ['last-update' => DateTimeFormat::utcNow()]); Contact::markForArchival($contact); return; } @@ -220,27 +234,24 @@ class OnePoll Logger::log('handshake with url ' . $url . ' returns xml: ' . $handshake_xml, Logger::DATA); if (!strlen($handshake_xml) || ($html_code >= 400) || !$html_code) { - Logger::log("$url appears to be dead - marking for death "); - // dead connection - might be a transient event, or this might // mean the software was uninstalled or the domain expired. // Will keep trying for one month. - - Contact::markForArchival($contact); + Logger::log("$url appears to be dead - marking for death "); // set the last-update so we don't keep polling $fields = ['last-update' => DateTimeFormat::utcNow(), 'failure_update' => DateTimeFormat::utcNow()]; self::updateContact($contact, $fields); + Contact::markForArchival($contact); return; } if (!strstr($handshake_xml, '<')) { Logger::log('response from ' . $url . ' did not contain XML.'); - Contact::markForArchival($contact); - $fields = ['last-update' => DateTimeFormat::utcNow(), 'failure_update' => DateTimeFormat::utcNow()]; self::updateContact($contact, $fields); + Contact::markForArchival($contact); return; } @@ -248,16 +259,14 @@ class OnePoll $res = XML::parseString($handshake_xml); if (intval($res->status) == 1) { + // we may not be friends anymore. Will keep trying for one month. Logger::log("$url replied status 1 - marking for death "); - // we may not be friends anymore. Will keep trying for one month. // set the last-update so we don't keep polling $fields = ['last-update' => DateTimeFormat::utcNow(), 'failure_update' => DateTimeFormat::utcNow()]; self::updateContact($contact, $fields); - Contact::markForArchival($contact); } elseif ($contact['term-date'] > DBA::NULL_DATETIME) { - Logger::log("$url back from the dead - removing mark for death"); Contact::unmarkForArchival($contact); } @@ -330,7 +339,6 @@ class OnePoll } // Are we allowed to import from this person? - if ($contact['rel'] == Contact::FOLLOWER || $contact['blocked']) { // set the last-update so we don't keep polling DBA::update('contact', ['last-update' => DateTimeFormat::utcNow()], ['id' => $contact['id']]); @@ -343,7 +351,7 @@ class OnePoll if ($curlResult->isTimeout()) { // set the last-update so we don't keep polling - DBA::update('contact', ['last-update' => DateTimeFormat::utcNow()], ['id' => $contact['id']]); + self::updateContact($contact, ['last-update' => DateTimeFormat::utcNow()]); Contact::markForArchival($contact); return; } @@ -356,7 +364,7 @@ class OnePoll $mail_disabled = ((function_exists('imap_open') && !Config::get('system', 'imap_disabled')) ? 0 : 1); if ($mail_disabled) { // set the last-update so we don't keep polling - DBA::update('contact', ['last-update' => DateTimeFormat::utcNow()], ['id' => $contact['id']]); + self::updateContact($contact, ['last-update' => DateTimeFormat::utcNow()]); Contact::markForArchival($contact); return; } @@ -652,8 +660,7 @@ class OnePoll DBA::update('gcontact', ['last_failure' => $updated], ['nurl' => $contact['nurl']]); Contact::markForArchival($contact); } else { - $updated = DateTimeFormat::utcNow(); - DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]); + self::updateContact($contact, ['last-update' => DateTimeFormat::utcNow()]); } return;