Browse Source

Merge pull request #7704 from annando/gcontact-update

Replace obsolete functionality in "PortableContact"
tags/2019.12
Hypolite Petovan 1 year ago
committed by GitHub
parent
commit
69ea65c0f2
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 212 additions and 274 deletions
  1. +180
    -4
      src/Model/GContact.php
  2. +27
    -5
      src/Model/GServer.php
  3. +0
    -260
      src/Protocol/PortableContact.php
  4. +2
    -2
      src/Worker/CronJobs.php
  5. +2
    -2
      src/Worker/DiscoverPoCo.php
  6. +1
    -1
      src/Worker/GProbe.php

+ 180
- 4
src/Model/GContact.php View File

@@ -6,6 +6,8 @@
*/
namespace Friendica\Model;

use DOMDocument;
use DOMXPath;
use Exception;
use Friendica\Core\Config;
use Friendica\Core\Logger;
@@ -14,6 +16,7 @@ use Friendica\Core\System;
use Friendica\Core\Worker;
use Friendica\Database\DBA;
use Friendica\Network\Probe;
use Friendica\Protocol\ActivityPub;
use Friendica\Protocol\PortableContact;
use Friendica\Util\DateTimeFormat;
use Friendica\Util\Network;
@@ -190,7 +193,7 @@ class GContact
}

if ((!isset($gcontact['network']) || !isset($gcontact['name']) || !isset($gcontact['addr']) || !isset($gcontact['photo']) || !isset($gcontact['server_url']) || $alternate)
&& PortableContact::reachable($gcontact['url'], $gcontact['server_url'], $gcontact['network'], false)
&& GServer::reachable($gcontact['url'], $gcontact['server_url'], $gcontact['network'], false)
) {
$data = Probe::uri($gcontact['url']);

@@ -860,6 +863,170 @@ class GContact
return $gcontact_id;
}

/**
* Set the last date that the contact had posted something
*
* @param string $data Probing result
* @param bool $force force updating
*/
public static function setLastUpdate(array $data, bool $force = false)
{
// Fetch the global contact
$gcontact = DBA::selectFirst('gcontact', ['created', 'updated', 'last_contact', 'last_failure'],
['nurl' => Strings::normaliseLink($data['url'])]);
if (!DBA::isResult($gcontact)) {
return;
}

if (!$force && !PortableContact::updateNeeded($gcontact['created'], $gcontact['updated'], $gcontact['last_failure'], $gcontact['last_contact'])) {
Logger::info("Don't update profile", ['url' => $data['url'], 'updated' => $gcontact['updated']]);
return;
}

if (self::updateFromNoScrape($data)) {
return;
}

// When the profile doesn't have got a feed, then we exit here
if (empty($data['poll'])) {
return;
}

if ($data['network'] == Protocol::ACTIVITYPUB) {
self::updateFromOutbox($data['poll'], $data);
} else {
self::updateFromFeed($data);
}
}

/**
* Update a global contact via the "noscrape" endpoint
*
* @param string $data Probing result
*
* @return bool 'true' if update was successful or the server was unreachable
*/
private static function updateFromNoScrape(array $data)
{
// Check the 'noscrape' endpoint when it is a Friendica server
$gserver = DBA::selectFirst('gserver', ['noscrape'], ["`nurl` = ? AND `noscrape` != ''",
Strings::normaliseLink($data['baseurl'])]);
if (!DBA::isResult($gserver)) {
return false;
}

$curlResult = Network::curl($gserver['noscrape'] . '/' . $data['nick']);

if ($curlResult->isSuccess() && !empty($curlResult->getBody())) {
$noscrape = json_decode($curlResult->getBody(), true);
if (!empty($noscrape)) {
$noscrape['updated'] = DateTimeFormat::utc($noscrape['updated'], DateTimeFormat::MYSQL);
$fields = ['last_contact' => DateTimeFormat::utcNow(), 'updated' => $noscrape['updated']];
DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($data['url'])]);
return true;
}
} elseif ($curlResult->isTimeout()) {
// On a timeout return the existing value, but mark the contact as failure
$fields = ['last_failure' => DateTimeFormat::utcNow()];
DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($data['url'])]);
return true;
}
return false;
}

/**
* Update a global contact via an ActivityPub Outbox
*
* @param string $data Probing result
*/
private static function updateFromOutbox(string $feed, array $data)
{
$outbox = ActivityPub::fetchContent($feed);
if (empty($outbox)) {
return;
}

if (!empty($outbox['orderedItems'])) {
$items = $outbox['orderedItems'];
} elseif (!empty($outbox['first']['orderedItems'])) {
$items = $outbox['first']['orderedItems'];
} elseif (!empty($outbox['first'])) {
self::updateFromOutbox($outbox['first'], $data);
return;
} else {
$items = [];
}

$last_updated = '';

foreach ($items as $activity) {
if ($last_updated < $activity['published']) {
$last_updated = $activity['published'];
}
}

if (empty($last_updated)) {
return;
}

$fields = ['last_contact' => DateTimeFormat::utcNow(), 'updated' => $last_updated];
DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($data['url'])]);
}

/**
* Update a global contact via an XML feed
*
* @param string $data Probing result
*/
private static function updateFromFeed(array $data)
{
// Search for the newest entry in the feed
$curlResult = Network::curl($data['poll']);
if (!$curlResult->isSuccess()) {
$fields = ['last_failure' => DateTimeFormat::utcNow()];
DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($profile)]);

Logger::info("Profile wasn't reachable (no feed)", ['url' => $data['url']]);
return;
}

$doc = new DOMDocument();
@$doc->loadXML($curlResult->getBody());

$xpath = new DOMXPath($doc);
$xpath->registerNamespace('atom', 'http://www.w3.org/2005/Atom');

$entries = $xpath->query('/atom:feed/atom:entry');

$last_updated = '';

foreach ($entries as $entry) {
$published_item = $xpath->query('atom:published/text()', $entry)->item(0);
$updated_item = $xpath->query('atom:updated/text()' , $entry)->item(0);
$published = !empty($published_item->nodeValue) ? DateTimeFormat::utc($published_item->nodeValue) : null;
$updated = !empty($updated_item->nodeValue) ? DateTimeFormat::utc($updated_item->nodeValue) : null;

if (empty($published) || empty($updated)) {
Logger::notice('Invalid entry for XPath.', ['entry' => $entry, 'url' => $data['url']]);
continue;
}

if ($last_updated < $published) {
$last_updated = $published;
}

if ($last_updated < $updated) {
$last_updated = $updated;
}
}

if (empty($last_updated)) {
return;
}

$fields = ['last_contact' => DateTimeFormat::utcNow(), 'updated' => $last_updated];
DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($data['url'])]);
}
/**
* @brief Updates the gcontact entry from a given public contact id
*
@@ -976,7 +1143,9 @@ class GContact
*
* @param string $url profile link
* @param boolean $force Optional forcing of network probing (otherwise we use the cached data)
* @return void
*
* @return boolean 'true' when contact had been updated
*
* @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @throws \ImagickException
*/
@@ -985,13 +1154,20 @@ class GContact
$data = Probe::uri($url, $force);

if (in_array($data["network"], [Protocol::PHANTOM])) {
Logger::log("Invalid network for contact url ".$data["url"]." - Called by: ".System::callstack(), Logger::DEBUG);
return;
$fields = ['last_failure' => DateTimeFormat::utcNow()];
DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($url)]);
Logger::info('Invalid network for contact', ['url' => $data['url'], 'callstack' => System::callstack()]);
return false;
}

$data["server_url"] = $data["baseurl"];

self::update($data);

// Set the date of the latest post
self::setLastUpdate($data, $force);

return true;
}

/**


+ 27
- 5
src/Model/GServer.php View File

@@ -26,6 +26,29 @@ use Friendica\Network\Probe;
*/
class GServer
{
/**
* Checks if the given server is reachable
*
* @param string $profile URL of the given profile
* @param string $server URL of the given server (If empty, taken from profile)
* @param string $network Network value that is used, when detection failed
* @param boolean $force Force an update.
*
* @return boolean 'true' if server seems vital
*/
public static function reachable(string $profile, string $server = '', string $network = '', bool $force = false)
{
if ($server == '') {
$server = Contact::getBasepath($profile);
}

if ($server == '') {
return true;
}

return self::check($server, $network, $force);
}

/**
* Checks the state of the given server.
*
@@ -661,21 +684,20 @@ class GServer
private static function detectNetworkViaContacts(string $url, array $serverdata)
{
$contacts = '';
$fields = ['nurl', 'url'];

$gcontacts = DBA::select('gcontact', $fields, ['server_url' => [$url, $serverdata['nurl']]]);
$gcontacts = DBA::select('gcontact', ['url', 'nurl'], ['server_url' => [$url, $serverdata['nurl']]]);
while ($gcontact = DBA::fetch($gcontacts)) {
$contacts[$gcontact['nurl']] = $gcontact['url'];
}
DBA::close($gcontacts);

$apcontacts = DBA::select('apcontact', $fields, ['baseurl' => [$url, $serverdata['nurl']]]);
$apcontacts = DBA::select('apcontact', ['url'], ['baseurl' => [$url, $serverdata['nurl']]]);
while ($gcontact = DBA::fetch($gcontacts)) {
$contacts[$apcontact['nurl']] = $apcontact['url'];
$contacts[Strings::normaliseLink($apcontact['url'])] = $apcontact['url'];
}
DBA::close($apcontacts);

$pcontacts = DBA::select('contact', $fields, ['uid' => 0, 'baseurl' => [$url, $serverdata['nurl']]]);
$pcontacts = DBA::select('contact', ['url', 'nurl'], ['uid' => 0, 'baseurl' => [$url, $serverdata['nurl']]]);
while ($gcontact = DBA::fetch($gcontacts)) {
$contacts[$pcontact['nurl']] = $pcontact['url'];
}


+ 0
- 260
src/Protocol/PortableContact.php View File

@@ -214,271 +214,11 @@ class PortableContact
DBA::delete('glink', $condition);
}

public static function reachable($profile, $server = "", $network = "", $force = false)
{
if ($server == "") {
$server = Contact::getBasepath($profile);
}

if ($server == "") {
return true;
}

return GServer::check($server, $network, $force);
}

public static function alternateOStatusUrl($url)
{
return(preg_match("=https?://.+/user/\d+=ism", $url, $matches));
}

public static function lastUpdated($profile, $force = false)
{
$gcontacts = q(
"SELECT * FROM `gcontact` WHERE `nurl` = '%s'",
DBA::escape(Strings::normaliseLink($profile))
);

if (!DBA::isResult($gcontacts)) {
return false;
}

$contact = ["url" => $profile];

if ($gcontacts[0]["created"] <= DBA::NULL_DATETIME) {
$contact['created'] = DateTimeFormat::utcNow();
}

$server_url = '';
if ($force) {
$server_url = Strings::normaliseLink(Contact::getBasepath($profile));
}

if (($server_url == '') && ($gcontacts[0]["server_url"] != "")) {
$server_url = $gcontacts[0]["server_url"];
}

if (!$force && (($server_url == '') || ($gcontacts[0]["server_url"] == $gcontacts[0]["nurl"]))) {
$server_url = Strings::normaliseLink(Contact::getBasepath($profile));
}

if (!in_array($gcontacts[0]["network"], [Protocol::DFRN, Protocol::DIASPORA, Protocol::FEED, Protocol::OSTATUS, ""])) {
Logger::log("Profile ".$profile.": Network type ".$gcontacts[0]["network"]." can't be checked", Logger::DEBUG);
return false;
}

if ($server_url != "") {
if (!GServer::check($server_url, $gcontacts[0]["network"], $force)) {
if ($force) {
$fields = ['last_failure' => DateTimeFormat::utcNow()];
DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($profile)]);
}

Logger::log("Profile ".$profile.": Server ".$server_url." wasn't reachable.", Logger::DEBUG);
return false;
}
$contact['server_url'] = $server_url;
}

if (in_array($gcontacts[0]["network"], ["", Protocol::FEED])) {
$server = q(
"SELECT `network` FROM `gserver` WHERE `nurl` = '%s' AND `network` != ''",
DBA::escape(Strings::normaliseLink($server_url))
);

if ($server) {
$contact['network'] = $server[0]["network"];
} else {
return false;
}
}

// noscrape is really fast so we don't cache the call.
if (($server_url != "") && ($gcontacts[0]["nick"] != "")) {
// Use noscrape if possible
$server = q("SELECT `noscrape`, `network` FROM `gserver` WHERE `nurl` = '%s' AND `noscrape` != ''", DBA::escape(Strings::normaliseLink($server_url)));

if ($server) {
$curlResult = Network::curl($server[0]["noscrape"]."/".$gcontacts[0]["nick"]);

if ($curlResult->isSuccess() && ($curlResult->getBody() != "")) {
$noscrape = json_decode($curlResult->getBody(), true);

if (is_array($noscrape)) {
$contact["network"] = $server[0]["network"];

if (isset($noscrape["fn"])) {
$contact["name"] = $noscrape["fn"];
}
if (isset($noscrape["comm"])) {
$contact["community"] = $noscrape["comm"];
}
if (isset($noscrape["tags"])) {
$keywords = implode(" ", $noscrape["tags"]);
if ($keywords != "") {
$contact["keywords"] = $keywords;
}
}

$location = Profile::formatLocation($noscrape);
if ($location) {
$contact["location"] = $location;
}
if (isset($noscrape["dfrn-notify"])) {
$contact["notify"] = $noscrape["dfrn-notify"];
}
// Remove all fields that are not present in the gcontact table
unset($noscrape["fn"]);
unset($noscrape["key"]);
unset($noscrape["homepage"]);
unset($noscrape["comm"]);
unset($noscrape["tags"]);
unset($noscrape["locality"]);
unset($noscrape["region"]);
unset($noscrape["country-name"]);
unset($noscrape["contacts"]);
unset($noscrape["dfrn-request"]);
unset($noscrape["dfrn-confirm"]);
unset($noscrape["dfrn-notify"]);
unset($noscrape["dfrn-poll"]);

// Set the date of the last contact
/// @todo By now the function "update_gcontact" doesn't work with this field
//$contact["last_contact"] = DateTimeFormat::utcNow();

$contact = array_merge($contact, $noscrape);

GContact::update($contact);

if (!empty($noscrape["updated"])) {
$fields = ['last_contact' => DateTimeFormat::utcNow()];
DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($profile)]);

Logger::log("Profile ".$profile." was last updated at ".$noscrape["updated"]." (noscrape)", Logger::DEBUG);

return $noscrape["updated"];
}
}
}
}
}

// If we only can poll the feed, then we only do this once a while
if (!$force && !self::updateNeeded($gcontacts[0]["created"], $gcontacts[0]["updated"], $gcontacts[0]["last_failure"], $gcontacts[0]["last_contact"])) {
Logger::log("Profile ".$profile." was last updated at ".$gcontacts[0]["updated"]." (cached)", Logger::DEBUG);

GContact::update($contact);
return $gcontacts[0]["updated"];
}

$data = Probe::uri($profile);

// Is the profile link the alternate OStatus link notation? (http://domain.tld/user/4711)
// Then check the other link and delete this one
if (($data["network"] == Protocol::OSTATUS) && self::alternateOStatusUrl($profile)
&& (Strings::normaliseLink($profile) == Strings::normaliseLink($data["alias"]))
&& (Strings::normaliseLink($profile) != Strings::normaliseLink($data["url"]))
) {
// Delete the old entry
DBA::delete('gcontact', ['nurl' => Strings::normaliseLink($profile)]);

$gcontact = array_merge($gcontacts[0], $data);

$gcontact["server_url"] = $data["baseurl"];

try {
$gcontact = GContact::sanitize($gcontact);
GContact::update($gcontact);

self::lastUpdated($data["url"], $force);
} catch (Exception $e) {
Logger::log($e->getMessage(), Logger::DEBUG);
}

Logger::log("Profile ".$profile." was deleted", Logger::DEBUG);
return false;
}

if (($data["poll"] == "") || (in_array($data["network"], [Protocol::FEED, Protocol::PHANTOM]))) {
$fields = ['last_failure' => DateTimeFormat::utcNow()];
DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($profile)]);

Logger::log("Profile ".$profile." wasn't reachable (profile)", Logger::DEBUG);
return false;
}

$contact = array_merge($contact, $data);

$contact["server_url"] = $data["baseurl"];

GContact::update($contact);

$curlResult = Network::curl($data["poll"]);

if (!$curlResult->isSuccess()) {
$fields = ['last_failure' => DateTimeFormat::utcNow()];
DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($profile)]);

Logger::log("Profile ".$profile." wasn't reachable (no feed)", Logger::DEBUG);
return false;
}

$doc = new DOMDocument();
/// @TODO Avoid error supression here
@$doc->loadXML($curlResult->getBody());

$xpath = new DOMXPath($doc);
$xpath->registerNamespace('atom', "http://www.w3.org/2005/Atom");

$entries = $xpath->query('/atom:feed/atom:entry');

$last_updated = "";

foreach ($entries as $entry) {
$published_item = $xpath->query('atom:published/text()', $entry)->item(0);
$updated_item = $xpath->query('atom:updated/text()' , $entry)->item(0);
$published = isset($published_item->nodeValue) ? DateTimeFormat::utc($published_item->nodeValue) : null;
$updated = isset($updated_item->nodeValue) ? DateTimeFormat::utc($updated_item->nodeValue) : null;

if (!isset($published) || !isset($updated)) {
Logger::notice('Invalid entry for XPath.', ['entry' => $entry, 'profile' => $profile]);
continue;
}

if ($last_updated < $published) {
$last_updated = $published;
}

if ($last_updated < $updated) {
$last_updated = $updated;
}
}

// Maybe there aren't any entries. Then check if it is a valid feed
if ($last_updated == "") {
if ($xpath->query('/atom:feed')->length > 0) {
$last_updated = DBA::NULL_DATETIME;
}
}

$fields = ['last_contact' => DateTimeFormat::utcNow()];

if (!empty($last_updated)) {
$fields['updated'] = $last_updated;
}

DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($profile)]);

if (($gcontacts[0]["generation"] == 0)) {
$fields = ['generation' => 9];
DBA::update('gcontact', $fields, ['nurl' => Strings::normaliseLink($profile)]);
}

Logger::log("Profile ".$profile." was last updated at ".$last_updated, Logger::DEBUG);

return $last_updated;
}

public static function updateNeeded($created, $updated, $last_failure, $last_contact)
{
$now = strtotime(DateTimeFormat::utcNow());


+ 2
- 2
src/Worker/CronJobs.php View File

@@ -16,11 +16,11 @@ use Friendica\Database\DBA;
use Friendica\Database\PostUpdate;
use Friendica\Model\Contact;
use Friendica\Model\GContact;
use Friendica\Model\GServer;
use Friendica\Model\Nodeinfo;
use Friendica\Model\Photo;
use Friendica\Model\User;
use Friendica\Network\Probe;
use Friendica\Protocol\PortableContact;
use Friendica\Util\Network;
use Friendica\Util\Proxy as ProxyUtils;
use Friendica\Util\Strings;
@@ -254,7 +254,7 @@ class CronJobs
return;
}

if (!PortableContact::reachable($contact["url"])) {
if (!GServer::reachable($contact["url"])) {
continue;
}



+ 2
- 2
src/Worker/DiscoverPoCo.php View File

@@ -64,7 +64,7 @@ class DiscoverPoCo

if ($mode == 8) {
if ($param1 != "") {
PortableContact::lastUpdated($param1, true);
GContact::updateFromProbe($param1, true);
}
} elseif ($mode == 7) {
if (!empty($param4)) {
@@ -232,7 +232,7 @@ class DiscoverPoCo
continue;
}
// Update the contact
PortableContact::lastUpdated($jj->url);
GContact::updateFromProbe($jj->url);
continue;
}



+ 1
- 1
src/Worker/GProbe.php View File

@@ -57,7 +57,7 @@ class GProbe {
}
if (DBA::isResult($r)) {
// Check for accessibility and do a poco discovery
if (PortableContact::lastUpdated($r[0]['url'], true) && ($r[0]["network"] == Protocol::DFRN)) {
if (GContact::updateFromProbe($r[0]['url'], true) && ($r[0]["network"] == Protocol::DFRN)) {
PortableContact::loadWorker(0, 0, $r[0]['id'], str_replace('/profile/', '/poco/', $r[0]['url']));
}
}


Loading…
Cancel
Save