From 1d7154a7d17c6034ba257b3e17609a7da9e981bc Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Tue, 2 Jul 2019 09:06:48 +0000
Subject: [PATCH 1/8] Update the gcontact entry when the public contact entry
 has changed

---
 config/dbstructure.config.php |   4 +-
 database.sql                  |   4 +-
 src/Model/Contact.php         |  11 ++-
 src/Model/GContact.php        | 129 +++++++++++++++++++++++-----------
 4 files changed, 103 insertions(+), 45 deletions(-)

diff --git a/config/dbstructure.config.php b/config/dbstructure.config.php
index 5cf9d8597..415e389b6 100755
--- a/config/dbstructure.config.php
+++ b/config/dbstructure.config.php
@@ -34,7 +34,7 @@
 use Friendica\Database\DBA;
 
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1315);
+	define('DB_UPDATE_VERSION', 1316);
 }
 
 return [
@@ -423,6 +423,8 @@ return [
 			"updated" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""],
 			"last_contact" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""],
 			"last_failure" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""],
+			"archive_date" => ["type" => "datetime", "default" => DBA::NULL_DATETIME, "comment" => ""],
+			"archived" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""],
 			"location" => ["type" => "varchar(255)", "not null" => "1", "default" => "", "comment" => ""],
 			"about" => ["type" => "text", "comment" => ""],
 			"keywords" => ["type" => "text", "comment" => "puplic keywords (interests)"],
diff --git a/database.sql b/database.sql
index 67ae8b4c2..07d4e2f28 100644
--- a/database.sql
+++ b/database.sql
@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2019.09-dev (Dalmatian Bellflower)
--- DB_UPDATE_VERSION 1314
+-- DB_UPDATE_VERSION 1316
 -- ------------------------------------------
 
 
@@ -372,6 +372,8 @@ CREATE TABLE IF NOT EXISTS `gcontact` (
 	`updated` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
 	`last_contact` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
 	`last_failure` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
+	`archive_date` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
+	`archived` boolean NOT NULL DEFAULT '0' COMMENT '',
 	`location` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	`about` text COMMENT '',
 	`keywords` text COMMENT 'puplic keywords (interests)',
diff --git a/src/Model/Contact.php b/src/Model/Contact.php
index a10c0d05e..77726e6d4 100644
--- a/src/Model/Contact.php
+++ b/src/Model/Contact.php
@@ -1516,6 +1516,11 @@ class Contact extends BaseObject
 
 		DBA::update('contact', $updated, ['id' => $contact_id], $contact);
 
+		if (!$background_update && ($uid == 0)) {
+			// Update the gcontact entry
+			GContact::updateFromPublicContact($contact_id);
+		}
+
 		return $contact_id;
 	}
 
@@ -1770,6 +1775,9 @@ class Contact extends BaseObject
 			return;
 		}
 
+		// Update the corresponding gcontact entry
+		GContact::updateFromPublicContact($id);
+
 		// Archive or unarchive the contact. We only need to do this for the public contact.
 		// The archive/unarchive function will update the personal contacts by themselves.
 		$contact = DBA::selectFirst('contact', [], ['id' => $id]);
@@ -1876,9 +1884,6 @@ class Contact extends BaseObject
 
 		self::updateContact($id, $uid, $ret['url'], $ret);
 
-		// Update the corresponding gcontact entry
-		GContact::updateFromProbe($ret['url']);
-
 		return true;
 	}
 
diff --git a/src/Model/GContact.php b/src/Model/GContact.php
index 8c6471281..dd1e4285c 100644
--- a/src/Model/GContact.php
+++ b/src/Model/GContact.php
@@ -140,7 +140,7 @@ class GContact
 		}
 
 		// Assure that there are no parameter fragments in the profile url
-		if (in_array($gcontact['network'], [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, ""])) {
+		if (empty($gcontact["network"]) || in_array($gcontact["network"], Protocol::FEDERATED)) {
 			$gcontact['url'] = self::cleanContactUrl($gcontact['url']);
 		}
 
@@ -216,7 +216,7 @@ class GContact
 			throw new Exception('No name and photo for URL '.$gcontact['url']);
 		}
 
-		if (!in_array($gcontact['network'], [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::OSTATUS, Protocol::DIASPORA])) {
+		if (!in_array($gcontact['network'], Protocol::FEDERATED)) {
 			throw new Exception('No federated network ('.$gcontact['network'].') detected for URL '.$gcontact['url']);
 		}
 
@@ -664,7 +664,7 @@ class GContact
 		self::fixAlternateContactAddress($contact);
 
 		// Remove unwanted parts from the contact url (e.g. "?zrl=...")
-		if (in_array($contact["network"], [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS])) {
+		if (in_array($contact["network"], Protocol::FEDERATED)) {
 			$contact["url"] = self::cleanContactUrl($contact["url"]);
 		}
 
@@ -851,55 +851,104 @@ class GContact
 					'server_url' => $contact['server_url'], 'connect' => $contact['connect']];
 
 			DBA::update('gcontact', $updated, $condition, $fields);
-
-			// Now update the contact entry with the user id "0" as well.
-			// This is used for the shadow copies of public items.
-			/// @todo Check if we really should do this.
-			// The quality of the gcontact table is mostly lower than the public contact
-			$public_contact = DBA::selectFirst('contact', ['id'], ['nurl' => Strings::normaliseLink($contact["url"]), 'uid' => 0]);
-			if (DBA::isResult($public_contact)) {
-				Logger::log("Update public contact ".$public_contact["id"], Logger::DEBUG);
-
-				Contact::updateAvatar($contact["photo"], 0, $public_contact["id"]);
-
-				$fields = ['name', 'nick', 'addr',
-						'network', 'bd', 'gender',
-						'keywords', 'alias', 'contact-type',
-						'url', 'location', 'about'];
-				$old_contact = DBA::selectFirst('contact', $fields, ['id' => $public_contact["id"]]);
-
-				// Update it with the current values
-				$fields = ['name' => $contact['name'], 'nick' => $contact['nick'],
-						'addr' => $contact['addr'], 'network' => $contact['network'],
-						'bd' => $contact['birthday'], 'gender' => $contact['gender'],
-						'keywords' => $contact['keywords'], 'alias' => $contact['alias'],
-						'contact-type' => $contact['contact-type'], 'url' => $contact['url'],
-						'location' => $contact['location'], 'about' => $contact['about']];
-
-				// Don't update the birthday field if not set or invalid
-				if (empty($contact['birthday']) || ($contact['birthday'] <= DBA::NULL_DATE)) {
-					unset($fields['bd']);
-				}
-
-
-				DBA::update('contact', $fields, ['id' => $public_contact["id"]], $old_contact);
-			}
 		}
 
 		return $gcontact_id;
 	}
 
 	/**
-	 * @brief Updates the gcontact entry from probe
+	 * @brief Updates the gcontact entry from a given public contact id
 	 *
-	 * @param string $url profile link
+	 * @param integer $cid contact id
 	 * @return void
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 * @throws \ImagickException
 	 */
-	public static function updateFromProbe($url)
+	public static function updateFromPublicContact($cid)
 	{
-		$data = Probe::uri($url);
+		$fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords', 'gender',
+			'bd', 'contact-type', 'network', 'addr', 'notify', 'alias', 'archive',
+			'created', 'updated', 'avatar', 'success_update', 'failure_update', 'forum', 'prv'];
+		$contact = DBA::selectFirst('contact', $fields, ['id' => $cid, 'uid' => 0, 'network' => Protocol::FEDERATED]);
+		if (!DBA::isResult($contact)) {
+			return;
+		}
+
+		// These fields cannot be updated, since they don't exist in the contact table
+		// hide, nsfw, server_url
+		// "connect" does exist, but seems to contain the same as "addr"
+
+		$fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords', 'gender', 'generation',
+			'birthday', 'contact-type', 'network', 'addr', 'notify', 'alias', 'archived', 'archive_date',
+			'created', 'updated', 'photo', 'last_contact', 'last_failure', 'community', 'connect'];
+
+		$old_gcontact = DBA::selectFirst('gcontact', $fields, ['nurl' => $contact['nurl']]);
+		$do_insert = !DBA::isResult($old_gcontact);
+		if ($do_insert) {
+			$old_gcontact = [];
+		}
+
+		$gcontact = $contact;
+
+		// These fields are having different names but the same content
+		$gcontact['archived'] = $gcontact['archive'];
+		unset($gcontact['archive']);
+		$gcontact['birthday'] = $gcontact['bd'];
+		unset($gcontact['bd']);
+		$gcontact['photo'] = $gcontact['avatar'];
+		unset($gcontact['avatar']);
+		$gcontact['last_contact'] = $gcontact['success_update'];
+		unset($gcontact['success_update']);
+		$gcontact['last_failure'] = $gcontact['failure_update'];
+		unset($gcontact['failure_update']);
+		$gcontact['community'] = ($gcontact['forum'] || $gcontact['prv']);
+		unset($gcontact['forum']);
+		unset($gcontact['prv']);
+
+		foreach (['last_contact', 'last_failure', 'updated'] as $field) {
+			if (!empty($old_gcontact[$field]) && ($old_gcontact[$field] >= $gcontact[$field])) {
+				unset($gcontact[$field]);
+			}
+		}
+
+		if ($gcontact['archived'] && (empty($old_gcontact['archive_date']) || ($old_gcontact['archive_date'] <= DBA::NULL_DATETIME))) {
+			$gcontact['archive_date'] = DateTimeFormat::utcNow();
+		} elseif (!$gcontact['archived']) {
+			$gcontact['archive_date'] = DBA::NULL_DATETIME;
+		}
+
+		if (!empty($old_gcontact['created']) && ($old_gcontact['created'] > DBA::NULL_DATETIME)
+			&& ($old_gcontact['created'] <= $gcontact['created'])) {
+			unset($gcontact['created']);
+		}
+
+		if (empty($gcontact['birthday']) && ($gcontact['birthday'] <= DBA::NULL_DATETIME)) {
+			unset($gcontact['birthday']);
+		}
+
+		if (empty($old_gcontact['generation']) || ($old_gcontact['generation'] > 2)) {
+			$gcontact['generation'] = 2; // We fetched the data directly from the other server
+		}
+
+		if (!$do_insert) {
+			DBA::update('gcontact', $gcontact, ['nurl' => $contact['nurl']], $old_gcontact);
+		} elseif (!$gcontact['archived']) {
+			DBA::insert('gcontact', $gcontact);
+		}
+	}
+
+	/**
+	 * @brief Updates the gcontact entry from probe
+	 *
+	 * @param string  $url   profile link
+	 * @param boolean $force Optional forcing of network probing (otherwise we use the cached data)
+	 * @return void
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 * @throws \ImagickException
+	 */
+	public static function updateFromProbe($url, $force = false)
+	{
+		$data = Probe::uri($url, $force);
 
 		if (in_array($data["network"], [Protocol::PHANTOM])) {
 			Logger::log("Invalid network for contact url ".$data["url"]." - Called by: ".System::callstack(), Logger::DEBUG);

From 80c3a81618b34a7d3981c9276fb94a3e4f029c8b Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Tue, 2 Jul 2019 14:46:04 +0000
Subject: [PATCH 2/8] Use the term-date as archive date

---
 src/Model/GContact.php | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Model/GContact.php b/src/Model/GContact.php
index dd1e4285c..60cea8b5a 100644
--- a/src/Model/GContact.php
+++ b/src/Model/GContact.php
@@ -867,7 +867,7 @@ class GContact
 	public static function updateFromPublicContact($cid)
 	{
 		$fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords', 'gender',
-			'bd', 'contact-type', 'network', 'addr', 'notify', 'alias', 'archive',
+			'bd', 'contact-type', 'network', 'addr', 'notify', 'alias', 'archive', 'term-date',
 			'created', 'updated', 'avatar', 'success_update', 'failure_update', 'forum', 'prv'];
 		$contact = DBA::selectFirst('contact', $fields, ['id' => $cid, 'uid' => 0, 'network' => Protocol::FEDERATED]);
 		if (!DBA::isResult($contact)) {
@@ -893,6 +893,8 @@ class GContact
 		// These fields are having different names but the same content
 		$gcontact['archived'] = $gcontact['archive'];
 		unset($gcontact['archive']);
+		$gcontact['archive_date'] = $gcontact['term-date'];
+		unset($gcontact['term-date']);
 		$gcontact['birthday'] = $gcontact['bd'];
 		unset($gcontact['bd']);
 		$gcontact['photo'] = $gcontact['avatar'];
@@ -911,9 +913,7 @@ class GContact
 			}
 		}
 
-		if ($gcontact['archived'] && (empty($old_gcontact['archive_date']) || ($old_gcontact['archive_date'] <= DBA::NULL_DATETIME))) {
-			$gcontact['archive_date'] = DateTimeFormat::utcNow();
-		} elseif (!$gcontact['archived']) {
+		if (!$gcontact['archived']) {
 			$gcontact['archive_date'] = DBA::NULL_DATETIME;
 		}
 

From 7b0a9ffd8923a26f416ce67a285b692c8f84ad3f Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Wed, 3 Jul 2019 05:46:35 +0000
Subject: [PATCH 3/8] Update gcontact when public contact is updated

---
 src/Model/Contact.php         | 29 ++++++-------
 src/Model/GContact.php        | 30 +++++++++++++-
 src/Worker/OnePoll.php        |  2 -
 src/Worker/UpdateGContact.php | 78 +++--------------------------------
 4 files changed, 46 insertions(+), 93 deletions(-)

diff --git a/src/Model/Contact.php b/src/Model/Contact.php
index 77726e6d4..c980482de 100644
--- a/src/Model/Contact.php
+++ b/src/Model/Contact.php
@@ -792,6 +792,7 @@ class Contact extends BaseObject
 				 */
 				DBA::update('contact', ['archive' => 1], ['id' => $contact['id']]);
 				DBA::update('contact', ['archive' => 1], ['nurl' => Strings::normaliseLink($contact['url']), 'self' => false]);
+				GContact::updateFromPublicContactURL($contact['url']);
 			}
 		}
 	}
@@ -829,6 +830,7 @@ class Contact extends BaseObject
 		$fields = ['term-date' => DBA::NULL_DATETIME, 'archive' => false];
 		DBA::update('contact', $fields, ['id' => $contact['id']]);
 		DBA::update('contact', $fields, ['nurl' => Strings::normaliseLink($contact['url'])]);
+		GContact::updateFromPublicContactURL($contact['url']);
 
 		if (!empty($contact['batch'])) {
 			$condition = ['batch' => $contact['batch'], 'contact-type' => self::TYPE_RELAY];
@@ -965,7 +967,7 @@ class Contact extends BaseObject
 		if ((empty($profile["addr"]) || empty($profile["name"])) && (defaults($profile, "gid", 0) != 0)
 			&& in_array($profile["network"], Protocol::FEDERATED)
 		) {
-			Worker::add(PRIORITY_LOW, "UpdateGContact", $profile["gid"]);
+			Worker::add(PRIORITY_LOW, "UpdateGContact", $url);
 		}
 
 		// Show contact details of Diaspora contacts only if connected
@@ -1339,10 +1341,14 @@ class Contact extends BaseObject
 			return 0;
 		}
 
-		// When we don't want to update, we look if we know this contact in any way
 		if ($no_update && empty($default)) {
+			// When we don't want to update, we look if we know this contact in any way
 			$data = self::getProbeDataFromDatabase($url, $contact_id);
 			$background_update = true;
+		} elseif ($no_update && !empty($default)) {
+			// If there are default values, take these
+			$data = $default;
+			$background_update = false;
 		} else {
 			$data = [];
 			$background_update = false;
@@ -1357,18 +1363,9 @@ class Contact extends BaseObject
 			}
 		}
 
-		// Last try in gcontact for unsupported networks
-		if (!in_array($data["network"], [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::OSTATUS, Protocol::DIASPORA, Protocol::PUMPIO, Protocol::MAIL, Protocol::FEED])) {
-			if ($uid != 0) {
-				return 0;
-			}
-
-			$contact = array_merge(self::getProbeDataFromDatabase($url, $contact_id), $default);
-			if (empty($contact)) {
-				return 0;
-			}
-
-			$data = array_merge($data, $contact);
+		// Take the default values when probing failed
+		if (!empty($default) && !in_array($data["network"], array_merge(Protocol::NATIVE_SUPPORT, [Protocol::PUMPIO]))) {
+			$data = array_merge($data, $default);
 		}
 
 		if (empty($data)) {
@@ -1518,7 +1515,7 @@ class Contact extends BaseObject
 
 		if (!$background_update && ($uid == 0)) {
 			// Update the gcontact entry
-			GContact::updateFromPublicContact($contact_id);
+			GContact::updateFromPublicContactID($contact_id);
 		}
 
 		return $contact_id;
@@ -1776,7 +1773,7 @@ class Contact extends BaseObject
 		}
 
 		// Update the corresponding gcontact entry
-		GContact::updateFromPublicContact($id);
+		GContact::updateFromPublicContactID($id);
 
 		// Archive or unarchive the contact. We only need to do this for the public contact.
 		// The archive/unarchive function will update the personal contacts by themselves.
diff --git a/src/Model/GContact.php b/src/Model/GContact.php
index 60cea8b5a..4fb766193 100644
--- a/src/Model/GContact.php
+++ b/src/Model/GContact.php
@@ -864,12 +864,38 @@ class GContact
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 * @throws \ImagickException
 	 */
-	public static function updateFromPublicContact($cid)
+	public static function updateFromPublicContactID($cid)
+	{
+		self::updateFromPublicContact(['id' => $cid]);
+	}
+
+	/**
+	 * @brief Updates the gcontact entry from a given public contact url
+	 *
+	 * @param string $url contact url
+	 * @return void
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 * @throws \ImagickException
+	 */
+	public static function updateFromPublicContactURL($url)
+	{
+		self::updateFromPublicContact(['nurl' => Strings::normaliseLink($url)]);
+	}
+
+	/**
+	 * @brief Helper function for updateFromPublicContactID and updateFromPublicContactURL
+	 *
+	 * @param array $condition contact condition
+	 * @return void
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+	 * @throws \ImagickException
+	 */
+	private static function updateFromPublicContact($condition)
 	{
 		$fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords', 'gender',
 			'bd', 'contact-type', 'network', 'addr', 'notify', 'alias', 'archive', 'term-date',
 			'created', 'updated', 'avatar', 'success_update', 'failure_update', 'forum', 'prv'];
-		$contact = DBA::selectFirst('contact', $fields, ['id' => $cid, 'uid' => 0, 'network' => Protocol::FEDERATED]);
+		$contact = DBA::selectFirst('contact', $fields, array_merge($condition, ['uid' => 0, 'network' => Protocol::FEDERATED]));
 		if (!DBA::isResult($contact)) {
 			return;
 		}
diff --git a/src/Worker/OnePoll.php b/src/Worker/OnePoll.php
index 33a9a51a1..9af4be9be 100644
--- a/src/Worker/OnePoll.php
+++ b/src/Worker/OnePoll.php
@@ -191,11 +191,9 @@ class OnePoll
 			}
 
 			self::updateContact($contact, ['last-update' => $updated, 'success_update' => $updated]);
-			DBA::update('gcontact', ['last_contact' => $updated], ['nurl' => $contact['nurl']]);
 			Contact::unmarkForArchival($contact);
 		} elseif (in_array($contact["network"], [Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::FEED])) {
 			self::updateContact($contact, ['last-update' => $updated, 'failure_update' => $updated]);
-			DBA::update('gcontact', ['last_failure' => $updated], ['nurl' => $contact['nurl']]);
 			Contact::markForArchival($contact);
 		} else {
 			self::updateContact($contact, ['last-update' => $updated]);
diff --git a/src/Worker/UpdateGContact.php b/src/Worker/UpdateGContact.php
index dd344baf9..aacebcb80 100644
--- a/src/Worker/UpdateGContact.php
+++ b/src/Worker/UpdateGContact.php
@@ -7,85 +7,17 @@
 namespace Friendica\Worker;
 
 use Friendica\Core\Logger;
-use Friendica\Core\Protocol;
+use Friendica\Model\GContact;
 use Friendica\Database\DBA;
-use Friendica\Network\Probe;
-use Friendica\Protocol\PortableContact;
-use Friendica\Util\DateTimeFormat;
-use Friendica\Util\Strings;
 
 class UpdateGContact
 {
-	public static function execute($contact_id)
+	public static function execute($url, $command = '')
 	{
-		Logger::log('update_gcontact: start');
+		$force = ($command == "force");
 
-		if (empty($contact_id)) {
-			Logger::log('update_gcontact: no contact');
-			return;
-		}
+		$success = GContact::updateFromProbe($url, $force);
 
-		$r = q("SELECT * FROM `gcontact` WHERE `id` = %d", intval($contact_id));
-
-		if (!DBA::isResult($r)) {
-			return;
-		}
-
-		if (!in_array($r[0]["network"], Protocol::FEDERATED)) {
-			return;
-		}
-
-		$data = Probe::uri($r[0]["url"]);
-
-		if (!in_array($data["network"], Protocol::FEDERATED)) {
-			if ($r[0]["server_url"] != "") {
-				PortableContact::checkServer($r[0]["server_url"], $r[0]["network"]);
-			}
-
-			q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `id` = %d",
-				DBA::escape(DateTimeFormat::utcNow()), intval($contact_id));
-			return;
-		}
-
-		if (($data["name"] == "") && ($r[0]['name'] != "")) {
-			$data["name"] = $r[0]['name'];
-		}
-
-		if (($data["nick"] == "") && ($r[0]['nick'] != "")) {
-			$data["nick"] = $r[0]['nick'];
-		}
-
-		if (($data["addr"] == "") && ($r[0]['addr'] != "")) {
-			$data["addr"] = $r[0]['addr'];
-		}
-
-		if (($data["photo"] == "") && ($r[0]['photo'] != "")) {
-			$data["photo"] = $r[0]['photo'];
-		}
-
-
-		q("UPDATE `gcontact` SET `name` = '%s', `nick` = '%s', `addr` = '%s', `photo` = '%s'
-					WHERE `id` = %d",
-					DBA::escape($data["name"]),
-					DBA::escape($data["nick"]),
-					DBA::escape($data["addr"]),
-					DBA::escape($data["photo"]),
-			intval($contact_id)
-		);
-
-		q("UPDATE `contact` SET `name` = '%s', `nick` = '%s', `addr` = '%s', `photo` = '%s'
-					WHERE `uid` = 0 AND `addr` = '' AND `nurl` = '%s'",
-					DBA::escape($data["name"]),
-					DBA::escape($data["nick"]),
-					DBA::escape($data["addr"]),
-					DBA::escape($data["photo"]),
-					DBA::escape(Strings::normaliseLink($data["url"]))
-		);
-
-		q("UPDATE `contact` SET `addr` = '%s'
-					WHERE `uid` != 0 AND `addr` = '' AND `nurl` = '%s'",
-					DBA::escape($data["addr"]),
-					DBA::escape(Strings::normaliseLink($data["url"]))
-		);
+		Logger::info('Updated from probe', ['url' => $url, 'force' => $force, 'success' => $success]);
 	}
 }

From ef5be9668fd969d52c9aa135d4724093f960d5c6 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Thu, 4 Jul 2019 04:08:55 +0000
Subject: [PATCH 4/8] New fields in contact table / don't update on probe

---
 config/dbstructure.config.php          |   3 +
 database.sql                           |   3 +
 src/Model/APContact.php                |   5 +-
 src/Model/Contact.php                  |  11 ++-
 src/Model/GContact.php                 |  27 +++---
 src/Network/Probe.php                  | 109 -------------------------
 src/Protocol/ActivityPub/Processor.php |   1 +
 src/Protocol/DFRN.php                  |   5 ++
 src/Protocol/Diaspora.php              |   4 +-
 9 files changed, 39 insertions(+), 129 deletions(-)

diff --git a/config/dbstructure.config.php b/config/dbstructure.config.php
index 415e389b6..3c2b81352 100755
--- a/config/dbstructure.config.php
+++ b/config/dbstructure.config.php
@@ -253,6 +253,9 @@ return [
 			"pending" => ["type" => "boolean", "not null" => "1", "default" => "1", "comment" => ""],
 			"deleted" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Contact has been deleted"],
 			"rating" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => ""],
+			"unsearchable" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Contact prefers to not be searchable"],
+			"sensitive" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Contact posts sensitive content"],
+			"baseurl" => ["type" => "varchar(255)", "default" => "", "comment" => "baseurl of the contact"],
 			"reason" => ["type" => "text", "comment" => ""],
 			"closeness" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "99", "comment" => ""],
 			"info" => ["type" => "mediumtext", "comment" => ""],
diff --git a/database.sql b/database.sql
index 07d4e2f28..7dc24b603 100644
--- a/database.sql
+++ b/database.sql
@@ -210,6 +210,9 @@ CREATE TABLE IF NOT EXISTS `contact` (
 	`pending` boolean NOT NULL DEFAULT '1' COMMENT '',
 	`deleted` boolean NOT NULL DEFAULT '0' COMMENT 'Contact has been deleted',
 	`rating` tinyint NOT NULL DEFAULT 0 COMMENT '',
+	`unsearchable` boolean NOT NULL DEFAULT '0' COMMENT 'Contact prefers to not be searchable',
+	`sensitive` boolean NOT NULL DEFAULT '0' COMMENT 'Contact posts sensitive content',
+	`baseurl` varchar(255) DEFAULT '' COMMENT 'baseurl of the contact',
 	`reason` text COMMENT '',
 	`closeness` tinyint unsigned NOT NULL DEFAULT 99 COMMENT '',
 	`info` mediumtext COMMENT '',
diff --git a/src/Model/APContact.php b/src/Model/APContact.php
index b027d6c47..6d50582e6 100644
--- a/src/Model/APContact.php
+++ b/src/Model/APContact.php
@@ -270,10 +270,7 @@ class APContact extends BaseObject
 		}
 
 		// Update the gcontact table
-		// These two fields don't exist in the gcontact table
-		unset($contact_fields['forum']);
-		unset($contact_fields['prv']);
-		DBA::update('gcontact', $contact_fields, ['nurl' => Strings::normaliseLink($url)]);
+		GContact::updateFromPublicContactURL($url);
 
 		Logger::log('Updated profile for ' . $url, Logger::DEBUG);
 
diff --git a/src/Model/Contact.php b/src/Model/Contact.php
index c980482de..20dfc4d35 100644
--- a/src/Model/Contact.php
+++ b/src/Model/Contact.php
@@ -1400,6 +1400,7 @@ class Contact extends BaseObject
 				'request'   => defaults($data, 'request', ''),
 				'confirm'   => defaults($data, 'confirm', ''),
 				'poco'      => defaults($data, 'poco', ''),
+				'baseurl'   => defaults($data, 'baseurl', ''),
 				'name-date' => DateTimeFormat::utcNow(),
 				'uri-date'  => DateTimeFormat::utcNow(),
 				'avatar-date' => DateTimeFormat::utcNow(),
@@ -1453,7 +1454,7 @@ class Contact extends BaseObject
 			self::updateAvatar($data['photo'], $uid, $contact_id);
 		}
 
-		$fields = ['url', 'nurl', 'addr', 'alias', 'name', 'nick', 'keywords', 'location', 'about', 'avatar-date', 'pubkey'];
+		$fields = ['url', 'nurl', 'addr', 'alias', 'name', 'nick', 'keywords', 'location', 'about', 'avatar-date', 'pubkey', 'baseurl'];
 		$contact = DBA::selectFirst('contact', $fields, ['id' => $contact_id]);
 
 		// This condition should always be true
@@ -1467,7 +1468,8 @@ class Contact extends BaseObject
 			'url' => $data['url'],
 			'nurl' => Strings::normaliseLink($data['url']),
 			'name' => $data['name'],
-			'nick' => $data['nick']
+			'nick' => $data['nick'],
+			'baseurl' => $data['baseurl']
 		];
 
 		if (!empty($data['keywords'])) {
@@ -1820,7 +1822,7 @@ class Contact extends BaseObject
 		 */
 
 		$fields = ['avatar', 'uid', 'name', 'nick', 'url', 'addr', 'batch', 'notify',
-			'poll', 'request', 'confirm', 'poco', 'network', 'alias'];
+			'poll', 'request', 'confirm', 'poco', 'network', 'alias', 'baseurl'];
 		$contact = DBA::selectFirst('contact', $fields, ['id' => $id]);
 		if (!DBA::isResult($contact)) {
 			return false;
@@ -1849,7 +1851,7 @@ class Contact extends BaseObject
 
 		// make sure to not overwrite existing values with blank entries
 		foreach ($ret as $key => $val) {
-			if (!isset($contact[$key])) {
+			if (!array_key_exists($key, $contact)) {
 				unset($ret[$key]);
 			} elseif (($contact[$key] != '') && ($val == '')) {
 				$ret[$key] = $contact[$key];
@@ -2082,6 +2084,7 @@ class Contact extends BaseObject
 				'name'    => $ret['name'],
 				'nick'    => $ret['nick'],
 				'network' => $ret['network'],
+				'baseurl' => $ret['baseurl'],
 				'protocol' => $protocol,
 				'pubkey'  => $ret['pubkey'],
 				'rel'     => $new_relation,
diff --git a/src/Model/GContact.php b/src/Model/GContact.php
index 4fb766193..7da5f18e9 100644
--- a/src/Model/GContact.php
+++ b/src/Model/GContact.php
@@ -873,20 +873,20 @@ class GContact
 	 * @brief Updates the gcontact entry from a given public contact url
 	 *
 	 * @param string $url contact url
-	 * @return void
+	 * @return integer gcontact id
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 * @throws \ImagickException
 	 */
 	public static function updateFromPublicContactURL($url)
 	{
-		self::updateFromPublicContact(['nurl' => Strings::normaliseLink($url)]);
+		return self::updateFromPublicContact(['nurl' => Strings::normaliseLink($url)]);
 	}
 
 	/**
 	 * @brief Helper function for updateFromPublicContactID and updateFromPublicContactURL
 	 *
 	 * @param array $condition contact condition
-	 * @return void
+	 * @return integer gcontact id
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 * @throws \ImagickException
 	 */
@@ -894,19 +894,18 @@ class GContact
 	{
 		$fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords', 'gender',
 			'bd', 'contact-type', 'network', 'addr', 'notify', 'alias', 'archive', 'term-date',
-			'created', 'updated', 'avatar', 'success_update', 'failure_update', 'forum', 'prv'];
+			'created', 'updated', 'avatar', 'success_update', 'failure_update', 'forum', 'prv',
+			'baseurl', 'sensitive', 'unsearchable'];
+
 		$contact = DBA::selectFirst('contact', $fields, array_merge($condition, ['uid' => 0, 'network' => Protocol::FEDERATED]));
 		if (!DBA::isResult($contact)) {
-			return;
+			return 0;
 		}
 
-		// These fields cannot be updated, since they don't exist in the contact table
-		// hide, nsfw, server_url
-		// "connect" does exist, but seems to contain the same as "addr"
-
 		$fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords', 'gender', 'generation',
 			'birthday', 'contact-type', 'network', 'addr', 'notify', 'alias', 'archived', 'archive_date',
-			'created', 'updated', 'photo', 'last_contact', 'last_failure', 'community', 'connect'];
+			'created', 'updated', 'photo', 'last_contact', 'last_failure', 'community', 'connect',
+			'server_url', 'nsfw', 'hide', 'id'];
 
 		$old_gcontact = DBA::selectFirst('gcontact', $fields, ['nurl' => $contact['nurl']]);
 		$do_insert = !DBA::isResult($old_gcontact);
@@ -917,6 +916,12 @@ class GContact
 		$gcontact = $contact;
 
 		// These fields are having different names but the same content
+		$gcontact['server_url'] = $gcontact['baseurl'];
+		unset($gcontact['baseurl']);
+		$gcontact['nsfw'] = $gcontact['sensitive'];
+		unset($gcontact['sensitive']);
+		$gcontact['hide'] = $gcontact['unsearchable'];
+		unset($gcontact['unsearchable']);
 		$gcontact['archived'] = $gcontact['archive'];
 		unset($gcontact['archive']);
 		$gcontact['archive_date'] = $gcontact['term-date'];
@@ -958,8 +963,10 @@ class GContact
 
 		if (!$do_insert) {
 			DBA::update('gcontact', $gcontact, ['nurl' => $contact['nurl']], $old_gcontact);
+			return $old_gcontact['id'];
 		} elseif (!$gcontact['archived']) {
 			DBA::insert('gcontact', $gcontact);
+			return DBA::lastInsertId();
 		}
 	}
 
diff --git a/src/Network/Probe.php b/src/Network/Probe.php
index a3fe3ca33..5c20cc071 100644
--- a/src/Network/Probe.php
+++ b/src/Network/Probe.php
@@ -17,7 +17,6 @@ use Friendica\Core\Logger;
 use Friendica\Core\Protocol;
 use Friendica\Core\System;
 use Friendica\Database\DBA;
-use Friendica\Model\Contact;
 use Friendica\Model\Profile;
 use Friendica\Protocol\ActivityPub;
 use Friendica\Protocol\Email;
@@ -396,114 +395,6 @@ class Probe
 		// Only store into the cache if the value seems to be valid
 		if (!in_array($data['network'], [Protocol::PHANTOM, Protocol::MAIL])) {
 			Cache::set('Probe::uri:' . $network . ':' . $uri, $data, Cache::DAY);
-
-			/// @todo temporary fix - we need a real contact update function that updates only changing fields
-			/// The biggest problem is the avatar picture that could have a reduced image size.
-			/// It should only be updated if the existing picture isn't existing anymore.
-			/// We only update the contact when it is no probing for a specific network.
-			if (($data['network'] != Protocol::FEED)
-				&& ($network == '')
-				&& $data['name']
-				&& $data['nick']
-				&& $data['url']
-				&& $data['addr']
-				&& $data['poll']
-			) {
-				$fields = [
-					'name' => $data['name'],
-					'nick' => $data['nick'],
-					'url' => $data['url'],
-					'addr' => $data['addr'],
-					'photo' => $data['photo'],
-					'keywords' => $data['keywords'],
-					'location' => $data['location'],
-					'about' => $data['about'],
-					'notify' => $data['notify'],
-					'network' => $data['network'],
-					'server_url' => $data['baseurl']
-				];
-
-				// This doesn't cover the case when a community isn't a community anymore
-				if (!empty($data['community']) && $data['community']) {
-					$fields['community'] = $data['community'];
-					$fields['contact-type'] = Contact::TYPE_COMMUNITY;
-				}
-
-				$fieldnames = [];
-
-				foreach ($fields as $key => $val) {
-					if (empty($val)) {
-						unset($fields[$key]);
-					} else {
-						$fieldnames[] = $key;
-					}
-				}
-
-				$fields['updated'] = DateTimeFormat::utcNow();
-
-				$condition = ['nurl' => Strings::normaliseLink($data['url'])];
-
-				$old_fields = DBA::selectFirst('gcontact', $fieldnames, $condition);
-
-				// When the gcontact doesn't exist, the value "true" will trigger an insert.
-				// In difference to the public contacts we want to have every contact
-				// in the world in our global contacts.
-				if (!$old_fields) {
-					$old_fields = true;
-
-					// These values have to be set only on insert
-					$fields['photo'] = $data['photo'];
-					$fields['created'] = DateTimeFormat::utcNow();
-				}
-
-				DBA::update('gcontact', $fields, $condition, $old_fields);
-
-				$fields = [
-					'name' => $data['name'],
-					'nick' => $data['nick'],
-					'url' => $data['url'],
-					'addr' => $data['addr'],
-					'alias' => $data['alias'],
-					'keywords' => $data['keywords'],
-					'location' => $data['location'],
-					'about' => $data['about'],
-					'batch' => $data['batch'],
-					'notify' => $data['notify'],
-					'poll' => $data['poll'],
-					'request' => $data['request'],
-					'confirm' => $data['confirm'],
-					'poco' => $data['poco'],
-					'network' => $data['network'],
-					'pubkey' => $data['pubkey'],
-					'priority' => $data['priority'],
-					'writable' => true,
-					'rel' => Contact::SHARING
-				];
-
-				$fieldnames = [];
-
-				foreach ($fields as $key => $val) {
-					if (empty($val)) {
-						unset($fields[$key]);
-					} else {
-						$fieldnames[] = $key;
-					}
-				}
-
-				$condition = ['nurl' => Strings::normaliseLink($data['url']), 'self' => false, 'uid' => 0];
-
-				// "$old_fields" will return a "false" when the contact doesn't exist.
-				// This won't trigger an insert. This is intended, since we only need
-				// public contacts for everyone we store items from.
-				// We don't need to store every contact on the planet.
-				$old_fields = DBA::selectFirst('contact', $fieldnames, $condition);
-
-				$fields['name-date'] = DateTimeFormat::utcNow();
-				$fields['uri-date'] = DateTimeFormat::utcNow();
-				$fields['success_update'] = DateTimeFormat::utcNow();
-
-				DBA::update('contact', $fields, $condition, $old_fields);
-			}
 		}
 
 		return $data;
diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php
index 41aed2fe3..101942642 100644
--- a/src/Protocol/ActivityPub/Processor.php
+++ b/src/Protocol/ActivityPub/Processor.php
@@ -612,6 +612,7 @@ class Processor
 
 		Logger::log('Updating profile for ' . $activity['object_id'], Logger::DEBUG);
 		APContact::getByURL($activity['object_id'], true);
+//		Contact::updateFromProbe($activity['object_id'], $network = '', $force = false)
 	}
 
 	/**
diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php
index a5deb490b..4c5f20c63 100644
--- a/src/Protocol/DFRN.php
+++ b/src/Protocol/DFRN.php
@@ -1742,6 +1742,11 @@ class DFRN
 				(strtotime($contact['avatar-date']) > strtotime($contact_old['avatar-date']) || ($author['avatar'] != $contact_old['avatar']))
 			);
 
+			// Update the "hidden" status in the public contact
+			// @todo Updating the contact with all fields and update the gcontact from that
+			$condition = ['uid' => 0, 'nurl' => Strings::normaliseLink($contact_old['url'])];
+			DBA::update('contact', ['unsearchable' => $hide], $condition, true);
+
 			/*
 			 * The generation is a sign for the reliability of the provided data.
 			 * It is used in the socgraph.php to prevent that old contact data
diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php
index f7d94256e..31d9efa18 100644
--- a/src/Protocol/Diaspora.php
+++ b/src/Protocol/Diaspora.php
@@ -2258,8 +2258,8 @@ class Diaspora
 		$fields = ['name' => $name, 'location' => $location,
 			'name-date' => DateTimeFormat::utcNow(),
 			'about' => $about, 'gender' => $gender,
-			'addr' => $author, 'nick' => $nick,
-			'keywords' => $keywords];
+			'addr' => $author, 'nick' => $nick, 'keywords' => $keywords,
+			'unsearchable' => !$searchable, 'sensitive' => $nsfw];
 
 		if (!empty($birthday)) {
 			$fields['bd'] = $birthday;

From 8d6aa9914cc5db822d3a336090987535dc7413d0 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Thu, 4 Jul 2019 19:31:42 +0000
Subject: [PATCH 5/8] We now handling the contact type

---
 src/Model/Contact.php        | 32 ++++++++++++++++++++++++++--
 src/Model/GContact.php       | 41 +++++++++++++++++-------------------
 src/Network/Probe.php        |  6 ++++--
 src/Protocol/ActivityPub.php | 29 ++++++++++++++++++++++++-
 4 files changed, 81 insertions(+), 27 deletions(-)

diff --git a/src/Model/Contact.php b/src/Model/Contact.php
index 20dfc4d35..bb0ba32c3 100644
--- a/src/Model/Contact.php
+++ b/src/Model/Contact.php
@@ -1821,8 +1821,12 @@ class Contact extends BaseObject
 		  This will reliably kill your communication with old Friendica contacts.
 		 */
 
+		// These fields aren't updated by this routine:
+		// 'location', 'about', 'keywords', 'gender', 'xmpp', 'unsearchable', 'sensitive'];
+
 		$fields = ['avatar', 'uid', 'name', 'nick', 'url', 'addr', 'batch', 'notify',
-			'poll', 'request', 'confirm', 'poco', 'network', 'alias', 'baseurl'];
+			'poll', 'request', 'confirm', 'poco', 'network', 'alias', 'baseurl',
+			'forum', 'prv', 'contact-type'];
 		$contact = DBA::selectFirst('contact', $fields, ['id' => $id]);
 		if (!DBA::isResult($contact)) {
 			return false;
@@ -1847,13 +1851,26 @@ class Contact extends BaseObject
 			return false;
 		}
 
+		if (isset($ret['account-type'])) {
+			$ret['forum'] = false;
+			$ret['prv'] = false;
+			$ret['contact-type'] = $ret['account-type'];
+			if ($ret['contact-type'] == User::ACCOUNT_TYPE_COMMUNITY) {
+				$apcontact = APContact::getByURL($ret['url'], false);
+				if (isset($apcontact['manually-approve'])) {
+					$ret['forum'] = (bool)!$apcontact['manually-approve'];
+					$ret['prv'] = (bool)!$ret['forum'];
+				}
+			}
+		}
+
 		$update = false;
 
 		// make sure to not overwrite existing values with blank entries
 		foreach ($ret as $key => $val) {
 			if (!array_key_exists($key, $contact)) {
 				unset($ret[$key]);
-			} elseif (($contact[$key] != '') && ($val == '')) {
+			} elseif (($contact[$key] != '') && ($val == '') && !is_bool($ret[$key])) {
 				$ret[$key] = $contact[$key];
 			} elseif ($ret[$key] != $contact[$key]) {
 				$update = true;
@@ -1886,6 +1903,17 @@ class Contact extends BaseObject
 		return true;
 	}
 
+	public static function updateFromProbeByURL($url, $force = false)
+	{
+		$id = self::getIdForURL($url);
+
+		if (empty($id)) {
+			return;
+		}
+
+		self::updateFromProbe($id, '', $force);
+	}
+
 	/**
 	 * Detects if a given contact array belongs to a legacy DFRN connection
 	 *
diff --git a/src/Model/GContact.php b/src/Model/GContact.php
index 7da5f18e9..0fae0da9d 100644
--- a/src/Model/GContact.php
+++ b/src/Model/GContact.php
@@ -913,30 +913,27 @@ class GContact
 			$old_gcontact = [];
 		}
 
-		$gcontact = $contact;
+		$gcontact = [];
+
+		// These fields are identical in both contact and gcontact
+		$fields = ['name', 'nick', 'url', 'nurl', 'location', 'about', 'keywords', 'gender',
+			'contact-type', 'network', 'addr', 'notify', 'alias', 'created', 'updated'];
+
+		foreach ($fields as $field) {
+			$gcontact[$field] = $contact[$field];
+		}
 
 		// These fields are having different names but the same content
-		$gcontact['server_url'] = $gcontact['baseurl'];
-		unset($gcontact['baseurl']);
-		$gcontact['nsfw'] = $gcontact['sensitive'];
-		unset($gcontact['sensitive']);
-		$gcontact['hide'] = $gcontact['unsearchable'];
-		unset($gcontact['unsearchable']);
-		$gcontact['archived'] = $gcontact['archive'];
-		unset($gcontact['archive']);
-		$gcontact['archive_date'] = $gcontact['term-date'];
-		unset($gcontact['term-date']);
-		$gcontact['birthday'] = $gcontact['bd'];
-		unset($gcontact['bd']);
-		$gcontact['photo'] = $gcontact['avatar'];
-		unset($gcontact['avatar']);
-		$gcontact['last_contact'] = $gcontact['success_update'];
-		unset($gcontact['success_update']);
-		$gcontact['last_failure'] = $gcontact['failure_update'];
-		unset($gcontact['failure_update']);
-		$gcontact['community'] = ($gcontact['forum'] || $gcontact['prv']);
-		unset($gcontact['forum']);
-		unset($gcontact['prv']);
+		$gcontact['server_url'] = $contact['baseurl'];
+		$gcontact['nsfw'] = $contact['sensitive'];
+		$gcontact['hide'] = $contact['unsearchable'];
+		$gcontact['archived'] = $contact['archive'];
+		$gcontact['archive_date'] = $contact['term-date'];
+		$gcontact['birthday'] = $contact['bd'];
+		$gcontact['photo'] = $contact['avatar'];
+		$gcontact['last_contact'] = $contact['success_update'];
+		$gcontact['last_failure'] = $contact['failure_update'];
+		$gcontact['community'] = ($contact['forum'] || $contact['prv']);
 
 		foreach (['last_contact', 'last_failure', 'updated'] as $field) {
 			if (!empty($old_gcontact[$field]) && ($old_gcontact[$field] >= $gcontact[$field])) {
diff --git a/src/Network/Probe.php b/src/Network/Probe.php
index 5c20cc071..fae6f8c93 100644
--- a/src/Network/Probe.php
+++ b/src/Network/Probe.php
@@ -45,8 +45,8 @@ class Probe
 	 */
 	private static function rearrangeData($data)
 	{
-		$fields = ["name", "nick", "guid", "url", "addr", "alias",
-				"photo", "community", "keywords", "location", "about",
+		$fields = ["name", "nick", "guid", "url", "addr", "alias", "photo",
+				"account-type", "community", "keywords", "location", "about",
 				"batch", "notify", "poll", "request", "confirm", "poco",
 				"priority", "network", "pubkey", "baseurl"];
 
@@ -349,6 +349,8 @@ class Probe
 
 			if (!empty($ap_profile) && empty($network) && (defaults($data, 'network', '') != Protocol::DFRN)) {
 				$data = $ap_profile;
+			} elseif (!empty($ap_profile)) {
+				$data = array_merge($ap_profile, $data);
 			}
 		} else {
 			Logger::notice('Time out detected. AP will not be probed.', ['uri' => $uri]);
diff --git a/src/Protocol/ActivityPub.php b/src/Protocol/ActivityPub.php
index 8caf7ac64..42b5de9e2 100644
--- a/src/Protocol/ActivityPub.php
+++ b/src/Protocol/ActivityPub.php
@@ -8,6 +8,7 @@ use Friendica\Util\JsonLD;
 use Friendica\Util\Network;
 use Friendica\Core\Protocol;
 use Friendica\Model\APContact;
+use Friendica\Model\User;
 use Friendica\Util\HTTPSignature;
 
 /**
@@ -88,6 +89,31 @@ class ActivityPub
 		return $content;
 	}
 
+	private static function getAccountType($apcontact)
+	{
+		$accounttype = -1;
+
+		switch($apcontact['type']) {
+			case 'Person':
+				$accounttype = User::ACCOUNT_TYPE_PERSON;
+				break;
+			case 'Organization':
+				$accounttype = User::ACCOUNT_TYPE_ORGANISATION;
+				break;
+			case 'Service':
+				$accounttype = User::ACCOUNT_TYPE_NEWS;
+				break;
+			case 'Group':
+				$accounttype = User::ACCOUNT_TYPE_COMMUNITY;
+				break;
+			case 'Application':
+				$accounttype = User::ACCOUNT_TYPE_RELAY;
+				break;
+		}
+
+		return $accounttype;
+	}
+
 	/**
 	 * Fetches a profile from the given url into an array that is compatible to Probe::uri
 	 *
@@ -112,7 +138,8 @@ class ActivityPub
 		$profile['addr'] = $apcontact['addr'];
 		$profile['alias'] = $apcontact['alias'];
 		$profile['photo'] = $apcontact['photo'];
-		// $profile['community']
+		$profile['account-type'] = self::getAccountType($apcontact);
+		$profile['community'] = ($profile['account-type'] == User::ACCOUNT_TYPE_COMMUNITY);
 		// $profile['keywords']
 		// $profile['location']
 		$profile['about'] = $apcontact['about'];

From c9666a9f1dcd9bc2388b3a3044070b5145c76c7f Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Thu, 4 Jul 2019 19:40:48 +0000
Subject: [PATCH 6/8] We don't update inside the probing anymore

---
 src/Model/APContact.php                | 35 --------------------------
 src/Protocol/ActivityPub/Processor.php |  3 +--
 2 files changed, 1 insertion(+), 37 deletions(-)

diff --git a/src/Model/APContact.php b/src/Model/APContact.php
index 6d50582e6..82afc0769 100644
--- a/src/Model/APContact.php
+++ b/src/Model/APContact.php
@@ -237,41 +237,6 @@ class APContact extends BaseObject
 			DBA::delete('apcontact', ['url' => $url]);
 		}
 
-		// Update some data in the contact table with various ways to catch them all
-		$contact_fields = ['name' => $apcontact['name'], 'about' => $apcontact['about'], 'alias' => $apcontact['alias']];
-
-		// Fetch the type and match it with the contact type
-		$contact_types = array_keys(ActivityPub::ACCOUNT_TYPES, $apcontact['type']);
-		if (!empty($contact_types)) {
-			$contact_type = array_pop($contact_types);
-			if (is_int($contact_type)) {
-				$contact_fields['contact-type'] = $contact_type;
-
-				if ($contact_fields['contact-type'] != User::ACCOUNT_TYPE_COMMUNITY) {
-					// Resetting the 'forum' and 'prv' field when it isn't a forum
-					$contact_fields['forum'] = false;
-					$contact_fields['prv'] = false;
-				} else {
-					// Otherwise set the corresponding forum type
-					$contact_fields['forum'] = !$apcontact['manually-approve'];
-					$contact_fields['prv'] = $apcontact['manually-approve'];
-				}
-			}
-		}
-
-		DBA::update('contact', $contact_fields, ['nurl' => Strings::normaliseLink($url)]);
-
-		if (!empty($apcontact['photo'])) {
-			$contacts = DBA::select('contact', ['uid', 'id'], ['nurl' => Strings::normaliseLink($url)]);
-			while ($contact = DBA::fetch($contacts)) {
-				Contact::updateAvatar($apcontact['photo'], $contact['uid'], $contact['id']);
-			}
-			DBA::close($contacts);
-		}
-
-		// Update the gcontact table
-		GContact::updateFromPublicContactURL($url);
-
 		Logger::log('Updated profile for ' . $url, Logger::DEBUG);
 
 		return $apcontact;
diff --git a/src/Protocol/ActivityPub/Processor.php b/src/Protocol/ActivityPub/Processor.php
index 101942642..6ba0c8f84 100644
--- a/src/Protocol/ActivityPub/Processor.php
+++ b/src/Protocol/ActivityPub/Processor.php
@@ -611,8 +611,7 @@ class Processor
 		}
 
 		Logger::log('Updating profile for ' . $activity['object_id'], Logger::DEBUG);
-		APContact::getByURL($activity['object_id'], true);
-//		Contact::updateFromProbe($activity['object_id'], $network = '', $force = false)
+		Contact::updateFromProbeByURL($activity['object_id'], true);
 	}
 
 	/**

From be0a1023335327e27e463e97156e6b84f325f96f Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Thu, 4 Jul 2019 21:19:23 +0000
Subject: [PATCH 7/8] Simplified the Contact update

---
 src/Model/Contact.php |  8 +++--
 src/Protocol/DFRN.php | 76 ++++++++++---------------------------------
 2 files changed, 22 insertions(+), 62 deletions(-)

diff --git a/src/Model/Contact.php b/src/Model/Contact.php
index bb0ba32c3..3f37d4ec9 100644
--- a/src/Model/Contact.php
+++ b/src/Model/Contact.php
@@ -1468,10 +1468,12 @@ class Contact extends BaseObject
 			'url' => $data['url'],
 			'nurl' => Strings::normaliseLink($data['url']),
 			'name' => $data['name'],
-			'nick' => $data['nick'],
-			'baseurl' => $data['baseurl']
+			'nick' => $data['nick']
 		];
 
+		if (!empty($data['baseurl'])) {
+			$updated['baseurl'] = $data['baseurl'];
+		}
 		if (!empty($data['keywords'])) {
 			$updated['keywords'] = $data['keywords'];
 		}
@@ -1500,7 +1502,7 @@ class Contact extends BaseObject
 		}
 
 		// Only fill the pubkey if it had been empty before. We have to prevent identity theft.
-		if (empty($contact['pubkey'])) {
+		if (empty($contact['pubkey']) && !empty($data['pubkey'])) {
 			$updated['pubkey'] = $data['pubkey'];
 		}
 
diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php
index 4c5f20c63..31d363a11 100644
--- a/src/Protocol/DFRN.php
+++ b/src/Protocol/DFRN.php
@@ -1684,68 +1684,26 @@ class DFRN
 				Event::createBirthday($contact, $birthday);
 			}
 
-			// Get all field names
-			$fields = [];
-			foreach ($contact_old as $field => $data) {
-				$fields[$field] = $data;
-			}
+			$fields = ['name' => $contact['name'], 'nick' => $contact['nick'], 'about' => $contact['about'],
+				'location' => $contact['location'], 'addr' => $contact['addr'], 'keywords' => $contact['keywords'],
+				'bdyear' => $contact['bdyear'], 'bd' => $contact['bd'], 'hidden' => $contact['hidden'],
+				'xmpp' => $contact['xmpp'], 'name-date' => DateTimeFormat::utc($contact['name-date']),
+				'uri-date' => DateTimeFormat::utc($contact['uri-date'])];
 
-			unset($fields["id"]);
-			unset($fields["uid"]);
-			unset($fields["url"]);
-			unset($fields["avatar-date"]);
-			unset($fields["avatar"]);
-			unset($fields["name-date"]);
-			unset($fields["uri-date"]);
+			DBA::update('contact', $fields, ['id' => $contact['id'], 'network' => $contact['network']], $contact_old);
 
-			$update = false;
-			// Update check for this field has to be done differently
-			$datefields = ["name-date", "uri-date"];
-			foreach ($datefields as $field) {
-				// The date fields arrives as '2018-07-17T10:44:45Z' - the database return '2018-07-17 10:44:45'
-				// The fields have to be in the same format to be comparable, since strtotime does add timezones.
-				$contact[$field] = DateTimeFormat::utc($contact[$field]);
-
-				if (strtotime($contact[$field]) > strtotime($contact_old[$field])) {
-					Logger::log("Difference for contact " . $contact["id"] . " in field '" . $field . "'. New value: '" . $contact[$field] . "', old value '" . $contact_old[$field] . "'", Logger::DEBUG);
-					$update = true;
-				}
-			}
-
-			foreach ($fields as $field => $data) {
-				if ($contact[$field] != $contact_old[$field]) {
-					Logger::log("Difference for contact " . $contact["id"] . " in field '" . $field . "'. New value: '" . $contact[$field] . "', old value '" . $contact_old[$field] . "'", Logger::DEBUG);
-					$update = true;
-				}
-			}
-
-			if ($update) {
-				Logger::log("Update contact data for contact " . $contact["id"] . " (" . $contact["nick"] . ")", Logger::DEBUG);
-
-				q(
-					"UPDATE `contact` SET `name` = '%s', `nick` = '%s', `about` = '%s', `location` = '%s',
-					`addr` = '%s', `keywords` = '%s', `bdyear` = '%s', `bd` = '%s', `hidden` = %d,
-					`xmpp` = '%s', `name-date`  = '%s', `uri-date` = '%s'
-					WHERE `id` = %d AND `network` = '%s'",
-					DBA::escape($contact["name"]), DBA::escape($contact["nick"]), DBA::escape($contact["about"]),	DBA::escape($contact["location"]),
-					DBA::escape($contact["addr"]), DBA::escape($contact["keywords"]), DBA::escape($contact["bdyear"]),
-					DBA::escape($contact["bd"]), intval($contact["hidden"]), DBA::escape($contact["xmpp"]),
-					DBA::escape(DateTimeFormat::utc($contact["name-date"])), DBA::escape(DateTimeFormat::utc($contact["uri-date"])),
-					intval($contact["id"]),	DBA::escape($contact["network"])
-				);
-			}
-
-			Contact::updateAvatar(
-				$author['avatar'],
-				$importer['importer_uid'],
-				$contact['id'],
-				(strtotime($contact['avatar-date']) > strtotime($contact_old['avatar-date']) || ($author['avatar'] != $contact_old['avatar']))
-			);
-
-			// Update the "hidden" status in the public contact
-			// @todo Updating the contact with all fields and update the gcontact from that
+			// Update the public contact. Don't set the "hidden" value, this is used differently for public contacts
+			unset($fields['hidden']);
+			$fields['unsearchable'] = $hide;
 			$condition = ['uid' => 0, 'nurl' => Strings::normaliseLink($contact_old['url'])];
-			DBA::update('contact', ['unsearchable' => $hide], $condition, true);
+			DBA::update('contact', $fields, $condition, true);
+
+			Contact::updateAvatar($author['avatar'], $importer['importer_uid'], $contact['id']);
+
+			$pcid = Contact::getIdForURL($contact_old['url']);
+			if (!empty($pcid)) {
+				Contact::updateAvatar($author['avatar'], 0, $pcid);
+			}
 
 			/*
 			 * The generation is a sign for the reliability of the provided data.

From dda86f6dfc6ac74c17b05b93cd55def76dafb1ea Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Fri, 5 Jul 2019 06:41:48 +0000
Subject: [PATCH 8/8] Basepath function now uses the contact field

---
 mod/redir.php         |  2 +-
 src/Model/Contact.php | 17 +++++++++++------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/mod/redir.php b/mod/redir.php
index 233ec9b00..c99e1823c 100644
--- a/mod/redir.php
+++ b/mod/redir.php
@@ -85,7 +85,7 @@ function redir_init(App $a) {
 
 		// When the remote page does support OWA, then we enforce the use of it
 		$basepath = Contact::getBasepath($contact_url);
-		if ($basepath == System::baseUrl()) {
+		if (Strings::compareLink($basepath, System::baseUrl())) {
 			$use_magic = true;
 		} else {
 			$serverret = Network::curl($basepath . '/magic');
diff --git a/src/Model/Contact.php b/src/Model/Contact.php
index 3f37d4ec9..775cc3fda 100644
--- a/src/Model/Contact.php
+++ b/src/Model/Contact.php
@@ -176,7 +176,6 @@ class Contact extends BaseObject
 
 	/**
 	 * @brief Get the basepath for a given contact link
-	 * @todo  Add functionality to store this value in the contact table
 	 *
 	 * @param string $url The contact link
 	 *
@@ -186,13 +185,19 @@ class Contact extends BaseObject
 	 */
 	public static function getBasepath($url)
 	{
-		$data = Probe::uri($url);
-		if (!empty($data['baseurl'])) {
-			return $data['baseurl'];
+		$contact = DBA::selectFirst('contact', ['baseurl'], ['uid' => 0, 'nurl' => Strings::normaliseLink($url)]);
+		if (!empty($contact['baseurl'])) {
+			return $contact['baseurl'];
 		}
 
-		// When we can't probe the server, we use some ugly function that does some pattern matching
-		return PortableContact::detectServer($url);
+		self::updateFromProbeByURL($url, true);
+
+		$contact = DBA::selectFirst('contact', ['baseurl'], ['uid' => 0, 'nurl' => Strings::normaliseLink($url)]);
+		if (!empty($contact['baseurl'])) {
+			return $contact['baseurl'];
+		}
+
+		return '';
 	}
 
 	/**