From 8cbdc7939e8fc8466ea282d58afb9de564f7a9b6 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Fri, 12 Jul 2019 14:55:23 +0000
Subject: [PATCH 1/8] Fetch more comtact data from probing, remove duplicated
 contacts

---
 src/Model/Contact.php     | 83 +++++++++++++++++++++++++++++++++------
 src/Network/Probe.php     | 17 ++++++--
 src/Protocol/DFRN.php     |  3 +-
 src/Protocol/Diaspora.php |  2 +
 4 files changed, 88 insertions(+), 17 deletions(-)

diff --git a/src/Model/Contact.php b/src/Model/Contact.php
index 757508f26..17506a7d3 100644
--- a/src/Model/Contact.php
+++ b/src/Model/Contact.php
@@ -1307,11 +1307,13 @@ class Contact extends BaseObject
 
 		/// @todo Verify if we can't use Contact::getDetailsByUrl instead of the following
 		// We first try the nurl (http://server.tld/nick), most common case
-		$contact = DBA::selectFirst('contact', ['id', 'avatar', 'updated', 'network'], ['nurl' => Strings::normaliseLink($url), 'uid' => $uid, 'deleted' => false]);
+		$fields = ['id', 'avatar', 'updated', 'network'];
+		$options = ['order' => ['id']];
+		$contact = DBA::selectFirst('contact', $fields, ['nurl' => Strings::normaliseLink($url), 'uid' => $uid, 'deleted' => false], $options);
 
 		// Then the addr (nick@server.tld)
 		if (!DBA::isResult($contact)) {
-			$contact = DBA::selectFirst('contact', ['id', 'avatar', 'updated', 'network'], ['addr' => $url, 'uid' => $uid, 'deleted' => false]);
+			$contact = DBA::selectFirst('contact', $fields, ['addr' => $url, 'uid' => $uid, 'deleted' => false], $options);
 		}
 
 		// Then the alias (which could be anything)
@@ -1319,7 +1321,7 @@ class Contact extends BaseObject
 			// The link could be provided as http although we stored it as https
 			$ssl_url = str_replace('http://', 'https://', $url);
 			$condition = ['`alias` IN (?, ?, ?) AND `uid` = ? AND NOT `deleted`', $url, Strings::normaliseLink($url), $ssl_url, $uid];
-			$contact = DBA::selectFirst('contact', ['id', 'avatar', 'updated', 'network'], $condition);
+			$contact = DBA::selectFirst('contact', $fields, $condition, $options);
 		}
 
 		if (DBA::isResult($contact)) {
@@ -1777,7 +1779,8 @@ class Contact extends BaseObject
 	{
 		DBA::update('contact', $fields, ['id' => $id]);
 
-		if ($uid != 0) {
+		// Search for duplicated contacts and get rid of them
+		if (self::handleDuplicates(Strings::normaliseLink($url), $uid, $id) || ($uid != 0)) {
 			return;
 		}
 
@@ -1813,6 +1816,55 @@ class Contact extends BaseObject
 		DBA::update('contact', $fields, $condition);
 	}
 
+        /**
+	 * @brief Helper function for "updateFromProbe". Remove duplicated contacts
+	 *
+	 * @param string  $nurl  Normalised contact url
+	 * @param integer $uid   User id
+	 * @param integer $id    Contact id of a duplicate
+	 * @throws \Exception
+	 */
+	private static function handleDuplicates($nurl, $uid, $id)
+	{
+		$condition = ['nurl' => $nurl, 'uid' => $uid, 'deleted' => false];
+		$count = DBA::count('contact', $condition);
+		if ($count <= 1) {
+			return false;
+		}
+
+		$first_contact = DBA::selectFirst('contact', ['id'], $condition, ['order' => ['id']]);
+		if (!DBA::isResult($first_contact)) {
+			// Shouldn't happen - so we handle it
+			return false;
+		}
+
+		$first = $first_contact['id'];
+		Logger::info('Found duplicates', ['count' => $count, 'id' => $id, 'first' => $first, 'uid' => $uid, 'nurl' => $nurl, 'callstack' => System::callstack(20)]);
+		if ($uid != 0) {
+			// Don't handle non public duplicates by now
+			Logger::info('Not handling non public duplicate', ['uid' => $uid, 'nurl' => $nurl]);
+			return false;
+		}
+
+		// Find all duplicates
+		$condition = ["`nurl` = ? AND `uid` = ? AND `id` != ? AND NOT `self` AND NOT `deleted`", $nurl, $uid, $first];
+		$duplicates = DBA::select('contact', ['id'], $condition);
+		while ($duplicate = DBA::fetch($duplicates)) {
+			$dup_id = $duplicate['id'];
+			Logger::info('Handling duplicate', ['search' => $dup_id, 'replace' => $first]);
+
+			// Search and replace
+			DBA::update('item',['author-id' => $first], ['author-id' => $dup_id]);
+			DBA::update('item',['owner-id' => $first], ['owner-id' => $dup_id]);
+			DBA::update('item',['contact-id' => $first], ['contact-id' => $dup_id]);
+
+			// Remove the duplicate
+			DBA::delete('contact', ['id' => $dup_id]);
+		}
+		Logger::info('Duplicates handled', ['uid' => $uid, 'nurl' => $nurl]);
+		return true;
+	}
+
 	/**
 	 * @param integer $id      contact id
 	 * @param string  $network Optional network we are probing for
@@ -1829,11 +1881,11 @@ class Contact extends BaseObject
 		 */
 
 		// These fields aren't updated by this routine:
-		// 'location', 'about', 'keywords', 'gender', 'xmpp', 'unsearchable', 'sensitive'];
+		// 'xmpp', 'sensitive'
 
-		$fields = ['avatar', 'uid', 'name', 'nick', 'url', 'addr', 'batch', 'notify',
-			'poll', 'request', 'confirm', 'poco', 'network', 'alias', 'baseurl',
-			'forum', 'prv', 'contact-type'];
+		$fields = ['uid', 'avatar', 'name', 'nick', 'location', 'keywords', 'about', 'gender',
+			'unsearchable', 'url', 'addr', 'batch', 'notify', 'poll', 'request', 'confirm', 'poco',
+			'network', 'alias', 'baseurl', 'forum', 'prv', 'contact-type'];
 		$contact = DBA::selectFirst('contact', $fields, ['id' => $id]);
 		if (!DBA::isResult($contact)) {
 			return false;
@@ -1858,7 +1910,11 @@ class Contact extends BaseObject
 			return false;
 		}
 
-		if (isset($ret['account-type'])) {
+		if (isset($ret['hide']) && is_bool($ret['hide'])) {
+			$ret['unsearchable'] = $ret['hide'];
+		}
+
+		if (isset($ret['account-type']) && is_int($ret['account-type'])) {
 			$ret['forum'] = false;
 			$ret['prv'] = false;
 			$ret['contact-type'] = $ret['account-type'];
@@ -1873,11 +1929,12 @@ class Contact extends BaseObject
 
 		$update = false;
 
-		// make sure to not overwrite existing values with blank entries
+		// make sure to not overwrite existing values with blank entries except some technical fields
+		$keep = ['batch', 'notify', 'poll', 'request', 'confirm', 'poco', 'baseurl'];
 		foreach ($ret as $key => $val) {
 			if (!array_key_exists($key, $contact)) {
 				unset($ret[$key]);
-			} elseif (($contact[$key] != '') && ($val == '') && !is_bool($ret[$key])) {
+			} elseif (($contact[$key] != '') && ($val == '') && !is_bool($ret[$key]) && !in_array($key, $keep)) {
 				$ret[$key] = $contact[$key];
 			} elseif ($ret[$key] != $contact[$key]) {
 				$update = true;
@@ -1915,10 +1972,12 @@ class Contact extends BaseObject
 		$id = self::getIdForURL($url);
 
 		if (empty($id)) {
-			return;
+			return $id;
 		}
 
 		self::updateFromProbe($id, '', $force);
+
+		return $id;
 	}
 
 	/**
diff --git a/src/Network/Probe.php b/src/Network/Probe.php
index 0094609b2..c38c43c09 100644
--- a/src/Network/Probe.php
+++ b/src/Network/Probe.php
@@ -45,8 +45,8 @@ class Probe
 	 */
 	private static function rearrangeData($data)
 	{
-		$fields = ["name", "nick", "guid", "url", "addr", "alias", "photo",
-				"account-type", "community", "keywords", "location", "about",
+		$fields = ["name", "nick", "guid", "url", "addr", "alias", "photo", "account-type",
+				"community", "keywords", "location", "about", "gender", "hide",
 				"batch", "notify", "poll", "request", "confirm", "poco",
 				"following", "followers", "inbox", "outbox", "sharedinbox",
 				"priority", "network", "pubkey", "baseurl"];
@@ -351,6 +351,7 @@ class Probe
 			if (!empty($ap_profile) && empty($network) && (defaults($data, 'network', '') != Protocol::DFRN)) {
 				$data = $ap_profile;
 			} elseif (!empty($ap_profile)) {
+				$ap_profile['batch'] = '';
 				$data = array_merge($ap_profile, $data);
 			}
 		} else {
@@ -739,7 +740,7 @@ class Probe
 		}
 
 		if (!empty($json["tags"])) {
-			$keywords = implode(" ", $json["tags"]);
+			$keywords = implode(", ", $json["tags"]);
 			if ($keywords != "") {
 				$data["keywords"] = $keywords;
 			}
@@ -754,6 +755,10 @@ class Probe
 			$data["about"] = $json["about"];
 		}
 
+		if (!empty($json["gender"])) {
+			$data["gender"] = $json["gender"];
+		}
+
 		if (!empty($json["key"])) {
 			$data["pubkey"] = $json["key"];
 		}
@@ -778,6 +783,12 @@ class Probe
 			$data["poll"] = $json["dfrn-poll"];
 		}
 
+		if (isset($json["hide"])) {
+			$data["hide"] = (bool)$json["hide"];
+		} else {
+			$data["hide"] = false;
+		}
+
 		return $data;
 	}
 
diff --git a/src/Protocol/DFRN.php b/src/Protocol/DFRN.php
index 84c4affc7..5fceab326 100644
--- a/src/Protocol/DFRN.php
+++ b/src/Protocol/DFRN.php
@@ -1693,13 +1693,12 @@ class DFRN
 				'location' => $contact['location'], 'addr' => $contact['addr'], 'keywords' => $contact['keywords'],
 				'bdyear' => $contact['bdyear'], 'bd' => $contact['bd'], 'hidden' => $contact['hidden'],
 				'xmpp' => $contact['xmpp'], 'name-date' => DateTimeFormat::utc($contact['name-date']),
-				'uri-date' => DateTimeFormat::utc($contact['uri-date'])];
+				'unsearchable' => $contact['hidden'], 'uri-date' => DateTimeFormat::utc($contact['uri-date'])];
 
 			DBA::update('contact', $fields, ['id' => $contact['id'], 'network' => $contact['network']], $contact_old);
 
 			// Update the public contact. Don't set the "hidden" value, this is used differently for public contacts
 			unset($fields['hidden']);
-			$fields['unsearchable'] = $hide;
 			$condition = ['uid' => 0, 'nurl' => Strings::normaliseLink($contact_old['url'])];
 			DBA::update('contact', $fields, $condition, true);
 
diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php
index 31d9efa18..a264398ef 100644
--- a/src/Protocol/Diaspora.php
+++ b/src/Protocol/Diaspora.php
@@ -2267,6 +2267,8 @@ class Diaspora
 
 		DBA::update('contact', $fields, ['id' => $contact['id']]);
 
+		// @todo Update the public contact, then update the gcontact from that
+
 		$gcontact = ["url" => $contact["url"], "network" => Protocol::DIASPORA, "generation" => 2,
 					"photo" => $image_url, "name" => $name, "location" => $location,
 					"about" => $about, "birthday" => $birthday, "gender" => $gender,

From 7dfadf7e7ec6f6d3b18fb4553492298491529f65 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Fri, 12 Jul 2019 21:07:47 +0000
Subject: [PATCH 2/8] Simplify the contact update in "getIdForURL"

---
 src/Model/Contact.php | 156 +++++++++++++++++-------------------------
 src/Network/Probe.php |   2 +-
 2 files changed, 63 insertions(+), 95 deletions(-)

diff --git a/src/Model/Contact.php b/src/Model/Contact.php
index 17506a7d3..764e29008 100644
--- a/src/Model/Contact.php
+++ b/src/Model/Contact.php
@@ -1418,115 +1418,65 @@ class Contact extends BaseObject
 
 			$condition = ['nurl' => Strings::normaliseLink($data["url"]), 'uid' => $uid, 'deleted' => false];
 
+			// This does an insert but should most likely prevent duplicates
 			DBA::update('contact', $fields, $condition, true);
 
-			$s = DBA::select('contact', ['id'], $condition, ['order' => ['id'], 'limit' => 2]);
-			$contacts = DBA::toArray($s);
-			if (!DBA::isResult($contacts)) {
+			$contact = DBA::selectFirst('contact', ['id'], $condition, ['order' => ['id']]);
+			if (!DBA::isResult($contact)) {
+				// Shouldn't happen
 				return 0;
 			}
 
-			$contact_id = $contacts[0]["id"];
-
-			// Update in the background when we fetched the data solely from the database
-			if ($background_update) {
-				Worker::add(PRIORITY_LOW, "UpdateContact", $contact_id, ($uid == 0 ? 'force' : ''));
-			}
-
-			// Update the newly created contact from data in the gcontact table
-			$gcontact = DBA::selectFirst('gcontact', ['location', 'about', 'keywords', 'gender'], ['nurl' => Strings::normaliseLink($data["url"])]);
-			if (DBA::isResult($gcontact)) {
-				// Only use the information when the probing hadn't fetched these values
-				if (!empty($data['keywords'])) {
-					unset($gcontact['keywords']);
-				}
-				if (!empty($data['location'])) {
-					unset($gcontact['location']);
-				}
-				if (!empty($data['about'])) {
-					unset($gcontact['about']);
-				}
-				DBA::update('contact', $gcontact, ['id' => $contact_id]);
-			}
-
-			if (count($contacts) > 1 && $uid == 0 && $contact_id != 0 && $data["url"] != "") {
-				$condition = ["`nurl` = ? AND `uid` = ? AND `id` != ? AND NOT `self`",
-					Strings::normaliseLink($data["url"]), 0, $contact_id];
-				Logger::log('Deleting duplicate contact ' . json_encode($condition), Logger::DEBUG);
-				DBA::delete('contact', $condition);
-			}
+			$contact_id = $contact["id"];
 		}
 
 		if (!empty($data['photo']) && ($data['network'] != Protocol::FEED)) {
 			self::updateAvatar($data['photo'], $uid, $contact_id);
 		}
 
-		$fields = ['url', 'nurl', 'addr', 'alias', 'name', 'nick', 'keywords', 'location', 'about', 'avatar-date', 'pubkey', 'baseurl'];
-		$contact = DBA::selectFirst('contact', $fields, ['id' => $contact_id]);
+		if (in_array($data["network"], array_merge(Protocol::NATIVE_SUPPORT, [Protocol::PUMPIO]))) {
+			if ($background_update) {
+				// Update in the background when we fetched the data solely from the database
+				Worker::add(PRIORITY_MEDIUM, "UpdateContact", $contact_id, ($uid == 0 ? 'force' : ''));
+			} else {
+				// Else do a direct update
+				self::updateFromProbe($contact_id, '', false);
 
-		// This condition should always be true
-		if (!DBA::isResult($contact)) {
-			return $contact_id;
-		}
+				// Update the gcontact entry
+				if ($uid == 0) {
+					GContact::updateFromPublicContactID($contact_id);
+				}
+			}
+		} else {
+			$fields = ['url', 'nurl', 'addr', 'alias', 'name', 'nick', 'keywords', 'location', 'about', 'avatar-date', 'baseurl'];
+			$contact = DBA::selectFirst('contact', $fields, ['id' => $contact_id]);
 
-		$updated = [
-			'addr' => $data['addr'] ?? '',
-			'alias' => defaults($data, 'alias', ''),
-			'url' => $data['url'],
-			'nurl' => Strings::normaliseLink($data['url']),
-			'name' => $data['name'],
-			'nick' => $data['nick']
-		];
+			// This condition should always be true
+			if (!DBA::isResult($contact)) {
+				return $contact_id;
+			}
 
-		if (!empty($data['baseurl'])) {
-			$updated['baseurl'] = $data['baseurl'];
-		}
-		if (!empty($data['keywords'])) {
-			$updated['keywords'] = $data['keywords'];
-		}
-		if (!empty($data['location'])) {
-			$updated['location'] = $data['location'];
-		}
+			$updated = [
+				'url' => $data['url'],
+				'nurl' => Strings::normaliseLink($data['url']),
+				'updated' => DateTimeFormat::utcNow()
+			];
 
-		// Update the technical stuff as well - if filled
-		if (!empty($data['notify'])) {
-			$updated['notify'] = $data['notify'];
-		}
-		if (!empty($data['poll'])) {
-			$updated['poll'] = $data['poll'];
-		}
-		if (!empty($data['batch'])) {
-			$updated['batch'] = $data['batch'];
-		}
-		if (!empty($data['request'])) {
-			$updated['request'] = $data['request'];
-		}
-		if (!empty($data['confirm'])) {
-			$updated['confirm'] = $data['confirm'];
-		}
-		if (!empty($data['poco'])) {
-			$updated['poco'] = $data['poco'];
-		}
+			$fields = ['addr', 'alias', 'name', 'nick', 'keywords', 'location', 'about', 'baseurl'];
 
-		// Only fill the pubkey if it had been empty before. We have to prevent identity theft.
-		if (empty($contact['pubkey']) && !empty($data['pubkey'])) {
-			$updated['pubkey'] = $data['pubkey'];
-		}
+			foreach ($fields as $field) {
+				$updated[$field] = defaults($data, $field, $contact[$fields]);
+			}
 
-		if (($updated['addr'] != $contact['addr']) || (!empty($data['alias']) && ($data['alias'] != $contact['alias']))) {
-			$updated['uri-date'] = DateTimeFormat::utcNow();
-		}
-		if (($data["name"] != $contact["name"]) || ($data["nick"] != $contact["nick"])) {
-			$updated['name-date'] = DateTimeFormat::utcNow();
-		}
+			if (($updated['addr'] != $contact['addr']) || (!empty($data['alias']) && ($data['alias'] != $contact['alias']))) {
+				$updated['uri-date'] = DateTimeFormat::utcNow();
+			}
 
-		$updated['updated'] = DateTimeFormat::utcNow();
+			if (($data['name'] != $contact['name']) || ($data['nick'] != $contact['nick'])) {
+				$updated['name-date'] = DateTimeFormat::utcNow();
+			}
 
-		DBA::update('contact', $updated, ['id' => $contact_id], $contact);
-
-		if (!$background_update && ($uid == 0)) {
-			// Update the gcontact entry
-			GContact::updateFromPublicContactID($contact_id);
+			DBA::update('contact', $updated, ['id' => $contact_id], $contact);
 		}
 
 		return $contact_id;
@@ -1854,9 +1804,9 @@ class Contact extends BaseObject
 			Logger::info('Handling duplicate', ['search' => $dup_id, 'replace' => $first]);
 
 			// Search and replace
-			DBA::update('item',['author-id' => $first], ['author-id' => $dup_id]);
-			DBA::update('item',['owner-id' => $first], ['owner-id' => $dup_id]);
-			DBA::update('item',['contact-id' => $first], ['contact-id' => $dup_id]);
+			DBA::update('item', ['author-id' => $first], ['author-id' => $dup_id]);
+			DBA::update('item', ['owner-id' => $first], ['owner-id' => $dup_id]);
+			DBA::update('item', ['contact-id' => $first], ['contact-id' => $dup_id]);
 
 			// Remove the duplicate
 			DBA::delete('contact', ['id' => $dup_id]);
@@ -1885,7 +1835,7 @@ class Contact extends BaseObject
 
 		$fields = ['uid', 'avatar', 'name', 'nick', 'location', 'keywords', 'about', 'gender',
 			'unsearchable', 'url', 'addr', 'batch', 'notify', 'poll', 'request', 'confirm', 'poco',
-			'network', 'alias', 'baseurl', 'forum', 'prv', 'contact-type'];
+			'network', 'alias', 'baseurl', 'forum', 'prv', 'contact-type', 'pubkey'];
 		$contact = DBA::selectFirst('contact', $fields, ['id' => $id]);
 		if (!DBA::isResult($contact)) {
 			return false;
@@ -1894,6 +1844,9 @@ class Contact extends BaseObject
 		$uid = $contact['uid'];
 		unset($contact['uid']);
 
+		$pubkey = $contact['pubkey'];
+		unset($contact['pubkey']);
+
 		$contact['photo'] = $contact['avatar'];
 		unset($contact['avatar']);
 
@@ -1927,6 +1880,8 @@ class Contact extends BaseObject
 			}
 		}
 
+		$new_pubkey = $ret['pubkey'];
+
 		$update = false;
 
 		// make sure to not overwrite existing values with blank entries except some technical fields
@@ -1955,6 +1910,19 @@ class Contact extends BaseObject
 		$ret['nurl'] = Strings::normaliseLink($ret['url']);
 		$ret['updated'] = $updated;
 
+		// Only fill the pubkey if it had been empty before. We have to prevent identity theft.
+		if (empty($pubkey) && !empty($new_pubkey)) {
+			$ret['pubkey'] = $new_pubkey;
+		}
+
+		if (($ret['addr'] != $contact['addr']) || (!empty($ret['alias']) && ($ret['alias'] != $contact['alias']))) {
+			$ret['uri-date'] = DateTimeFormat::utcNow();
+		}
+
+		if (($ret['name'] != $contact['name']) || ($ret['nick'] != $contact['nick'])) {
+			$ret['name-date'] = $updated;
+		}
+
 		if ($force && ($uid == 0)) {
 			$ret['last-update'] = $updated;
 			$ret['success_update'] = $updated;
diff --git a/src/Network/Probe.php b/src/Network/Probe.php
index c38c43c09..7a8e8de35 100644
--- a/src/Network/Probe.php
+++ b/src/Network/Probe.php
@@ -348,7 +348,7 @@ class Probe
 		if (!self::$istimeout) {
 			$ap_profile = ActivityPub::probeProfile($uri);
 
-			if (!empty($ap_profile) && empty($network) && (defaults($data, 'network', '') != Protocol::DFRN)) {
+			if (empty($data) || (!empty($ap_profile) && empty($network) && (defaults($data, 'network', '') != Protocol::DFRN))) {
 				$data = $ap_profile;
 			} elseif (!empty($ap_profile)) {
 				$ap_profile['batch'] = '';

From 9b8396620bd34a032f7d9e93e6a69719eff499ed Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Sat, 13 Jul 2019 07:25:01 +0000
Subject: [PATCH 3/8] Fix duplicated contacts due to relay probing

---
 src/Model/Contact.php | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/Model/Contact.php b/src/Model/Contact.php
index 764e29008..64afa91e3 100644
--- a/src/Model/Contact.php
+++ b/src/Model/Contact.php
@@ -1313,7 +1313,7 @@ class Contact extends BaseObject
 
 		// Then the addr (nick@server.tld)
 		if (!DBA::isResult($contact)) {
-			$contact = DBA::selectFirst('contact', $fields, ['addr' => $url, 'uid' => $uid, 'deleted' => false], $options);
+			$contact = DBA::selectFirst('contact', $fields, ['addr' => str_replace('acct:', '', $url), 'uid' => $uid, 'deleted' => false], $options);
 		}
 
 		// Then the alias (which could be anything)
@@ -1772,6 +1772,7 @@ class Contact extends BaseObject
 	 * @param string  $nurl  Normalised contact url
 	 * @param integer $uid   User id
 	 * @param integer $id    Contact id of a duplicate
+	 * @return boolean
 	 * @throws \Exception
 	 */
 	private static function handleDuplicates($nurl, $uid, $id)
@@ -1789,7 +1790,7 @@ class Contact extends BaseObject
 		}
 
 		$first = $first_contact['id'];
-		Logger::info('Found duplicates', ['count' => $count, 'id' => $id, 'first' => $first, 'uid' => $uid, 'nurl' => $nurl, 'callstack' => System::callstack(20)]);
+		Logger::info('Found duplicates', ['count' => $count, 'id' => $id, 'first' => $first, 'uid' => $uid, 'nurl' => $nurl]);
 		if ($uid != 0) {
 			// Don't handle non public duplicates by now
 			Logger::info('Not handling non public duplicate', ['uid' => $uid, 'nurl' => $nurl]);
@@ -1854,6 +1855,15 @@ class Contact extends BaseObject
 
 		$updated = DateTimeFormat::utcNow();
 
+		// We must not try to update relay contacts via probe. They are no real contacts.
+		// We check after the probing to be able to correct falsely detected contact types.
+		if (($contact['contact-type'] == self::TYPE_RELAY) &&
+			(!Strings::compareLink($ret['url'], $contact['url']) || in_array($ret['network'], [Protocol::FEED, Protocol::PHANTOM]))) {
+			self::updateContact($id, $uid, $contact['url'], ['last-update' => $updated, 'success_update' => $updated]);
+			Logger::info('Not updating relais', ['id' => $id, 'url' => $contact['url']]);
+			return true;
+		}
+
 		// If Probe::uri fails the network code will be different (mostly "feed" or "unkn")
 		if (!in_array($ret['network'], Protocol::NATIVE_SUPPORT) ||
 			(in_array($ret['network'], [Protocol::FEED, Protocol::PHANTOM]) && ($ret['network'] != $contact['network']))) {
@@ -1889,7 +1899,7 @@ class Contact extends BaseObject
 		foreach ($ret as $key => $val) {
 			if (!array_key_exists($key, $contact)) {
 				unset($ret[$key]);
-			} elseif (($contact[$key] != '') && ($val == '') && !is_bool($ret[$key]) && !in_array($key, $keep)) {
+			} elseif (($contact[$key] != '') && ($val === '') && !is_bool($ret[$key]) && !in_array($key, $keep)) {
 				$ret[$key] = $contact[$key];
 			} elseif ($ret[$key] != $contact[$key]) {
 				$update = true;

From 002963f1982940fda537b1007e5a6bf12b75c719 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Sat, 13 Jul 2019 07:29:56 +0000
Subject: [PATCH 4/8] Fix wrong variable

---
 src/Model/Contact.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Model/Contact.php b/src/Model/Contact.php
index 64afa91e3..e892c99f2 100644
--- a/src/Model/Contact.php
+++ b/src/Model/Contact.php
@@ -1465,7 +1465,7 @@ class Contact extends BaseObject
 			$fields = ['addr', 'alias', 'name', 'nick', 'keywords', 'location', 'about', 'baseurl'];
 
 			foreach ($fields as $field) {
-				$updated[$field] = defaults($data, $field, $contact[$fields]);
+				$updated[$field] = defaults($data, $field, $contact[$field]);
 			}
 
 			if (($updated['addr'] != $contact['addr']) || (!empty($data['alias']) && ($data['alias'] != $contact['alias']))) {

From 2df17fc0c02998f525d611ab3d022b82ec7c65eb Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Sat, 13 Jul 2019 07:39:53 +0000
Subject: [PATCH 5/8] Fix SQL problem with "not null"

---
 src/Model/GContact.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Model/GContact.php b/src/Model/GContact.php
index 0fae0da9d..b301c0efb 100644
--- a/src/Model/GContact.php
+++ b/src/Model/GContact.php
@@ -924,7 +924,7 @@ class GContact
 		}
 
 		// These fields are having different names but the same content
-		$gcontact['server_url'] = $contact['baseurl'];
+		$gcontact['server_url'] = defaults($contact, 'baseurl', ''); // "baseurl" can be null, "server_url" not
 		$gcontact['nsfw'] = $contact['sensitive'];
 		$gcontact['hide'] = $contact['unsearchable'];
 		$gcontact['archived'] = $contact['archive'];

From 5bba0e7d3941ad87c4f3fd9e0d90b42e70850b2d Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Sat, 13 Jul 2019 23:53:15 +0000
Subject: [PATCH 6/8] Hopefully avoiding duplicated relay entries

---
 src/Protocol/Diaspora.php | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php
index a264398ef..5f37684ad 100644
--- a/src/Protocol/Diaspora.php
+++ b/src/Protocol/Diaspora.php
@@ -180,21 +180,25 @@ class Diaspora
 	public static function setRelayContact($server_url, array $network_fields = [])
 	{
 		$fields = ['created' => DateTimeFormat::utcNow(),
-			'name' => 'relay', 'nick' => 'relay',
-			'url' => $server_url, 'network' => Protocol::DIASPORA,
+			'name' => 'relay', 'nick' => 'relay', 'url' => $server_url,
+			'nurl' => Strings::normaliseLink($server_url),
+			'network' => Protocol::DIASPORA, 'uid' => 0,
 			'batch' => $server_url . '/receive/public',
 			'rel' => Contact::FOLLOWER, 'blocked' => false,
-			'pending' => false, 'writable' => true];
+			'pending' => false, 'writable' => true,
+			'baseurl' => $server_url, 'contact-type' => Contact::TYPE_RELAY];
 
 		$fields = array_merge($fields, $network_fields);
 
-		$condition = ['uid' => 0, 'nurl' => Strings::normaliseLink($server_url),
-			'contact-type' => Contact::TYPE_RELAY];
-
-		if (DBA::exists('contact', $condition)) {
+		$condition = ['uid' => 0, 'nurl' => Strings::normaliseLink($server_url)];
+		$relay = DBA::selectFirst('contact', ['id'], $condition);
+		if (DBA::isResult($relay)) {
 			unset($fields['created']);
+			$condition = ['id' => $relay['id']];
 		}
 
+		Logger::info('Set relay contact', ['fields' => $fields, 'condition' => $condition]);
+
 		DBA::update('contact', $fields, $condition, true);
 	}
 

From 9ba3ee13a80ad9db6cad6a0c52dcafb6177ae72d Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Sun, 14 Jul 2019 10:22:19 +0000
Subject: [PATCH 7/8] Use an insert to avoid duplicates and for analyzing

---
 src/Model/Contact.php     | 19 ++++++++++++++-----
 src/Protocol/Diaspora.php | 16 +++++++++-------
 2 files changed, 23 insertions(+), 12 deletions(-)

diff --git a/src/Model/Contact.php b/src/Model/Contact.php
index e892c99f2..9f6fa6264 100644
--- a/src/Model/Contact.php
+++ b/src/Model/Contact.php
@@ -1418,13 +1418,22 @@ class Contact extends BaseObject
 
 			$condition = ['nurl' => Strings::normaliseLink($data["url"]), 'uid' => $uid, 'deleted' => false];
 
-			// This does an insert but should most likely prevent duplicates
-			DBA::update('contact', $fields, $condition, true);
-
+			// Before inserting we do check if the entry does exist now.
 			$contact = DBA::selectFirst('contact', ['id'], $condition, ['order' => ['id']]);
 			if (!DBA::isResult($contact)) {
-				// Shouldn't happen
-				return 0;
+				Logger::info('Create new contact', $fields);
+
+				DBA::insert('contact', $fields);
+
+				// We intentionally aren't using lastInsertId here. There is a chance for duplicates.
+				$contact = DBA::selectFirst('contact', ['id'], $condition, ['order' => ['id']]);
+				if (!DBA::isResult($contact)) {
+					Logger::info('Contact creation failed', $fields);
+					// Shouldn't happen
+					return 0;
+				}
+			} else {
+				Logger::info('Contact had been created before', ['id' => $contact["id"], 'url' => $url, 'contact' => $fields]);
 			}
 
 			$contact_id = $contact["id"];
diff --git a/src/Protocol/Diaspora.php b/src/Protocol/Diaspora.php
index 5f37684ad..8e2e48779 100644
--- a/src/Protocol/Diaspora.php
+++ b/src/Protocol/Diaspora.php
@@ -191,15 +191,17 @@ class Diaspora
 		$fields = array_merge($fields, $network_fields);
 
 		$condition = ['uid' => 0, 'nurl' => Strings::normaliseLink($server_url)];
-		$relay = DBA::selectFirst('contact', ['id'], $condition);
-		if (DBA::isResult($relay)) {
+		$old = DBA::selectFirst('contact', [], $condition);
+		if (DBA::isResult($old)) {
 			unset($fields['created']);
-			$condition = ['id' => $relay['id']];
+			$condition = ['id' => $old['id']];
+
+			Logger::info('Update relay contact', ['fields' => $fields, 'condition' => $condition]);
+			DBA::update('contact', $fields, $condition, $old);
+		} else {
+			Logger::info('Create relay contact', ['fields' => $fields]);
+			DBA::insert('contact', $fields);
 		}
-
-		Logger::info('Set relay contact', ['fields' => $fields, 'condition' => $condition]);
-
-		DBA::update('contact', $fields, $condition, true);
 	}
 
 	/**

From 83729b8c00a7fec7037d82a10e2b8cabc25c4ce7 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Sun, 14 Jul 2019 16:04:52 +0000
Subject: [PATCH 8/8] "defaults" is replaced

---
 src/Model/GContact.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Model/GContact.php b/src/Model/GContact.php
index b301c0efb..6ffa0b2e7 100644
--- a/src/Model/GContact.php
+++ b/src/Model/GContact.php
@@ -924,7 +924,7 @@ class GContact
 		}
 
 		// These fields are having different names but the same content
-		$gcontact['server_url'] = defaults($contact, 'baseurl', ''); // "baseurl" can be null, "server_url" not
+		$gcontact['server_url'] = $contact['baseurl'] ?? ''; // "baseurl" can be null, "server_url" not
 		$gcontact['nsfw'] = $contact['sensitive'];
 		$gcontact['hide'] = $contact['unsearchable'];
 		$gcontact['archived'] = $contact['archive'];