Merge pull request #2349 from annando/1602-gcontact-cleanup

Updating of gcontact data has improved

boot.php

@@ -1039,19 +1039,29 @@ class App {
 		$this->performance[$value] += (float)$duration;
 		$this->performance["marktime"] += (float)$duration;
 
-		// Trace the different functions with their timestamps
+		$callstack = $this->callstack();
-		$trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 5);
 
+		$this->callstack[$value][$callstack] += (float)$duration;
+
+	}
+
+	/**
+	 * @brief Returns a string with a callstack. Can be used for logging.
+	 *
+	 * @return string
+	 */
+	function callstack() {
+		$trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 6);
+
+		// We remove the first two items from the list since they contain data that we don't need.
+		array_shift($trace);
 		array_shift($trace);
 
-		$function = array();
+		$callstack = array();
 		foreach ($trace AS $func)
-			$function[] = $func["function"];
+			$callstack[] = $func["function"];
-
-		$function = implode(", ", $function);
-
-		$this->callstack[$value][$function] += (float)$duration;
 
+		return implode(", ", $callstack);
 	}
 
 	function mark_timestamp($mark) {

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 3.5-dev (Asparagus)
--- DB_UPDATE_VERSION 1193
+-- DB_UPDATE_VERSION 1194
 -- ------------------------------------------
 
 
@@ -119,6 +119,7 @@ CREATE TABLE IF NOT EXISTS `contact` (
 	`keywords` text NOT NULL,
 	`gender` varchar(32) NOT NULL DEFAULT '',
 	`attag` varchar(255) NOT NULL DEFAULT '',
+	`avatar` varchar(255) NOT NULL DEFAULT '',
 	`photo` text NOT NULL,
 	`thumb` text NOT NULL,
 	`micro` text NOT NULL,
@@ -411,21 +412,6 @@ CREATE TABLE IF NOT EXISTS `gserver` (
 	 INDEX `nurl` (`nurl`)
 ) DEFAULT CHARSET=utf8;
 
---
--- TABLE guid
---
-CREATE TABLE IF NOT EXISTS `guid` (
-	`id` int(10) unsigned NOT NULL auto_increment,
-	`guid` varchar(255) NOT NULL DEFAULT '',
-	`plink` varchar(255) NOT NULL DEFAULT '',
-	`uri` varchar(255) NOT NULL DEFAULT '',
-	`network` varchar(32) NOT NULL DEFAULT '',
-	 PRIMARY KEY(`id`),
-	 INDEX `guid` (`guid`),
-	 INDEX `plink` (`plink`),
-	 INDEX `uri` (`uri`)
-) DEFAULT CHARSET=utf8;
-
 --
 -- TABLE hook
 --

include/Scrape.php

@@ -12,6 +12,20 @@ function scrape_dfrn($url, $dont_probe = false) {
 
 	logger('scrape_dfrn: url=' . $url);
 
+	// Try to fetch the data from noscrape. This is faster than parsing the HTML
+	$noscrape = str_replace("/hcard/", "/noscrape/", $url);
+	$noscrapejson = fetch_url($noscrape);
+	$noscrapedata = array();
+	if ($noscrapejson) {
+		$noscrapedata = json_decode($noscrapejson, true);
+
+		if (is_array($noscrapedata)) {
+			if ($noscrapedata["nick"] != "")
+				return($noscrapedata);
+		} else
+			$noscrapedata = array();
+	}
+
 	$s = fetch_url($url);
 
 	if(! $s)
@@ -91,8 +105,7 @@ function scrape_dfrn($url, $dont_probe = false) {
 			}
 		}
 	}
-
+	return array_merge($ret, $noscrapedata);
-	return $ret;
 }}
 
 
@@ -828,18 +841,18 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 		$vcard['fn'] = $url;
 
 	if (($notify != "") AND ($poll != "")) {
-		$baseurl = matching(normalise_link($notify), normalise_link($poll));
+		$baseurl = matching_url(normalise_link($notify), normalise_link($poll));
 
-		$baseurl2 = matching($baseurl, normalise_link($profile));
+		$baseurl2 = matching_url($baseurl, normalise_link($profile));
 		if ($baseurl2 != "")
 			$baseurl = $baseurl2;
 	}
 
 	if (($baseurl == "") AND ($notify != ""))
-		$baseurl = matching(normalise_link($profile), normalise_link($notify));
+		$baseurl = matching_url(normalise_link($profile), normalise_link($notify));
 
 	if (($baseurl == "") AND ($poll != ""))
-		$baseurl = matching(normalise_link($profile), normalise_link($poll));
+		$baseurl = matching_url(normalise_link($profile), normalise_link($poll));
 
 	$baseurl = rtrim($baseurl, "/");
 
@@ -894,19 +907,56 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 	return $result;
 }
 
-function matching($part1, $part2) {
+/**
-	$len = min(strlen($part1), strlen($part2));
+ * @brief Find the matching part between two url
+ *
+ * @param string $url1
+ * @param string $url2
+ * @return string The matching part
+ */
+function matching_url($url1, $url2) {
+
+	if (($url1 == "") OR ($url2 == ""))
+		return "";
+
+	$url1 = normalise_link($url1);
+	$url2 = normalise_link($url2);
+
+	$parts1 = parse_url($url1);
+	$parts2 = parse_url($url2);
+
+	if (!isset($parts1["host"]) OR !isset($parts2["host"]))
+		return "";
+
+	if ($parts1["scheme"] != $parts2["scheme"])
+		return "";
+
+	if ($parts1["host"] != $parts2["host"])
+		return "";
+
+	if ($parts1["port"] != $parts2["port"])
+		return "";
+
+	$match = $parts1["scheme"]."://".$parts1["host"];
+
+	if ($parts1["port"])
+		$match .= ":".$parts1["port"];
+
+	$pathparts1 = explode("/", $parts1["path"]);
+	$pathparts2 = explode("/", $parts2["path"]);
 
-	$match = "";
-	$matching = true;
 	$i = 0;
-	while (($i <= $len) AND $matching) {
+	$path = "";
-		if (substr($part1, $i, 1) == substr($part2, $i, 1))
+	do {
-			$match .= substr($part1, $i, 1);
+		$path1 = $pathparts1[$i];
-		else
+		$path2 = $pathparts2[$i];
-			$matching = false;
 
-		$i++;
+		if ($path1 == $path2)
-	}
+			$path .= $path1."/";
-	return($match);
+
+	} while (($path1 == $path2) AND ($i++ <= count($pathparts1)));
+
+	$match .= $path;
+
+	return normalise_link($match);
 }

include/dfrn.php

@@ -95,7 +95,7 @@ class dfrn {
 
 		$sql_extra = " AND `item`.`allow_cid` = '' AND `item`.`allow_gid` = '' AND `item`.`deny_cid`  = '' AND `item`.`deny_gid`  = '' ";
 
-		$r = q("SELECT `contact`.*, `user`.`uid` AS `user_uid`, `user`.`nickname`, `user`.`timezone`, `user`.`page-flags`
+		$r = q("SELECT `contact`.*, `user`.`nickname`, `user`.`timezone`, `user`.`page-flags`
 			FROM `contact` INNER JOIN `user` ON `user`.`uid` = `contact`.`uid`
 			WHERE `contact`.`self` = 1 AND `user`.`nickname` = '%s' LIMIT 1",
 			dbesc($owner_nick)
@@ -105,7 +105,7 @@ class dfrn {
 			killme();
 
 		$owner = $r[0];
-		$owner_id = $owner['user_uid'];
+		$owner_id = $owner['uid'];
 		$owner_nick = $owner['nickname'];
 
 		$sql_post_table = "";
@@ -483,7 +483,7 @@ class dfrn {
 					"media:width" => 175, "media:height" => 175, "href" => $owner['photo']);
 		xml_add_element($doc, $author, "link", "", $attributes);
 
-		$birthday = feed_birthday($owner['user_uid'], $owner['timezone']);
+		$birthday = feed_birthday($owner['uid'], $owner['timezone']);
 
 		if ($birthday)
 			xml_add_element($doc, $author, "dfrn:birthday", $birthday);
@@ -498,7 +498,7 @@ class dfrn {
 			FROM `profile`
 				INNER JOIN `user` ON `user`.`uid` = `profile`.`uid`
 				WHERE `profile`.`is-default` AND NOT `user`.`hidewall` AND `user`.`uid` = %d",
-			intval($owner['user_uid']));
+			intval($owner['uid']));
 		if ($r) {
 			$profile = $r[0];
 			xml_add_element($doc, $author, "poco:displayName", $profile["name"]);
@@ -1114,13 +1114,13 @@ class dfrn {
 	 *
 	 * @return Returns an array with relevant data of the author
 	 */
-	private function fetchauthor($xpath, $context, $importer, $element, $onlyfetch) {
+	private function fetchauthor($xpath, $context, $importer, $element, $onlyfetch, $xml = "") {
 
 		$author = array();
 		$author["name"] = $xpath->evaluate($element."/atom:name/text()", $context)->item(0)->nodeValue;
 		$author["link"] = $xpath->evaluate($element."/atom:uri/text()", $context)->item(0)->nodeValue;
 
-		$r = q("SELECT `id`, `uid`, `network`, `avatar-date`, `name-date`, `uri-date`, `addr`,
+		$r = q("SELECT `id`, `uid`, `url`, `network`, `avatar-date`, `name-date`, `uri-date`, `addr`,
 				`name`, `nick`, `about`, `location`, `keywords`, `bdyear`, `bd`
 				FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' AND `network` != '%s'",
 			intval($importer["uid"]), dbesc(normalise_link($author["link"])), dbesc(NETWORK_STATUSNET));
@@ -1129,6 +1129,9 @@ class dfrn {
 			$author["contact-id"] = $r[0]["id"];
 			$author["network"] = $r[0]["network"];
 		} else {
+			if (!$onlyfetch)
+				logger("Contact ".$author["link"]." wasn't found for user ".$importer["uid"]." XML: ".$xml, LOGGER_DEBUG);
+
 			$author["contact-id"] = $importer["id"];
 			$author["network"] = $importer["network"];
 			$onlyfetch = true;
@@ -1158,38 +1161,41 @@ class dfrn {
 		}
 
 		if ($r AND !$onlyfetch) {
+			logger("Check if contact details for contact ".$r[0]["id"]." (".$r[0]["nick"].") have to be updated.", LOGGER_DEBUG);
+
+			$poco = array("url" => $contact["url"]);
 
 			// When was the last change to name or uri?
 			$name_element = $xpath->query($element."/atom:name", $context)->item(0);
 			foreach($name_element->attributes AS $attributes)
 				if ($attributes->name == "updated")
-					$contact["name-date"] = $attributes->textContent;
+					$poco["name-date"] = $attributes->textContent;
 
 			$link_element = $xpath->query($element."/atom:link", $context)->item(0);
 			foreach($link_element->attributes AS $attributes)
 				if ($attributes->name == "updated")
-					$contact["uri-date"] = $attributes->textContent;
+					$poco["uri-date"] = $attributes->textContent;
 
 			// Update contact data
 			$value = $xpath->evaluate($element."/dfrn:handle/text()", $context)->item(0)->nodeValue;
 			if ($value != "")
-				$contact["addr"] = $value;
+				$poco["addr"] = $value;
 
 			$value = $xpath->evaluate($element."/poco:displayName/text()", $context)->item(0)->nodeValue;
 			if ($value != "")
-				$contact["name"] = $value;
+				$poco["name"] = $value;
 
 			$value = $xpath->evaluate($element."/poco:preferredUsername/text()", $context)->item(0)->nodeValue;
 			if ($value != "")
-				$contact["nick"] = $value;
+				$poco["nick"] = $value;
 
 			$value = $xpath->evaluate($element."/poco:note/text()", $context)->item(0)->nodeValue;
 			if ($value != "")
-				$contact["about"] = $value;
+				$poco["about"] = $value;
 
 			$value = $xpath->evaluate($element."/poco:address/poco:formatted/text()", $context)->item(0)->nodeValue;
 			if ($value != "")
-				$contact["location"] = $value;
+				$poco["location"] = $value;
 
 			/// @todo Add support for the following fields that we don't support by now in the contact table:
 			/// - poco:utcOffset
@@ -1206,7 +1212,7 @@ class dfrn {
 				$tags[$tag->nodeValue] = $tag->nodeValue;
 
 			if (count($tags))
-				$contact["keywords"] = implode(", ", $tags);
+				$poco["keywords"] = implode(", ", $tags);
 
 			// "dfrn:birthday" contains the birthday converted to UTC
 			$old_bdyear = $contact["bdyear"];
@@ -1216,7 +1222,7 @@ class dfrn {
 			if (strtotime($birthday) > time()) {
 				$bd_timestamp = strtotime($birthday);
 
-				$contact["bdyear"] = date("Y", $bd_timestamp);
+				$poco["bdyear"] = date("Y", $bd_timestamp);
 			}
 
 			// "poco:birthday" is the birthday in the format "yyyy-mm-dd"
@@ -1231,9 +1237,11 @@ class dfrn {
 					$bdyear = $bdyear + 1;
 				}
 
-				$contact["bd"] = $value;
+				$poco["bd"] = $value;
 			}
 
+			$contact = array_merge($contact, $poco);
+
 			if ($old_bdyear != $contact["bdyear"])
 				self::birthday_event($contact, $birthday);
 
@@ -1244,6 +1252,7 @@ class dfrn {
 
 			unset($fields["id"]);
 			unset($fields["uid"]);
+			unset($fields["url"]);
 			unset($fields["avatar-date"]);
 			unset($fields["name-date"]);
 			unset($fields["uri-date"]);
@@ -1263,7 +1272,7 @@ class dfrn {
 				}
 
 			if ($update) {
-				logger("Update contact data for contact ".$contact["id"], LOGGER_DEBUG);
+				logger("Update contact data for contact ".$contact["id"]." (".$contact["nick"].")", LOGGER_DEBUG);
 
 				q("UPDATE `contact` SET `name` = '%s', `nick` = '%s', `about` = '%s', `location` = '%s',
 					`addr` = '%s', `keywords` = '%s', `bdyear` = '%s', `bd` = '%s',
@@ -1282,9 +1291,10 @@ class dfrn {
 			// It is used in the socgraph.php to prevent that old contact data
 			// that was relayed over several servers can overwrite contact
 			// data that we received directly.
-			$contact["generation"] = 2;
+
-			$contact["photo"] = $author["avatar"];
+			$poco["generation"] = 2;
-			update_gcontact($contact);
+			$poco["photo"] = $author["avatar"];
+			update_gcontact($poco);
 		}
 
 		return($author);
@@ -1954,6 +1964,8 @@ class dfrn {
 			$item['body'] = @html2bbcode($item['body']);
 		}
 
+		/// @todo We should check for a repeated post and if we know the repeated author.
+
 		// We don't need the content element since "dfrn:env" is always present
 		//$item["body"] = $xpath->query("atom:content/text()", $entry)->item(0)->nodeValue;
 
@@ -2368,8 +2380,14 @@ class dfrn {
 		$header["contact-id"] = $importer["id"];
 
 		// Update the contact table if the data has changed
+
+		// The "atom:author" is only present in feeds
+		if ($xpath->query("/atom:feed/atom:author")->length > 0)
+			self::fetchauthor($xpath, $doc->firstChild, $importer, "atom:author", false, $xml);
+
 		// Only the "dfrn:owner" in the head section contains all data
-		self::fetchauthor($xpath, $doc->firstChild, $importer, "dfrn:owner", false);
+		if ($xpath->query("/atom:feed/dfrn:owner")->length > 0)
+			self::fetchauthor($xpath, $doc->firstChild, $importer, "dfrn:owner", false, $xml);
 
 		logger("Import DFRN message for user ".$importer["uid"]." from contact ".$importer["id"], LOGGER_DEBUG);
 

include/follow.php

@@ -1,5 +1,6 @@
 <?php
 require_once("include/Scrape.php");
+require_once("include/socgraph.php");
 
 function update_contact($id) {
 	/*
@@ -43,6 +44,9 @@ function update_contact($id) {
 		intval($id)
 	);
 
+	// Update the corresponding gcontact entry
+	poco_last_updated($ret["url"]);
+
 	return true;
 }
 

include/items.php

@@ -500,14 +500,8 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
 	$arr['file']          = ((x($arr,'file'))          ? trim($arr['file'])                  : '');
 
 
-	if (($arr['author-link'] == "") AND ($arr['owner-link'] == "")) {
+	if (($arr['author-link'] == "") AND ($arr['owner-link'] == ""))
-		$trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 5);
+		logger("Both author-link and owner-link are empty. Called by: ".App::callstack(), LOGGER_DEBUG);
-		foreach ($trace AS $func)
-		        $function[] = $func["function"];
-
-		$function = implode(", ", $function);
-		logger("Both author-link and owner-link are empty. Called by: ".$function, LOGGER_DEBUG);
-	}
 
 	if ($arr['plink'] == "") {
 		$a = get_app();
@@ -888,9 +882,6 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
 			logger('item_store: new item not found in DB, id ' . $current_post);
 	}
 
-	// Add every contact of the post to the global contact table
-	poco_store($arr);
-
 	create_tags_from_item($current_post);
 	create_files_from_item($current_post);
 

include/ostatus.php

@@ -164,8 +164,6 @@ function ostatus_fetchauthor($xpath, $context, $importer, &$contact, $onlyfetch)
 			update_contact_avatar($author["author-avatar"], $importer["uid"], $contact["id"]);
 		}
 
-
-		/// @todo Add the "addr" field
 		$contact["generation"] = 2;
 		$contact["photo"] = $author["author-avatar"];
 		update_gcontact($contact);
@@ -626,6 +624,59 @@ function check_conversations($mentions = false, $override = false) {
 	set_config('system','ostatus_last_poll', time());
 }
 
+/**
+ * @brief Updates the gcontact table with actor data from the conversation
+ *
+ * @param object $actor The actor object that contains the contact data
+ */
+function ostatus_conv_fetch_actor($actor) {
+
+	// We set the generation to "3" since the data here is not as reliable as the data we get on other occasions
+	$contact = array("network" => NETWORK_OSTATUS, "generation" => 3);
+
+	if (isset($actor->url))
+		$contact["url"] = $actor->url;
+
+	if (isset($actor->displayName))
+		$contact["name"] = $actor->displayName;
+
+	if (isset($actor->portablecontacts_net->displayName))
+		$contact["name"] = $actor->portablecontacts_net->displayName;
+
+	if (isset($actor->portablecontacts_net->preferredUsername))
+		$contact["nick"] = $actor->portablecontacts_net->preferredUsername;
+
+	if (isset($actor->id))
+		$contact["alias"] = $actor->id;
+
+	if (isset($actor->summary))
+		$contact["about"] = $actor->summary;
+
+	if (isset($actor->portablecontacts_net->note))
+		$contact["about"] = $actor->portablecontacts_net->note;
+
+	if (isset($actor->portablecontacts_net->addresses->formatted))
+		$contact["location"] = $actor->portablecontacts_net->addresses->formatted;
+
+
+	if (isset($actor->image->url))
+		$contact["photo"] = $actor->image->url;
+
+	if (isset($actor->image->width))
+		$avatarwidth = $actor->image->width;
+
+	if (is_array($actor->status_net->avatarLinks))
+		foreach ($actor->status_net->avatarLinks AS $avatar) {
+			if ($avatarsize < $avatar->width) {
+				$contact["photo"] = $avatar->url;
+				$avatarsize = $avatar->width;
+			}
+		}
+
+	update_gcontact($contact);
+}
+
+
 function ostatus_completion($conversation_url, $uid, $item = array()) {
 
 	$a = get_app();
@@ -729,6 +780,9 @@ function ostatus_completion($conversation_url, $uid, $item = array()) {
 
 	foreach ($items as $single_conv) {
 
+		// Update the gcontact table
+		ostatus_conv_fetch_actor($single_conv->actor);
+
 		// Test - remove before flight
 		//$tempfile = tempnam(get_temppath(), "conversation");
 		//file_put_contents($tempfile, json_encode($single_conv));

include/socgraph.php

@@ -10,7 +10,8 @@
 require_once('include/datetime.php');
 require_once("include/Scrape.php");
 require_once("include/html2bbcode.php");
-
+require_once("include/Contact.php");
+require_once("include/Photo.php");
 
 /*
  * poco_load
@@ -428,7 +429,7 @@ function poco_last_updated($profile, $force = false) {
 	if (($gcontacts[0]["server_url"] != "") AND ($gcontacts[0]["nick"] != "")) {
 
 		//  Use noscrape if possible
-		$server = q("SELECT `noscrape` FROM `gserver` WHERE `nurl` = '%s' AND `noscrape` != ''", dbesc(normalise_link($gcontacts[0]["server_url"])));
+		$server = q("SELECT `noscrape`, `network` FROM `gserver` WHERE `nurl` = '%s' AND `noscrape` != ''", dbesc(normalise_link($gcontacts[0]["server_url"])));
 
 		if ($server) {
 			$noscraperet = z_fetch_url($server[0]["noscrape"]."/".$gcontacts[0]["nick"]);
@@ -437,67 +438,42 @@ function poco_last_updated($profile, $force = false) {
 
 				$noscrape = json_decode($noscraperet["body"], true);
 
-				if (($noscrape["fn"] != "") AND ($noscrape["fn"] != $gcontacts[0]["name"]))
+				$contact = array("url" => $profile,
-					q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'",
+						"network" => $server[0]["network"],
-						dbesc($noscrape["fn"]), dbesc(normalise_link($profile)));
+						"generation" => $gcontacts[0]["generation"]);
 
-				if (($noscrape["photo"] != "") AND ($noscrape["photo"] != $gcontacts[0]["photo"]))
+				$contact["name"] = $noscrape["fn"];
-					q("UPDATE `gcontact` SET `photo` = '%s' WHERE `nurl` = '%s'",
+				$contact["community"] = $noscrape["comm"];
-						dbesc($noscrape["photo"]), dbesc(normalise_link($profile)));
 
-				if (($noscrape["updated"] != "") AND ($noscrape["updated"] != $gcontacts[0]["updated"]))
+				if (isset($noscrape["tags"])) {
-					q("UPDATE `gcontact` SET `updated` = '%s' WHERE `nurl` = '%s'",
-						dbesc($noscrape["updated"]), dbesc(normalise_link($profile)));
-
-				if (($noscrape["gender"] != "") AND ($noscrape["gender"] != $gcontacts[0]["gender"]))
-					q("UPDATE `gcontact` SET `gender` = '%s' WHERE `nurl` = '%s'",
-						dbesc($noscrape["gender"]), dbesc(normalise_link($profile)));
-
-				if (($noscrape["pdesc"] != "") AND ($noscrape["pdesc"] != $gcontacts[0]["about"]))
-					q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'",
-						dbesc($noscrape["pdesc"]), dbesc(normalise_link($profile)));
-
-				if (($noscrape["about"] != "") AND ($noscrape["about"] != $gcontacts[0]["about"]))
-					q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'",
-						dbesc($noscrape["about"]), dbesc(normalise_link($profile)));
-
-				if (isset($noscrape["comm"]) AND ($noscrape["comm"] != $gcontacts[0]["community"]))
-					q("UPDATE `gcontact` SET `community` = %d WHERE `nurl` = '%s'",
-						intval($noscrape["comm"]), dbesc(normalise_link($profile)));
-
-				if (isset($noscrape["tags"]))
 					$keywords = implode(" ", $noscrape["tags"]);
-				else
+					if ($keywords != "")
-					$keywords = "";
+						$contact["keywords"] = $keywords;
-
-				if (($keywords != "") AND ($keywords != $gcontacts[0]["keywords"]))
-					q("UPDATE `gcontact` SET `keywords` = '%s' WHERE `nurl` = '%s'",
-						dbesc($keywords), dbesc(normalise_link($profile)));
-
-				$location = $noscrape["locality"];
-
-				if ($noscrape["region"] != "") {
-					if ($location != "")
-						$location .= ", ";
-
-					$location .= $noscrape["region"];
 				}
 
-				if ($noscrape["country-name"] != "") {
+				$location = formatted_location($noscrape);
-					if ($location != "")
+				if ($location)
-						$location .= ", ";
+					$contact["location"] = $location;
 
-					$location .= $noscrape["country-name"];
+				$contact["notify"] = $noscrape["dfrn-notify"];
-				}
 
-				if (($location != "") AND ($location != $gcontacts[0]["location"]))
+				// Remove all fields that are not present in the gcontact table
-					q("UPDATE `gcontact` SET `location` = '%s' WHERE `nurl` = '%s'",
+				unset($noscrape["fn"]);
-						dbesc($location), dbesc(normalise_link($profile)));
+				unset($noscrape["key"]);
+				unset($noscrape["homepage"]);
+				unset($noscrape["comm"]);
+				unset($noscrape["tags"]);
+				unset($noscrape["locality"]);
+				unset($noscrape["region"]);
+				unset($noscrape["country-name"]);
+				unset($noscrape["contacts"]);
+				unset($noscrape["dfrn-request"]);
+				unset($noscrape["dfrn-confirm"]);
+				unset($noscrape["dfrn-notify"]);
+				unset($noscrape["dfrn-poll"]);
 
-				// If we got data from noscrape then mark the contact as reachable
+				$contact = array_merge($contact, $noscrape);
-				if (is_array($noscrape) AND count($noscrape))
+				update_gcontact($contact);
-					q("UPDATE `gcontact` SET `last_contact` = '%s' WHERE `nurl` = '%s'",
-						dbesc(datetime_convert()), dbesc(normalise_link($profile)));
 
 				return $noscrape["updated"];
 			}
@@ -534,25 +510,22 @@ function poco_last_updated($profile, $force = false) {
 		return false;
 	}
 
-	if (($data["name"] != "") AND ($data["name"] != $gcontacts[0]["name"]))
+	$contact = array("generation" => $gcontacts[0]["generation"]);
-		q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'",
-			dbesc($data["name"]), dbesc(normalise_link($profile)));
 
-	if (($data["nick"] != "") AND ($data["nick"] != $gcontacts[0]["nick"]))
+	$contact = array_merge($contact, $data);
-		q("UPDATE `gcontact` SET `nick` = '%s' WHERE `nurl` = '%s'",
-			dbesc($data["nick"]), dbesc(normalise_link($profile)));
 
-	if (($data["addr"] != "") AND ($data["addr"] != $gcontacts[0]["connect"]))
+	$contact["server_url"] = $data["baseurl"];
-		q("UPDATE `gcontact` SET `connect` = '%s' WHERE `nurl` = '%s'",
-			dbesc($data["addr"]), dbesc(normalise_link($profile)));
 
-	if (($data["photo"] != "") AND ($data["photo"] != $gcontacts[0]["photo"]))
+	unset($contact["batch"]);
-		q("UPDATE `gcontact` SET `photo` = '%s' WHERE `nurl` = '%s'",
+	unset($contact["poll"]);
-			dbesc($data["photo"]), dbesc(normalise_link($profile)));
+	unset($contact["request"]);
+	unset($contact["confirm"]);
+	unset($contact["poco"]);
+	unset($contact["priority"]);
+	unset($contact["pubkey"]);
+	unset($contact["baseurl"]);
 
-	if (($data["baseurl"] != "") AND ($data["baseurl"] != $gcontacts[0]["server_url"]))
+	update_gcontact($contact);
-		q("UPDATE `gcontact` SET `server_url` = '%s' WHERE `nurl` = '%s'",
-			dbesc($data["baseurl"]), dbesc(normalise_link($profile)));
 
 	$feedret = z_fetch_url($data["poll"]);
 
@@ -921,88 +894,6 @@ function poco_check_server($server_url, $network = "", $force = false) {
 	return !$failure;
 }
 
-function poco_contact_from_body($body, $created, $cid, $uid) {
-	preg_replace_callback("/\[share(.*?)\].*?\[\/share\]/ism",
-		function ($match) use ($created, $cid, $uid){
-			return(sub_poco_from_share($match, $created, $cid, $uid));
-		}, $body);
-}
-
-function sub_poco_from_share($share, $created, $cid, $uid) {
-	$profile = "";
-	preg_match("/profile='(.*?)'/ism", $share[1], $matches);
-	if ($matches[1] != "")
-		$profile = $matches[1];
-
-	preg_match('/profile="(.*?)"/ism', $share[1], $matches);
-	if ($matches[1] != "")
-		$profile = $matches[1];
-
-	if ($profile == "")
-		return;
-
-	logger("prepare poco_check for profile ".$profile, LOGGER_DEBUG);
-	poco_check($profile, "", "", "", "", "", "", "", "", $created, 3, $cid, $uid);
-}
-
-function poco_store($item) {
-
-	// Isn't it public?
-	if ($item['private'])
-		return;
-
-	// Or is it from a network where we don't store the global contacts?
-	if (!in_array($item["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, NETWORK_STATUSNET, "")))
-		return;
-
-	// Is it a global copy?
-	$store_gcontact = ($item["uid"] == 0);
-
-	// Is it a comment on a global copy?
-	if (!$store_gcontact AND ($item["uri"] != $item["parent-uri"])) {
-		$q = q("SELECT `id` FROM `item` WHERE `uri`='%s' AND `uid` = 0", $item["parent-uri"]);
-		$store_gcontact = count($q);
-	}
-
-	if (!$store_gcontact)
-		return;
-
-	// "3" means: We don't know this contact directly (Maybe a reshared item)
-	$generation = 3;
-	$network = "";
-	$profile_url = $item["author-link"];
-
-	// Is it a user from our server?
-	$q = q("SELECT `id` FROM `contact` WHERE `self` AND `nurl` = '%s' LIMIT 1",
-		dbesc(normalise_link($item["author-link"])));
-	if (count($q)) {
-		logger("Our user (generation 1): ".$item["author-link"], LOGGER_DEBUG);
-		$generation = 1;
-		$network = NETWORK_DFRN;
-	} else { // Is it a contact from a user on our server?
-		$q = q("SELECT `network`, `url` FROM `contact` WHERE `uid` != 0 AND `network` != ''
-			AND (`nurl` = '%s' OR `alias` IN ('%s', '%s')) AND `network` != '%s' LIMIT 1",
-			dbesc(normalise_link($item["author-link"])),
-			dbesc(normalise_link($item["author-link"])),
-			dbesc($item["author-link"]),
-			dbesc(NETWORK_STATUSNET));
-		if (count($q)) {
-			$generation = 2;
-			$network = $q[0]["network"];
-			$profile_url = $q[0]["url"];
-			logger("Known contact (generation 2): ".$profile_url, LOGGER_DEBUG);
-		}
-	}
-
-	if ($generation == 3)
-		logger("Unknown contact (generation 3): ".$item["author-link"], LOGGER_DEBUG);
-
-	poco_check($profile_url, $item["author-name"], $network, $item["author-avatar"], "", "", "", "", "", $item["received"], $generation, $item["contact-id"], $item["uid"]);
-
-	// Maybe its a body with a shared item? Then extract a global contact from it.
-	poco_contact_from_body($item["body"], $item["received"], $item["contact-id"], $item["uid"]);
-}
-
 function count_common_friends($uid,$cid) {
 
 	$r = q("SELECT count(*) as `total`
@@ -1531,9 +1422,17 @@ function update_gcontact($contact) {
 	unset($fields["url"]);
 	unset($fields["updated"]);
 
+	// Bugfix: We had an error in the storing of keywords which lead to the "0"
+	// This value is still transmitted via poco.
+	if ($contact["keywords"] == "0")
+		unset($contact["keywords"]);
+
+	if ($r[0]["keywords"] == "0")
+		$r[0]["keywords"] = "";
+
 	// assign all unassigned fields from the database entry
 	foreach ($fields AS $field => $data)
-		if (!isset($contact[$field]))
+		if (!isset($contact[$field]) OR ($contact[$field] == ""))
 			$contact[$field] = $r[0][$field];
 
 	if ($contact["network"] == NETWORK_STATUSNET)
@@ -1542,18 +1441,48 @@ function update_gcontact($contact) {
 	if (!isset($contact["updated"]))
 		$contact["updated"] = datetime_convert();
 
+	if ($contact["server_url"] == "") {
+		$server_url = $contact["url"];
+
+		$server_url = matching_url($server_url, $contact["alias"]);
+		if ($server_url != "")
+			$contact["server_url"] = $server_url;
+
+		$server_url = matching_url($server_url, $contact["photo"]);
+		if ($server_url != "")
+			$contact["server_url"] = $server_url;
+
+		$server_url = matching_url($server_url, $contact["notify"]);
+		if ($server_url != "")
+			$contact["server_url"] = $server_url;
+	} else
+		$contact["server_url"] = normalise_link($contact["server_url"]);
+
+	if (($contact["addr"] == "") AND ($contact["server_url"] != "") AND ($contact["nick"] != "")) {
+		$hostname = str_replace("http://", "", $contact["server_url"]);
+		$contact["addr"] = $contact["nick"]."@".$hostname;
+	}
+
 	// Check if any field changed
 	$update = false;
 	unset($fields["generation"]);
 
+	if ((($contact["generation"] > 0) AND ($contact["generation"] <= $r[0]["generation"])) OR ($r[0]["generation"] == 0)) {
 		foreach ($fields AS $field => $data)
-		if ($contact[$field] != $r[0][$field])
+			if ($contact[$field] != $r[0][$field]) {
+				logger("Difference for contact ".$contact["url"]." in field '".$field."'. New value: '".$contact[$field]."', old value '".$r[0][$field]."'", LOGGER_DEBUG);
 				$update = true;
+			}
 
-	if ($contact["generation"] < $r[0]["generation"])
+		if ($contact["generation"] < $r[0]["generation"]) {
+			logger("Difference for contact ".$contact["url"]." in field 'generation'. new value: '".$contact["generation"]."', old value '".$r[0]["generation"]."'", LOGGER_DEBUG);
 			$update = true;
+		}
+	}
 
 	if ($update) {
+		logger("Update gcontact for ".$contact["url"]." Callstack: ".App::callstack(), LOGGER_DEBUG);
+
 		q("UPDATE `gcontact` SET `photo` = '%s', `name` = '%s', `nick` = '%s', `addr` = '%s', `network` = '%s',
 					`birthday` = '%s', `gender` = '%s', `keywords` = '%s', `hide` = %d, `nsfw` = %d,
 					`alias` = '%s', `notify` = '%s', `url` = '%s',
@@ -1568,6 +1497,28 @@ function update_gcontact($contact) {
 			intval($contact["generation"]), dbesc($contact["updated"]),
 			dbesc($contact["server_url"]), dbesc($contact["connect"]),
 			dbesc(normalise_link($contact["url"])), intval($contact["generation"]));
+
+
+		// Now update the contact entry with the user id "0" as well.
+		// This is used for the shadow copies of public items.
+		$r = q("SELECT `id` FROM `contact` WHERE `nurl` = '%s' AND `uid` = 0 ORDER BY `id` LIMIT 1",
+			dbesc(normalise_link($contact["url"])));
+
+		if ($r) {
+			logger("Update shadow contact ".$r[0]["id"], LOGGER_DEBUG);
+
+			update_contact_avatar($contact["photo"], 0, $r[0]["id"]);
+
+			q("UPDATE `contact` SET `name` = '%s', `nick` = '%s', `addr` = '%s',
+						`network` = '%s', `bd` = '%s', `gender` = '%s',
+						`keywords` = '%s', `alias` = '%s', `url` = '%s',
+						`location` = '%s', `about` = '%s'
+					WHERE `id` = %d",
+				dbesc($contact["name"]), dbesc($contact["nick"]), dbesc($contact["addr"]),
+				dbesc($contact["network"]), dbesc($contact["birthday"]), dbesc($contact["gender"]),
+				dbesc($contact["keywords"]), dbesc($contact["alias"]), dbesc($contact["url"]),
+				dbesc($contact["location"]), dbesc($contact["about"]), intval($r[0]["id"]));
+		}
 	}
 
 	return $gcontact_id;
@@ -1581,7 +1532,9 @@ function update_gcontact($contact) {
 function update_gcontact_from_probe($url) {
 	$data = probe_url($url);
 
-	if ($data["network"] != NETWORK_PHANTOM)
+	if ($data["network"] == NETWORK_PHANTOM)
+		return;
+
 	update_gcontact($data);
 }
 

mod/item.php

@@ -160,6 +160,9 @@ function item_post(&$a) {
 				logger('no contact found: '.print_r($thrparent, true), LOGGER_DEBUG);
 			} else
 				logger('parent contact: '.print_r($parent_contact, true), LOGGER_DEBUG);
+
+			if ($parent_contact["nick"] == "")
+				$parent_contact["nick"] = $parent_contact["name"];
 		}
 	}
 

mod/noscrape.php

@@ -22,13 +22,17 @@ function noscrape_init(&$a) {
 	$keywords = str_replace(array('#',',',' ',',,'),array('',' ',',',','),$keywords);
 	$keywords = explode(',', $keywords);
 
+	$r = q("SELECT `photo` FROM `contact` WHERE `self` AND `uid` = %d",
+		intval($a->profile['uid']));
+
 	$json_info = array(
 		'fn' => $a->profile['name'],
 		'addr' => $a->profile['addr'],
+		'nick' => $a->user['nickname'],
 		'key' => $a->profile['pubkey'],
 		'homepage' => $a->get_baseurl()."/profile/{$which}",
 		'comm' => (x($a->profile,'page-flags')) && ($a->profile['page-flags'] == PAGE_COMMUNITY),
-		'photo' => $a->profile['photo'],
+		'photo' => $r[0]["photo"],
 		'tags' => $keywords
 	);