From 2ca6cdf6b6e0fa4a04fb1d1fb6bf5d83f9ba74e8 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 13 Feb 2016 12:26:58 +0100 Subject: [PATCH 01/10] Improvements how gcontact entries are updated --- boot.php | 26 +++-- database.sql | 18 +--- include/Scrape.php | 15 ++- include/dfrn.php | 50 ++++++---- include/items.php | 13 +-- include/socgraph.php | 219 ++++++++++++------------------------------- mod/noscrape.php | 6 +- 7 files changed, 133 insertions(+), 214 deletions(-) diff --git a/boot.php b/boot.php index 4ef30eadac..2b7d814123 100644 --- a/boot.php +++ b/boot.php @@ -1037,19 +1037,29 @@ class App { $this->performance[$value] += (float)$duration; $this->performance["marktime"] += (float)$duration; - // Trace the different functions with their timestamps - $trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 5); + $callstack = $this->callstack(); + $this->callstack[$value][$callstack] += (float)$duration; + + } + + /** + * @brief Returns a string with a callstack. Can be used for logging. + * + * @return string + */ + function callstack() { + $trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 6); + + // We remove the first two items from the list since they contain data that we don't need. + array_shift($trace); array_shift($trace); - $function = array(); + $callstack = array(); foreach ($trace AS $func) - $function[] = $func["function"]; - - $function = implode(", ", $function); - - $this->callstack[$value][$function] += (float)$duration; + $callstack[] = $func["function"]; + return implode(", ", $callstack); } function mark_timestamp($mark) { diff --git a/database.sql b/database.sql index 70b315ea24..25faf0f4c0 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 3.5-dev (Asparagus) --- DB_UPDATE_VERSION 1193 +-- DB_UPDATE_VERSION 1194 -- ------------------------------------------ @@ -119,6 +119,7 @@ CREATE TABLE IF NOT EXISTS `contact` ( `keywords` text NOT NULL, `gender` varchar(32) NOT NULL DEFAULT '', `attag` varchar(255) NOT NULL DEFAULT '', + `avatar` varchar(255) NOT NULL DEFAULT '', `photo` text NOT NULL, `thumb` text NOT NULL, `micro` text NOT NULL, @@ -411,21 +412,6 @@ CREATE TABLE IF NOT EXISTS `gserver` ( INDEX `nurl` (`nurl`) ) DEFAULT CHARSET=utf8; --- --- TABLE guid --- -CREATE TABLE IF NOT EXISTS `guid` ( - `id` int(10) unsigned NOT NULL auto_increment, - `guid` varchar(255) NOT NULL DEFAULT '', - `plink` varchar(255) NOT NULL DEFAULT '', - `uri` varchar(255) NOT NULL DEFAULT '', - `network` varchar(32) NOT NULL DEFAULT '', - PRIMARY KEY(`id`), - INDEX `guid` (`guid`), - INDEX `plink` (`plink`), - INDEX `uri` (`uri`) -) DEFAULT CHARSET=utf8; - -- -- TABLE hook -- diff --git a/include/Scrape.php b/include/Scrape.php index ca6489b16a..bc6aebbdcf 100644 --- a/include/Scrape.php +++ b/include/Scrape.php @@ -12,6 +12,18 @@ function scrape_dfrn($url, $dont_probe = false) { logger('scrape_dfrn: url=' . $url); + // Try to fetch the data from noscrape. This is faster than parsing the HTML + $noscrape = str_replace("/hcard/", "/noscrape/", $url); + $noscrapejson = fetch_url($noscrape); + $noscrapedata = array(); + if ($noscrapejson) { + $noscrapedata = json_decode($noscrapejson, true); + + if (is_array($noscrapedata)) + if ($noscrapedata["nick"] != "") + return($noscrapedata); + } + $s = fetch_url($url); if(! $s) @@ -91,8 +103,7 @@ function scrape_dfrn($url, $dont_probe = false) { } } } - - return $ret; + return array_merge($ret, $noscrapedata); }} diff --git a/include/dfrn.php b/include/dfrn.php index 8938a9b6f7..a37aaf29ce 100644 --- a/include/dfrn.php +++ b/include/dfrn.php @@ -1115,13 +1115,13 @@ class dfrn { * * @return Returns an array with relevant data of the author */ - private function fetchauthor($xpath, $context, $importer, $element, $onlyfetch) { + private function fetchauthor($xpath, $context, $importer, $element, $onlyfetch, $xml = "") { $author = array(); $author["name"] = $xpath->evaluate($element."/atom:name/text()", $context)->item(0)->nodeValue; $author["link"] = $xpath->evaluate($element."/atom:uri/text()", $context)->item(0)->nodeValue; - $r = q("SELECT `id`, `uid`, `network`, `avatar-date`, `name-date`, `uri-date`, `addr`, + $r = q("SELECT `id`, `uid`, `url`, `network`, `avatar-date`, `name-date`, `uri-date`, `addr`, `name`, `nick`, `about`, `location`, `keywords`, `bdyear`, `bd` FROM `contact` WHERE `uid` = %d AND `nurl` = '%s' AND `network` != '%s'", intval($importer["uid"]), dbesc(normalise_link($author["link"])), dbesc(NETWORK_STATUSNET)); @@ -1130,6 +1130,9 @@ class dfrn { $author["contact-id"] = $r[0]["id"]; $author["network"] = $r[0]["network"]; } else { + if (!$onlyfetch) + logger("Contact ".$author["link"]." wasn't found for user ".$importer["uid"]." XML: ".$xml, LOGGER_DEBUG); + $author["contact-id"] = $importer["id"]; $author["network"] = $importer["network"]; $onlyfetch = true; @@ -1159,38 +1162,41 @@ class dfrn { } if ($r AND !$onlyfetch) { + logger("Check if contact details for contact ".$r[0]["id"]." (".$r[0]["nick"].") have to be updated.", LOGGER_DEBUG); + + $poco = array("url" => $contact["url"]); // When was the last change to name or uri? $name_element = $xpath->query($element."/atom:name", $context)->item(0); foreach($name_element->attributes AS $attributes) if ($attributes->name == "updated") - $contact["name-date"] = $attributes->textContent; + $poco["name-date"] = $attributes->textContent; $link_element = $xpath->query($element."/atom:link", $context)->item(0); foreach($link_element->attributes AS $attributes) if ($attributes->name == "updated") - $contact["uri-date"] = $attributes->textContent; + $poco["uri-date"] = $attributes->textContent; // Update contact data $value = $xpath->evaluate($element."/dfrn:handle/text()", $context)->item(0)->nodeValue; if ($value != "") - $contact["addr"] = $value; + $poco["addr"] = $value; $value = $xpath->evaluate($element."/poco:displayName/text()", $context)->item(0)->nodeValue; if ($value != "") - $contact["name"] = $value; + $poco["name"] = $value; $value = $xpath->evaluate($element."/poco:preferredUsername/text()", $context)->item(0)->nodeValue; if ($value != "") - $contact["nick"] = $value; + $poco["nick"] = $value; $value = $xpath->evaluate($element."/poco:note/text()", $context)->item(0)->nodeValue; if ($value != "") - $contact["about"] = $value; + $poco["about"] = $value; $value = $xpath->evaluate($element."/poco:address/poco:formatted/text()", $context)->item(0)->nodeValue; if ($value != "") - $contact["location"] = $value; + $poco["location"] = $value; /// @todo Add support for the following fields that we don't support by now in the contact table: /// - poco:utcOffset @@ -1207,7 +1213,7 @@ class dfrn { $tags[$tag->nodeValue] = $tag->nodeValue; if (count($tags)) - $contact["keywords"] = implode(", ", $tags); + $poco["keywords"] = implode(", ", $tags); // "dfrn:birthday" contains the birthday converted to UTC $old_bdyear = $contact["bdyear"]; @@ -1217,7 +1223,7 @@ class dfrn { if (strtotime($birthday) > time()) { $bd_timestamp = strtotime($birthday); - $contact["bdyear"] = date("Y", $bd_timestamp); + $poco["bdyear"] = date("Y", $bd_timestamp); } // "poco:birthday" is the birthday in the format "yyyy-mm-dd" @@ -1232,9 +1238,11 @@ class dfrn { $bdyear = $bdyear + 1; } - $contact["bd"] = $value; + $poco["bd"] = $value; } + $contact = array_merge($contact, $poco); + if ($old_bdyear != $contact["bdyear"]) self::birthday_event($contact, $birthday); @@ -1245,6 +1253,7 @@ class dfrn { unset($fields["id"]); unset($fields["uid"]); + unset($fields["url"]); unset($fields["avatar-date"]); unset($fields["name-date"]); unset($fields["uri-date"]); @@ -1264,7 +1273,7 @@ class dfrn { } if ($update) { - logger("Update contact data for contact ".$contact["id"], LOGGER_DEBUG); + logger("Update contact data for contact ".$contact["id"]." (".$contact["nick"].")", LOGGER_DEBUG); q("UPDATE `contact` SET `name` = '%s', `nick` = '%s', `about` = '%s', `location` = '%s', `addr` = '%s', `keywords` = '%s', `bdyear` = '%s', `bd` = '%s', @@ -1283,9 +1292,10 @@ class dfrn { // It is used in the socgraph.php to prevent that old contact data // that was relayed over several servers can overwrite contact // data that we received directly. - $contact["generation"] = 2; - $contact["photo"] = $author["avatar"]; - update_gcontact($contact); + + $poco["generation"] = 2; + $poco["photo"] = $author["avatar"]; + update_gcontact($poco); } return($author); @@ -2369,8 +2379,14 @@ class dfrn { $header["contact-id"] = $importer["id"]; // Update the contact table if the data has changed + + // The "atom:author" is only present in feeds + if ($xpath->query("/atom:feed/atom:author")->length > 0) + self::fetchauthor($xpath, $doc->firstChild, $importer, "atom:author", false, $xml); + // Only the "dfrn:owner" in the head section contains all data - self::fetchauthor($xpath, $doc->firstChild, $importer, "dfrn:owner", false); + if ($xpath->query("/atom:feed/dfrn:owner")->length > 0) + self::fetchauthor($xpath, $doc->firstChild, $importer, "dfrn:owner", false, $xml); logger("Import DFRN message for user ".$importer["uid"]." from contact ".$importer["id"], LOGGER_DEBUG); diff --git a/include/items.php b/include/items.php index 1af6fe1b52..8d6b5b471c 100644 --- a/include/items.php +++ b/include/items.php @@ -500,14 +500,8 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa $arr['file'] = ((x($arr,'file')) ? trim($arr['file']) : ''); - if (($arr['author-link'] == "") AND ($arr['owner-link'] == "")) { - $trace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS, 5); - foreach ($trace AS $func) - $function[] = $func["function"]; - - $function = implode(", ", $function); - logger("Both author-link and owner-link are empty. Called by: ".$function, LOGGER_DEBUG); - } + if (($arr['author-link'] == "") AND ($arr['owner-link'] == "")) + logger("Both author-link and owner-link are empty. Called by: ".App::callstack(), LOGGER_DEBUG); if ($arr['plink'] == "") { $a = get_app(); @@ -888,9 +882,6 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa logger('item_store: new item not found in DB, id ' . $current_post); } - // Add every contact of the post to the global contact table - poco_store($arr); - create_tags_from_item($current_post); create_files_from_item($current_post); diff --git a/include/socgraph.php b/include/socgraph.php index 3b8e9140f8..8ad7a4cd31 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -10,6 +10,7 @@ require_once('include/datetime.php'); require_once("include/Scrape.php"); require_once("include/html2bbcode.php"); +require_once("include/Contact.php"); /* @@ -428,7 +429,7 @@ function poco_last_updated($profile, $force = false) { if (($gcontacts[0]["server_url"] != "") AND ($gcontacts[0]["nick"] != "")) { // Use noscrape if possible - $server = q("SELECT `noscrape` FROM `gserver` WHERE `nurl` = '%s' AND `noscrape` != ''", dbesc(normalise_link($gcontacts[0]["server_url"]))); + $server = q("SELECT `noscrape`, `network` FROM `gserver` WHERE `nurl` = '%s' AND `noscrape` != ''", dbesc(normalise_link($gcontacts[0]["server_url"]))); if ($server) { $noscraperet = z_fetch_url($server[0]["noscrape"]."/".$gcontacts[0]["nick"]); @@ -437,67 +438,42 @@ function poco_last_updated($profile, $force = false) { $noscrape = json_decode($noscraperet["body"], true); - if (($noscrape["fn"] != "") AND ($noscrape["fn"] != $gcontacts[0]["name"])) - q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'", - dbesc($noscrape["fn"]), dbesc(normalise_link($profile))); + $contact = array("url" => $profile, + "network" => $server[0]["network"], + "generation" => $gcontacts[0]["generation"]); - if (($noscrape["photo"] != "") AND ($noscrape["photo"] != $gcontacts[0]["photo"])) - q("UPDATE `gcontact` SET `photo` = '%s' WHERE `nurl` = '%s'", - dbesc($noscrape["photo"]), dbesc(normalise_link($profile))); + $contact["name"] = $noscrape["fn"]; + $contact["community"] = $noscrape["comm"]; - if (($noscrape["updated"] != "") AND ($noscrape["updated"] != $gcontacts[0]["updated"])) - q("UPDATE `gcontact` SET `updated` = '%s' WHERE `nurl` = '%s'", - dbesc($noscrape["updated"]), dbesc(normalise_link($profile))); - - if (($noscrape["gender"] != "") AND ($noscrape["gender"] != $gcontacts[0]["gender"])) - q("UPDATE `gcontact` SET `gender` = '%s' WHERE `nurl` = '%s'", - dbesc($noscrape["gender"]), dbesc(normalise_link($profile))); - - if (($noscrape["pdesc"] != "") AND ($noscrape["pdesc"] != $gcontacts[0]["about"])) - q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'", - dbesc($noscrape["pdesc"]), dbesc(normalise_link($profile))); - - if (($noscrape["about"] != "") AND ($noscrape["about"] != $gcontacts[0]["about"])) - q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'", - dbesc($noscrape["about"]), dbesc(normalise_link($profile))); - - if (isset($noscrape["comm"]) AND ($noscrape["comm"] != $gcontacts[0]["community"])) - q("UPDATE `gcontact` SET `community` = %d WHERE `nurl` = '%s'", - intval($noscrape["comm"]), dbesc(normalise_link($profile))); - - if (isset($noscrape["tags"])) + if (isset($noscrape["tags"])) { $keywords = implode(" ", $noscrape["tags"]); - else - $keywords = ""; - - if (($keywords != "") AND ($keywords != $gcontacts[0]["keywords"])) - q("UPDATE `gcontact` SET `keywords` = '%s' WHERE `nurl` = '%s'", - dbesc($keywords), dbesc(normalise_link($profile))); - - $location = $noscrape["locality"]; - - if ($noscrape["region"] != "") { - if ($location != "") - $location .= ", "; - - $location .= $noscrape["region"]; + if ($keywords != "") + $contact["keywords"] = $keywords; } - if ($noscrape["country-name"] != "") { - if ($location != "") - $location .= ", "; + $location = formatted_location($noscrape); + if ($location) + $contact["location"] = $location; - $location .= $noscrape["country-name"]; - } + $contact["notify"] = $noscrape["dfrn-notify"]; - if (($location != "") AND ($location != $gcontacts[0]["location"])) - q("UPDATE `gcontact` SET `location` = '%s' WHERE `nurl` = '%s'", - dbesc($location), dbesc(normalise_link($profile))); + // Remove all fields that are not present in the gcontact table + unset($noscrape["fn"]); + unset($noscrape["key"]); + unset($noscrape["homepage"]); + unset($noscrape["comm"]); + unset($noscrape["tags"]); + unset($noscrape["locality"]); + unset($noscrape["region"]); + unset($noscrape["country-name"]); + unset($noscrape["contacts"]); + unset($noscrape["dfrn-request"]); + unset($noscrape["dfrn-confirm"]); + unset($noscrape["dfrn-notify"]); + unset($noscrape["dfrn-poll"]); - // If we got data from noscrape then mark the contact as reachable - if (is_array($noscrape) AND count($noscrape)) - q("UPDATE `gcontact` SET `last_contact` = '%s' WHERE `nurl` = '%s'", - dbesc(datetime_convert()), dbesc(normalise_link($profile))); + $contact = array_merge($contact, $noscrape); + update_gcontact($contact); return $noscrape["updated"]; } @@ -534,25 +510,22 @@ function poco_last_updated($profile, $force = false) { return false; } - if (($data["name"] != "") AND ($data["name"] != $gcontacts[0]["name"])) - q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'", - dbesc($data["name"]), dbesc(normalise_link($profile))); + $contact = array("generation" => $gcontacts[0]["generation"]); - if (($data["nick"] != "") AND ($data["nick"] != $gcontacts[0]["nick"])) - q("UPDATE `gcontact` SET `nick` = '%s' WHERE `nurl` = '%s'", - dbesc($data["nick"]), dbesc(normalise_link($profile))); + $contact = array_merge($contact, $data); - if (($data["addr"] != "") AND ($data["addr"] != $gcontacts[0]["connect"])) - q("UPDATE `gcontact` SET `connect` = '%s' WHERE `nurl` = '%s'", - dbesc($data["addr"]), dbesc(normalise_link($profile))); + $contact["server_url"] = $data["baseurl"]; - if (($data["photo"] != "") AND ($data["photo"] != $gcontacts[0]["photo"])) - q("UPDATE `gcontact` SET `photo` = '%s' WHERE `nurl` = '%s'", - dbesc($data["photo"]), dbesc(normalise_link($profile))); + unset($contact["batch"]); + unset($contact["poll"]); + unset($contact["request"]); + unset($contact["confirm"]); + unset($contact["poco"]); + unset($contact["priority"]); + unset($contact["pubkey"]); + unset($contact["baseurl"]); - if (($data["baseurl"] != "") AND ($data["baseurl"] != $gcontacts[0]["server_url"])) - q("UPDATE `gcontact` SET `server_url` = '%s' WHERE `nurl` = '%s'", - dbesc($data["baseurl"]), dbesc(normalise_link($profile))); + update_gcontact($contact); $feedret = z_fetch_url($data["poll"]); @@ -921,88 +894,6 @@ function poco_check_server($server_url, $network = "", $force = false) { return !$failure; } -function poco_contact_from_body($body, $created, $cid, $uid) { - preg_replace_callback("/\[share(.*?)\].*?\[\/share\]/ism", - function ($match) use ($created, $cid, $uid){ - return(sub_poco_from_share($match, $created, $cid, $uid)); - }, $body); -} - -function sub_poco_from_share($share, $created, $cid, $uid) { - $profile = ""; - preg_match("/profile='(.*?)'/ism", $share[1], $matches); - if ($matches[1] != "") - $profile = $matches[1]; - - preg_match('/profile="(.*?)"/ism', $share[1], $matches); - if ($matches[1] != "") - $profile = $matches[1]; - - if ($profile == "") - return; - - logger("prepare poco_check for profile ".$profile, LOGGER_DEBUG); - poco_check($profile, "", "", "", "", "", "", "", "", $created, 3, $cid, $uid); -} - -function poco_store($item) { - - // Isn't it public? - if ($item['private']) - return; - - // Or is it from a network where we don't store the global contacts? - if (!in_array($item["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, NETWORK_STATUSNET, ""))) - return; - - // Is it a global copy? - $store_gcontact = ($item["uid"] == 0); - - // Is it a comment on a global copy? - if (!$store_gcontact AND ($item["uri"] != $item["parent-uri"])) { - $q = q("SELECT `id` FROM `item` WHERE `uri`='%s' AND `uid` = 0", $item["parent-uri"]); - $store_gcontact = count($q); - } - - if (!$store_gcontact) - return; - - // "3" means: We don't know this contact directly (Maybe a reshared item) - $generation = 3; - $network = ""; - $profile_url = $item["author-link"]; - - // Is it a user from our server? - $q = q("SELECT `id` FROM `contact` WHERE `self` AND `nurl` = '%s' LIMIT 1", - dbesc(normalise_link($item["author-link"]))); - if (count($q)) { - logger("Our user (generation 1): ".$item["author-link"], LOGGER_DEBUG); - $generation = 1; - $network = NETWORK_DFRN; - } else { // Is it a contact from a user on our server? - $q = q("SELECT `network`, `url` FROM `contact` WHERE `uid` != 0 AND `network` != '' - AND (`nurl` = '%s' OR `alias` IN ('%s', '%s')) AND `network` != '%s' LIMIT 1", - dbesc(normalise_link($item["author-link"])), - dbesc(normalise_link($item["author-link"])), - dbesc($item["author-link"]), - dbesc(NETWORK_STATUSNET)); - if (count($q)) { - $generation = 2; - $network = $q[0]["network"]; - $profile_url = $q[0]["url"]; - logger("Known contact (generation 2): ".$profile_url, LOGGER_DEBUG); - } - } - - if ($generation == 3) - logger("Unknown contact (generation 3): ".$item["author-link"], LOGGER_DEBUG); - - poco_check($profile_url, $item["author-name"], $network, $item["author-avatar"], "", "", "", "", "", $item["received"], $generation, $item["contact-id"], $item["uid"]); - - // Maybe its a body with a shared item? Then extract a global contact from it. - poco_contact_from_body($item["body"], $item["received"], $item["contact-id"], $item["uid"]); -} - function count_common_friends($uid,$cid) { $r = q("SELECT count(*) as `total` @@ -1533,7 +1424,7 @@ function update_gcontact($contact) { // assign all unassigned fields from the database entry foreach ($fields AS $field => $data) - if (!isset($contact[$field])) + if (!isset($contact[$field]) OR ($contact[$field] == "")) $contact[$field] = $r[0][$field]; if ($contact["network"] == NETWORK_STATUSNET) @@ -1546,14 +1437,22 @@ function update_gcontact($contact) { $update = false; unset($fields["generation"]); - foreach ($fields AS $field => $data) - if ($contact[$field] != $r[0][$field]) - $update = true; + if ((($contact["generation"] > 0) AND ($contact["generation"] <= $r[0]["generation"])) OR ($r[0]["generation"] == 0)) { + foreach ($fields AS $field => $data) + if ($contact[$field] != $r[0][$field]) { + logger("Difference for contact ".$contact["url"]." in field '".$field."'. New value: '".$contact[$field]."', old value '".$r[0][$field]."'", LOGGER_DEBUG); + $update = true; + } - if ($contact["generation"] < $r[0]["generation"]) - $update = true; + if ($contact["generation"] < $r[0]["generation"]) { + logger("Difference for contact ".$contact["url"]." in field 'generation'. new value: '".$contact["generation"]."', old value '".$r[0]["generation"]."'", LOGGER_DEBUG); + $update = true; + } + } if ($update) { + logger("Update gcontact for ".$contact["url"]." Callstack: ".App::callstack(), LOGGER_DEBUG); + q("UPDATE `gcontact` SET `photo` = '%s', `name` = '%s', `nick` = '%s', `addr` = '%s', `network` = '%s', `birthday` = '%s', `gender` = '%s', `keywords` = '%s', `hide` = %d, `nsfw` = %d, `alias` = '%s', `notify` = '%s', `url` = '%s', @@ -1581,8 +1480,10 @@ function update_gcontact($contact) { function update_gcontact_from_probe($url) { $data = probe_url($url); - if ($data["network"] != NETWORK_PHANTOM) - update_gcontact($data); + if ($data["network"] == NETWORK_PHANTOM) + return; + + update_gcontact($data); } /** diff --git a/mod/noscrape.php b/mod/noscrape.php index 51bd7234cf..1f7105b769 100644 --- a/mod/noscrape.php +++ b/mod/noscrape.php @@ -22,13 +22,17 @@ function noscrape_init(&$a) { $keywords = str_replace(array('#',',',' ',',,'),array('',' ',',',','),$keywords); $keywords = explode(',', $keywords); + $r = q("SELECT `photo` FROM `contact` WHERE `self` AND `uid` = %d", + intval($a->profile['uid'])); + $json_info = array( 'fn' => $a->profile['name'], 'addr' => $a->profile['addr'], + 'nick' => $a->user['nickname'], 'key' => $a->profile['pubkey'], 'homepage' => $a->get_baseurl()."/profile/{$which}", 'comm' => (x($a->profile,'page-flags')) && ($a->profile['page-flags'] == PAGE_COMMUNITY), - 'photo' => $a->profile['photo'], + 'photo' => $r[0]["photo"], 'tags' => $keywords ); From 253ba45c1a9f080b5086decc9696c9cda89c7ca3 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 13 Feb 2016 13:30:53 +0100 Subject: [PATCH 02/10] Added a to-do --- include/dfrn.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/dfrn.php b/include/dfrn.php index a37aaf29ce..1db2c6b33d 100644 --- a/include/dfrn.php +++ b/include/dfrn.php @@ -1965,6 +1965,8 @@ class dfrn { $item['body'] = @html2bbcode($item['body']); } + /// @todo We should check for a repeated post and if we know the repeated author. + // We don't need the content element since "dfrn:env" is always present //$item["body"] = $xpath->query("atom:content/text()", $entry)->item(0)->nodeValue; From a4da9fb55db84fe8a88129fe542422b70ae8e68e Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 13 Feb 2016 14:09:08 +0100 Subject: [PATCH 03/10] Update the gcontact entry when the contact entry is checked for updates --- include/follow.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/follow.php b/include/follow.php index 22ff079b63..410e0e58aa 100644 --- a/include/follow.php +++ b/include/follow.php @@ -1,5 +1,6 @@ Date: Sat, 13 Feb 2016 21:35:05 +0100 Subject: [PATCH 04/10] OStatus: The gcontact table will now be updated from the conversation as well --- include/ostatus.php | 74 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/include/ostatus.php b/include/ostatus.php index 00022f8c6c..983ed6e1b4 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -626,6 +626,77 @@ function check_conversations($mentions = false, $override = false) { set_config('system','ostatus_last_poll', time()); } +/** + * @brief Updates the gcontact table with actor data from the conversation + * + * @param object $actor The actor object that contains the contact data + */ +function ostatus_conv_fetch_actor($actor) { + + // We set the generation to "3" since the data here is not as reliable as the data we get on other occasions + $contact = array("network" => NETWORK_OSTATUS, "generation" => 3); + + if (isset($actor->url)) + $contact["url"] = $actor->url; + + if (isset($actor->displayName)) + $contact["name"] = $actor->displayName; + + if (isset($actor->portablecontacts_net->displayName)) + $contact["name"] = $actor->portablecontacts_net->displayName; + + if (isset($actor->portablecontacts_net->preferredUsername)) + $contact["nick"] = $actor->portablecontacts_net->preferredUsername; + + if (isset($actor->id)) + $contact["alias"] = $actor->id; + + if (isset($actor->summary)) + $contact["about"] = $actor->summary; + + if (isset($actor->portablecontacts_net->note)) + $contact["about"] = $actor->portablecontacts_net->note; + + if (isset($actor->portablecontacts_net->addresses->formatted)) + $contact["location"] = $actor->portablecontacts_net->addresses->formatted; + + + if (isset($actor->image->url)) + $contact["photo"] = $actor->image->url; + + if (isset($actor->image->width)) + $avatarwidth = $actor->image->width; + + if (is_array($actor->status_net->avatarLinks)) + foreach ($actor->status_net->avatarLinks AS $avatar) { + if ($avatarsize < $avatar->width) { + $contact["photo"] = $avatar->url; + $avatarsize = $avatar->width; + } + } + + $contact["server_url"] = $contact["url"]; + + $server_url = matching($contact["server_url"], $contact["alias"]); + if (strlen($server_url) > 8) + $contact["server_url"] = $server_url; + + $server_url = matching($contact["server_url"], $contact["photo"]); + if (strlen($server_url) > 8) + $contact["server_url"] = $server_url; + + if (($contact["server_url"] == $contact["url"]) OR ($contact["server_url"] == $contact["alias"])) + unset($contact["server_url"]); + else { + $hostname = str_replace("http://", "", normalise_link($contact["server_url"])); + if ($hostname AND $contact["nick"]) + $contact["addr"] = $contact["nick"]."@".$hostname; + } + + update_gcontact($contact); +} + + function ostatus_completion($conversation_url, $uid, $item = array()) { $a = get_app(); @@ -729,6 +800,9 @@ function ostatus_completion($conversation_url, $uid, $item = array()) { foreach ($items as $single_conv) { + // Update the gcontact table + ostatus_conv_fetch_actor($single_conv->actor); + // Test - remove before flight //$tempfile = tempnam(get_temppath(), "conversation"); //file_put_contents($tempfile, json_encode($single_conv)); From 15296b8036a670fd32891d71927df81d9db6293a Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 13 Feb 2016 22:20:00 +0100 Subject: [PATCH 05/10] Avoid errors when noscrape data can't be fetched. --- include/Scrape.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/Scrape.php b/include/Scrape.php index bc6aebbdcf..d54a838afd 100644 --- a/include/Scrape.php +++ b/include/Scrape.php @@ -19,9 +19,11 @@ function scrape_dfrn($url, $dont_probe = false) { if ($noscrapejson) { $noscrapedata = json_decode($noscrapejson, true); - if (is_array($noscrapedata)) + if (is_array($noscrapedata)) { if ($noscrapedata["nick"] != "") return($noscrapedata); + } else + $noscrapedata = array(); } $s = fetch_url($url); From 5086b8b2a77c0d20898dd56e720a73f8228672fb Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 14 Feb 2016 00:14:03 +0100 Subject: [PATCH 06/10] DFRN Bugfix: The poco data wasn't sent --- include/dfrn.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/dfrn.php b/include/dfrn.php index 17ef541cd3..f7a05bdb63 100644 --- a/include/dfrn.php +++ b/include/dfrn.php @@ -95,7 +95,7 @@ class dfrn { $sql_extra = " AND `item`.`allow_cid` = '' AND `item`.`allow_gid` = '' AND `item`.`deny_cid` = '' AND `item`.`deny_gid` = '' "; - $r = q("SELECT `contact`.*, `user`.`uid` AS `user_uid`, `user`.`nickname`, `user`.`timezone`, `user`.`page-flags` + $r = q("SELECT `contact`.*, `user`.`nickname`, `user`.`timezone`, `user`.`page-flags` FROM `contact` INNER JOIN `user` ON `user`.`uid` = `contact`.`uid` WHERE `contact`.`self` = 1 AND `user`.`nickname` = '%s' LIMIT 1", dbesc($owner_nick) @@ -105,7 +105,7 @@ class dfrn { killme(); $owner = $r[0]; - $owner_id = $owner['user_uid']; + $owner_id = $owner['uid']; $owner_nick = $owner['nickname']; $sql_post_table = ""; @@ -483,7 +483,7 @@ class dfrn { "media:width" => 175, "media:height" => 175, "href" => $owner['photo']); xml_add_element($doc, $author, "link", "", $attributes); - $birthday = feed_birthday($owner['user_uid'], $owner['timezone']); + $birthday = feed_birthday($owner['uid'], $owner['timezone']); if ($birthday) xml_add_element($doc, $author, "dfrn:birthday", $birthday); @@ -498,7 +498,7 @@ class dfrn { FROM `profile` INNER JOIN `user` ON `user`.`uid` = `profile`.`uid` WHERE `profile`.`is-default` AND NOT `user`.`hidewall` AND `user`.`uid` = %d", - intval($owner['user_uid'])); + intval($owner['uid'])); if ($r) { $profile = $r[0]; xml_add_element($doc, $author, "poco:displayName", $profile["name"]); From b9e179207678576e6ade8243a4868b523c19c919 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 14 Feb 2016 07:31:57 +0100 Subject: [PATCH 07/10] The shadow contact (contact with uid=0) will be updated now as well. --- include/socgraph.php | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/include/socgraph.php b/include/socgraph.php index 8ad7a4cd31..ce5b08e641 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -11,7 +11,7 @@ require_once('include/datetime.php'); require_once("include/Scrape.php"); require_once("include/html2bbcode.php"); require_once("include/Contact.php"); - +require_once("include/Photo.php"); /* * poco_load @@ -1467,6 +1467,28 @@ function update_gcontact($contact) { intval($contact["generation"]), dbesc($contact["updated"]), dbesc($contact["server_url"]), dbesc($contact["connect"]), dbesc(normalise_link($contact["url"])), intval($contact["generation"])); + + + // Now update the contact entry with the user id "0" as well. + // This is used for the shadow copies of public items. + $r = q("SELECT `id` FROM `contact` WHERE `nurl` = '%s' AND `uid` = 0 ORDER BY `id` LIMIT 1", + dbesc(normalise_link($contact["url"]))); + + if ($r) { + logger("Update shadow contact ".$r[0]["id"], LOGGER_DEBUG); + + update_contact_avatar($contact["photo"], 0, $r[0]["id"]); + + q("UPDATE `contact` SET `name` = '%s', `nick` = '%s', `addr` = '%s', + `network` = '%s', `bd` = '%s', `gender` = '%s', + `keywords` = '%s', `alias` = '%s', `url` = '%s', + `location` = '%s', `about` = '%s' + WHERE `id` = %d", + dbesc($contact["name"]), dbesc($contact["nick"]), dbesc($contact["addr"]), + dbesc($contact["network"]), dbesc($contact["birthday"]), dbesc($contact["gender"]), + dbesc($contact["keywords"]), dbesc($contact["alias"]), dbesc($contact["url"]), + dbesc($contact["location"]), dbesc($contact["about"]), intval($r[0]["id"])); + } } return $gcontact_id; From 5d35974c19b50c98695c57cc1ad9195a36e7c396 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 14 Feb 2016 11:56:23 +0100 Subject: [PATCH 08/10] "addr" and "server_url" are now generated directly in "update_gcontact" if not given. --- include/Scrape.php | 69 ++++++++++++++++++++++++++++++++++---------- include/ostatus.php | 20 ------------- include/socgraph.php | 22 ++++++++++++++ 3 files changed, 75 insertions(+), 36 deletions(-) diff --git a/include/Scrape.php b/include/Scrape.php index d54a838afd..ef1db0312d 100644 --- a/include/Scrape.php +++ b/include/Scrape.php @@ -841,18 +841,18 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) { $vcard['fn'] = $url; if (($notify != "") AND ($poll != "")) { - $baseurl = matching(normalise_link($notify), normalise_link($poll)); + $baseurl = matching_url(normalise_link($notify), normalise_link($poll)); - $baseurl2 = matching($baseurl, normalise_link($profile)); + $baseurl2 = matching_url($baseurl, normalise_link($profile)); if ($baseurl2 != "") $baseurl = $baseurl2; } if (($baseurl == "") AND ($notify != "")) - $baseurl = matching(normalise_link($profile), normalise_link($notify)); + $baseurl = matching_url(normalise_link($profile), normalise_link($notify)); if (($baseurl == "") AND ($poll != "")) - $baseurl = matching(normalise_link($profile), normalise_link($poll)); + $baseurl = matching_url(normalise_link($profile), normalise_link($poll)); $baseurl = rtrim($baseurl, "/"); @@ -907,19 +907,56 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) { return $result; } -function matching($part1, $part2) { - $len = min(strlen($part1), strlen($part2)); +/** + * @brief Find the matching part between two url + * + * @param string $url1 + * @param string $url2 + * @return string The matching part + */ +function matching_url($url1, $url2) { + + if (($url1 == "") OR ($url2 == "")) + return ""; + + $url1 = normalise_link($url1); + $url2 = normalise_link($url2); + + $parts1 = parse_url($url1); + $parts2 = parse_url($url2); + + if (!isset($parts1["host"]) OR !isset($parts2["host"])) + return ""; + + if ($parts1["scheme"] != $parts2["scheme"]) + return ""; + + if ($parts1["host"] != $parts2["host"]) + return ""; + + if ($parts1["port"] != $parts2["port"]) + return ""; + + $match = $parts1["scheme"]."://".$parts1["host"]; + + if ($parts1["port"]) + $match .= ":".$parts1["port"]; + + $pathparts1 = explode("/", $parts1["path"]); + $pathparts2 = explode("/", $parts2["path"]); - $match = ""; - $matching = true; $i = 0; - while (($i <= $len) AND $matching) { - if (substr($part1, $i, 1) == substr($part2, $i, 1)) - $match .= substr($part1, $i, 1); - else - $matching = false; + $path = ""; + do { + $path1 = $pathparts1[$i]; + $path2 = $pathparts2[$i]; - $i++; - } - return($match); + if ($path1 == $path2) + $path .= $path1."/"; + + } while (($path1 == $path2) AND ($i++ <= count($pathparts1))); + + $match .= $path; + + return normalise_link($match); } diff --git a/include/ostatus.php b/include/ostatus.php index 983ed6e1b4..5c5016d0fc 100644 --- a/include/ostatus.php +++ b/include/ostatus.php @@ -164,8 +164,6 @@ function ostatus_fetchauthor($xpath, $context, $importer, &$contact, $onlyfetch) update_contact_avatar($author["author-avatar"], $importer["uid"], $contact["id"]); } - - /// @todo Add the "addr" field $contact["generation"] = 2; $contact["photo"] = $author["author-avatar"]; update_gcontact($contact); @@ -675,24 +673,6 @@ function ostatus_conv_fetch_actor($actor) { } } - $contact["server_url"] = $contact["url"]; - - $server_url = matching($contact["server_url"], $contact["alias"]); - if (strlen($server_url) > 8) - $contact["server_url"] = $server_url; - - $server_url = matching($contact["server_url"], $contact["photo"]); - if (strlen($server_url) > 8) - $contact["server_url"] = $server_url; - - if (($contact["server_url"] == $contact["url"]) OR ($contact["server_url"] == $contact["alias"])) - unset($contact["server_url"]); - else { - $hostname = str_replace("http://", "", normalise_link($contact["server_url"])); - if ($hostname AND $contact["nick"]) - $contact["addr"] = $contact["nick"]."@".$hostname; - } - update_gcontact($contact); } diff --git a/include/socgraph.php b/include/socgraph.php index ce5b08e641..e07e0d3533 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -1433,6 +1433,28 @@ function update_gcontact($contact) { if (!isset($contact["updated"])) $contact["updated"] = datetime_convert(); + if ($contact["server_url"] == "") { + $server_url = $contact["url"]; + + $server_url = matching_url($server_url, $contact["alias"]); + if ($server_url != "") + $contact["server_url"] = $server_url; + + $server_url = matching_url($server_url, $contact["photo"]); + if ($server_url != "") + $contact["server_url"] = $server_url; + + $server_url = matching_url($server_url, $contact["notify"]); + if ($server_url != "") + $contact["server_url"] = $server_url; + } else + $contact["server_url"] = normalise_link($contact["server_url"]); + + if (($contact["addr"] == "") AND ($contact["server_url"] != "") AND ($contact["nick"] != "")) { + $hostname = str_replace("http://", "", $contact["server_url"]); + $contact["addr"] = $contact["nick"]."@".$hostname; + } + // Check if any field changed $update = false; unset($fields["generation"]); From 922186bdd1a09ddc45b346e46aef4762f9b64283 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 14 Feb 2016 12:39:57 +0100 Subject: [PATCH 09/10] Small bugfix for the keyword bug in the gcontact table --- include/socgraph.php | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/socgraph.php b/include/socgraph.php index e07e0d3533..bd5b1817f0 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -1422,6 +1422,14 @@ function update_gcontact($contact) { unset($fields["url"]); unset($fields["updated"]); + // Bugfix: We had an error in the storing of keywords which lead to the "0" + // This value is still transmitted via poco. + if ($contact["keywords"] == "0") + unset($contact["keywords"]); + + if ($r[0]["keywords"] == "0") + $r[0]["keywords"] = ""; + // assign all unassigned fields from the database entry foreach ($fields AS $field => $data) if (!isset($contact[$field]) OR ($contact[$field] == "")) From 7f1f549c6f435bfe29bf7d331748b2623f98e6ae Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 14 Feb 2016 15:08:49 +0100 Subject: [PATCH 10/10] Fallback when there is no nick name --- mod/item.php | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mod/item.php b/mod/item.php index 7e575a17e4..2ade524a05 100644 --- a/mod/item.php +++ b/mod/item.php @@ -160,6 +160,9 @@ function item_post(&$a) { logger('no contact found: '.print_r($thrparent, true), LOGGER_DEBUG); } else logger('parent contact: '.print_r($parent_contact, true), LOGGER_DEBUG); + + if ($parent_contact["nick"] == "") + $parent_contact["nick"] = $parent_contact["name"]; } }