From 54a5a80b2425fe575643530b00c28578f1610448 Mon Sep 17 00:00:00 2001
From: root <root@pirati.ca>
Date: Fri, 17 Jul 2015 01:08:28 +0200
Subject: [PATCH 01/47] Poco: New fields for last contact and last failure of a
 contact.

---
 boot.php                |   2 +-
 include/dbstructure.php |   2 +
 include/plaintext.php   |   4 +-
 include/socgraph.php    | 129 ++++++++++++++++++++++++++++++++++++----
 mod/parse_url.php       |  15 +++++
 update.php              |   2 +-
 6 files changed, 139 insertions(+), 15 deletions(-)

diff --git a/boot.php b/boot.php
index 42498a6cc6..ec803a1573 100644
--- a/boot.php
+++ b/boot.php
@@ -19,7 +19,7 @@ define ( 'FRIENDICA_PLATFORM',     'Friendica');
 define ( 'FRIENDICA_CODENAME',     'Lily of the valley');
 define ( 'FRIENDICA_VERSION',      '3.4.1' );
 define ( 'DFRN_PROTOCOL_VERSION',  '2.23'    );
-define ( 'DB_UPDATE_VERSION',      1185      );
+define ( 'DB_UPDATE_VERSION',      1186      );
 define ( 'EOL',                    "<br />\r\n"     );
 define ( 'ATOM_TIME',              'Y-m-d\TH:i:s\Z' );
 
diff --git a/include/dbstructure.php b/include/dbstructure.php
index 0f81ee6249..8688903ba3 100644
--- a/include/dbstructure.php
+++ b/include/dbstructure.php
@@ -630,6 +630,8 @@ function db_definition() {
 					"photo" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"connect" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"updated" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
+					"last_contact" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
+					"last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
 					"location" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"about" => array("type" => "text", "not null" => "1"),
 					"keywords" => array("type" => "text", "not null" => "1"),
diff --git a/include/plaintext.php b/include/plaintext.php
index c8cdfa57df..4f435fc6a2 100644
--- a/include/plaintext.php
+++ b/include/plaintext.php
@@ -52,7 +52,7 @@ function get_attached_data($body) {
 		if (preg_match_all("(\[url=([$URLSearchString]*)\]\s*\[img\]([$URLSearchString]*)\[\/img\]\s*\[\/url\])ism", $body, $pictures,  PREG_SET_ORDER)) {
 			if (count($pictures) == 1) {
 				// Checking, if the link goes to a picture
-				$data = parseurl_getsiteinfo($pictures[0][1], true);
+				$data = parseurl_getsiteinfo_cached($pictures[0][1], true);
 				if ($data["type"] == "photo") {
 					$post["type"] = "photo";
 					if (isset($data["images"][0]))
@@ -95,7 +95,7 @@ function get_attached_data($body) {
 		}
 	} elseif (isset($post["url"]) AND ($post["type"] == "video")) {
 		require_once("mod/parse_url.php");
-		$data = parseurl_getsiteinfo($post["url"], true);
+		$data = parseurl_getsiteinfo_cached($post["url"], true);
 
 		if (isset($data["images"][0]))
 			$post["image"] = $data["images"][0]["src"];
diff --git a/include/socgraph.php b/include/socgraph.php
index 1e1b6076f2..1e8bd97aae 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -195,8 +195,20 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	$x = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1",
 		dbesc(normalise_link($profile_url))
 	);
-	if(count($x) AND ($network == "") AND ($x[0]["network"] != NETWORK_STATUSNET))
-		$network = $x[0]["network"];
+
+	if (count($x)) {
+		if (($network == "") AND ($x[0]["network"] != NETWORK_STATUSNET))
+			$network = $x[0]["network"];
+
+		if ($updated = "0000-00-00 00:00:00")
+			$updated = $x[0]["updated"];
+
+		$last_contact = $x[0]["last_contact"];
+		$last_failure = $x[0]["last_failure"];
+	} else {
+		$last_contact = "0000-00-00 00:00:00";
+		$last_failure = "0000-00-00 00:00:00";
+	}
 
 	if (($network == "") OR ($name == "") OR ($profile_photo == "")) {
 		require_once("include/Scrape.php");
@@ -223,6 +235,23 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 
 	logger("profile-check generation: ".$generation." Network: ".$network." URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG);
 
+	if (poco_do_update($updated, $last_contact, $last_failure)) {
+		$last_updated = poco_last_updated($profile_url);
+		if ($last_updated) {
+			$updated = $last_updated;
+			$last_contact = datetime_convert();
+			logger("Last updated for profile ".$profile_url.": ".$updated, LOGGER_DEBUG);
+
+			if (count($x))
+				q("UPDATE `gcontact` SET `last_contact` = '%s' WHERE `nurl` = '%s'", dbesc($last_contact), dbesc(normalise_link($profile_url)));
+		} else {
+			$last_failure = datetime_convert();
+
+			if (count($x))
+				q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", dbesc($last_failure), dbesc(normalise_link($profile_url)));
+		}
+	}
+
 	if(count($x)) {
 		$gcid = $x[0]['id'];
 
@@ -241,6 +270,9 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 		if (($generation == 0) AND ($x[0]['generation'] > 0))
 			$generation = $x[0]['generation'];
 
+		if ($last_contact < $x[0]['last_contact'])
+			$last_contact = $x[0]['last_contact'];
+
 		if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo || $x[0]['updated'] < $updated) {
 			q("UPDATE `gcontact` SET `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s',
 				`updated` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d
@@ -261,7 +293,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 			);
 		}
 	} else {
-		q("INSERT INTO `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `updated`, `location`, `about`, `keywords`, `gender`, `generation`)
+		q("INSERT INTO `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`)
 			VALUES ('%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s', '%s', %d)",
 			dbesc($name),
 			dbesc($network),
@@ -270,6 +302,8 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 			dbesc($profile_photo),
 			dbesc($connect_url),
 			dbesc($updated),
+			dbesc($last_contact),
+			dbesc($last_failure),
 			dbesc($location),
 			dbesc($about),
 			dbesc($keywords),
@@ -320,6 +354,79 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	return $gcid;
 }
 
+function poco_last_updated($profile) {
+	$data = probe_url($profile);
+
+	if (($data["poll"] == "") OR ($data["network"] == NETWORK_FEED))
+		return false;
+
+	$feedret = z_fetch_url($data["poll"]);
+
+	if (!$feedret["success"])
+		return false;
+
+	$doc = new DOMDocument();
+	@$doc->loadXML($feedret["body"]);
+
+	$xpath = new DomXPath($doc);
+	$xpath->registerNamespace('atom', "http://www.w3.org/2005/Atom");
+
+	$entries = $xpath->query('/atom:feed/atom:entry');
+
+	$last_updated = "";
+
+	foreach ($entries AS $entry) {
+		$published = $xpath->query('atom:published/text()', $entry)->item(0)->nodeValue;
+		$updated = $xpath->query('atom:updated/text()', $entry)->item(0)->nodeValue;
+
+		if ($last_updated < $published)
+			$last_updated = $published;
+
+		if ($last_updated < $updated)
+			$last_updated = $updated;
+	}
+
+	// Maybe there aren't any entries. Then check if it is a valid feed
+	if ($last_updated == "")
+		if ($xpath->query('/atom:feed')->length > 0)
+			$last_updated = "0000-00-00 00:00:00";
+
+	return($last_updated);
+}
+
+function poco_do_update($updated, $last_contact, $last_failure) {
+	$now = strtotime(datetime_convert());
+
+	if ($updated > $last_contact)
+		$contact_time = strtotime($updated);
+	else
+		$contact_time = strtotime($last_contact);
+
+	$failure_time = strtotime($last_failure);
+
+	// If the last contact was less than 24 hours then don't update
+	if (($now - $contact_time) < (60 * 60 * 24))
+		return false;
+
+	// If the last failure was less than 24 hours then don't update
+	if (($now - $failure_time) < (60 * 60 * 24))
+		return false;
+
+	// If the last contact was less than a week ago and the last failure is older than a week then don't update
+	if ((($now - $contact_time) < (60 * 60 * 24 * 7)) AND ($contact_time > $failure_time))
+		return false;
+
+	// If the last contact time was more than a week ago, then only try once a week
+	if (($now - $contact_time) > (60 * 60 * 24 * 7) AND ($now - $failure_time) < (60 * 60 * 24 * 7))
+		return false;
+
+	// If the last contact time was more than a month ago, then only try once a month
+	if (($now - $contact_time) > (60 * 60 * 24 * 30) AND ($now - $failure_time) < (60 * 60 * 24 * 30))
+		return false;
+
+	return true;
+}
+
 function poco_contact_from_body($body, $created, $cid, $uid) {
 	preg_replace_callback("/\[share(.*?)\].*?\[\/share\]/ism",
 		function ($match) use ($created, $cid, $uid){
@@ -328,20 +435,20 @@ function poco_contact_from_body($body, $created, $cid, $uid) {
 }
 
 function sub_poco_from_share($share, $created, $cid, $uid) {
-        $profile = "";
-        preg_match("/profile='(.*?)'/ism", $share[1], $matches);
-        if ($matches[1] != "")
-                $profile = $matches[1];
+	$profile = "";
+	preg_match("/profile='(.*?)'/ism", $share[1], $matches);
+	if ($matches[1] != "")
+		$profile = $matches[1];
 
-        preg_match('/profile="(.*?)"/ism', $share[1], $matches);
-        if ($matches[1] != "")
-                $profile = $matches[1];
+	preg_match('/profile="(.*?)"/ism', $share[1], $matches);
+	if ($matches[1] != "")
+		$profile = $matches[1];
 
 	if ($profile == "")
 		return;
 
 	logger("prepare poco_check for profile ".$profile, LOGGER_DEBUG);
-        poco_check($profile, "", "", "", "", "", "", "", "", $created, 3, $cid, $uid);
+	poco_check($profile, "", "", "", "", "", "", "", "", $created, 3, $cid, $uid);
 }
 
 function poco_store($item) {
diff --git a/mod/parse_url.php b/mod/parse_url.php
index 71d767675f..50c6a15b0e 100644
--- a/mod/parse_url.php
+++ b/mod/parse_url.php
@@ -50,6 +50,21 @@ function completeurl($url, $scheme) {
 	return($complete);
 }
 
+function parseurl_getsiteinfo_cached($url, $no_guessing = false, $do_oembed = true) {
+
+	$data = Cache::get("parse_url:".$no_guessing.":".$do_oembed.":".$url);
+	if (!is_null($data)) {
+		$data = unserialize($data);
+		return $data;
+	}
+
+	$data = parseurl_getsiteinfo($url, $no_guessing, $do_oembed);
+
+	Cache::set("parse_url:".$no_guessing.":".$do_oembed.":".$url,serialize($data));
+
+	return $data;
+}
+
 function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $count = 1) {
 	require_once("include/network.php");
 
diff --git a/update.php b/update.php
index c182eb590e..4f762661a5 100644
--- a/update.php
+++ b/update.php
@@ -1,6 +1,6 @@
 <?php
 
-define( 'UPDATE_VERSION' , 1185 );
+define( 'UPDATE_VERSION' , 1186 );
 
 /**
  *

From fdfd524e7489ca2789daccd10944be45c85532d9 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Fri, 17 Jul 2015 08:38:20 +0200
Subject: [PATCH 02/47] Only suggest contacts that haven't failed recently

---
 include/socgraph.php | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/include/socgraph.php b/include/socgraph.php
index 1e8bd97aae..6d14df613e 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -2,6 +2,13 @@
 
 require_once('include/datetime.php');
 
+/*
+ To-Do:
+ - noscrape for updating contact fields and "last updated"
+ - use /poco/@global for discovering contacts from other servers
+ - Make search for last activity optional
+*/
+
 /*
  * poco_load
  *
@@ -270,9 +277,6 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 		if (($generation == 0) AND ($x[0]['generation'] > 0))
 			$generation = $x[0]['generation'];
 
-		if ($last_contact < $x[0]['last_contact'])
-			$last_contact = $x[0]['last_contact'];
-
 		if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo || $x[0]['updated'] < $updated) {
 			q("UPDATE `gcontact` SET `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s',
 				`updated` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d
@@ -360,6 +364,8 @@ function poco_last_updated($profile) {
 	if (($data["poll"] == "") OR ($data["network"] == NETWORK_FEED))
 		return false;
 
+	// To-Do: Use noscrape
+
 	$feedret = z_fetch_url($data["poll"]);
 
 	if (!$feedret["success"])
@@ -649,6 +655,7 @@ function suggestion_query($uid, $start = 0, $limit = 80) {
 		and not gcontact.name in ( select name from contact where uid = %d )
 		and not gcontact.id in ( select gcid from gcign where uid = %d )
 		AND `gcontact`.`updated` != '0000-00-00 00:00:00'
+		AND `gcontact`.`last_contact` >= `gcontact`.`last_failure`
 		AND `gcontact`.`network` IN (%s)
 		group by glink.gcid order by gcontact.updated desc,total desc limit %d, %d ",
 		intval($uid),

From 09d71abbfc507791124140ae6b21e92ffb050a5c Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Fri, 17 Jul 2015 08:58:42 +0200
Subject: [PATCH 03/47] poco: Only fetch last update manually if it wasn't
 provided

---
 include/socgraph.php | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/include/socgraph.php b/include/socgraph.php
index 6d14df613e..3ff934f724 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -174,6 +174,8 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	if ($profile_url == "")
 		return $gcid;
 
+	$orig_updated = $updated;
+
 	// Don't store the statusnet connector as network
 	// We can't simply set this to NETWORK_OSTATUS since the connector could have fetched posts from friendica as well
 	if ($network == NETWORK_STATUSNET)
@@ -207,7 +209,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 		if (($network == "") AND ($x[0]["network"] != NETWORK_STATUSNET))
 			$network = $x[0]["network"];
 
-		if ($updated = "0000-00-00 00:00:00")
+		if ($updated == "0000-00-00 00:00:00")
 			$updated = $x[0]["updated"];
 
 		$last_contact = $x[0]["last_contact"];
@@ -242,7 +244,8 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 
 	logger("profile-check generation: ".$generation." Network: ".$network." URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG);
 
-	if (poco_do_update($updated, $last_contact, $last_failure)) {
+	// Only fetch last update manually if it wasn't provided
+	if (($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) {
 		$last_updated = poco_last_updated($profile_url);
 		if ($last_updated) {
 			$updated = $last_updated;

From 639f94f8b90ce23420be40d9f183e57190a4ecf4 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Fri, 17 Jul 2015 23:05:50 +0200
Subject: [PATCH 04/47] Onepoll: mark the last request failure for this contact

---
 include/dbstructure.php |   1 +
 include/onepoll.php     | 117 +++++++++++++++++++++++++++++-----------
 include/socgraph.php    |   5 +-
 3 files changed, 89 insertions(+), 34 deletions(-)

diff --git a/include/dbstructure.php b/include/dbstructure.php
index 8688903ba3..98e225bf8f 100644
--- a/include/dbstructure.php
+++ b/include/dbstructure.php
@@ -451,6 +451,7 @@ function db_definition() {
 					"hub-verify" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"last-update" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
 					"success_update" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
+					"failure_update" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
 					"name-date" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
 					"uri-date" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
 					"avatar-date" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
diff --git a/include/onepoll.php b/include/onepoll.php
index 9723fadd51..73668126a1 100644
--- a/include/onepoll.php
+++ b/include/onepoll.php
@@ -79,28 +79,69 @@ function onepoll_run(&$argv, &$argc){
 
 	$contacts = q("SELECT `contact`.* FROM `contact`
 		WHERE ( `rel` = %d OR `rel` = %d ) AND `poll` != ''
-		AND NOT `network` IN ( '%s', '%s', '%s' )
+		AND NOT `network` IN ( '%s', '%s' )
 		AND `contact`.`id` = %d
 		AND `self` = 0 AND `contact`.`blocked` = 0 AND `contact`.`readonly` = 0
 		AND `contact`.`archive` = 0 LIMIT 1",
 		intval(CONTACT_IS_SHARING),
 		intval(CONTACT_IS_FRIEND),
-		dbesc(NETWORK_DIASPORA),
 		dbesc(NETWORK_FACEBOOK),
 		dbesc(NETWORK_PUMPIO),
 		intval($contact_id)
 	);
 
-	if(! count($contacts)) {
-		// Maybe it is a Redmatrix account. Then we can fetch their contacts via poco
-		$contacts = q("SELECT `id`, `poco` FROM `contact` WHERE `id` = %d AND `poco` != ''", intval($contact_id));
-		if ($contacts)
-			poco_load($contacts[0]['id'],$importer_uid,0,$contacts[0]['poco']);
+	if(! count($contacts))
 		return;
-	}
 
 	$contact = $contacts[0];
 
+	// load current friends if possible.
+	if (($contact['poco'] != "") AND ($contact['success_update'] > $contact['failure_update'])) {
+		$r = q("SELECT count(*) as total from glink
+			where `cid` = %d and updated > UTC_TIMESTAMP() - INTERVAL 1 DAY",
+			intval($contact['id'])
+		);
+		if (count($r))
+			if (!$r[0]['total'])
+				poco_load($contact['id'],$importer_uid,0,$contact['poco']);
+	}
+
+	// To-Do:
+	// - Check why we don't poll the Diaspora feed at the moment (some guid problem in the items?)
+	// - Check whether this is possible with Redmatrix
+	if ($contact["network"] == NETWORK_DIASPORA) {
+		if (poco_do_update($contact["last-item"], $contact["success_update"], $contact["failure_update"])) {
+			$last_updated = poco_last_updated($contact["url"]);
+			$updated = datetime_convert();
+			if ($last_updated) {
+				q("UPDATE `contact` SET `last-item` = '%s', `last-update` = '%s', `success_update` = '%s' WHERE `id` = %d",
+					dbesc($last_updated),
+					dbesc($updated),
+					dbesc($updated),
+					intval($contact['id'])
+				);
+
+				q("UPDATE `gcontact` SET `updated` = '%s', `last_contact` = '%s' WHERE `nurl` = '%s'",
+					dbesc($last_updated),
+					dbesc($updated),
+					dbesc($contact['nurl'])
+				);
+			} else {
+				q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d",
+					dbesc($updated),
+					dbesc($updated),
+					intval($contact['id'])
+				);
+
+				q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'",
+					dbesc($updated),
+					dbesc($contact['nurl'])
+				);
+			}
+		}
+		return;
+	}
+
 	$xml = false;
 
 	$t = $contact['last-update'];
@@ -177,7 +218,8 @@ function onepoll_run(&$argv, &$argc){
 			mark_for_death($contact);
 
 			// set the last-update so we don't keep polling
-			$r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d",
+			$r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d",
+				dbesc(datetime_convert()),
 				dbesc(datetime_convert()),
 				intval($contact['id'])
 			);
@@ -190,7 +232,8 @@ function onepoll_run(&$argv, &$argc){
 
 			mark_for_death($contact);
 
-			$r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d",
+			$r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d",
+				dbesc(datetime_convert()),
 				dbesc(datetime_convert()),
 				intval($contact['id'])
 			);
@@ -207,7 +250,8 @@ function onepoll_run(&$argv, &$argc){
 			// set the last-update so we don't keep polling
 
 
-			$r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d",
+			$r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d",
+				dbesc(datetime_convert()),
 				dbesc(datetime_convert()),
 				intval($contact['id'])
 			);
@@ -554,14 +598,14 @@ function onepoll_run(&$argv, &$argc){
 		logger('poller: received xml : ' . $xml, LOGGER_DATA);
 		if(! strstr($xml,'<')) {
 			logger('poller: post_handshake: response from ' . $url . ' did not contain XML.');
-			$r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d",
+			$r = q("UPDATE `contact` SET `last-update` = '%s',  `failure_update` = '%s' WHERE `id` = %d",
+				dbesc(datetime_convert()),
 				dbesc(datetime_convert()),
 				intval($contact['id'])
 			);
 			return;
 		}
 
-
 		consume_feed($xml,$importer,$contact,$hub,1,1);
 
 
@@ -588,29 +632,38 @@ function onepoll_run(&$argv, &$argc){
 				}
 			}
 		}
-	}
 
-	$updated = datetime_convert();
+		$updated = datetime_convert();
 
-	$r = q("UPDATE `contact` SET `last-update` = '%s', `success_update` = '%s' WHERE `id` = %d",
-		dbesc($updated),
-		dbesc($updated),
-		intval($contact['id'])
-	);
-
-
-	// load current friends if possible.
-
-	if($contact['poco']) {
-		$r = q("SELECT count(*) as total from glink
-			where `cid` = %d and updated > UTC_TIMESTAMP() - INTERVAL 1 DAY",
+		$r = q("UPDATE `contact` SET `last-update` = '%s', `success_update` = '%s' WHERE `id` = %d",
+			dbesc($updated),
+			dbesc($updated),
+			intval($contact['id'])
+		);
+
+		q("UPDATE `gcontact` SET `last_contact` = '%s' WHERE `nurl` = '%s'",
+			dbesc($updated),
+			dbesc($contact['nurl'])
+		);
+
+	} elseif (in_array($contact["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, NETWORK_FEED))) {
+		$updated = datetime_convert();
+
+		$r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d",
+			dbesc($updated),
+			dbesc($updated),
+			intval($contact['id'])
+		);
+
+		q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'",
+			dbesc($updated),
+			dbesc($contact['nurl'])
+		);
+	} else {
+		$r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d",
+			dbesc($updated),
 			intval($contact['id'])
 		);
-	}
-	if(count($r)) {
-		if(! $r[0]['total']) {
-			poco_load($contact['id'],$importer_uid,0,$contact['poco']);
-		}
 	}
 
 	return;
diff --git a/include/socgraph.php b/include/socgraph.php
index 3ff934f724..c676157eae 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -1,12 +1,15 @@
 <?php
 
 require_once('include/datetime.php');
+require_once("include/Scrape.php");
 
 /*
  To-Do:
  - noscrape for updating contact fields and "last updated"
  - use /poco/@global for discovering contacts from other servers
  - Make search for last activity optional
+ - only export contacts via poco where update is higher than failure
+ - check your own contacts in some way as well
 */
 
 /*
@@ -220,8 +223,6 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	}
 
 	if (($network == "") OR ($name == "") OR ($profile_photo == "")) {
-		require_once("include/Scrape.php");
-
 		$data = probe_url($profile_url);
 		$network = $data["network"];
 		$name = $data["name"];

From f1e7369283a603cd2509d368341caac1c6e4beb4 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sat, 18 Jul 2015 00:48:42 +0200
Subject: [PATCH 05/47] Poco: Option to activate and deactivate detection of
 last activity.

---
 include/socgraph.php          |  7 ++-----
 mod/admin.php                 |  5 +++++
 mod/poco.php                  | 10 ++++++----
 view/templates/admin_site.tpl | 11 +++++++----
 4 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/include/socgraph.php b/include/socgraph.php
index c676157eae..5c9b1d98ad 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -7,9 +7,6 @@ require_once("include/Scrape.php");
  To-Do:
  - noscrape for updating contact fields and "last updated"
  - use /poco/@global for discovering contacts from other servers
- - Make search for last activity optional
- - only export contacts via poco where update is higher than failure
- - check your own contacts in some way as well
 */
 
 /*
@@ -245,8 +242,8 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 
 	logger("profile-check generation: ".$generation." Network: ".$network." URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG);
 
-	// Only fetch last update manually if it wasn't provided
-	if (($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) {
+	// Only fetch last update manually if it wasn't provided and enabled in the system
+	if (get_config('system','ld_discover_activity') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) {
 		$last_updated = poco_last_updated($profile_url);
 		if ($last_updated) {
 			$updated = $last_updated;
diff --git a/mod/admin.php b/mod/admin.php
index 28a7a91e4b..214ce62ea9 100644
--- a/mod/admin.php
+++ b/mod/admin.php
@@ -358,6 +358,7 @@ function admin_page_site_post(&$a){
 	$poll_interval		=	((x($_POST,'poll_interval'))		? intval(trim($_POST['poll_interval']))		: 0);
 	$maxloadavg		=	((x($_POST,'maxloadavg'))		? intval(trim($_POST['maxloadavg']))		: 50);
 	$maxloadavg_frontend	=	((x($_POST,'maxloadavg_frontend'))	? intval(trim($_POST['maxloadavg_frontend']))	: 50);
+	$ld_discover_activity	=	((x($_POST,'ld_discover_activity'))	? intval(trim($_POST['ld_discover_activity']))	: false);
 	$dfrn_only		=	((x($_POST,'dfrn_only'))		? True						: False);
 	$ostatus_disabled	=	!((x($_POST,'ostatus_disabled'))	? True  					: False);
 	$ostatus_poll_interval	=	((x($_POST,'ostatus_poll_interval'))	? intval(trim($_POST['ostatus_poll_interval']))	:  0);
@@ -427,6 +428,7 @@ function admin_page_site_post(&$a){
 	set_config('system','poll_interval',$poll_interval);
 	set_config('system','maxloadavg',$maxloadavg);
 	set_config('system','maxloadavg_frontend',$maxloadavg_frontend);
+	set_config('system','ld_discover_activity',$ld_discover_activity);
 	set_config('config','sitename',$sitename);
 	set_config('config','hostname',$hostname);
 	set_config('config','sender_email', $sender_email);
@@ -630,6 +632,7 @@ function admin_page_site(&$a) {
 		'$upload' => t('File upload'),
 		'$corporate' => t('Policies'),
 		'$advanced' => t('Advanced'),
+		'$local_directory' => t('Local Directory (Portable Contacts)'),
 		'$performance' => t('Performance'),
 		'$relocate'=> t('Relocate - WARNING: advanced function. Could make this server unreachable.'),
 		'$baseurl' => $a->get_baseurl(true),
@@ -687,6 +690,8 @@ function admin_page_site(&$a) {
 		'$maxloadavg'		=> array('maxloadavg', t("Maximum Load Average"), ((intval(get_config('system','maxloadavg')) > 0)?get_config('system','maxloadavg'):50), t("Maximum system load before delivery and poll processes are deferred - default 50.")),
 		'$maxloadavg_frontend'	=> array('maxloadavg_frontend', t("Maximum Load Average (Frontend)"), ((intval(get_config('system','maxloadavg_frontend')) > 0)?get_config('system','maxloadavg_frontend'):50), t("Maximum system load before the frontend quits service - default 50.")),
 
+		'$ld_discover_activity'	=> array('ld_discover_activity', t("Discover last activity"), get_config('system','ld_discover_activity'), t("Update the last activity when this isn't provided via the 'portable contacts' functionality. (Useful for poco exchange with Redmatrix and friendica servers before 3.3)")),
+
 		'$use_fulltext_engine'	=> array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")),
 		'$suppress_language'	=> array('suppress_language', t("Suppress Language"), get_config('system','suppress_language'), t("Suppress language information in meta information about a posting.")),
 		'$suppress_tags'	=> array('suppress_tags', t("Suppress Tags"), get_config('system','suppress_tags'), t("Suppress showing a list of hashtags at the end of the posting.")),
diff --git a/mod/poco.php b/mod/poco.php
index 89f2f879a0..0b62e93e38 100644
--- a/mod/poco.php
+++ b/mod/poco.php
@@ -61,12 +61,13 @@ function poco_init(&$a) {
 		$update_limit =  date("Y-m-d H:i:s",strtotime($_GET['updatedSince']));
 
 	if ($global) {
-		$r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND `network` IN ('%s')",
+		$r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND `last_contact` >= `last_failure` AND `network` IN ('%s')",
 			dbesc($update_limit),
 			dbesc(NETWORK_DFRN)
 		);
 	} elseif($system_mode) {
 		$r = q("SELECT count(*) AS `total` FROM `contact` WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '')
+			AND `success_update` >= `failure_update`
 			AND `uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) ",
 			dbesc(NETWORK_DFRN),
 			dbesc(NETWORK_DIASPORA),
@@ -75,7 +76,7 @@ function poco_init(&$a) {
 			);
 	} else {
 		$r = q("SELECT count(*) AS `total` FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0
-			AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra",
+			AND `success_update` >= `failure_update` AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra",
 			intval($user['uid']),
 			dbesc(NETWORK_DFRN),
 			dbesc(NETWORK_DIASPORA),
@@ -95,7 +96,7 @@ function poco_init(&$a) {
 
 
 	if ($global) {
-		$r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') LIMIT %d, %d",
+		$r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') AND `last_contact` >= `last_failure` LIMIT %d, %d",
 			dbesc($update_limit),
 			dbesc(NETWORK_DFRN),
 			intval($startIndex),
@@ -106,6 +107,7 @@ function poco_init(&$a) {
 			`profile`.`address` AS `paddress`, `profile`.`region` AS `pregion`, `profile`.`postal-code` AS `ppostalcode`, `profile`.`country-name` AS `pcountry`
 			FROM `contact` INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid`
 			WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') AND `profile`.`is-default`
+			AND `contact`.`success_update` >= `contact`.`failure_update`
 			AND `contact`.`uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) LIMIT %d, %d",
 			dbesc(NETWORK_DFRN),
 			dbesc(NETWORK_DIASPORA),
@@ -116,7 +118,7 @@ function poco_init(&$a) {
 		);
 	} else {
 		$r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0
-			AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra LIMIT %d, %d",
+			AND `success_update` >= `failure_update` AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra LIMIT %d, %d",
 			intval($user['uid']),
 			dbesc(NETWORK_DFRN),
 			dbesc(NETWORK_DIASPORA),
diff --git a/view/templates/admin_site.tpl b/view/templates/admin_site.tpl
index 34c590b2ea..5809d5ac6f 100644
--- a/view/templates/admin_site.tpl
+++ b/view/templates/admin_site.tpl
@@ -59,8 +59,6 @@
 	{{include file="field_checkbox.tpl" field=$old_share}}
 	{{include file="field_checkbox.tpl" field=$hide_help}}
 	{{include file="field_select.tpl" field=$singleuser}}
-
-	
 	<div class="submit"><input type="submit" name="page_site" value="{{$submit|escape:'html'}}" /></div>
 	
 	<h3>{{$registration}}</h3>
@@ -70,13 +68,13 @@
 	{{include file="field_checkbox.tpl" field=$no_multi_reg}}
 	{{include file="field_checkbox.tpl" field=$no_openid}}
 	{{include file="field_checkbox.tpl" field=$no_regfullname}}
-	
 	<div class="submit"><input type="submit" name="page_site" value="{{$submit|escape:'html'}}" /></div>
 
 	<h3>{{$upload}}</h3>
 	{{include file="field_input.tpl" field=$maximagesize}}
 	{{include file="field_input.tpl" field=$maximagelength}}
 	{{include file="field_input.tpl" field=$jpegimagequality}}
+	<div class="submit"><input type="submit" name="page_site" value="{{$submit|escape:'html'}}" /></div>
 	
 	<h3>{{$corporate}}</h3>
 	{{include file="field_input.tpl" field=$allowed_sites}}
@@ -99,7 +97,7 @@
 	<div class="submit"><input type="submit" name="page_site" value="{{$submit|escape:'html'}}" /></div>
 	
 	<h3>{{$advanced}}</h3>
-    {{include file="field_select.tpl" field=$rino}}
+	{{include file="field_select.tpl" field=$rino}}
 	{{include file="field_checkbox.tpl" field=$no_utf}}
 	{{include file="field_checkbox.tpl" field=$verifyssl}}
 	{{include file="field_input.tpl" field=$proxy}}
@@ -115,6 +113,11 @@
 	{{include file="field_input.tpl" field=$basepath}}
 	{{include file="field_checkbox.tpl" field=$suppress_language}}
 	{{include file="field_checkbox.tpl" field=$suppress_tags}}
+	<div class="submit"><input type="submit" name="page_site" value="{{$submit|escape:'html'}}" /></div>
+
+	<h3>{{$local_directory}}</h3>
+	{{include file="field_checkbox.tpl" field=$ld_discover_activity}}
+	<div class="submit"><input type="submit" name="page_site" value="{{$submit|escape:'html'}}" /></div>
 
 	<h3>{{$performance}}</h3>
 	{{include file="field_checkbox.tpl" field=$use_fulltext_engine}}

From 38d7b5e32641cccfbdcfb9ed11c2ece0243f7698 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sat, 18 Jul 2015 20:15:21 +0200
Subject: [PATCH 06/47] New table "gserver" for server data of the global
 contacts

---
 database.sql            |  27 ++++-
 include/dbstructure.php |  21 ++++
 include/socgraph.php    | 251 +++++++++++++++++++++++++++++++++++++++-
 3 files changed, 293 insertions(+), 6 deletions(-)

diff --git a/database.sql b/database.sql
index b65dee657b..ce5f051cdc 100644
--- a/database.sql
+++ b/database.sql
@@ -1,6 +1,6 @@
 -- ------------------------------------------
--- Friendica 3.4.0 (Lily of the valley)
--- DB_UPDATE_VERSION 1185
+-- Friendica 3.4.1 (Lily of the valley)
+-- DB_UPDATE_VERSION 1186
 -- ------------------------------------------
 
 
@@ -136,6 +136,7 @@ CREATE TABLE IF NOT EXISTS `contact` (
 	`hub-verify` varchar(255) NOT NULL DEFAULT '',
 	`last-update` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
 	`success_update` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
+	`failure_update` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
 	`name-date` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
 	`uri-date` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
 	`avatar-date` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
@@ -305,12 +306,15 @@ CREATE TABLE IF NOT EXISTS `gcontact` (
 	`photo` varchar(255) NOT NULL DEFAULT '',
 	`connect` varchar(255) NOT NULL DEFAULT '',
 	`updated` datetime DEFAULT '0000-00-00 00:00:00',
+	`last_contact` datetime DEFAULT '0000-00-00 00:00:00',
+	`last_failure` datetime DEFAULT '0000-00-00 00:00:00',
 	`location` varchar(255) NOT NULL DEFAULT '',
 	`about` text NOT NULL,
 	`keywords` text NOT NULL,
 	`gender` varchar(32) NOT NULL DEFAULT '',
 	`network` varchar(255) NOT NULL DEFAULT '',
 	`generation` tinyint(3) NOT NULL DEFAULT 0,
+	`server_url` varchar(255) NOT NULL DEFAULT '',
 	 INDEX `nurl` (`nurl`)
 ) DEFAULT CHARSET=utf8;
 
@@ -352,6 +356,25 @@ CREATE TABLE IF NOT EXISTS `group_member` (
 	 INDEX `uid_gid_contactid` (`uid`,`gid`,`contact-id`)
 ) DEFAULT CHARSET=utf8;
 
+--
+-- TABLE gserver
+--
+CREATE TABLE IF NOT EXISTS `gserver` (
+	`url` varchar(255) NOT NULL DEFAULT '',
+	`nurl` varchar(255) NOT NULL DEFAULT '',
+	`version` varchar(255) NOT NULL DEFAULT '',
+	`site_name` varchar(255) NOT NULL DEFAULT '',
+	`info` text NOT NULL,
+	`register_policy` tinyint(1) NOT NULL DEFAULT 0,
+	`poco` varchar(255) NOT NULL DEFAULT '',
+	`noscrape` varchar(255) NOT NULL DEFAULT '',
+	`network` varchar(32) NOT NULL DEFAULT '',
+	`platform` varchar(255) NOT NULL DEFAULT '',
+	`last_poco_query` datetime DEFAULT '0000-00-00 00:00:00',
+	`last_contact` datetime DEFAULT '0000-00-00 00:00:00',
+	`last_failure` datetime DEFAULT '0000-00-00 00:00:00'
+) DEFAULT CHARSET=utf8;
+
 --
 -- TABLE guid
 --
diff --git a/include/dbstructure.php b/include/dbstructure.php
index 98e225bf8f..9d0856c255 100644
--- a/include/dbstructure.php
+++ b/include/dbstructure.php
@@ -639,6 +639,7 @@ function db_definition() {
 					"gender" => array("type" => "varchar(32)", "not null" => "1", "default" => ""),
 					"network" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"generation" => array("type" => "tinyint(3)", "not null" => "1", "default" => "0"),
+					"server_url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					),
 			"indexes" => array(
 					"PRIMARY" => array("id"),
@@ -686,6 +687,26 @@ function db_definition() {
 					"uid_gid_contactid" => array("uid","gid","contact-id"),
 					)
 			);
+	$database["gserver"] = array(
+			"fields" => array(
+					"url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"nurl" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"version" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"site_name" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"info" => array("type" => "text", "not null" => "1"),
+					"register_policy" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"),
+					"poco" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"noscrape" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"network" => array("type" => "varchar(32)", "not null" => "1", "default" => ""),
+					"platform" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"last_poco_query" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
+					"last_contact" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
+					"last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
+					),
+			"indexes" => array(
+					"PRIMARY" => array("nurl"),
+					)
+			);
 	$database["guid"] = array(
 			"fields" => array(
 					"id" => array("type" => "int(10) unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"),
diff --git a/include/socgraph.php b/include/socgraph.php
index 5c9b1d98ad..a6899e952c 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -214,17 +214,20 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 
 		$last_contact = $x[0]["last_contact"];
 		$last_failure = $x[0]["last_failure"];
+		$server_url = $x[0]["server_url"];
 	} else {
 		$last_contact = "0000-00-00 00:00:00";
 		$last_failure = "0000-00-00 00:00:00";
+		$server_url = "";
 	}
 
-	if (($network == "") OR ($name == "") OR ($profile_photo == "")) {
+	if (($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == "")) {
 		$data = probe_url($profile_url);
 		$network = $data["network"];
 		$name = $data["name"];
 		$profile_url = $data["url"];
 		$profile_photo = $data["photo"];
+		$server_url = $data["baseurl"];
 	}
 
 	if (count($x) AND ($x[0]["network"] == "") AND ($network != "")) {
@@ -260,6 +263,14 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 		}
 	}
 
+	poco_check_server($server_url, $network);
+
+	// Test - remove before flight
+	if ($last_contact > $last_failure)
+		q("UPDATE `gserver` SET `last_contact` = '%s' WHERE `nurl` = '%s'", dbesc($last_contact), dbesc(normalise_link($server_url)));
+	else
+		q("UPDATE `gserver` SET `last_failure` = '%s' WHERE `nurl` = '%s'", dbesc($last_failure), dbesc(normalise_link($server_url)));
+
 	if(count($x)) {
 		$gcid = $x[0]['id'];
 
@@ -279,7 +290,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 			$generation = $x[0]['generation'];
 
 		if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo || $x[0]['updated'] < $updated) {
-			q("UPDATE `gcontact` SET `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s',
+			q("UPDATE `gcontact` SET `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s', `server_url` = '%s',
 				`updated` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d
 				WHERE (`generation` >= %d OR `generation` = 0) AND `nurl` = '%s'",
 				dbesc($name),
@@ -287,6 +298,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 				dbesc($profile_photo),
 				dbesc($connect_url),
 				dbesc($profile_url),
+				dbesc($server_url),
 				dbesc($updated),
 				dbesc($location),
 				dbesc($about),
@@ -298,14 +310,15 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 			);
 		}
 	} else {
-		q("INSERT INTO `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`)
-			VALUES ('%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s', '%s', %d)",
+		q("INSERT INTO `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `server_url`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`)
+			VALUES ('%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)",
 			dbesc($name),
 			dbesc($network),
 			dbesc($profile_url),
 			dbesc(normalise_link($profile_url)),
 			dbesc($profile_photo),
 			dbesc($connect_url),
+			dbesc($server_url),
 			dbesc($updated),
 			dbesc($last_contact),
 			dbesc($last_failure),
@@ -434,6 +447,236 @@ function poco_do_update($updated, $last_contact, $last_failure) {
 	return true;
 }
 
+function poco_to_boolean($val) {
+        if (($val == "true") OR ($val == 1))
+                return(true);
+        if (($val == "false") OR ($val == 0))
+                return(false);
+
+        return ($val);
+}
+
+function poco_check_server($server_url, $network = "") {
+
+	if ($server_url == "")
+		return;
+
+	$servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url)));
+	if ($servers) {
+		$poco = $servers[0]["poco"];
+		$noscrape = $servers[0]["noscrape"];
+
+		if ($network == "")
+			$network = $servers[0]["network"];
+
+		$last_contact = $servers[0]["last_contact"];
+		$last_failure = $servers[0]["last_failure"];
+		$version = $servers[0]["version"];
+		$platform = $servers[0]["platform"];
+		$site_name = $servers[0]["site_name"];
+		$info = $servers[0]["info"];
+		$register_policy = $servers[0]["register_policy"];
+
+		// Only check the server once a week
+		if (strtotime(datetime_convert()) < (strtotime($last_contact) + (60 * 60 * 24 * 7)))
+			return;
+
+		if (strtotime(datetime_convert()) < (strtotime($last_failure) + (60 * 60 * 24 * 7)))
+			return;
+	} else {
+		$poco = "";
+		$noscrape = "";
+		$version = "";
+		$platform = "";
+		$site_name = "";
+		$info = "";
+		$register_policy = -1;
+
+		$last_contact = "0000-00-00 00:00:00";
+		$last_failure = "0000-00-00 00:00:00";
+	}
+
+	$failure = false;
+	$orig_last_failure = $last_failure;
+
+	// Check if the page is accessible via SSL.
+	$server_url = str_replace("http://", "https://", $server_url);
+	$serverret = z_fetch_url($server_url."/.well-known/host-meta");
+
+	// Maybe the page is unencrypted only?
+	$xmlobj = @simplexml_load_string($serverret["body"],'SimpleXMLElement',0, "http://docs.oasis-open.org/ns/xri/xrd-1.0");
+	if (!$serverret["success"] OR ($serverret["body"] == "") OR (@sizeof($xmlobj) == 0) OR !is_object($xmlobj)) {
+		$server_url = str_replace("https://", "http://", $server_url);
+		$serverret = z_fetch_url($server_url."/.well-known/host-meta");
+
+		$xmlobj = @simplexml_load_string($serverret["body"],'SimpleXMLElement',0, "http://docs.oasis-open.org/ns/xri/xrd-1.0");
+	}
+
+	if (!$serverret["success"] OR ($serverret["body"] == "") OR (sizeof($xmlobj) == 0) OR !is_object($xmlobj)) {
+		$last_failure = datetime_convert();
+		$failure = true;
+	} elseif ($network == NETWORK_DIASPORA)
+		$last_contact = datetime_convert();
+
+	if (!$failure) {
+		// Test for Statusnet
+		// Will also return data for Friendica and GNU Social - but it will be overwritten later
+		// The "not implemented" is a special treatment for really, really old Friendica versions
+		$serverret = z_fetch_url($server_url."/api/statusnet/version.json");
+		if ($serverret["success"] AND ($serverret["body"] != '{"error":"not implemented"}') AND ($serverret["body"] != '') AND (strlen($serverret["body"]) < 250)) {
+			$platform = "StatusNet";
+			$version = trim($serverret["body"], '"');
+			$network = NETWORK_OSTATUS;
+		}
+
+		// Test for GNU Social
+		$serverret = z_fetch_url($server_url."/api/gnusocial/version.json");
+		if ($serverret["success"] AND ($serverret["body"] != '{"error":"not implemented"}') AND ($serverret["body"] != '') AND (strlen($serverret["body"]) < 250)) {
+			$platform = "GNU Social";
+			$version = trim($serverret["body"], '"');
+			$network = NETWORK_OSTATUS;
+		}
+
+		$serverret = z_fetch_url($server_url."/api/statusnet/config.json");
+		if ($serverret["success"]) {
+			$data = json_decode($serverret["body"]);
+
+			if (isset($data->site->server)) {
+				$last_contact = datetime_convert();
+
+				if (isset($data->site->redmatrix)) {
+					if (isset($data->site->redmatrix->PLATFORM_NAME))
+						$platform = $data->site->redmatrix->PLATFORM_NAME;
+					elseif (isset($data->site->redmatrix->RED_PLATFORM))
+						$platform = $data->site->redmatrix->RED_PLATFORM;
+
+					$version = $data->site->redmatrix->RED_VERSION;
+					$network = NETWORK_DIASPORA;
+				}
+				if (isset($data->site->friendica)) {
+					$platform = $data->site->friendica->FRIENDICA_PLATFORM;
+					$version = $data->site->friendica->FRIENDICA_VERSION;
+					$network = NETWORK_DFRN;
+				}
+
+				$site_name = $data->site->name;
+
+				$data->site->closed = poco_to_boolean($data->site->closed);
+				$data->site->private = poco_to_boolean($data->site->private);
+				$data->site->inviteonly = poco_to_boolean($data->site->inviteonly);
+
+				if (!$data->site->closed AND !$data->site->private and $data->site->inviteonly)
+					$register_policy = REGISTER_APPROVE;
+				elseif (!$data->site->closed AND !$data->site->private)
+					$register_policy = REGISTER_OPEN;
+				else
+					$register_policy = REGISTER_CLOSED;
+			}
+		}
+	}
+
+	// Query statistics.json. Optional package for Diaspora, Friendica and Redmatrix
+	if (!$failure) {
+		$serverret = z_fetch_url($server_url."/statistics.json");
+		if ($serverret["success"]) {
+			$data = json_decode($serverret["body"]);
+			if ($version == "")
+				$version = $data->version;
+
+			$site_name = $data->name;
+
+			if (isset($data->network) AND ($platform == ""))
+				$platform = $data->network;
+
+			if ($data->registrations_open)
+				$register_policy = REGISTER_OPEN;
+			else
+				$register_policy = REGISTER_CLOSED;
+
+			if (isset($data->version))
+				$last_contact = datetime_convert();
+		}
+	}
+
+	// Check for noscrape
+	// Friendica servers could be detected as OStatus servers
+	if (!$failure AND in_array($network, array(NETWORK_DFRN, NETWORK_OSTATUS))) {
+		$serverret = z_fetch_url($server_url."/friendica/json");
+
+		if ($serverret["success"]) {
+			$data = json_decode($serverret["body"]);
+
+			if (isset($data->version)) {
+				$last_contact = datetime_convert();
+				$network = NETWORK_DFRN;
+
+				$noscrape = $data->no_scrape_url;
+				$version = $data->version;
+				$site_name = $data->site_name;
+				$info = $data->info;
+				$register_policy_str = $data->register_policy;
+				$platform = $data->platform;
+
+				switch ($register_policy_str) {
+					case "REGISTER_CLOSED":
+						$register_policy = REGISTER_CLOSED;
+						break;
+					case "REGISTER_APPROVE":
+						$register_policy = REGISTER_APPROVE;
+						break;
+					case "REGISTER_OPEN":
+						$register_policy = REGISTER_OPEN;
+						break;
+				}
+			}
+		}
+	}
+
+	// Look for poco
+	if (!$failure) {
+		$serverret = z_fetch_url($server_url."/poco");
+		if ($serverret["success"]) {
+			$data = json_decode($serverret["body"]);
+			if (isset($data->totalResults)) {
+				$poco = $server_url."/poco";
+				$last_contact = datetime_convert();
+			}
+		}
+	}
+
+	if ($servers)
+		 q("UPDATE `gserver` SET `url` = '%s', `version` = '%s', `site_name` = '%s', `info` = '%s', `register_policy` = %d, `poco` = '%s', `noscrape` = '%s',
+			`network` = '%s', `platform` = '%s', `last_contact` = '%s', `last_failure` = '%s' WHERE `nurl` = '%s'",
+			dbesc($server_url),
+			dbesc($version),
+			dbesc($site_name),
+			dbesc($info),
+			intval($register_policy),
+			dbesc($poco),
+			dbesc($noscrape),
+			dbesc($network),
+			dbesc($platform),
+			dbesc($last_contact),
+			dbesc($last_failure),
+			dbesc(normalise_link($server_url))
+		);
+	else
+		q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `last_contact`)
+					VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s')",
+				dbesc($server_url),
+				dbesc(normalise_link($server_url)),
+				dbesc($version),
+				dbesc($site_name),
+				dbesc($info),
+				intval($register_policy),
+				dbesc($poco),
+				dbesc($noscrape),
+				dbesc($network),
+				dbesc($platform),
+				dbesc(datetime_convert())
+		);
+}
+
 function poco_contact_from_body($body, $created, $cid, $uid) {
 	preg_replace_callback("/\[share(.*?)\].*?\[\/share\]/ism",
 		function ($match) use ($created, $cid, $uid){

From cf3214c9040d84102cfe30943ee42211305412d3 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sat, 18 Jul 2015 22:26:06 +0200
Subject: [PATCH 07/47] Query other servers periodically for their known
 contacts

---
 include/discover_poco.php     |  71 +++++++++++++++++++++
 include/poller.php            |   4 ++
 include/socgraph.php          | 115 ++++++++++++++++++++++++++++------
 mod/admin.php                 |  13 ++--
 view/templates/admin_site.tpl |   5 +-
 5 files changed, 183 insertions(+), 25 deletions(-)
 create mode 100644 include/discover_poco.php

diff --git a/include/discover_poco.php b/include/discover_poco.php
new file mode 100644
index 0000000000..5a493ab967
--- /dev/null
+++ b/include/discover_poco.php
@@ -0,0 +1,71 @@
+<?php
+
+require_once("boot.php");
+require_once("include/socgraph.php");
+
+
+function discover_poco_run(&$argv, &$argc){
+	global $a, $db;
+
+	if(is_null($a)) {
+		$a = new App;
+	}
+
+	if(is_null($db)) {
+	    @include(".htconfig.php");
+    	require_once("include/dba.php");
+	    $db = new dba($db_host, $db_user, $db_pass, $db_data);
+    	unset($db_host, $db_user, $db_pass, $db_data);
+  	};
+
+	require_once('include/session.php');
+	require_once('include/datetime.php');
+	require_once('include/pidfile.php');
+
+	load_config('config');
+	load_config('system');
+
+	$maxsysload = intval(get_config('system','maxloadavg'));
+	if($maxsysload < 1)
+		$maxsysload = 50;
+	if(function_exists('sys_getloadavg')) {
+		$load = sys_getloadavg();
+		if(intval($load[0]) > $maxsysload) {
+			logger('system: load ' . $load[0] . ' too high. discover_poco deferred to next scheduled run.');
+			return;
+		}
+	}
+
+	$lockpath = get_lockpath();
+	if ($lockpath != '') {
+		$pidfile = new pidfile($lockpath, 'discover_poco');
+		if($pidfile->is_already_running()) {
+			logger("discover_poco: Already running");
+			if ($pidfile->running_time() > 19*60) {
+                                $pidfile->kill();
+                                logger("discover_poco: killed stale process");
+				// Calling a new instance
+				proc_run('php','include/discover_poco.php');
+                        }
+			exit;
+		}
+	}
+
+	$a->set_baseurl(get_config('system','url'));
+
+	load_hooks();
+
+	logger('start');
+
+	if (get_config('system','poco_discovery'))
+		poco_discover();
+
+	logger('end');
+
+	return;
+}
+
+if (array_search(__file__,get_included_files())===0){
+  discover_poco_run($_SERVER["argv"],$_SERVER["argc"]);
+  killme();
+}
diff --git a/include/poller.php b/include/poller.php
index d971d4f004..44ac94daa4 100644
--- a/include/poller.php
+++ b/include/poller.php
@@ -82,6 +82,10 @@ function poller_run(&$argv, &$argc){
 
 	proc_run('php',"include/dsprphotoq.php");
 
+	// run the process to discover global contacts in the background
+
+	proc_run('php',"include/discover_poco.php");
+
 	// expire any expired accounts
 
 	q("UPDATE user SET `account_expired` = 1 where `account_expired` = 0
diff --git a/include/socgraph.php b/include/socgraph.php
index a6899e952c..1d0dfd6f79 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -2,12 +2,7 @@
 
 require_once('include/datetime.php');
 require_once("include/Scrape.php");
-
-/*
- To-Do:
- - noscrape for updating contact fields and "last updated"
- - use /poco/@global for discovering contacts from other servers
-*/
+require_once("include/html2bbcode.php");
 
 /*
  * poco_load
@@ -28,8 +23,6 @@ require_once("include/Scrape.php");
 
 function poco_load($cid,$uid = 0,$zcid = 0,$url = null) {
 
-	require_once("include/html2bbcode.php");
-
 	$a = get_app();
 
 	if($cid) {
@@ -246,7 +239,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	logger("profile-check generation: ".$generation." Network: ".$network." URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG);
 
 	// Only fetch last update manually if it wasn't provided and enabled in the system
-	if (get_config('system','ld_discover_activity') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) {
+	if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) {
 		$last_updated = poco_last_updated($profile_url);
 		if ($last_updated) {
 			$updated = $last_updated;
@@ -266,10 +259,10 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	poco_check_server($server_url, $network);
 
 	// Test - remove before flight
-	if ($last_contact > $last_failure)
-		q("UPDATE `gserver` SET `last_contact` = '%s' WHERE `nurl` = '%s'", dbesc($last_contact), dbesc(normalise_link($server_url)));
-	else
-		q("UPDATE `gserver` SET `last_failure` = '%s' WHERE `nurl` = '%s'", dbesc($last_failure), dbesc(normalise_link($server_url)));
+	//if ($last_contact > $last_failure)
+	//	q("UPDATE `gserver` SET `last_contact` = '%s' WHERE `nurl` = '%s'", dbesc($last_contact), dbesc(normalise_link($server_url)));
+	//else
+	//	q("UPDATE `gserver` SET `last_failure` = '%s' WHERE `nurl` = '%s'", dbesc($last_failure), dbesc(normalise_link($server_url)));
 
 	if(count($x)) {
 		$gcid = $x[0]['id'];
@@ -448,12 +441,12 @@ function poco_do_update($updated, $last_contact, $last_failure) {
 }
 
 function poco_to_boolean($val) {
-        if (($val == "true") OR ($val == 1))
-                return(true);
-        if (($val == "false") OR ($val == 0))
-                return(false);
+	if (($val == "true") OR ($val == 1))
+		return(true);
+	if (($val == "false") OR ($val == 0))
+		return(false);
 
-        return ($val);
+	return ($val);
 }
 
 function poco_check_server($server_url, $network = "") {
@@ -976,3 +969,89 @@ function update_suggestions() {
 		}
 	}
 }
+
+function poco_discover() {
+
+	$last_update = date("c", time() - (60 * 60 * 24));
+
+	$r = q("SELECT `poco`, `nurl` FROM `gserver` WHERE `last_contact` > `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update));
+	if ($r)
+		foreach ($r AS $server) {
+			$url = $server["poco"]."/@global?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
+
+			$retdata = z_fetch_url($url);
+			if ($retdata["success"]) {
+				poco_discover_server(json_decode($retdata["body"]));
+				q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"]));
+				break;
+			} else
+				q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"]));
+
+		}
+}
+
+function poco_discover_server($data) {
+
+	foreach ($data->entry AS $entry) {
+		$profile_url = '';
+		$profile_photo = '';
+		$connect_url = '';
+		$name = '';
+		$network = '';
+		$updated = '0000-00-00 00:00:00';
+		$location = '';
+		$about = '';
+		$keywords = '';
+		$gender = '';
+		$generation = 0;
+
+		$name = $entry->displayName;
+
+		if(isset($entry->urls)) {
+			foreach($entry->urls as $url) {
+				if($url->type == 'profile') {
+					$profile_url = $url->value;
+					continue;
+				}
+				if($url->type == 'webfinger') {
+					$connect_url = str_replace('acct:' , '', $url->value);
+					continue;
+				}
+			}
+		}
+		if(isset($entry->photos)) {
+			foreach($entry->photos as $photo) {
+				if($photo->type == 'profile') {
+					$profile_photo = $photo->value;
+					continue;
+				}
+			}
+		}
+
+		if(isset($entry->updated))
+			$updated = date("Y-m-d H:i:s", strtotime($entry->updated));
+
+		if(isset($entry->network))
+			$network = $entry->network;
+
+		if(isset($entry->currentLocation))
+			$location = $entry->currentLocation;
+
+		if(isset($entry->aboutMe))
+			$about = html2bbcode($entry->aboutMe);
+
+		if(isset($entry->gender))
+			$gender = $entry->gender;
+
+		if(isset($entry->generation) AND ($entry->generation > 0))
+			$generation = ++$entry->generation;
+
+		if(isset($entry->tags))
+			foreach($entry->tags as $tag)
+				$keywords = implode(", ", $tag);
+
+		if ($generation > 0)
+			poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $generation);
+	}
+}
+?>
diff --git a/mod/admin.php b/mod/admin.php
index 214ce62ea9..393fe16147 100644
--- a/mod/admin.php
+++ b/mod/admin.php
@@ -358,7 +358,8 @@ function admin_page_site_post(&$a){
 	$poll_interval		=	((x($_POST,'poll_interval'))		? intval(trim($_POST['poll_interval']))		: 0);
 	$maxloadavg		=	((x($_POST,'maxloadavg'))		? intval(trim($_POST['maxloadavg']))		: 50);
 	$maxloadavg_frontend	=	((x($_POST,'maxloadavg_frontend'))	? intval(trim($_POST['maxloadavg_frontend']))	: 50);
-	$ld_discover_activity	=	((x($_POST,'ld_discover_activity'))	? intval(trim($_POST['ld_discover_activity']))	: false);
+	$poco_completion	=	((x($_POST,'poco_completion'))		? intval(trim($_POST['poco_completion']))	: false);
+	$poco_discovery		=	((x($_POST,'poco_discovery'))		? intval(trim($_POST['poco_discovery']))	: false);
 	$dfrn_only		=	((x($_POST,'dfrn_only'))		? True						: False);
 	$ostatus_disabled	=	!((x($_POST,'ostatus_disabled'))	? True  					: False);
 	$ostatus_poll_interval	=	((x($_POST,'ostatus_poll_interval'))	? intval(trim($_POST['ostatus_poll_interval']))	:  0);
@@ -428,7 +429,8 @@ function admin_page_site_post(&$a){
 	set_config('system','poll_interval',$poll_interval);
 	set_config('system','maxloadavg',$maxloadavg);
 	set_config('system','maxloadavg_frontend',$maxloadavg_frontend);
-	set_config('system','ld_discover_activity',$ld_discover_activity);
+	set_config('system','poco_completion',$poco_completion);
+	set_config('system','poco_discovery',$poco_discovery);
 	set_config('config','sitename',$sitename);
 	set_config('config','hostname',$hostname);
 	set_config('config','sender_email', $sender_email);
@@ -436,7 +438,7 @@ function admin_page_site_post(&$a){
 	set_config('system','suppress_tags',$suppress_tags);
 	set_config('system','shortcut_icon',$shortcut_icon);
 	set_config('system','touch_icon',$touch_icon);
-	
+
 	if ($banner==""){
 		// don't know why, but del_config doesn't work...
 		q("DELETE FROM `config` WHERE `cat` = '%s' AND `k` = '%s' LIMIT 1",
@@ -632,7 +634,7 @@ function admin_page_site(&$a) {
 		'$upload' => t('File upload'),
 		'$corporate' => t('Policies'),
 		'$advanced' => t('Advanced'),
-		'$local_directory' => t('Local Directory (Portable Contacts)'),
+		'$portable_contacts' => t('Portable Contact Directory'),
 		'$performance' => t('Performance'),
 		'$relocate'=> t('Relocate - WARNING: advanced function. Could make this server unreachable.'),
 		'$baseurl' => $a->get_baseurl(true),
@@ -690,7 +692,8 @@ function admin_page_site(&$a) {
 		'$maxloadavg'		=> array('maxloadavg', t("Maximum Load Average"), ((intval(get_config('system','maxloadavg')) > 0)?get_config('system','maxloadavg'):50), t("Maximum system load before delivery and poll processes are deferred - default 50.")),
 		'$maxloadavg_frontend'	=> array('maxloadavg_frontend', t("Maximum Load Average (Frontend)"), ((intval(get_config('system','maxloadavg_frontend')) > 0)?get_config('system','maxloadavg_frontend'):50), t("Maximum system load before the frontend quits service - default 50.")),
 
-		'$ld_discover_activity'	=> array('ld_discover_activity', t("Discover last activity"), get_config('system','ld_discover_activity'), t("Update the last activity when this isn't provided via the 'portable contacts' functionality. (Useful for poco exchange with Redmatrix and friendica servers before 3.3)")),
+		'$poco_completion'	=> array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful for poco exchange with Redmatrix and friendica servers before 3.3)")),
+		'$poco_discovery'	=> array('poco_discovery', t("Discover contacts from other servers"), get_config('system','poco_discovery'), t("Periodically query other friendica servers for their recent contacts.")),
 
 		'$use_fulltext_engine'	=> array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")),
 		'$suppress_language'	=> array('suppress_language', t("Suppress Language"), get_config('system','suppress_language'), t("Suppress language information in meta information about a posting.")),
diff --git a/view/templates/admin_site.tpl b/view/templates/admin_site.tpl
index 5809d5ac6f..22d785f6ff 100644
--- a/view/templates/admin_site.tpl
+++ b/view/templates/admin_site.tpl
@@ -115,8 +115,9 @@
 	{{include file="field_checkbox.tpl" field=$suppress_tags}}
 	<div class="submit"><input type="submit" name="page_site" value="{{$submit|escape:'html'}}" /></div>
 
-	<h3>{{$local_directory}}</h3>
-	{{include file="field_checkbox.tpl" field=$ld_discover_activity}}
+	<h3>{{$portable_contacts}}</h3>
+	{{include file="field_checkbox.tpl" field=$poco_completion}}
+	{{include file="field_checkbox.tpl" field=$poco_discovery}}
 	<div class="submit"><input type="submit" name="page_site" value="{{$submit|escape:'html'}}" /></div>
 
 	<h3>{{$performance}}</h3>

From e47c65eb1357aeb4d0471914336574e02194c979 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sat, 18 Jul 2015 23:33:54 +0200
Subject: [PATCH 08/47] Query the other servers for their users.

---
 include/socgraph.php | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/include/socgraph.php b/include/socgraph.php
index 1d0dfd6f79..99d72d41ce 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -977,20 +977,27 @@ function poco_discover() {
 	$r = q("SELECT `poco`, `nurl` FROM `gserver` WHERE `last_contact` > `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update));
 	if ($r)
 		foreach ($r AS $server) {
-			$url = $server["poco"]."/@global?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
+			// Fetch all users from the other server
+			$url = $server["poco"]."/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
 
 			$retdata = z_fetch_url($url);
 			if ($retdata["success"]) {
-				poco_discover_server(json_decode($retdata["body"]));
+				poco_discover_server(json_decode($retdata["body"]), 2);
+
+				// Fetch all global contacts from the other server (Not working with Redmatrix and Friendica versions before 3.3)
+				$url = $server["poco"]."/@global?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
+
+				$retdata = z_fetch_url($url);
+				if ($retdata["success"])
+					poco_discover_server(json_decode($retdata["body"]));
+
 				q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"]));
 				break;
-			} else
-				q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"]));
-
+			}
 		}
 }
 
-function poco_discover_server($data) {
+function poco_discover_server($data, $default_generation = 0) {
 
 	foreach ($data->entry AS $entry) {
 		$profile_url = '';
@@ -1003,7 +1010,7 @@ function poco_discover_server($data) {
 		$about = '';
 		$keywords = '';
 		$gender = '';
-		$generation = 0;
+		$generation = $default_generation;
 
 		$name = $entry->displayName;
 

From 2a5fb7b175f487b277254a76a484687f7e491e05 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sun, 19 Jul 2015 09:41:36 +0200
Subject: [PATCH 09/47] Option to discover contacts from remote users (fallback
 option)

---
 include/discover_poco.php     |  2 +-
 include/socgraph.php          | 60 +++++++++++++++++++++++++++++------
 mod/admin.php                 | 15 ++++++---
 view/templates/admin_site.tpl |  2 +-
 4 files changed, 64 insertions(+), 15 deletions(-)

diff --git a/include/discover_poco.php b/include/discover_poco.php
index 5a493ab967..c69e27fef1 100644
--- a/include/discover_poco.php
+++ b/include/discover_poco.php
@@ -57,7 +57,7 @@ function discover_poco_run(&$argv, &$argc){
 
 	logger('start');
 
-	if (get_config('system','poco_discovery'))
+	if (get_config('system','poco_discovery') > 0)
 		poco_discover();
 
 	logger('end');
diff --git a/include/socgraph.php b/include/socgraph.php
index 99d72d41ce..18c2c7c4c1 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -970,7 +970,7 @@ function update_suggestions() {
 	}
 }
 
-function poco_discover() {
+function poco_discover($complete = false) {
 
 	$last_update = date("c", time() - (60 * 60 * 24));
 
@@ -980,25 +980,64 @@ function poco_discover() {
 			// Fetch all users from the other server
 			$url = $server["poco"]."/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
 
+			logger("Fetch all users from the server ".$server["nurl"], LOGGER_DEBUG);
+
 			$retdata = z_fetch_url($url);
 			if ($retdata["success"]) {
-				poco_discover_server(json_decode($retdata["body"]), 2);
+				$data = json_decode($retdata["body"]);
+				poco_discover_server($data, 2);
 
-				// Fetch all global contacts from the other server (Not working with Redmatrix and Friendica versions before 3.3)
-				$url = $server["poco"]."/@global?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
+				if (get_config('system','poco_discovery') > 1) {
 
-				$retdata = z_fetch_url($url);
-				if ($retdata["success"])
-					poco_discover_server(json_decode($retdata["body"]));
+					// Fetch all global contacts from the other server (Not working with Redmatrix and Friendica versions before 3.3)
+					$url = $server["poco"]."/@global?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
+
+					$retdata = z_fetch_url($url);
+					if ($retdata["success"]) {
+						logger("Fetch all global contacts from the server ".$server["nurl"], LOGGER_DEBUG);
+						poco_discover_server(json_decode($retdata["body"]));
+					} elseif (get_config('system','poco_discovery') > 2) {
+						logger("Fetch contacts from users of the server ".$server["nurl"], LOGGER_DEBUG);
+						poco_discover_server_users($data);
+					}
+				}
 
 				q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"]));
-				break;
+				if (!$complete)
+					break;
 			}
 		}
 }
 
+function poco_discover_server_users($data) {
+	foreach ($data->entry AS $entry) {
+		$username = "";
+		if (isset($entry->urls)) {
+			foreach($entry->urls as $url)
+				if($url->type == 'profile') {
+					$profile_url = $url->value;
+					$urlparts = parse_url($profile_url);
+					$username = end(explode("/", $urlparts["path"]));
+				}
+		}
+		if ($username != "") {
+			logger("Fetch contacts for the user ".$username." from the server ".$server["nurl"], LOGGER_DEBUG);
+
+			// Fetch all contacts from a given user from the other server
+			$url = $server["poco"]."/".$username."/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
+
+			$retdata = z_fetch_url($url);
+			if ($retdata["success"])
+				poco_discover_server(json_decode($retdata["body"]), 3);
+		}
+	}
+}
+
 function poco_discover_server($data, $default_generation = 0) {
 
+	if (!isset($data->entry) OR !count($data->entry))
+		return;
+
 	foreach ($data->entry AS $entry) {
 		$profile_url = '';
 		$profile_photo = '';
@@ -1057,8 +1096,11 @@ function poco_discover_server($data, $default_generation = 0) {
 			foreach($entry->tags as $tag)
 				$keywords = implode(", ", $tag);
 
-		if ($generation > 0)
+		if ($generation > 0) {
+			logger("Store profile ".$profile_url, LOGGER_DEBUG);
 			poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $generation);
+			logger("Done for profile ".$profile_url, LOGGER_DEBUG);
+		}
 	}
 }
 ?>
diff --git a/mod/admin.php b/mod/admin.php
index 393fe16147..8c3e2441bc 100644
--- a/mod/admin.php
+++ b/mod/admin.php
@@ -359,7 +359,7 @@ function admin_page_site_post(&$a){
 	$maxloadavg		=	((x($_POST,'maxloadavg'))		? intval(trim($_POST['maxloadavg']))		: 50);
 	$maxloadavg_frontend	=	((x($_POST,'maxloadavg_frontend'))	? intval(trim($_POST['maxloadavg_frontend']))	: 50);
 	$poco_completion	=	((x($_POST,'poco_completion'))		? intval(trim($_POST['poco_completion']))	: false);
-	$poco_discovery		=	((x($_POST,'poco_discovery'))		? intval(trim($_POST['poco_discovery']))	: false);
+	$poco_discovery		=	((x($_POST,'poco_discovery'))		? intval(trim($_POST['poco_discovery']))	: 0);
 	$dfrn_only		=	((x($_POST,'dfrn_only'))		? True						: False);
 	$ostatus_disabled	=	!((x($_POST,'ostatus_disabled'))	? True  					: False);
 	$ostatus_poll_interval	=	((x($_POST,'ostatus_poll_interval'))	? intval(trim($_POST['ostatus_poll_interval']))	:  0);
@@ -382,8 +382,8 @@ function admin_page_site_post(&$a){
 	$old_pager		=	((x($_POST,'old_pager'))		? True						: False);
 	$only_tag_search	=	((x($_POST,'only_tag_search'))		? True						: False);
 	$rino			=	((x($_POST,'rino'))				? intval($_POST['rino'])				: 0);
-	
-	
+
+
 	if($ssl_policy != intval(get_config('system','ssl_policy'))) {
 		if($ssl_policy == SSL_POLICY_FULL) {
 			q("update `contact` set
@@ -586,6 +586,13 @@ function admin_page_site(&$a) {
 			"1440" => t("Daily")
 			);
 
+		$poco_discovery_choices = array(
+			"0" => t("Disabled"),
+			"1" => t("Users"),
+			"2" => t("Users, Global Contacts"),
+			"3" => t("Users, Global Contacts/fallback"),
+			);
+
 		/* get user names to make the install a personal install of X */
 		$user_names = array();
 		$user_names['---'] = t('Multi user instance');
@@ -693,7 +700,7 @@ function admin_page_site(&$a) {
 		'$maxloadavg_frontend'	=> array('maxloadavg_frontend', t("Maximum Load Average (Frontend)"), ((intval(get_config('system','maxloadavg_frontend')) > 0)?get_config('system','maxloadavg_frontend'):50), t("Maximum system load before the frontend quits service - default 50.")),
 
 		'$poco_completion'	=> array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful for poco exchange with Redmatrix and friendica servers before 3.3)")),
-		'$poco_discovery'	=> array('poco_discovery', t("Discover contacts from other servers"), get_config('system','poco_discovery'), t("Periodically query other friendica servers for their recent contacts.")),
+		'$poco_discovery'	=> array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for profiles. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available."), $poco_discovery_choices),
 
 		'$use_fulltext_engine'	=> array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")),
 		'$suppress_language'	=> array('suppress_language', t("Suppress Language"), get_config('system','suppress_language'), t("Suppress language information in meta information about a posting.")),
diff --git a/view/templates/admin_site.tpl b/view/templates/admin_site.tpl
index 22d785f6ff..f0461e1b87 100644
--- a/view/templates/admin_site.tpl
+++ b/view/templates/admin_site.tpl
@@ -117,7 +117,7 @@
 
 	<h3>{{$portable_contacts}}</h3>
 	{{include file="field_checkbox.tpl" field=$poco_completion}}
-	{{include file="field_checkbox.tpl" field=$poco_discovery}}
+	{{include file="field_select.tpl" field=$poco_discovery}}
 	<div class="submit"><input type="submit" name="page_site" value="{{$submit|escape:'html'}}" /></div>
 
 	<h3>{{$performance}}</h3>

From 93e93f58bd703dc4dbe254e1e6ca80fc41739971 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sun, 19 Jul 2015 10:40:33 +0200
Subject: [PATCH 10/47] Query Redmatrix contacts for their contacts as well.

---
 include/socgraph.php | 9 +++++++--
 mod/admin.php        | 6 +++---
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/include/socgraph.php b/include/socgraph.php
index 18c2c7c4c1..c97a0f15cd 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -939,6 +939,7 @@ function update_suggestions() {
 
 	$done = array();
 
+	// To-Do: Check if it is really neccessary to poll the own server
 	poco_load(0,0,0,$a->get_baseurl() . '/poco');
 
 	$done[] = $a->get_baseurl() . '/poco';
@@ -949,6 +950,9 @@ function update_suggestions() {
 			$j = json_decode($x);
 			if($j->entries) {
 				foreach($j->entries as $entry) {
+
+					poco_check_server($entry->url);
+
 					$url = $entry->url . '/poco';
 					if(! in_array($url,$done))
 						poco_load(0,0,0,$entry->url . '/poco');
@@ -957,8 +961,9 @@ function update_suggestions() {
 		}
 	}
 
-	$r = q("select distinct(poco) as poco from contact where network = '%s'",
-		dbesc(NETWORK_DFRN)
+	// Query your contacts from Friendica and Redmatrix/Hubzilla for their contacts
+	$r = q("SELECT DISTINCT(`poco`) AS `poco` FROM `contact` WHERE `network` IN ('%s', '%s')",
+		dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA)
 	);
 
 	if(count($r)) {
diff --git a/mod/admin.php b/mod/admin.php
index 8c3e2441bc..a079bdabf8 100644
--- a/mod/admin.php
+++ b/mod/admin.php
@@ -641,7 +641,7 @@ function admin_page_site(&$a) {
 		'$upload' => t('File upload'),
 		'$corporate' => t('Policies'),
 		'$advanced' => t('Advanced'),
-		'$portable_contacts' => t('Portable Contact Directory'),
+		'$portable_contacts' => t('Auto Discovered Contact Directory'),
 		'$performance' => t('Performance'),
 		'$relocate'=> t('Relocate - WARNING: advanced function. Could make this server unreachable.'),
 		'$baseurl' => $a->get_baseurl(true),
@@ -699,8 +699,8 @@ function admin_page_site(&$a) {
 		'$maxloadavg'		=> array('maxloadavg', t("Maximum Load Average"), ((intval(get_config('system','maxloadavg')) > 0)?get_config('system','maxloadavg'):50), t("Maximum system load before delivery and poll processes are deferred - default 50.")),
 		'$maxloadavg_frontend'	=> array('maxloadavg_frontend', t("Maximum Load Average (Frontend)"), ((intval(get_config('system','maxloadavg_frontend')) > 0)?get_config('system','maxloadavg_frontend'):50), t("Maximum system load before the frontend quits service - default 50.")),
 
-		'$poco_completion'	=> array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful for poco exchange with Redmatrix and friendica servers before 3.3)")),
-		'$poco_discovery'	=> array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for profiles. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available."), $poco_discovery_choices),
+		'$poco_completion'	=> array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful when communicating with Redmatrix and friendica servers before 3.3)")),
+		'$poco_discovery'	=> array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available."), $poco_discovery_choices),
 
 		'$use_fulltext_engine'	=> array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")),
 		'$suppress_language'	=> array('suppress_language', t("Suppress Language"), get_config('system','suppress_language'), t("Suppress language information in meta information about a posting.")),

From 0bede65a6b26826158703a0a6a164c2e0dac9851 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sun, 19 Jul 2015 13:23:01 +0200
Subject: [PATCH 11/47] Use local search

---
 mod/dirfind.php               | 65 ++++++++++++++++++++++++++++-------
 view/templates/match.tpl      |  2 +-
 view/templates/peoplefind.tpl |  2 +-
 3 files changed, 54 insertions(+), 15 deletions(-)

diff --git a/mod/dirfind.php b/mod/dirfind.php
index 0c6d79480f..c49d37e48d 100644
--- a/mod/dirfind.php
+++ b/mod/dirfind.php
@@ -16,11 +16,13 @@ function dirfind_init(&$a) {
 
 function dirfind_content(&$a) {
 
+	$local = true;
+
 	$search = notags(trim($_REQUEST['search']));
 
 	if(strpos($search,'@') === 0)
 		$search = substr($search,1);
-	
+
 	$o = '';
 
 	$o .= replace_macros(get_markup_template("section_title.tpl"),array(
@@ -29,16 +31,53 @@ function dirfind_content(&$a) {
 
 	if($search) {
 
-		$p = (($a->pager['page'] != 1) ? '&p=' . $a->pager['page'] : '');
-			
-		if(strlen(get_config('system','directory_submit_url')))
-			$x = fetch_url('http://dir.friendica.com/lsearch?f=' . $p .  '&search=' . urlencode($search));
+		if ($local) {
 
-//TODO fallback local search if global dir not available.
-//		else
-//			$x = post_url($a->get_baseurl() . '/lsearch', $params);
+			$perpage = 80;
+			$startrec = (($a->pager['page']) * $perpage) - $perpage;
 
-		$j = json_decode($x);
+			$count = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `network` IN ('%s', '%s', '%s') AND
+					(`url` REGEXP '%s' OR `name` REGEXP '%s' OR `location` REGEXP '%s' OR
+						`about` REGEXP '%s' OR `keywords` REGEXP '%s')",
+					dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA),
+					dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)),
+					dbesc(escape_tags($search)), dbesc(escape_tags($search)));
+
+			$results = q("SELECT `url`, `name`, `photo`, `keywords` FROM `gcontact`WHERE `network` IN ('%s', '%s', '%s') AND
+					(`url` REGEXP '%s' OR `name` REGEXP '%s' OR `location` REGEXP '%s' OR `about` REGEXP '%s' OR
+						`keywords` REGEXP '%s') ORDER BY `name` ASC LIMIT %d, %d",
+					dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA),
+					dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)),
+					dbesc(escape_tags($search)), dbesc(escape_tags($search)),
+					intval($startrec), intval($perpage));
+
+			$j = new stdClass();
+			$j->total = $count[0]["total"];
+			$j->items_page = $perpage;
+			$j->page = $a->pager['page'];
+			foreach ($results AS $result) {
+				if ($result["name"] == "") {
+					$urlparts = parse_url($result["url"]);
+					$result["name"] = end(explode("/", $urlparts["path"]));
+				}
+
+				$objresult = new stdClass();
+				$objresult->name = $result["name"];
+				$objresult->url = $result["url"];
+				$objresult->photo = $result["photo"];
+				$objresult->tags = $result["keywords"];
+
+				$j->results[] = $objresult;
+			}
+		} else {
+
+			$p = (($a->pager['page'] != 1) ? '&p=' . $a->pager['page'] : '');
+
+			if(strlen(get_config('system','directory_submit_url')))
+				$x = fetch_url('http://dir.friendica.com/lsearch?f=' . $p .  '&search=' . urlencode($search));
+
+			$j = json_decode($x);
+		}
 
 		if($j->total) {
 			$a->set_pager_total($j->total);
@@ -46,21 +85,21 @@ function dirfind_content(&$a) {
 		}
 
 		if(count($j->results)) {
-			
+
 			$tpl = get_markup_template('match.tpl');
 			foreach($j->results as $jj) {
-				
+
 				$o .= replace_macros($tpl,array(
 					'$url' => zrl($jj->url),
 					'$name' => $jj->name,
-					'$photo' => $jj->photo,
+					'$photo' => proxy_url($jj->photo),
 					'$tags' => $jj->tags
 				));
 			}
 		}
 		else {
 			info( t('No matches') . EOL);
-		}		
+		}
 
 	}
 
diff --git a/view/templates/match.tpl b/view/templates/match.tpl
index f4bbc3c1bb..32f046e6aa 100644
--- a/view/templates/match.tpl
+++ b/view/templates/match.tpl
@@ -2,7 +2,7 @@
 <div class="profile-match-wrapper">
 	<div class="profile-match-photo">
 		<a href="{{$url}}">
-			<img src="{{$photo}}" alt="{{$name}}" title="{{$name}}[{{$tags}}]" />
+			<img width="80" height="80" src="{{$photo}}" alt="{{$name}}" title="{{$name}}[{{$tags}}]" />
 		</a>
 	</div>
 	<div class="profile-match-break"></div>
diff --git a/view/templates/peoplefind.tpl b/view/templates/peoplefind.tpl
index de8cd011b4..45c4c0b622 100644
--- a/view/templates/peoplefind.tpl
+++ b/view/templates/peoplefind.tpl
@@ -2,7 +2,7 @@
 <div id="peoplefind-sidebar" class="widget">
 	<h3>{{$findpeople}}</h3>
 	<div id="peoplefind-desc">{{$desc}}</div>
-	<form action="dirfind" method="post" />
+	<form action="dirfind" method="get" />
 		<input id="side-peoplefind-url" type="text" name="search" size="24" title="{{$hint|escape:'html'}}" /><input id="side-peoplefind-submit" type="submit" name="submit" value="{{$findthem|escape:'html'}}" />
 	</form>
 	<div class="side-link" id="side-match-link"><a href="match" >{{$similar}}</a></div>

From f3c83fb51479c1aca19fd2528dc6d68d1f160739 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sun, 19 Jul 2015 18:02:24 +0200
Subject: [PATCH 12/47] Automatically updating the searched contacts while
 searching it.

---
 include/discover_poco.php     | 68 ++++++++++++++++++++++++++++++++---
 mod/admin.php                 |  3 ++
 mod/dirfind.php               | 33 +++++++++++++----
 view/templates/admin_site.tpl |  1 +
 4 files changed, 94 insertions(+), 11 deletions(-)

diff --git a/include/discover_poco.php b/include/discover_poco.php
index c69e27fef1..504d972873 100644
--- a/include/discover_poco.php
+++ b/include/discover_poco.php
@@ -36,9 +36,18 @@ function discover_poco_run(&$argv, &$argc){
 		}
 	}
 
+	if(($argc > 2) && ($argv[1] == "dirsearch")) {
+		$search = urldecode($argv[2]);
+		$searchmode = 1;
+	} elseif ($argc == 1) {
+		$search = "";
+		$searchmode = 0;
+	} else
+		die("Unknown or missing parameter ".$argv[1]."\n");
+
 	$lockpath = get_lockpath();
 	if ($lockpath != '') {
-		$pidfile = new pidfile($lockpath, 'discover_poco');
+		$pidfile = new pidfile($lockpath, 'discover_poco'.urlencode($search));
 		if($pidfile->is_already_running()) {
 			logger("discover_poco: Already running");
 			if ($pidfile->running_time() > 19*60) {
@@ -55,16 +64,67 @@ function discover_poco_run(&$argv, &$argc){
 
 	load_hooks();
 
-	logger('start');
+	logger('start '.$search);
 
-	if (get_config('system','poco_discovery') > 0)
+	if (($search != "") and get_config('system','poco_local_search'))
+		discover_directory($search);
+	elseif (($search == "") and get_config('system','poco_discovery') > 0)
 		poco_discover();
 
-	logger('end');
+	logger('end '.$search);
 
 	return;
 }
 
+function discover_directory($search) {
+
+	$data = Cache::get("dirsearch:".$search);
+	if (!is_null($data)){
+		// Only search for the same item every 24 hours
+		if (time() < $data + (60 * 60 * 24)) {
+			logger("Already searched for ".$search." in the last 24 hours", LOGGER_DEBUG);
+			return;
+		}
+	}
+
+	$x = fetch_url("http://dir.friendica.com/lsearch?p=1&n=500&search=".urlencode($search));
+	$j = json_decode($x);
+
+	if(count($j->results))
+		foreach($j->results as $jj) {
+			// Check if the contact already exists
+			$exists = q("SELECT `id`, `last_contact`, `last_failure`  FROM `gcontact` WHERE `nurl` = '%s'", normalise_link($jj->url));
+			if ($exists) {
+				logger("Profile ".$jj->url." already exists (".$search.")", LOGGER_DEBUG);
+
+				if ($exists[0]["last_contact"] < $exists[0]["last_failure"])
+					continue;
+
+				$last_updated = poco_last_updated($jj->url);
+				$last_contact = datetime_convert();
+
+				if ($last_updated) {
+					logger("Mark profile ".$jj->url." as accessible (".$search.")", LOGGER_DEBUG);
+					q("UPDATE `gcontact` SET `updated` = '%s', `last_contact` = '%s' WHERE `nurl` = '%s'",
+						dbesc($last_updated), dbesc($last_contact), dbesc(normalise_link($jj->url)));
+				} else {
+					logger("Mark profile ".$jj->url." as unaccessible (".$search.")", LOGGER_DEBUG);
+					q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'",
+						dbesc($last_contact), dbesc(normalise_link($jj->url)));
+				}
+				continue;
+			}
+
+			logger("Check if profile ".$jj->url." is reachable (".$search.")", LOGGER_DEBUG);
+			$data = probe_url($jj->url);
+			if ($data["network"] == NETWORK_DFRN) {
+				logger("Add profile ".$jj->url." to local directory (".$search.")", LOGGER_DEBUG);
+				poco_check($data["url"], $data["name"], $data["network"], $data["photo"], "", "", "", $jj->tags, $data["addr"], "", 0);
+			}
+		}
+	Cache::set("dirsearch:".$search, time());
+}
+
 if (array_search(__file__,get_included_files())===0){
   discover_poco_run($_SERVER["argv"],$_SERVER["argc"]);
   killme();
diff --git a/mod/admin.php b/mod/admin.php
index a079bdabf8..ea0196db83 100644
--- a/mod/admin.php
+++ b/mod/admin.php
@@ -360,6 +360,7 @@ function admin_page_site_post(&$a){
 	$maxloadavg_frontend	=	((x($_POST,'maxloadavg_frontend'))	? intval(trim($_POST['maxloadavg_frontend']))	: 50);
 	$poco_completion	=	((x($_POST,'poco_completion'))		? intval(trim($_POST['poco_completion']))	: false);
 	$poco_discovery		=	((x($_POST,'poco_discovery'))		? intval(trim($_POST['poco_discovery']))	: 0);
+	$poco_local_search	=	((x($_POST,'poco_local_search'))	? intval(trim($_POST['poco_local_search']))	: false);
 	$dfrn_only		=	((x($_POST,'dfrn_only'))		? True						: False);
 	$ostatus_disabled	=	!((x($_POST,'ostatus_disabled'))	? True  					: False);
 	$ostatus_poll_interval	=	((x($_POST,'ostatus_poll_interval'))	? intval(trim($_POST['ostatus_poll_interval']))	:  0);
@@ -431,6 +432,7 @@ function admin_page_site_post(&$a){
 	set_config('system','maxloadavg_frontend',$maxloadavg_frontend);
 	set_config('system','poco_completion',$poco_completion);
 	set_config('system','poco_discovery',$poco_discovery);
+	set_config('system','poco_local_search',$poco_local_search);
 	set_config('config','sitename',$sitename);
 	set_config('config','hostname',$hostname);
 	set_config('config','sender_email', $sender_email);
@@ -701,6 +703,7 @@ function admin_page_site(&$a) {
 
 		'$poco_completion'	=> array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful when communicating with Redmatrix and friendica servers before 3.3)")),
 		'$poco_discovery'	=> array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available."), $poco_discovery_choices),
+		'$poco_local_search'	=> array('poco_local_search', t("Search the local directory"), get_config('system','poco_local_search'), t("Search the local directory instead of the global directory. When searching locally, every search will be executed on the global directory in the background. This improves the search results when the search is repeated.")),
 
 		'$use_fulltext_engine'	=> array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")),
 		'$suppress_language'	=> array('suppress_language', t("Suppress Language"), get_config('system','suppress_language'), t("Suppress language information in meta information about a posting.")),
diff --git a/mod/dirfind.php b/mod/dirfind.php
index c49d37e48d..996d2a4641 100644
--- a/mod/dirfind.php
+++ b/mod/dirfind.php
@@ -16,7 +16,7 @@ function dirfind_init(&$a) {
 
 function dirfind_content(&$a) {
 
-	$local = true;
+	$local = get_config('system','poco_local_search');
 
 	$search = notags(trim($_REQUEST['search']));
 
@@ -43,14 +43,18 @@ function dirfind_content(&$a) {
 					dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)),
 					dbesc(escape_tags($search)), dbesc(escape_tags($search)));
 
-			$results = q("SELECT `url`, `name`, `photo`, `keywords` FROM `gcontact`WHERE `network` IN ('%s', '%s', '%s') AND
-					(`url` REGEXP '%s' OR `name` REGEXP '%s' OR `location` REGEXP '%s' OR `about` REGEXP '%s' OR
-						`keywords` REGEXP '%s') ORDER BY `name` ASC LIMIT %d, %d",
-					dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA),
+			$results = q("SELECT `contact`.`id` AS `cid`, `gcontact`.`url`, `gcontact`.`name`, `gcontact`.`photo`, `gcontact`.`keywords`
+					FROM `gcontact`
+					LEFT JOIN `contact` ON `contact`.`nurl` = `gcontact`.`nurl` AND `contact`.`uid` = %d
+					WHERE `gcontact`.`network` IN ('%s', '%s', '%s') AND `gcontact`.`last_contact` >= `gcontact`.`last_failure` AND
+					(`gcontact`.`url` REGEXP '%s' OR `gcontact`.`name` REGEXP '%s' OR `gcontact`.`location` REGEXP '%s' OR
+						`gcontact`.`about` REGEXP '%s' OR `gcontact`.`keywords` REGEXP '%s')
+						GROUP BY `gcontact`.`nurl`
+						ORDER BY `gcontact`.`updated` DESC LIMIT %d, %d",
+					intval(local_user()), dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA),
 					dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)),
 					dbesc(escape_tags($search)), dbesc(escape_tags($search)),
 					intval($startrec), intval($perpage));
-
 			$j = new stdClass();
 			$j->total = $count[0]["total"];
 			$j->items_page = $perpage;
@@ -62,6 +66,7 @@ function dirfind_content(&$a) {
 				}
 
 				$objresult = new stdClass();
+				$objresult->cid = $result["cid"];
 				$objresult->name = $result["name"];
 				$objresult->url = $result["url"];
 				$objresult->photo = $result["photo"];
@@ -69,6 +74,9 @@ function dirfind_content(&$a) {
 
 				$j->results[] = $objresult;
 			}
+
+			// Add found profiles from the global directory to the local directory
+			proc_run('php','include/discover_poco.php', "dirsearch", urlencode($search));
 		} else {
 
 			$p = (($a->pager['page'] != 1) ? '&p=' . $a->pager['page'] : '');
@@ -89,11 +97,22 @@ function dirfind_content(&$a) {
 			$tpl = get_markup_template('match.tpl');
 			foreach($j->results as $jj) {
 
+				// If We already know this contact then don't show the "connect" button
+				if ($jj->cid > 0) {
+					$connlnk = "";
+					$conntxt = "";
+				} else {
+					$connlnk = $a->get_baseurl().'/follow/?url='.(($jj->connect) ? $jj->connect : $jj->url);
+					$conntxt = t('Connect');
+				}
+
 				$o .= replace_macros($tpl,array(
 					'$url' => zrl($jj->url),
 					'$name' => $jj->name,
 					'$photo' => proxy_url($jj->photo),
-					'$tags' => $jj->tags
+					'$tags' => $jj->tags,
+					'$conntxt' => $conntxt,
+					'$connlnk' => $connlnk,
 				));
 			}
 		}
diff --git a/view/templates/admin_site.tpl b/view/templates/admin_site.tpl
index f0461e1b87..2f448d1b91 100644
--- a/view/templates/admin_site.tpl
+++ b/view/templates/admin_site.tpl
@@ -118,6 +118,7 @@
 	<h3>{{$portable_contacts}}</h3>
 	{{include file="field_checkbox.tpl" field=$poco_completion}}
 	{{include file="field_select.tpl" field=$poco_discovery}}
+	{{include file="field_checkbox.tpl" field=$poco_local_search}}
 	<div class="submit"><input type="submit" name="page_site" value="{{$submit|escape:'html'}}" /></div>
 
 	<h3>{{$performance}}</h3>

From 3622f1a335998c8052be78d83b1e18708a509bbe Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sun, 19 Jul 2015 21:29:24 +0200
Subject: [PATCH 13/47] Do some caching

---
 include/discover_poco.php | 25 +++++++------
 include/onepoll.php       | 11 ------
 include/socgraph.php      | 79 +++++++++++++++++++++++++--------------
 3 files changed, 64 insertions(+), 51 deletions(-)

diff --git a/include/discover_poco.php b/include/discover_poco.php
index 504d972873..74ed83b473 100644
--- a/include/discover_poco.php
+++ b/include/discover_poco.php
@@ -100,21 +100,22 @@ function discover_directory($search) {
 				if ($exists[0]["last_contact"] < $exists[0]["last_failure"])
 					continue;
 
-				$last_updated = poco_last_updated($jj->url);
-				$last_contact = datetime_convert();
-
-				if ($last_updated) {
-					logger("Mark profile ".$jj->url." as accessible (".$search.")", LOGGER_DEBUG);
-					q("UPDATE `gcontact` SET `updated` = '%s', `last_contact` = '%s' WHERE `nurl` = '%s'",
-						dbesc($last_updated), dbesc($last_contact), dbesc(normalise_link($jj->url)));
-				} else {
-					logger("Mark profile ".$jj->url." as unaccessible (".$search.")", LOGGER_DEBUG);
-					q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'",
-						dbesc($last_contact), dbesc(normalise_link($jj->url)));
-				}
+				// Update the contact
+				poco_last_updated($jj->url);
 				continue;
 			}
 
+			// Harcoded paths aren't so good. But in this case it is okay.
+			// First: We only will get Friendica contacts (which always are using this url schema)
+			// Second: There will be no further problems if we are doing a mistake
+			$server_url = preg_replace("=(https?://)(.*)/profile/(.*)=ism", "$1$2", $jj->url);
+			if ($server_url != $jj->url)
+				if (!poco_check_server($server_url)) {
+					logger("Friendica server ".$server_url." doesn't answer.", LOGGER_DEBUG);
+					continue;
+				}
+					logger("Friendica server ".$server_url." seems to be okay.", LOGGER_DEBUG);
+
 			logger("Check if profile ".$jj->url." is reachable (".$search.")", LOGGER_DEBUG);
 			$data = probe_url($jj->url);
 			if ($data["network"] == NETWORK_DFRN) {
diff --git a/include/onepoll.php b/include/onepoll.php
index 73668126a1..c0c2f1a34b 100644
--- a/include/onepoll.php
+++ b/include/onepoll.php
@@ -120,23 +120,12 @@ function onepoll_run(&$argv, &$argc){
 					dbesc($updated),
 					intval($contact['id'])
 				);
-
-				q("UPDATE `gcontact` SET `updated` = '%s', `last_contact` = '%s' WHERE `nurl` = '%s'",
-					dbesc($last_updated),
-					dbesc($updated),
-					dbesc($contact['nurl'])
-				);
 			} else {
 				q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d",
 					dbesc($updated),
 					dbesc($updated),
 					intval($contact['id'])
 				);
-
-				q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'",
-					dbesc($updated),
-					dbesc($contact['nurl'])
-				);
 			}
 		}
 		return;
diff --git a/include/socgraph.php b/include/socgraph.php
index c97a0f15cd..9ccae50405 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -238,6 +238,8 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 
 	logger("profile-check generation: ".$generation." Network: ".$network." URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG);
 
+	poco_check_server($server_url, $network);
+
 	// Only fetch last update manually if it wasn't provided and enabled in the system
 	if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) {
 		$last_updated = poco_last_updated($profile_url);
@@ -245,25 +247,10 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 			$updated = $last_updated;
 			$last_contact = datetime_convert();
 			logger("Last updated for profile ".$profile_url.": ".$updated, LOGGER_DEBUG);
-
-			if (count($x))
-				q("UPDATE `gcontact` SET `last_contact` = '%s' WHERE `nurl` = '%s'", dbesc($last_contact), dbesc(normalise_link($profile_url)));
-		} else {
+		} else
 			$last_failure = datetime_convert();
-
-			if (count($x))
-				q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", dbesc($last_failure), dbesc(normalise_link($profile_url)));
-		}
 	}
 
-	poco_check_server($server_url, $network);
-
-	// Test - remove before flight
-	//if ($last_contact > $last_failure)
-	//	q("UPDATE `gserver` SET `last_contact` = '%s' WHERE `nurl` = '%s'", dbesc($last_contact), dbesc(normalise_link($server_url)));
-	//else
-	//	q("UPDATE `gserver` SET `last_failure` = '%s' WHERE `nurl` = '%s'", dbesc($last_failure), dbesc(normalise_link($server_url)));
-
 	if(count($x)) {
 		$gcid = $x[0]['id'];
 
@@ -366,17 +353,53 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 }
 
 function poco_last_updated($profile) {
+
+	$gcontacts = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s'",
+			dbesc(normalise_link($profile)));
+
+	if ($gcontacts[0]["server_url"] != "") {
+		$servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s' AND `last_contact` < `last_failure`", dbesc(normalise_link($gcontacts[0]["server_url"])));
+
+		if ($servers)
+			return false;
+	}
+
 	$data = probe_url($profile);
 
-	if (($data["poll"] == "") OR ($data["network"] == NETWORK_FEED))
+	if (($data["poll"] == "") OR ($data["network"] == NETWORK_FEED)) {
+		q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'",
+			dbesc(datetime_convert()), dbesc(normalise_link($profile)));
 		return false;
+	}
+
+	if (($data["name"] != "") AND ($data["name"] != $gcontacts[0]["name"]))
+		q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'",
+			dbesc($data["name"]), dbesc(normalise_link($profile)));
+
+	if (($data["addr"] != "") AND ($data["addr"] != $gcontacts[0]["connect"]))
+		q("UPDATE `gcontact` SET `connect` = '%s' WHERE `nurl` = '%s'",
+			dbesc($data["addr"]), dbesc(normalise_link($profile)));
+
+	if (($data["photo"] != "") AND ($data["photo"] != $gcontacts[0]["photo"]))
+		q("UPDATE `gcontact` SET `photo` = '%s' WHERE `nurl` = '%s'",
+			dbesc($data["photo"]), dbesc(normalise_link($profile)));
+
+	if (($data["baseurl"] != "") AND ($data["baseurl"] != $gcontacts[0]["server_url"]))
+		q("UPDATE `gcontact` SET `server_url` = '%s' WHERE `nurl` = '%s'",
+			dbesc($data["baseurl"]), dbesc(normalise_link($profile)));
+
+	if ($data["baseurl"] != "")
+		poco_check_server($data["baseurl"], $data["network"]);
 
 	// To-Do: Use noscrape
 
 	$feedret = z_fetch_url($data["poll"]);
 
-	if (!$feedret["success"])
+	if (!$feedret["success"]) {
+		q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'",
+			dbesc(datetime_convert()), dbesc(normalise_link($profile)));
 		return false;
+	}
 
 	$doc = new DOMDocument();
 	@$doc->loadXML($feedret["body"]);
@@ -404,6 +427,9 @@ function poco_last_updated($profile) {
 		if ($xpath->query('/atom:feed')->length > 0)
 			$last_updated = "0000-00-00 00:00:00";
 
+	q("UPDATE `gcontact` SET `updated` = '%s', `last_contact` = '%s' WHERE `nurl` = '%s'",
+		dbesc($last_updated), dbesc(datetime_convert()), dbesc(normalise_link($profile)));
+
 	return($last_updated);
 }
 
@@ -430,11 +456,11 @@ function poco_do_update($updated, $last_contact, $last_failure) {
 		return false;
 
 	// If the last contact time was more than a week ago, then only try once a week
-	if (($now - $contact_time) > (60 * 60 * 24 * 7) AND ($now - $failure_time) < (60 * 60 * 24 * 7))
+	if ((($now - $contact_time) > (60 * 60 * 24 * 7)) AND (($now - $failure_time) < (60 * 60 * 24 * 7)))
 		return false;
 
-	// If the last contact time was more than a month ago, then only try once a month
-	if (($now - $contact_time) > (60 * 60 * 24 * 30) AND ($now - $failure_time) < (60 * 60 * 24 * 30))
+	// If the last contact time was more than a month ago, then only try once a month - but only if there ever was a contact time
+	if (($contact_time > 0) AND (($now - $contact_time) > (60 * 60 * 24 * 30)) AND (($now - $failure_time) < (60 * 60 * 24 * 30)))
 		return false;
 
 	return true;
@@ -452,7 +478,7 @@ function poco_to_boolean($val) {
 function poco_check_server($server_url, $network = "") {
 
 	if ($server_url == "")
-		return;
+		return false;
 
 	$servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url)));
 	if ($servers) {
@@ -470,12 +496,8 @@ function poco_check_server($server_url, $network = "") {
 		$info = $servers[0]["info"];
 		$register_policy = $servers[0]["register_policy"];
 
-		// Only check the server once a week
-		if (strtotime(datetime_convert()) < (strtotime($last_contact) + (60 * 60 * 24 * 7)))
-			return;
-
-		if (strtotime(datetime_convert()) < (strtotime($last_failure) + (60 * 60 * 24 * 7)))
-			return;
+		if (!poco_do_update("", $last_contact, $last_failure))
+			return ($last_contact >= $last_failure);
 	} else {
 		$poco = "";
 		$noscrape = "";
@@ -668,6 +690,7 @@ function poco_check_server($server_url, $network = "") {
 				dbesc($platform),
 				dbesc(datetime_convert())
 		);
+	return $failure;
 }
 
 function poco_contact_from_body($body, $created, $cid, $uid) {

From a4aec06e5b7064433bff87e115ad7b63107c6d0a Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Mon, 20 Jul 2015 00:14:14 +0200
Subject: [PATCH 14/47] Support for detecting "Hubzilla" (and some bugfixing in
 the server detection)

---
 include/socgraph.php | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/include/socgraph.php b/include/socgraph.php
index 9ccae50405..064fc182bd 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -430,6 +430,10 @@ function poco_last_updated($profile) {
 	q("UPDATE `gcontact` SET `updated` = '%s', `last_contact` = '%s' WHERE `nurl` = '%s'",
 		dbesc($last_updated), dbesc(datetime_convert()), dbesc(normalise_link($profile)));
 
+	if (($gcontacts[0]["generation"] == 0))
+		q("UPDATE `gcontact` SET `generation` = 9 WHERE `nurl` = '%s'",
+			dbesc(normalise_link($profile)));
+
 	return($last_updated);
 }
 
@@ -559,6 +563,11 @@ function poco_check_server($server_url, $network = "") {
 			if (isset($data->site->server)) {
 				$last_contact = datetime_convert();
 
+				if (isset($data->site->hubzilla)) {
+					$platform = $data->site->hubzilla->PLATFORM_NAME;
+					$version = $data->site->hubzilla->RED_VERSION;
+					$network = NETWORK_DFRN;
+				}
 				if (isset($data->site->redmatrix)) {
 					if (isset($data->site->redmatrix->PLATFORM_NAME))
 						$platform = $data->site->redmatrix->PLATFORM_NAME;
@@ -676,8 +685,8 @@ function poco_check_server($server_url, $network = "") {
 			dbesc(normalise_link($server_url))
 		);
 	else
-		q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `last_contact`)
-					VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s')",
+		q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `last_contact`, `last_failure`)
+					VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s')",
 				dbesc($server_url),
 				dbesc(normalise_link($server_url)),
 				dbesc($version),
@@ -688,9 +697,11 @@ function poco_check_server($server_url, $network = "") {
 				dbesc($noscrape),
 				dbesc($network),
 				dbesc($platform),
+				dbesc($last_contact),
+				dbesc($last_failure),
 				dbesc(datetime_convert())
 		);
-	return $failure;
+	return !$failure;
 }
 
 function poco_contact_from_body($body, $created, $cid, $uid) {

From 9717fa82fe02413bf007cff4698538121d71b611 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Tue, 21 Jul 2015 00:05:44 +0200
Subject: [PATCH 15/47] New field "created" in gcontacts. New option for
 timeframe when polling global contacts

---
 include/dbstructure.php       |   3 +
 include/onepoll.php           |   2 +-
 include/socgraph.php          | 131 ++++++++++++++++++++++++++++------
 mod/admin.php                 |  10 +++
 view/templates/admin_site.tpl |   1 +
 5 files changed, 126 insertions(+), 21 deletions(-)

diff --git a/include/dbstructure.php b/include/dbstructure.php
index 9d0856c255..c079efeb55 100644
--- a/include/dbstructure.php
+++ b/include/dbstructure.php
@@ -626,10 +626,12 @@ function db_definition() {
 			"fields" => array(
 					"id" => array("type" => "int(10) unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"),
 					"name" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"nick" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"nurl" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"photo" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"connect" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
 					"updated" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
 					"last_contact" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
 					"last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
@@ -699,6 +701,7 @@ function db_definition() {
 					"noscrape" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"network" => array("type" => "varchar(32)", "not null" => "1", "default" => ""),
 					"platform" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
 					"last_poco_query" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
 					"last_contact" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
 					"last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
diff --git a/include/onepoll.php b/include/onepoll.php
index c0c2f1a34b..464a50fb9b 100644
--- a/include/onepoll.php
+++ b/include/onepoll.php
@@ -110,7 +110,7 @@ function onepoll_run(&$argv, &$argc){
 	// - Check why we don't poll the Diaspora feed at the moment (some guid problem in the items?)
 	// - Check whether this is possible with Redmatrix
 	if ($contact["network"] == NETWORK_DIASPORA) {
-		if (poco_do_update($contact["last-item"], $contact["success_update"], $contact["failure_update"])) {
+		if (poco_do_update($contact["created"], $contact["last-item"], $contact["failure_update"], $contact["success_update"])) {
 			$last_updated = poco_last_updated($contact["url"]);
 			$updated = datetime_convert();
 			if ($last_updated) {
diff --git a/include/socgraph.php b/include/socgraph.php
index 064fc182bd..45cf3a650a 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -205,19 +205,26 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 		if ($updated == "0000-00-00 00:00:00")
 			$updated = $x[0]["updated"];
 
+		$created = $x[0]["created"];
 		$last_contact = $x[0]["last_contact"];
 		$last_failure = $x[0]["last_failure"];
 		$server_url = $x[0]["server_url"];
+		$nick = $x[0]["nick"];
 	} else {
+		$created = "0000-00-00 00:00:00";
 		$last_contact = "0000-00-00 00:00:00";
 		$last_failure = "0000-00-00 00:00:00";
 		$server_url = "";
+
+		$urlparts = parse_url($profile_url);
+		$nick = end(explode("/", $urlparts["path"]));
 	}
 
 	if (($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == "")) {
 		$data = probe_url($profile_url);
 		$network = $data["network"];
 		$name = $data["name"];
+		$nick = $data["nick"];
 		$profile_url = $data["url"];
 		$profile_photo = $data["photo"];
 		$server_url = $data["baseurl"];
@@ -241,7 +248,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	poco_check_server($server_url, $network);
 
 	// Only fetch last update manually if it wasn't provided and enabled in the system
-	if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) {
+	if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($created, $updated, $last_failure, $last_contact)) {
 		$last_updated = poco_last_updated($profile_url);
 		if ($last_updated) {
 			$updated = $last_updated;
@@ -290,15 +297,17 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 			);
 		}
 	} else {
-		q("INSERT INTO `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `server_url`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`)
-			VALUES ('%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)",
+		q("INSERT INTO `gcontact` (`name`, `nick`, `network`, `url`, `nurl`, `photo`, `connect`, `server_url`, `created`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`)
+			VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)",
 			dbesc($name),
+			dbesc($nick),
 			dbesc($network),
 			dbesc($profile_url),
 			dbesc(normalise_link($profile_url)),
 			dbesc($profile_photo),
 			dbesc($connect_url),
 			dbesc($server_url),
+			dbesc(datetime_convert()),
 			dbesc($updated),
 			dbesc($last_contact),
 			dbesc($last_failure),
@@ -357,13 +366,79 @@ function poco_last_updated($profile) {
 	$gcontacts = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s'",
 			dbesc(normalise_link($profile)));
 
-	if ($gcontacts[0]["server_url"] != "") {
-		$servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s' AND `last_contact` < `last_failure`", dbesc(normalise_link($gcontacts[0]["server_url"])));
+	if ($gcontacts[0]["created"] == "0000-00-00 00:00:00")
+		q("UPDATE `gcontact` SET `created` = '%s' WHERE `nurl` = '%s'",
+			dbesc(datetime_convert()), dbesc(normalise_link($profile)));
 
-		if ($servers)
+	if (($gcontacts[0]["server_url"] != "") AND ($gcontacts[0]["network"] != ""))
+		if (!poco_check_server($gcontacts[0]["server_url"], $gcontacts[0]["network"]))
 			return false;
+
+	// noscrape is really fast so we don't cache the call.
+	if (($gcontacts[0]["server_url"] != "") AND ($gcontacts[0]["nick"] != "")) {
+
+		//  Use noscrape if possible
+		$server = q("SELECT `noscrape` FROM `gserver` WHERE `nurl` = '%s' AND `noscrape` != ''", dbesc(normalise_link($gcontacts[0]["server_url"])));
+
+		if ($server) {
+			$noscraperet = z_fetch_url($server[0]["noscrape"]."/".$gcontacts[0]["nick"]);
+			 if ($noscraperet["success"]) {
+				$noscrape = json_decode($noscraperet["body"], true);
+
+				if (($noscrape["name"] != "") AND ($noscrape["name"] != $gcontacts[0]["name"]))
+					q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'",
+						dbesc($noscrape["name"]), dbesc(normalise_link($profile)));
+
+				if (($noscrape["photo"] != "") AND ($noscrape["photo"] != $gcontacts[0]["photo"]))
+					q("UPDATE `gcontact` SET `photo` = '%s' WHERE `nurl` = '%s'",
+						dbesc($noscrape["photo"]), dbesc(normalise_link($profile)));
+
+				if (($noscrape["updated"] != "") AND ($noscrape["updated"] != $gcontacts[0]["updated"]))
+					q("UPDATE `gcontact` SET `updated` = '%s' WHERE `nurl` = '%s'",
+						dbesc($noscrape["updated"]), dbesc(normalise_link($profile)));
+
+				if (($noscrape["gender"] != "") AND ($noscrape["gender"] != $gcontacts[0]["gender"]))
+					q("UPDATE `gcontact` SET `gender` = '%s' WHERE `nurl` = '%s'",
+						dbesc($noscrape["gender"]), dbesc(normalise_link($profile)));
+
+				if (($noscrape["about"] != "") AND ($noscrape["about"] != $gcontacts[0]["name"]))
+					q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'",
+						dbesc($noscrape["about"]), dbesc(normalise_link($profile)));
+
+				$keywords = implode(" ", $noscrape["tags"]);
+				if (($keywords != "") AND ($keywords != $gcontacts[0]["keywords"]))
+					q("UPDATE `gcontact` SET `keywords` = '%s' WHERE `nurl` = '%s'",
+						dbesc($keywords), dbesc(normalise_link($profile)));
+
+				$location = $noscrape["locality"];
+
+				if ($noscrape["region"] != "") {
+					if ($location != "")
+						$location .= ", ";
+
+					$location .= $noscrape["region"];
+				}
+
+				if ($noscrape["country-name"] != "") {
+					if ($location != "")
+						$location .= ", ";
+
+					$location .= $noscrape["country-name"];
+				}
+
+				if (($location != "") AND ($location != $gcontacts[0]["location"]))
+					q("UPDATE `gcontact` SET `location` = '%s' WHERE `nurl` = '%s'",
+						dbesc($location), dbesc(normalise_link($profile)));
+
+				return $noscrape["updated"];
+			}
+		}
 	}
 
+	// If we only can poll the feed, then we only do this once a while
+	if (!poco_do_update($gcontacts[0]["created"], $gcontacts[0]["updated"], $gcontacts[0]["last_failure"],  $gcontacts[0]["last_contact"]))
+/		return $gcontacts[0]["updated"];
+
 	$data = probe_url($profile);
 
 	if (($data["poll"] == "") OR ($data["network"] == NETWORK_FEED)) {
@@ -376,6 +451,10 @@ function poco_last_updated($profile) {
 		q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'",
 			dbesc($data["name"]), dbesc(normalise_link($profile)));
 
+	if (($data["nick"] != "") AND ($data["nick"] != $gcontacts[0]["nick"]))
+		q("UPDATE `gcontact` SET `nick` = '%s' WHERE `nurl` = '%s'",
+			dbesc($data["nick"]), dbesc(normalise_link($profile)));
+
 	if (($data["addr"] != "") AND ($data["addr"] != $gcontacts[0]["connect"]))
 		q("UPDATE `gcontact` SET `connect` = '%s' WHERE `nurl` = '%s'",
 			dbesc($data["addr"]), dbesc(normalise_link($profile)));
@@ -388,11 +467,6 @@ function poco_last_updated($profile) {
 		q("UPDATE `gcontact` SET `server_url` = '%s' WHERE `nurl` = '%s'",
 			dbesc($data["baseurl"]), dbesc(normalise_link($profile)));
 
-	if ($data["baseurl"] != "")
-		poco_check_server($data["baseurl"], $data["network"]);
-
-	// To-Do: Use noscrape
-
 	$feedret = z_fetch_url($data["poll"]);
 
 	if (!$feedret["success"]) {
@@ -437,7 +511,7 @@ function poco_last_updated($profile) {
 	return($last_updated);
 }
 
-function poco_do_update($updated, $last_contact, $last_failure) {
+function poco_do_update($created, $updated, $last_failure,  $last_contact) {
 	$now = strtotime(datetime_convert());
 
 	if ($updated > $last_contact)
@@ -446,6 +520,11 @@ function poco_do_update($updated, $last_contact, $last_failure) {
 		$contact_time = strtotime($last_contact);
 
 	$failure_time = strtotime($last_failure);
+	$created_time = strtotime($created);
+
+	// If there is no "created" time then use the current time
+	if ($created_time <= 0)
+		$created_time = $now;
 
 	// If the last contact was less than 24 hours then don't update
 	if (($now - $contact_time) < (60 * 60 * 24))
@@ -459,12 +538,12 @@ function poco_do_update($updated, $last_contact, $last_failure) {
 	if ((($now - $contact_time) < (60 * 60 * 24 * 7)) AND ($contact_time > $failure_time))
 		return false;
 
-	// If the last contact time was more than a week ago, then only try once a week
-	if ((($now - $contact_time) > (60 * 60 * 24 * 7)) AND (($now - $failure_time) < (60 * 60 * 24 * 7)))
+	// If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week
+	if ((($now - $contact_time) > (60 * 60 * 24 * 7)) AND (($now - $created_time) > (60 * 60 * 24 * 7)) AND (($now - $failure_time) < (60 * 60 * 24 * 7)))
 		return false;
 
-	// If the last contact time was more than a month ago, then only try once a month - but only if there ever was a contact time
-	if (($contact_time > 0) AND (($now - $contact_time) > (60 * 60 * 24 * 30)) AND (($now - $failure_time) < (60 * 60 * 24 * 30)))
+	// If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month
+	if ((($now - $contact_time) > (60 * 60 * 24 * 30)) AND (($now - $created_time) > (60 * 60 * 24 * 30)) AND (($now - $failure_time) < (60 * 60 * 24 * 30)))
 		return false;
 
 	return true;
@@ -486,6 +565,11 @@ function poco_check_server($server_url, $network = "") {
 
 	$servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url)));
 	if ($servers) {
+
+		if ($servers[0]["created"] == "0000-00-00 00:00:00")
+			q("UPDATE `gserver` SET `created` = '%s' WHERE `nurl` = '%s'",
+				dbesc(datetime_convert()), dbesc(normalise_link($server_url)));
+
 		$poco = $servers[0]["poco"];
 		$noscrape = $servers[0]["noscrape"];
 
@@ -500,7 +584,7 @@ function poco_check_server($server_url, $network = "") {
 		$info = $servers[0]["info"];
 		$register_policy = $servers[0]["register_policy"];
 
-		if (!poco_do_update("", $last_contact, $last_failure))
+		if (!poco_do_update($servers[0]["created"], "", $last_failure, $last_contact))
 			return ($last_contact >= $last_failure);
 	} else {
 		$poco = "";
@@ -566,7 +650,7 @@ function poco_check_server($server_url, $network = "") {
 				if (isset($data->site->hubzilla)) {
 					$platform = $data->site->hubzilla->PLATFORM_NAME;
 					$version = $data->site->hubzilla->RED_VERSION;
-					$network = NETWORK_DFRN;
+					$network = NETWORK_DIASPORA;
 				}
 				if (isset($data->site->redmatrix)) {
 					if (isset($data->site->redmatrix->PLATFORM_NAME))
@@ -685,7 +769,7 @@ function poco_check_server($server_url, $network = "") {
 			dbesc(normalise_link($server_url))
 		);
 	else
-		q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `last_contact`, `last_failure`)
+		q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `created`, `last_contact`, `last_failure`)
 					VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s')",
 				dbesc($server_url),
 				dbesc(normalise_link($server_url)),
@@ -697,6 +781,7 @@ function poco_check_server($server_url, $network = "") {
 				dbesc($noscrape),
 				dbesc($network),
 				dbesc($platform),
+				dbesc(datetime_convert()),
 				dbesc($last_contact),
 				dbesc($last_failure),
 				dbesc(datetime_convert())
@@ -1028,8 +1113,14 @@ function poco_discover($complete = false) {
 
 				if (get_config('system','poco_discovery') > 1) {
 
+					$timeframe = get_config('system','poco_discovery_since');
+					if ($timeframe == 0)
+						$timeframe = 30;
+
+					$updatedSince = date("Y-m-d H:i:s", time() - $timeframe * 86400);
+
 					// Fetch all global contacts from the other server (Not working with Redmatrix and Friendica versions before 3.3)
-					$url = $server["poco"]."/@global?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
+					$url = $server["poco"]."/@global?updatedSince=".$updatedSince."&fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
 
 					$retdata = z_fetch_url($url);
 					if ($retdata["success"]) {
diff --git a/mod/admin.php b/mod/admin.php
index ea0196db83..cdfd3b5a68 100644
--- a/mod/admin.php
+++ b/mod/admin.php
@@ -360,6 +360,7 @@ function admin_page_site_post(&$a){
 	$maxloadavg_frontend	=	((x($_POST,'maxloadavg_frontend'))	? intval(trim($_POST['maxloadavg_frontend']))	: 50);
 	$poco_completion	=	((x($_POST,'poco_completion'))		? intval(trim($_POST['poco_completion']))	: false);
 	$poco_discovery		=	((x($_POST,'poco_discovery'))		? intval(trim($_POST['poco_discovery']))	: 0);
+	$poco_discovery_since	=	((x($_POST,'poco_discovery_since'))	? intval(trim($_POST['poco_discovery_since']))	: 30);
 	$poco_local_search	=	((x($_POST,'poco_local_search'))	? intval(trim($_POST['poco_local_search']))	: false);
 	$dfrn_only		=	((x($_POST,'dfrn_only'))		? True						: False);
 	$ostatus_disabled	=	!((x($_POST,'ostatus_disabled'))	? True  					: False);
@@ -432,6 +433,7 @@ function admin_page_site_post(&$a){
 	set_config('system','maxloadavg_frontend',$maxloadavg_frontend);
 	set_config('system','poco_completion',$poco_completion);
 	set_config('system','poco_discovery',$poco_discovery);
+	set_config('system','poco_discovery_since',$poco_discovery_since);
 	set_config('system','poco_local_search',$poco_local_search);
 	set_config('config','sitename',$sitename);
 	set_config('config','hostname',$hostname);
@@ -595,6 +597,13 @@ function admin_page_site(&$a) {
 			"3" => t("Users, Global Contacts/fallback"),
 			);
 
+		$poco_discovery_since_choices = array(
+			"30" => t("One month"),
+			"91" => t("Three months"),
+			"182" => t("Half a year"),
+			"365" => t("One year"),
+			);
+
 		/* get user names to make the install a personal install of X */
 		$user_names = array();
 		$user_names['---'] = t('Multi user instance');
@@ -703,6 +712,7 @@ function admin_page_site(&$a) {
 
 		'$poco_completion'	=> array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful when communicating with Redmatrix and friendica servers before 3.3)")),
 		'$poco_discovery'	=> array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available."), $poco_discovery_choices),
+		'$poco_discovery_since'	=> array('poco_discovery_since', t("Timeframe for fetching global contacts"), (string) intval(get_config('system','poco_discovery_since')), t("When the discovery is activated, this value defines the timeframe for the global contacts that are fetched from other servers."), $poco_discovery_since_choices),
 		'$poco_local_search'	=> array('poco_local_search', t("Search the local directory"), get_config('system','poco_local_search'), t("Search the local directory instead of the global directory. When searching locally, every search will be executed on the global directory in the background. This improves the search results when the search is repeated.")),
 
 		'$use_fulltext_engine'	=> array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")),
diff --git a/view/templates/admin_site.tpl b/view/templates/admin_site.tpl
index 2f448d1b91..9b3c75a0e6 100644
--- a/view/templates/admin_site.tpl
+++ b/view/templates/admin_site.tpl
@@ -118,6 +118,7 @@
 	<h3>{{$portable_contacts}}</h3>
 	{{include file="field_checkbox.tpl" field=$poco_completion}}
 	{{include file="field_select.tpl" field=$poco_discovery}}
+	{{include file="field_select.tpl" field=$poco_discovery_since}}
 	{{include file="field_checkbox.tpl" field=$poco_local_search}}
 	<div class="submit"><input type="submit" name="page_site" value="{{$submit|escape:'html'}}" /></div>
 

From 749a646480f6352af3b9a912811fdc182c0b7c4e Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Wed, 22 Jul 2015 11:51:37 +0200
Subject: [PATCH 16/47] Improved scraping and poco discovery.

---
 include/Scrape.php   |  74 +++++++++++++--------------
 include/socgraph.php | 118 +++++++++++++++++++++++++++++++++++++------
 2 files changed, 137 insertions(+), 55 deletions(-)

diff --git a/include/Scrape.php b/include/Scrape.php
index 90fb1a9e3d..83c099769d 100644
--- a/include/Scrape.php
+++ b/include/Scrape.php
@@ -249,7 +249,7 @@ function scrape_feed($url) {
 					$ret['feed_atom'] = $url;
 					return $ret;
 				}
- 				if(stristr($line,'application/rss+xml') || stristr($s,'<rss')) {
+				if(stristr($line,'application/rss+xml') || stristr($s,'<rss')) {
 					$ret['feed_rss'] = $url;
 					return $ret;
 				}
@@ -262,47 +262,40 @@ function scrape_feed($url) {
 		}
 	}
 
-	try {
-		$dom = HTML5_Parser::parse($s);
-	} catch (DOMException $e) {
-		logger('scrape_feed: parse error: ' . $e);
+	$basename = implode('/', array_slice(explode('/',$url),0,3)) . '/';
+
+	$doc = new DOMDocument();
+	@$doc->loadHTML($s);
+	$xpath = new DomXPath($doc);
+
+	$base = $xpath->query("//base");
+	foreach ($base as $node) {
+		$attr = array();
+
+		if ($node->attributes->length)
+			foreach ($node->attributes as $attribute)
+				$attr[$attribute->name] = $attribute->value;
+
+		if ($attr["href"] != "")
+			$basename = $attr["href"] ;
 	}
 
-	if(! $dom) {
-		logger('scrape_feed: failed to parse.');
-		return $ret;
+	$list = $xpath->query("//link");
+	foreach ($list as $node) {
+		$attr = array();
+
+		if ($node->attributes->length)
+			foreach ($node->attributes as $attribute)
+				$attr[$attribute->name] = $attribute->value;
+
+		if (($attr["rel"] == "alternate") AND ($attr["type"] == "application/atom+xml"))
+			$ret["feed_atom"] = $attr["href"];
+
+		if (($attr["rel"] == "alternate") AND ($attr["type"] == "application/rss+xml"))
+			$ret["feed_rss"] = $attr["href"];
 	}
 
-
-	$head = $dom->getElementsByTagName('base');
-	if($head) {
-		foreach($head as $head0) {
-			$basename = $head0->getAttribute('href');
-			break;
-		}
-	}
-	if(! $basename)
-		$basename = implode('/', array_slice(explode('/',$url),0,3)) . '/';
-
-	$items = $dom->getElementsByTagName('link');
-
-	// get Atom/RSS link elements, take the first one of either.
-
-	if($items) {
-		foreach($items as $item) {
-			$x = $item->getAttribute('rel');
-			if(($x === 'alternate') && ($item->getAttribute('type') === 'application/atom+xml')) {
-				if(! x($ret,'feed_atom'))
-					$ret['feed_atom'] = $item->getAttribute('href');
-			}
-			if(($x === 'alternate') && ($item->getAttribute('type') === 'application/rss+xml')) {
-				if(! x($ret,'feed_rss'))
-					$ret['feed_rss'] = $item->getAttribute('href');
-			}
-		}
-	}
-
-	// Drupal and perhaps others only provide relative URL's. Turn them into absolute.
+	// Drupal and perhaps others only provide relative URLs. Turn them into absolute.
 
 	if(x($ret,'feed_atom') && (! strstr($ret['feed_atom'],'://')))
 		$ret['feed_atom'] = $basename . $ret['feed_atom'];
@@ -509,6 +502,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 	}
 
 	if($mode == PROBE_NORMAL) {
+
 		if(strlen($zot)) {
 			$s = fetch_url($zot);
 			if($s) {
@@ -528,6 +522,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 			}
 		}
 
+
 		if(strlen($dfrn)) {
 			$ret = scrape_dfrn(($hcard) ? $hcard : $dfrn);
 			if(is_array($ret) && x($ret,'dfrn-request')) {
@@ -634,6 +629,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 		if($check_feed) {
 
 			$feedret = scrape_feed(($poll) ? $poll : $url);
+
 			logger('probe_url: scrape_feed ' . (($poll)? $poll : $url) . ' returns: ' . print_r($feedret,true), LOGGER_DATA);
 			if(count($feedret) && ($feedret['feed_atom'] || $feedret['feed_rss'])) {
 				$poll = ((x($feedret,'feed_atom')) ? unamp($feedret['feed_atom']) : unamp($feedret['feed_rss']));
@@ -653,7 +649,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) {
 			logger('probe_url: scrape_feed: headers: ' . $a->get_curl_headers(), LOGGER_DATA);
 
 			// Don't try and parse an empty string
-   			$feed->set_raw_data(($xml) ? $xml : '<?xml version="1.0" encoding="utf-8" ?><xml></xml>');
+			$feed->set_raw_data(($xml) ? $xml : '<?xml version="1.0" encoding="utf-8" ?><xml></xml>');
 
 			$feed->init();
 			if($feed->error())
diff --git a/include/socgraph.php b/include/socgraph.php
index 45cf3a650a..9348cb67b5 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -4,6 +4,11 @@ require_once('include/datetime.php');
 require_once("include/Scrape.php");
 require_once("include/html2bbcode.php");
 
+/*
+ To-Do:
+ - Move GNU Social URL schemata (http://server.tld/user/number) to http://server.tld/username
+*/
+
 /*
  * poco_load
  *
@@ -167,6 +172,14 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	if ($profile_url == "")
 		return $gcid;
 
+	$urlparts = parse_url($profile_url);
+	if (!isset($urlparts["scheme"]))
+		return $gcid;
+
+	if (in_array($urlparts["host"], array("facebook.com", "twitter.com", "www.twitter-rss.com",
+						"identi.ca", "alpha.app.net")))
+		return $gcid;
+
 	$orig_updated = $updated;
 
 	// Don't store the statusnet connector as network
@@ -220,7 +233,8 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 		$nick = end(explode("/", $urlparts["path"]));
 	}
 
-	if (($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == "")) {
+	if ((($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == ""))
+		AND poco_reachable($profile_url, $server_url, $network)) {
 		$data = probe_url($profile_url);
 		$network = $data["network"];
 		$name = $data["name"];
@@ -248,7 +262,9 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	poco_check_server($server_url, $network);
 
 	// Only fetch last update manually if it wasn't provided and enabled in the system
-	if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($created, $updated, $last_failure, $last_contact)) {
+	if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00")
+		AND poco_do_update($created, $updated, $last_failure, $last_contact)
+		AND poco_reachable($profile_url, $server_url, $network)) {
 		$last_updated = poco_last_updated($profile_url);
 		if ($last_updated) {
 			$updated = $last_updated;
@@ -361,6 +377,49 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	return $gcid;
 }
 
+function poco_reachable($profile, $server = "", $network = "") {
+
+	if ($server == "")
+		$server = poco_detect_server($profile);
+
+	if ($server == "")
+		return true;
+
+	return poco_check_server($server, $network);
+}
+
+function poco_detect_server($profile) {
+
+	// Try to detect the server path based upon some known standard paths
+	$server_url = "";
+
+	if ($server_url == "") {
+		$friendica = preg_replace("=(https?://)(.*)/profile/(.*)=ism", "$1$2", $profile);
+		if ($friendica != $profile) {
+			$server_url = $friendica;
+			$network = NETWORK_DFRN;
+		}
+	}
+
+	if ($server_url == "") {
+		$diaspora = preg_replace("=(https?://)(.*)/u/(.*)=ism", "$1$2", $profile);
+		if ($diaspora != $profile) {
+			$server_url = $diaspora;
+			$network = NETWORK_DIASPORA;
+		}
+	}
+
+	if ($server_url == "") {
+		$red = preg_replace("=(https?://)(.*)/channel/(.*)=ism", "$1$2", $profile);
+		if ($red != $profile) {
+			$server_url = $red;
+			$network = NETWORK_DIASPORA;
+		}
+	}
+
+	return $server_url;
+}
+
 function poco_last_updated($profile) {
 
 	$gcontacts = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s'",
@@ -437,7 +496,7 @@ function poco_last_updated($profile) {
 
 	// If we only can poll the feed, then we only do this once a while
 	if (!poco_do_update($gcontacts[0]["created"], $gcontacts[0]["updated"], $gcontacts[0]["last_failure"],  $gcontacts[0]["last_contact"]))
-/		return $gcontacts[0]["updated"];
+		return $gcontacts[0]["updated"];
 
 	$data = probe_url($profile);
 
@@ -558,7 +617,7 @@ function poco_to_boolean($val) {
 	return ($val);
 }
 
-function poco_check_server($server_url, $network = "") {
+function poco_check_server($server_url, $network = "", $force = false) {
 
 	if ($server_url == "")
 		return false;
@@ -584,8 +643,10 @@ function poco_check_server($server_url, $network = "") {
 		$info = $servers[0]["info"];
 		$register_policy = $servers[0]["register_policy"];
 
-		if (!poco_do_update($servers[0]["created"], "", $last_failure, $last_contact))
+		if (!$force AND !poco_do_update($servers[0]["created"], "", $last_failure, $last_contact)) {
+			logger("Use cached data for server ".$server_url, LOGGER_DEBUG);
 			return ($last_contact >= $last_failure);
+		}
 	} else {
 		$poco = "";
 		$noscrape = "";
@@ -598,6 +659,7 @@ function poco_check_server($server_url, $network = "") {
 		$last_contact = "0000-00-00 00:00:00";
 		$last_failure = "0000-00-00 00:00:00";
 	}
+	logger("Server ".$server_url." is unknown. Start discovery.", LOGGER_DEBUG);
 
 	$failure = false;
 	$orig_last_failure = $last_failure;
@@ -752,6 +814,9 @@ function poco_check_server($server_url, $network = "") {
 		}
 	}
 
+	// Check again if the server exists
+	$servers = q("SELECT `nurl` FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url)));
+
 	if ($servers)
 		 q("UPDATE `gserver` SET `url` = '%s', `version` = '%s', `site_name` = '%s', `info` = '%s', `register_policy` = %d, `poco` = '%s', `noscrape` = '%s',
 			`network` = '%s', `platform` = '%s', `last_contact` = '%s', `last_failure` = '%s' WHERE `nurl` = '%s'",
@@ -770,7 +835,7 @@ function poco_check_server($server_url, $network = "") {
 		);
 	else
 		q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `created`, `last_contact`, `last_failure`)
-					VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s')",
+					VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s')",
 				dbesc($server_url),
 				dbesc(normalise_link($server_url)),
 				dbesc($version),
@@ -786,6 +851,9 @@ function poco_check_server($server_url, $network = "") {
 				dbesc($last_failure),
 				dbesc(datetime_convert())
 		);
+
+	logger("End discovery for server ".$server_url, LOGGER_DEBUG);
+
 	return !$failure;
 }
 
@@ -1096,11 +1164,17 @@ function update_suggestions() {
 
 function poco_discover($complete = false) {
 
-	$last_update = date("c", time() - (60 * 60 * 24));
+	$no_of_queries = 5;
 
-	$r = q("SELECT `poco`, `nurl` FROM `gserver` WHERE `last_contact` > `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update));
+	$last_update = date("c", time() - (60 * 60 * 6)); // 24
+
+	$r = q("SELECT `poco`, `nurl`, `url`, `network` FROM `gserver` WHERE `last_contact` > `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update));
 	if ($r)
 		foreach ($r AS $server) {
+
+			if (!poco_check_server($server["url"], $server["network"]))
+				continue;
+
 			// Fetch all users from the other server
 			$url = $server["poco"]."/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
 
@@ -1109,6 +1183,7 @@ function poco_discover($complete = false) {
 			$retdata = z_fetch_url($url);
 			if ($retdata["success"]) {
 				$data = json_decode($retdata["body"]);
+
 				poco_discover_server($data, 2);
 
 				if (get_config('system','poco_discovery') > 1) {
@@ -1122,24 +1197,29 @@ function poco_discover($complete = false) {
 					// Fetch all global contacts from the other server (Not working with Redmatrix and Friendica versions before 3.3)
 					$url = $server["poco"]."/@global?updatedSince=".$updatedSince."&fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
 
+					$success = false;
+
 					$retdata = z_fetch_url($url);
 					if ($retdata["success"]) {
 						logger("Fetch all global contacts from the server ".$server["nurl"], LOGGER_DEBUG);
-						poco_discover_server(json_decode($retdata["body"]));
-					} elseif (get_config('system','poco_discovery') > 2) {
+						$success = poco_discover_server(json_decode($retdata["body"]));
+					}
+
+					if (!$success AND (get_config('system','poco_discovery') > 2)) {
 						logger("Fetch contacts from users of the server ".$server["nurl"], LOGGER_DEBUG);
-						poco_discover_server_users($data);
+						poco_discover_server_users($data, $server);
 					}
 				}
 
 				q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"]));
-				if (!$complete)
+				if (!$complete AND (--$no_of_queries == 0))
 					break;
-			}
+			} else	// If the server hadn't replied correctly, then force a sanity check
+				poco_check_server($server["url"], $server["network"], true);
 		}
 }
 
-function poco_discover_server_users($data) {
+function poco_discover_server_users($data, $server) {
 	foreach ($data->entry AS $entry) {
 		$username = "";
 		if (isset($entry->urls)) {
@@ -1166,7 +1246,9 @@ function poco_discover_server_users($data) {
 function poco_discover_server($data, $default_generation = 0) {
 
 	if (!isset($data->entry) OR !count($data->entry))
-		return;
+		return false;
+
+	$success = false;
 
 	foreach ($data->entry AS $entry) {
 		$profile_url = '';
@@ -1195,6 +1277,7 @@ function poco_discover_server($data, $default_generation = 0) {
 				}
 			}
 		}
+
 		if(isset($entry->photos)) {
 			foreach($entry->photos as $photo) {
 				if($photo->type == 'profile') {
@@ -1227,10 +1310,13 @@ function poco_discover_server($data, $default_generation = 0) {
 				$keywords = implode(", ", $tag);
 
 		if ($generation > 0) {
+			$success = true;
+
 			logger("Store profile ".$profile_url, LOGGER_DEBUG);
-			poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $generation);
+			poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $generation, 0, 0, 0);
 			logger("Done for profile ".$profile_url, LOGGER_DEBUG);
 		}
 	}
+	return $success;
 }
 ?>

From 56a43dcbdf1cfb6d45acc1eb6f96c6183467acb4 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Thu, 23 Jul 2015 00:32:41 +0200
Subject: [PATCH 17/47] New Diaspora detection

---
 include/socgraph.php | 33 ++++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/include/socgraph.php b/include/socgraph.php
index 9348cb67b5..f72fe5f6f7 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -464,7 +464,11 @@ function poco_last_updated($profile) {
 					q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'",
 						dbesc($noscrape["about"]), dbesc(normalise_link($profile)));
 
-				$keywords = implode(" ", $noscrape["tags"]);
+				if (isset($noscrape["tags"]))
+					$keywords = implode(" ", $noscrape["tags"]);
+				else
+					$keywords = "";
+
 				if (($keywords != "") AND ($keywords != $gcontacts[0]["keywords"]))
 					q("UPDATE `gcontact` SET `keywords` = '%s' WHERE `nurl` = '%s'",
 						dbesc($keywords), dbesc(normalise_link($profile)));
@@ -683,6 +687,23 @@ function poco_check_server($server_url, $network = "", $force = false) {
 	} elseif ($network == NETWORK_DIASPORA)
 		$last_contact = datetime_convert();
 
+	if (!$failure) {
+		// Test for Diaspora
+		$serverret = z_fetch_url($server_url);
+
+		$lines = explode("\n",$serverret["header"]);
+		if(count($lines))
+			foreach($lines as $line) {
+				$line = trim($line);
+				if(stristr($line,'X-Diaspora-Version:')) {
+					$platform = "Diaspora";
+					$version = trim(str_replace("X-Diaspora-Version:", "", $line));
+					$version = trim(str_replace("x-diaspora-version:", "", $version));
+					$network = NETWORK_DIASPORA;
+				}
+			}
+	}
+
 	if (!$failure) {
 		// Test for Statusnet
 		// Will also return data for Friendica and GNU Social - but it will be overwritten later
@@ -758,6 +779,9 @@ function poco_check_server($server_url, $network = "", $force = false) {
 			if (isset($data->network) AND ($platform == ""))
 				$platform = $data->network;
 
+			if ($platform == "Diaspora")
+				$network = NETWORK_DIASPORA;
+
 			if ($data->registrations_open)
 				$register_policy = REGISTER_OPEN;
 			else
@@ -773,6 +797,9 @@ function poco_check_server($server_url, $network = "", $force = false) {
 	if (!$failure AND in_array($network, array(NETWORK_DFRN, NETWORK_OSTATUS))) {
 		$serverret = z_fetch_url($server_url."/friendica/json");
 
+		if (!$serverret["success"])
+			$serverret = z_fetch_url($server_url."/friendika/json");
+
 		if ($serverret["success"]) {
 			$data = json_decode($serverret["body"]);
 
@@ -1220,6 +1247,10 @@ function poco_discover($complete = false) {
 }
 
 function poco_discover_server_users($data, $server) {
+
+	if (!isset($data->entry))
+		return;
+
 	foreach ($data->entry AS $entry) {
 		$username = "";
 		if (isset($entry->urls)) {

From bb5186ae277c500614fab54df3255c869af9ec53 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Thu, 23 Jul 2015 08:35:45 +0200
Subject: [PATCH 18/47] Use an ID as a primary field.

---
 include/dbstructure.php | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/include/dbstructure.php b/include/dbstructure.php
index c079efeb55..7d6d015a3f 100644
--- a/include/dbstructure.php
+++ b/include/dbstructure.php
@@ -691,6 +691,7 @@ function db_definition() {
 			);
 	$database["gserver"] = array(
 			"fields" => array(
+					"id" => array("type" => "int(10) unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"),
 					"url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"nurl" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"version" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
@@ -707,7 +708,8 @@ function db_definition() {
 					"last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
 					),
 			"indexes" => array(
-					"PRIMARY" => array("nurl"),
+					"PRIMARY" => array("id"),
+					"nurl" => array("nurl"),
 					)
 			);
 	$database["guid"] = array(

From 5989b5dd68d380f22df25dc324f75b504ccf5760 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Thu, 23 Jul 2015 08:37:10 +0200
Subject: [PATCH 19/47] Adopted changes to database.sql

---
 database.sql | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/database.sql b/database.sql
index ce5f051cdc..67bc718221 100644
--- a/database.sql
+++ b/database.sql
@@ -301,10 +301,12 @@ CREATE TABLE IF NOT EXISTS `gcign` (
 CREATE TABLE IF NOT EXISTS `gcontact` (
 	`id` int(10) unsigned NOT NULL auto_increment PRIMARY KEY,
 	`name` varchar(255) NOT NULL DEFAULT '',
+	`nick` varchar(255) NOT NULL DEFAULT '',
 	`url` varchar(255) NOT NULL DEFAULT '',
 	`nurl` varchar(255) NOT NULL DEFAULT '',
 	`photo` varchar(255) NOT NULL DEFAULT '',
 	`connect` varchar(255) NOT NULL DEFAULT '',
+	`created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
 	`updated` datetime DEFAULT '0000-00-00 00:00:00',
 	`last_contact` datetime DEFAULT '0000-00-00 00:00:00',
 	`last_failure` datetime DEFAULT '0000-00-00 00:00:00',
@@ -360,6 +362,7 @@ CREATE TABLE IF NOT EXISTS `group_member` (
 -- TABLE gserver
 --
 CREATE TABLE IF NOT EXISTS `gserver` (
+	`id` int(10) unsigned NOT NULL auto_increment PRIMARY KEY,
 	`url` varchar(255) NOT NULL DEFAULT '',
 	`nurl` varchar(255) NOT NULL DEFAULT '',
 	`version` varchar(255) NOT NULL DEFAULT '',
@@ -370,9 +373,11 @@ CREATE TABLE IF NOT EXISTS `gserver` (
 	`noscrape` varchar(255) NOT NULL DEFAULT '',
 	`network` varchar(32) NOT NULL DEFAULT '',
 	`platform` varchar(255) NOT NULL DEFAULT '',
+	`created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00',
 	`last_poco_query` datetime DEFAULT '0000-00-00 00:00:00',
 	`last_contact` datetime DEFAULT '0000-00-00 00:00:00',
-	`last_failure` datetime DEFAULT '0000-00-00 00:00:00'
+	`last_failure` datetime DEFAULT '0000-00-00 00:00:00',
+	 INDEX `nurl` (`nurl`)
 ) DEFAULT CHARSET=utf8;
 
 --

From 201d95a8558506ede370e3f4d1986428fd6826bb Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Thu, 23 Jul 2015 10:28:40 +0200
Subject: [PATCH 20/47] Set contact as reachable when there was data from
 noscrape

---
 include/socgraph.php | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/include/socgraph.php b/include/socgraph.php
index f72fe5f6f7..272fcea994 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -429,8 +429,13 @@ function poco_last_updated($profile) {
 		q("UPDATE `gcontact` SET `created` = '%s' WHERE `nurl` = '%s'",
 			dbesc(datetime_convert()), dbesc(normalise_link($profile)));
 
-	if (($gcontacts[0]["server_url"] != "") AND ($gcontacts[0]["network"] != ""))
-		if (!poco_check_server($gcontacts[0]["server_url"], $gcontacts[0]["network"]))
+	if ($gcontacts[0]["server_url"] != "")
+		$server_url = $gcontacts[0]["server_url"];
+	else
+		$server_url = poco_detect_server($profile);
+
+	if ($server_url != "")
+		if (!poco_check_server($pserver_url, $gcontacts[0]["network"]))
 			return false;
 
 	// noscrape is really fast so we don't cache the call.
@@ -493,6 +498,11 @@ function poco_last_updated($profile) {
 					q("UPDATE `gcontact` SET `location` = '%s' WHERE `nurl` = '%s'",
 						dbesc($location), dbesc(normalise_link($profile)));
 
+				// If we got data from noscrape then mark the contact as reachable
+				if (is_array($noscrape) AND count($noscrape))
+					q("UPDATE `gcontact` SET `last_contact` = '%s' WHERE `nurl` = '%s'",
+						dbesc(datetime_convert()), dbesc(normalise_link($profile)));
+
 				return $noscrape["updated"];
 			}
 		}

From cbd2fc1572028a4b7e6753d4abebb46995328145 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Thu, 23 Jul 2015 20:57:08 +0200
Subject: [PATCH 21/47] "last_contact" is now set automatically if it is lower
 than the update value.

---
 include/socgraph.php | 6 +++++-
 mod/poco.php         | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/include/socgraph.php b/include/socgraph.php
index 272fcea994..b9028724a2 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -223,6 +223,9 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 		$last_failure = $x[0]["last_failure"];
 		$server_url = $x[0]["server_url"];
 		$nick = $x[0]["nick"];
+
+		if ($updated > $last_contact)
+			$last_contact = $updated;
 	} else {
 		$created = "0000-00-00 00:00:00";
 		$last_contact = "0000-00-00 00:00:00";
@@ -294,7 +297,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 
 		if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo || $x[0]['updated'] < $updated) {
 			q("UPDATE `gcontact` SET `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s', `server_url` = '%s',
-				`updated` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d
+				`updated` = '%s', `last_contact` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d
 				WHERE (`generation` >= %d OR `generation` = 0) AND `nurl` = '%s'",
 				dbesc($name),
 				dbesc($network),
@@ -303,6 +306,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 				dbesc($profile_url),
 				dbesc($server_url),
 				dbesc($updated),
+				dbesc($last_contact),
 				dbesc($location),
 				dbesc($about),
 				dbesc($keywords),
diff --git a/mod/poco.php b/mod/poco.php
index 0b62e93e38..250add179f 100644
--- a/mod/poco.php
+++ b/mod/poco.php
@@ -61,7 +61,7 @@ function poco_init(&$a) {
 		$update_limit =  date("Y-m-d H:i:s",strtotime($_GET['updatedSince']));
 
 	if ($global) {
-		$r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND `last_contact` >= `last_failure` AND `network` IN ('%s')",
+		$r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND ((`last_contact` >= `last_failure`) OR (`updated` > `last_failure`))  AND `network` IN ('%s')",
 			dbesc($update_limit),
 			dbesc(NETWORK_DFRN)
 		);

From 880f486044a519e4dd22c7e7f52ebf6e5f50b723 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Thu, 23 Jul 2015 22:21:22 +0200
Subject: [PATCH 22/47] Check not only for the last contact but also for the
 last update for the sanity check.

---
 include/discover_poco.php | 5 +++--
 mod/dirfind.php           | 3 ++-
 mod/poco.php              | 6 +++---
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/include/discover_poco.php b/include/discover_poco.php
index 74ed83b473..7dbafcafde 100644
--- a/include/discover_poco.php
+++ b/include/discover_poco.php
@@ -93,11 +93,12 @@ function discover_directory($search) {
 	if(count($j->results))
 		foreach($j->results as $jj) {
 			// Check if the contact already exists
-			$exists = q("SELECT `id`, `last_contact`, `last_failure`  FROM `gcontact` WHERE `nurl` = '%s'", normalise_link($jj->url));
+			$exists = q("SELECT `id`, `last_contact`, `last_failure`, `updated` FROM `gcontact` WHERE `nurl` = '%s'", normalise_link($jj->url));
 			if ($exists) {
 				logger("Profile ".$jj->url." already exists (".$search.")", LOGGER_DEBUG);
 
-				if ($exists[0]["last_contact"] < $exists[0]["last_failure"])
+				if (($exists[0]["last_contact"] < $exists[0]["last_failure"]) AND
+					($exists[0]["updated"] < $exists[0]["last_failure"]))
 					continue;
 
 				// Update the contact
diff --git a/mod/dirfind.php b/mod/dirfind.php
index 996d2a4641..45fc93b7e8 100644
--- a/mod/dirfind.php
+++ b/mod/dirfind.php
@@ -46,7 +46,8 @@ function dirfind_content(&$a) {
 			$results = q("SELECT `contact`.`id` AS `cid`, `gcontact`.`url`, `gcontact`.`name`, `gcontact`.`photo`, `gcontact`.`keywords`
 					FROM `gcontact`
 					LEFT JOIN `contact` ON `contact`.`nurl` = `gcontact`.`nurl` AND `contact`.`uid` = %d
-					WHERE `gcontact`.`network` IN ('%s', '%s', '%s') AND `gcontact`.`last_contact` >= `gcontact`.`last_failure` AND
+					WHERE `gcontact`.`network` IN ('%s', '%s', '%s') AND
+					((`gcontact`.`last_contact` >= `gcontact`.`last_failure`) OR (`gcontact`.`updated` >= `gcontact`.`last_failure`)) AND
 					(`gcontact`.`url` REGEXP '%s' OR `gcontact`.`name` REGEXP '%s' OR `gcontact`.`location` REGEXP '%s' OR
 						`gcontact`.`about` REGEXP '%s' OR `gcontact`.`keywords` REGEXP '%s')
 						GROUP BY `gcontact`.`nurl`
diff --git a/mod/poco.php b/mod/poco.php
index 250add179f..0b08e30bf0 100644
--- a/mod/poco.php
+++ b/mod/poco.php
@@ -61,7 +61,7 @@ function poco_init(&$a) {
 		$update_limit =  date("Y-m-d H:i:s",strtotime($_GET['updatedSince']));
 
 	if ($global) {
-		$r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND ((`last_contact` >= `last_failure`) OR (`updated` > `last_failure`))  AND `network` IN ('%s')",
+		$r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND ((`last_contact` >= `last_failure`) OR (`updated` >= `last_failure`))  AND `network` IN ('%s')",
 			dbesc($update_limit),
 			dbesc(NETWORK_DFRN)
 		);
@@ -96,7 +96,7 @@ function poco_init(&$a) {
 
 
 	if ($global) {
-		$r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') AND `last_contact` >= `last_failure` LIMIT %d, %d",
+		$r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') AND ((`last_contact` >= `last_failure`) OR (`updated` > `last_failure`)) LIMIT %d, %d",
 			dbesc($update_limit),
 			dbesc(NETWORK_DFRN),
 			intval($startIndex),
@@ -107,7 +107,7 @@ function poco_init(&$a) {
 			`profile`.`address` AS `paddress`, `profile`.`region` AS `pregion`, `profile`.`postal-code` AS `ppostalcode`, `profile`.`country-name` AS `pcountry`
 			FROM `contact` INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid`
 			WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') AND `profile`.`is-default`
-			AND `contact`.`success_update` >= `contact`.`failure_update`
+			AND ((`contact`.`success_update` >= `contact`.`failure_update`) OR (`contact`.`last-item` >= `contact`.`failure_update`))
 			AND `contact`.`uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) LIMIT %d, %d",
 			dbesc(NETWORK_DFRN),
 			dbesc(NETWORK_DIASPORA),

From 39d2720e7b9d7674cbdf4c76bc579f3819691528 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Thu, 23 Jul 2015 23:27:15 +0200
Subject: [PATCH 23/47] Update the contact when activated

---
 include/socgraph.php | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/include/socgraph.php b/include/socgraph.php
index b9028724a2..7d4cca18a9 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -264,8 +264,9 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 
 	poco_check_server($server_url, $network);
 
-	// Only fetch last update manually if it wasn't provided and enabled in the system
-	if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00")
+	// Fetch last update manually if it is enabled in the system
+	// AND ($orig_updated == "0000-00-00 00:00:00")
+	if (get_config('system','poco_completion')
 		AND poco_do_update($created, $updated, $last_failure, $last_contact)
 		AND poco_reachable($profile_url, $server_url, $network)) {
 		$last_updated = poco_last_updated($profile_url);

From e3e9de2e74b623a6843ee684a7978d7aa51e34b6 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Fri, 24 Jul 2015 07:23:57 +0200
Subject: [PATCH 24/47] Export all federation contacts through poco

---
 include/dbstructure.php |  1 +
 include/socgraph.php    |  4 ++--
 mod/poco.php            | 12 +++++++++---
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/include/dbstructure.php b/include/dbstructure.php
index 7d6d015a3f..e14ce64d81 100644
--- a/include/dbstructure.php
+++ b/include/dbstructure.php
@@ -646,6 +646,7 @@ function db_definition() {
 			"indexes" => array(
 					"PRIMARY" => array("id"),
 					"nurl" => array("nurl"),
+					"updated" => array("updated"),
 					)
 			);
 	$database["glink"] = array(
diff --git a/include/socgraph.php b/include/socgraph.php
index 7d4cca18a9..1f3536a9f2 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -613,8 +613,8 @@ function poco_do_update($created, $updated, $last_failure,  $last_contact) {
 		return false;
 
 	// If the last contact was less than a week ago and the last failure is older than a week then don't update
-	if ((($now - $contact_time) < (60 * 60 * 24 * 7)) AND ($contact_time > $failure_time))
-		return false;
+	//if ((($now - $contact_time) < (60 * 60 * 24 * 7)) AND ($contact_time > $failure_time))
+	//	return false;
 
 	// If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week
 	if ((($now - $contact_time) > (60 * 60 * 24 * 7)) AND (($now - $created_time) > (60 * 60 * 24 * 7)) AND (($now - $failure_time) < (60 * 60 * 24 * 7)))
diff --git a/mod/poco.php b/mod/poco.php
index 0b08e30bf0..53c7f5b0b7 100644
--- a/mod/poco.php
+++ b/mod/poco.php
@@ -61,9 +61,12 @@ function poco_init(&$a) {
 		$update_limit =  date("Y-m-d H:i:s",strtotime($_GET['updatedSince']));
 
 	if ($global) {
-		$r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND ((`last_contact` >= `last_failure`) OR (`updated` >= `last_failure`))  AND `network` IN ('%s')",
+		//$r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND ((`last_contact` >= `last_failure`) OR (`updated` >= `last_failure`))  AND `network` IN ('%s')",
+		$r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND `updated` >= `last_failure`  AND `network` IN ('%s', '%s', '%s')",
 			dbesc($update_limit),
-			dbesc(NETWORK_DFRN)
+			dbesc(NETWORK_DFRN),
+			dbesc(NETWORK_DIASPORA),
+			dbesc(NETWORK_OSTATUS)
 		);
 	} elseif($system_mode) {
 		$r = q("SELECT count(*) AS `total` FROM `contact` WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '')
@@ -96,9 +99,12 @@ function poco_init(&$a) {
 
 
 	if ($global) {
-		$r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') AND ((`last_contact` >= `last_failure`) OR (`updated` > `last_failure`)) LIMIT %d, %d",
+		//$r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') AND ((`last_contact` >= `last_failure`) OR (`updated` > `last_failure`)) LIMIT %d, %d",
+		$r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s', '%1', '%1') and `updated` > `last_failure` LIMIT %d, %d",
 			dbesc($update_limit),
 			dbesc(NETWORK_DFRN),
+			dbesc(NETWORK_DIASPORA),
+			dbesc(NETWORK_OSTATUS),
 			intval($startIndex),
 			intval($itemsPerPage)
 		);

From 68e318e3162e992acbc31fe4d2efeff26ae6d3d5 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sat, 25 Jul 2015 10:50:00 +0200
Subject: [PATCH 25/47] Don't export contacts with empty network

---
 mod/poco.php | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/mod/poco.php b/mod/poco.php
index 53c7f5b0b7..d180650a42 100644
--- a/mod/poco.php
+++ b/mod/poco.php
@@ -69,8 +69,8 @@ function poco_init(&$a) {
 			dbesc(NETWORK_OSTATUS)
 		);
 	} elseif($system_mode) {
-		$r = q("SELECT count(*) AS `total` FROM `contact` WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '')
-			AND `success_update` >= `failure_update`
+		$r = q("SELECT count(*) AS `total` FROM `contact` WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s')
+			AND (`success_update` >= `failure_update` OR `last-item` >= `failure_update`)
 			AND `uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) ",
 			dbesc(NETWORK_DFRN),
 			dbesc(NETWORK_DIASPORA),
@@ -79,7 +79,8 @@ function poco_init(&$a) {
 			);
 	} else {
 		$r = q("SELECT count(*) AS `total` FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0
-			AND `success_update` >= `failure_update` AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra",
+			AND (`success_update` >= `failure_update` OR `last-item` >= `failure_update`)
+			AND `network` IN ('%s', '%s', '%s', '%s') $sql_extra",
 			intval($user['uid']),
 			dbesc(NETWORK_DFRN),
 			dbesc(NETWORK_DIASPORA),
@@ -112,7 +113,7 @@ function poco_init(&$a) {
 		$r = q("SELECT `contact`.*, `profile`.`about` AS `pabout`, `profile`.`locality` AS `plocation`, `profile`.`pub_keywords`, `profile`.`gender` AS `pgender`,
 			`profile`.`address` AS `paddress`, `profile`.`region` AS `pregion`, `profile`.`postal-code` AS `ppostalcode`, `profile`.`country-name` AS `pcountry`
 			FROM `contact` INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid`
-			WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') AND `profile`.`is-default`
+			WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s') AND `profile`.`is-default`
 			AND ((`contact`.`success_update` >= `contact`.`failure_update`) OR (`contact`.`last-item` >= `contact`.`failure_update`))
 			AND `contact`.`uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) LIMIT %d, %d",
 			dbesc(NETWORK_DFRN),
@@ -124,7 +125,8 @@ function poco_init(&$a) {
 		);
 	} else {
 		$r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0
-			AND `success_update` >= `failure_update` AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra LIMIT %d, %d",
+			AND (`success_update` >= `failure_update` OR `last-item` >= `failure_update`)
+			AND `network` IN ('%s', '%s', '%s', '%s') $sql_extra LIMIT %d, %d",
 			intval($user['uid']),
 			dbesc(NETWORK_DFRN),
 			dbesc(NETWORK_DIASPORA),

From dcfcd6f906e9b7a10d4e733ff53879149c05e80a Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sat, 25 Jul 2015 12:05:27 +0200
Subject: [PATCH 26/47] Poco: added caching

---
 mod/poco.php | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/mod/poco.php b/mod/poco.php
index d180650a42..f84fc964d9 100644
--- a/mod/poco.php
+++ b/mod/poco.php
@@ -98,10 +98,11 @@ function poco_init(&$a) {
 		$startIndex = 0;
 	$itemsPerPage = ((x($_GET,'count') && intval($_GET['count'])) ? intval($_GET['count']) : $totalResults);
 
-
 	if ($global) {
+		logger("Start global query", LOGGER_DEBUG);
 		//$r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') AND ((`last_contact` >= `last_failure`) OR (`updated` > `last_failure`)) LIMIT %d, %d",
-		$r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s', '%1', '%1') and `updated` > `last_failure` LIMIT %d, %d",
+		$r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s', '%s', '%s') AND `updated` > `last_failure`
+			ORDER BY `updated` DESC LIMIT %d, %d",
 			dbesc($update_limit),
 			dbesc(NETWORK_DFRN),
 			dbesc(NETWORK_DIASPORA),
@@ -110,6 +111,7 @@ function poco_init(&$a) {
 			intval($itemsPerPage)
 		);
 	} elseif($system_mode) {
+		logger("Start system mode query", LOGGER_DEBUG);
 		$r = q("SELECT `contact`.*, `profile`.`about` AS `pabout`, `profile`.`locality` AS `plocation`, `profile`.`pub_keywords`, `profile`.`gender` AS `pgender`,
 			`profile`.`address` AS `paddress`, `profile`.`region` AS `pregion`, `profile`.`postal-code` AS `ppostalcode`, `profile`.`country-name` AS `pcountry`
 			FROM `contact` INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid`
@@ -124,6 +126,7 @@ function poco_init(&$a) {
 			intval($itemsPerPage)
 		);
 	} else {
+		logger("Start query for user ".$user['nickname'], LOGGER_DEBUG);
 		$r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0
 			AND (`success_update` >= `failure_update` OR `last-item` >= `failure_update`)
 			AND `network` IN ('%s', '%s', '%s', '%s') $sql_extra LIMIT %d, %d",
@@ -136,6 +139,7 @@ function poco_init(&$a) {
 			intval($itemsPerPage)
 		);
 	}
+	logger("Query done", LOGGER_DEBUG);
 
 	$ret = array();
 	if(x($_GET,'sorted'))
@@ -216,13 +220,19 @@ function poco_init(&$a) {
 				if (($rr['keywords'] == "") AND isset($rr['pub_keywords']))
 					$rr['keywords'] = $rr['pub_keywords'];
 
+				$about = Cache::get("about:".$rr['updated'].":".$rr['nurl']);
+				if (is_null($about)) {
+					$about = bbcode($rr['about'], false, false);
+					Cache::set("about:".$rr['updated'].":".$rr['nurl'],$about);
+				}
+
 				$entry = array();
 				if($fields_ret['id'])
 					$entry['id'] = (int)$rr['id'];
 				if($fields_ret['displayName'])
 					$entry['displayName'] = $rr['name'];
 				if($fields_ret['aboutMe'])
-					$entry['aboutMe'] = bbcode($rr['about'], false, false);
+					$entry['aboutMe'] = $about;
 				if($fields_ret['currentLocation'])
 					$entry['currentLocation'] = $rr['location'];
 				if($fields_ret['gender'])
@@ -305,6 +315,8 @@ function poco_init(&$a) {
 	else
 		http_status_exit(500);
 
+	logger("End of poco", LOGGER_DEBUG);
+
 	if($format === 'xml') {
 		header('Content-type: text/xml');
 		echo replace_macros(get_markup_template('poco_xml.tpl'),array_xmlify(array('$response' => $ret)));

From 5f524006dd058dd913510bfb6604aefaf1f27c9b Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sat, 25 Jul 2015 14:51:56 +0200
Subject: [PATCH 27/47] Poco: Only fetch the latest post if it wasn't provided.

---
 include/socgraph.php | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/include/socgraph.php b/include/socgraph.php
index 1f3536a9f2..6111e07790 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -265,8 +265,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	poco_check_server($server_url, $network);
 
 	// Fetch last update manually if it is enabled in the system
-	// AND ($orig_updated == "0000-00-00 00:00:00")
-	if (get_config('system','poco_completion')
+	if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00")
 		AND poco_do_update($created, $updated, $last_failure, $last_contact)
 		AND poco_reachable($profile_url, $server_url, $network)) {
 		$last_updated = poco_last_updated($profile_url);

From 072d7f6b3c9ee2b3a01f91044dfa95771320a633 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sat, 25 Jul 2015 19:59:27 +0200
Subject: [PATCH 28/47] Do a recheck when the server isn't available.

---
 include/socgraph.php | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/include/socgraph.php b/include/socgraph.php
index 6111e07790..a84d2d7be4 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -239,6 +239,18 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	if ((($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == ""))
 		AND poco_reachable($profile_url, $server_url, $network)) {
 		$data = probe_url($profile_url);
+
+		// If the system doesn't seem to react, recheck the server
+		if ($data["network"] == NETWORK_FEED) {
+			logger("Recheck the server for profile ".$profile_url, LOGGER_DEBUG);
+			if ($server_url == "")
+				$url_check = poco_detect_server($profile_url);
+			else
+				$url_check = $server_url;
+
+			poco_check_server($url_check, $network, true);
+		}
+
 		$network = $data["network"];
 		$name = $data["name"];
 		$nick = $data["nick"];
@@ -1208,6 +1220,7 @@ function poco_discover($complete = false) {
 	$no_of_queries = 5;
 
 	$last_update = date("c", time() - (60 * 60 * 6)); // 24
+	$last_update = date("c", time() - (60 * 60 * 24)); // 24
 
 	$r = q("SELECT `poco`, `nurl`, `url`, `network` FROM `gserver` WHERE `last_contact` > `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update));
 	if ($r)

From fc9c6e6bfd1cf31959d79077e701b086c5d8c713 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sat, 25 Jul 2015 20:10:42 +0200
Subject: [PATCH 29/47] Poco: always do a forced check for availability of the
 server

---
 include/socgraph.php | 17 +++--------------
 1 file changed, 3 insertions(+), 14 deletions(-)

diff --git a/include/socgraph.php b/include/socgraph.php
index a84d2d7be4..0c78cfe235 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -237,20 +237,9 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	}
 
 	if ((($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == ""))
-		AND poco_reachable($profile_url, $server_url, $network)) {
+		AND poco_reachable($profile_url, $server_url, $network, true)) {
 		$data = probe_url($profile_url);
 
-		// If the system doesn't seem to react, recheck the server
-		if ($data["network"] == NETWORK_FEED) {
-			logger("Recheck the server for profile ".$profile_url, LOGGER_DEBUG);
-			if ($server_url == "")
-				$url_check = poco_detect_server($profile_url);
-			else
-				$url_check = $server_url;
-
-			poco_check_server($url_check, $network, true);
-		}
-
 		$network = $data["network"];
 		$name = $data["name"];
 		$nick = $data["nick"];
@@ -393,7 +382,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	return $gcid;
 }
 
-function poco_reachable($profile, $server = "", $network = "") {
+function poco_reachable($profile, $server = "", $network = "", $force = false) {
 
 	if ($server == "")
 		$server = poco_detect_server($profile);
@@ -401,7 +390,7 @@ function poco_reachable($profile, $server = "", $network = "") {
 	if ($server == "")
 		return true;
 
-	return poco_check_server($server, $network);
+	return poco_check_server($server, $network, $force);
 }
 
 function poco_detect_server($profile) {

From 166519fc4db8eec57ce17c4ab7b094d2252b65e1 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sun, 26 Jul 2015 14:41:34 +0200
Subject: [PATCH 30/47] Move the contact discovery into the background

---
 include/discover_poco.php | 32 ++++++++++++++++++++++++++------
 include/poller.php        |  4 ++++
 include/socgraph.php      | 22 +++++++++++-----------
 3 files changed, 41 insertions(+), 17 deletions(-)

diff --git a/include/discover_poco.php b/include/discover_poco.php
index 7dbafcafde..68f483dab5 100644
--- a/include/discover_poco.php
+++ b/include/discover_poco.php
@@ -38,23 +38,26 @@ function discover_poco_run(&$argv, &$argc){
 
 	if(($argc > 2) && ($argv[1] == "dirsearch")) {
 		$search = urldecode($argv[2]);
-		$searchmode = 1;
+		$mode = 1;
+	} elseif(($argc == 2) && ($argv[1] == "checkcontact")) {
+		$mode = 2;
 	} elseif ($argc == 1) {
 		$search = "";
-		$searchmode = 0;
+		$mode = 0;
 	} else
 		die("Unknown or missing parameter ".$argv[1]."\n");
 
 	$lockpath = get_lockpath();
 	if ($lockpath != '') {
-		$pidfile = new pidfile($lockpath, 'discover_poco'.urlencode($search));
+		$pidfile = new pidfile($lockpath, 'discover_poco'.$mode.urlencode($search));
 		if($pidfile->is_already_running()) {
 			logger("discover_poco: Already running");
 			if ($pidfile->running_time() > 19*60) {
                                 $pidfile->kill();
                                 logger("discover_poco: killed stale process");
 				// Calling a new instance
-				proc_run('php','include/discover_poco.php');
+				if ($mode == 0)
+					proc_run('php','include/discover_poco.php');
                         }
 			exit;
 		}
@@ -66,9 +69,11 @@ function discover_poco_run(&$argv, &$argc){
 
 	logger('start '.$search);
 
-	if (($search != "") and get_config('system','poco_local_search'))
+	if (($mode == 2) AND get_config('system','poco_completion'))
+		discover_users();
+	elseif (($mode == 1) AND ($search != "") and get_config('system','poco_local_search'))
 		discover_directory($search);
-	elseif (($search == "") and get_config('system','poco_discovery') > 0)
+	elseif (($mode == 0) AND ($search == "") and (get_config('system','poco_discovery') > 0))
 		poco_discover();
 
 	logger('end '.$search);
@@ -76,6 +81,21 @@ function discover_poco_run(&$argv, &$argc){
 	return;
 }
 
+function discover_users() {
+	$users = q("SELECT `url` FROM `gcontact` WHERE `updated` = '0000-00-00 00:00:00' AND
+			`last_contact` = '0000-00-00 00:00:00' AND `last_failure` = '0000-00-00 00:00:00' AND
+			`network` IN ('%s', '%s', '%s') ORDER BY rand() LIMIT 50",
+			dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS));
+
+	if (!$users)
+		return;
+
+	foreach ($users AS $user) {
+		logger('Check user '.$user["url"]);
+		poco_last_updated($user["url"]);
+	}
+}
+
 function discover_directory($search) {
 
 	$data = Cache::get("dirsearch:".$search);
diff --git a/include/poller.php b/include/poller.php
index 44ac94daa4..7cc1a28527 100644
--- a/include/poller.php
+++ b/include/poller.php
@@ -86,6 +86,10 @@ function poller_run(&$argv, &$argc){
 
 	proc_run('php',"include/discover_poco.php");
 
+	// run the process to update locally stored global contacts in the background
+
+	proc_run('php',"include/discover_poco.php", "checkcontact");
+
 	// expire any expired accounts
 
 	q("UPDATE user SET `account_expired` = 1 where `account_expired` = 0
diff --git a/include/socgraph.php b/include/socgraph.php
index 0c78cfe235..53763626d6 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -266,17 +266,17 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	poco_check_server($server_url, $network);
 
 	// Fetch last update manually if it is enabled in the system
-	if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00")
-		AND poco_do_update($created, $updated, $last_failure, $last_contact)
-		AND poco_reachable($profile_url, $server_url, $network)) {
-		$last_updated = poco_last_updated($profile_url);
-		if ($last_updated) {
-			$updated = $last_updated;
-			$last_contact = datetime_convert();
-			logger("Last updated for profile ".$profile_url.": ".$updated, LOGGER_DEBUG);
-		} else
-			$last_failure = datetime_convert();
-	}
+	//if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00")
+	//	AND poco_do_update($created, $updated, $last_failure, $last_contact)
+	//	AND poco_reachable($profile_url, $server_url, $network)) {
+	//	$last_updated = poco_last_updated($profile_url);
+	//	if ($last_updated) {
+	//		$updated = $last_updated;
+	//		$last_contact = datetime_convert();
+	//		logger("Last updated for profile ".$profile_url.": ".$updated, LOGGER_DEBUG);
+	//	} else
+	//		$last_failure = datetime_convert();
+	//}
 
 	if(count($x)) {
 		$gcid = $x[0]['id'];

From 7136a39f9e4dd7adce5adb3bc13ec3ccd4c3483d Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sun, 26 Jul 2015 16:37:05 +0200
Subject: [PATCH 31/47] Discovery for unchecked contacts

---
 include/discover_poco.php | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/include/discover_poco.php b/include/discover_poco.php
index 68f483dab5..bc5ee5d321 100644
--- a/include/discover_poco.php
+++ b/include/discover_poco.php
@@ -82,17 +82,21 @@ function discover_poco_run(&$argv, &$argc){
 }
 
 function discover_users() {
-	$users = q("SELECT `url` FROM `gcontact` WHERE `updated` = '0000-00-00 00:00:00' AND
-			`last_contact` = '0000-00-00 00:00:00' AND `last_failure` = '0000-00-00 00:00:00' AND
-			`network` IN ('%s', '%s', '%s') ORDER BY rand() LIMIT 50",
+	// To-Do: Maybe we should check old contact as well.
+	$users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact` FROM `gcontact`
+			WHERE `updated` = '0000-00-00 00:00:00' AND `last_contact` = '0000-00-00 00:00:00' AND
+				`last_failure` = '0000-00-00 00:00:00' AND `network` IN ('%s', '%s', '%s')
+				ORDER BY rand() LIMIT 100",
 			dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS));
 
 	if (!$users)
 		return;
 
 	foreach ($users AS $user) {
-		logger('Check user '.$user["url"]);
-		poco_last_updated($user["url"]);
+		if (poco_do_update($user["created"], $user["updated"], $user["last_failure"], $user["last_contact"])) {
+			logger('Check user '.$user["url"]);
+			poco_last_updated($user["url"]);
+		}
 	}
 }
 

From e5bf7bae7b2e2d670744e4b5ec6fa82b28ca59e2 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sun, 26 Jul 2015 16:52:37 +0200
Subject: [PATCH 32/47] Just some improvements ...

---
 include/discover_poco.php | 21 +++++++++++++++++----
 include/socgraph.php      |  2 +-
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/include/discover_poco.php b/include/discover_poco.php
index bc5ee5d321..481bedcf92 100644
--- a/include/discover_poco.php
+++ b/include/discover_poco.php
@@ -83,19 +83,32 @@ function discover_poco_run(&$argv, &$argc){
 
 function discover_users() {
 	// To-Do: Maybe we should check old contact as well.
-	$users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact` FROM `gcontact`
+	$users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url` FROM `gcontact`
 			WHERE `updated` = '0000-00-00 00:00:00' AND `last_contact` = '0000-00-00 00:00:00' AND
 				`last_failure` = '0000-00-00 00:00:00' AND `network` IN ('%s', '%s', '%s')
-				ORDER BY rand() LIMIT 100",
+				ORDER BY rand()",
 			dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS));
 
 	if (!$users)
 		return;
 
+	$checked = 0;
+
 	foreach ($users AS $user) {
 		if (poco_do_update($user["created"], $user["updated"], $user["last_failure"], $user["last_contact"])) {
-			logger('Check user '.$user["url"]);
-			poco_last_updated($user["url"]);
+
+			if ($user[0]["server_url"] != "")
+                		$server_url = $user[0]["server_url"];
+        		else
+                		$server_url = poco_detect_server($user["url"]);
+
+			if (poco_check_server($server_url, $gcontacts[0]["network"])) {
+				logger('Check user '.$user["url"]);
+				poco_last_updated($user["url"]);
+
+				if (++$checked > 100)
+					return;
+			}
 		}
 	}
 }
diff --git a/include/socgraph.php b/include/socgraph.php
index 53763626d6..04374f3e54 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -440,7 +440,7 @@ function poco_last_updated($profile) {
 		$server_url = poco_detect_server($profile);
 
 	if ($server_url != "")
-		if (!poco_check_server($pserver_url, $gcontacts[0]["network"]))
+		if (!poco_check_server($server_url, $gcontacts[0]["network"]))
 			return false;
 
 	// noscrape is really fast so we don't cache the call.

From a0011ba58e7503a90e643268b63db173e9b1a268 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sun, 26 Jul 2015 17:14:32 +0200
Subject: [PATCH 33/47] Discover users who never are contacted.

---
 include/discover_poco.php | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/include/discover_poco.php b/include/discover_poco.php
index 481bedcf92..bd3483307d 100644
--- a/include/discover_poco.php
+++ b/include/discover_poco.php
@@ -82,11 +82,11 @@ function discover_poco_run(&$argv, &$argc){
 }
 
 function discover_users() {
+	logger("Discover users", LOGGER_DEBUG);
 	// To-Do: Maybe we should check old contact as well.
 	$users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url` FROM `gcontact`
-			WHERE `updated` = '0000-00-00 00:00:00' AND `last_contact` = '0000-00-00 00:00:00' AND
-				`last_failure` = '0000-00-00 00:00:00' AND `network` IN ('%s', '%s', '%s')
-				ORDER BY rand()",
+			WHERE `last_contact` = '0000-00-00 00:00:00' AND `last_failure` = '0000-00-00 00:00:00' AND
+				`network` IN ('%s', '%s', '%s') ORDER BY rand()",
 			dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS));
 
 	if (!$users)

From e0955535902b13982f58c063d47ab00a9d4d1db9 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Mon, 27 Jul 2015 07:16:06 +0200
Subject: [PATCH 34/47] Mark contacts from not reachable servers as unreachable

---
 include/discover_poco.php | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/include/discover_poco.php b/include/discover_poco.php
index bd3483307d..687698b44f 100644
--- a/include/discover_poco.php
+++ b/include/discover_poco.php
@@ -108,7 +108,9 @@ function discover_users() {
 
 				if (++$checked > 100)
 					return;
-			}
+			} else
+				q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'",
+					dbesc(datetime_convert()), dbesc(normalise_link($user["url"])));
 		}
 	}
 }

From 4b2c2c480231cdccc40fea0e31aaac5811507ae7 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Mon, 27 Jul 2015 08:14:04 +0200
Subject: [PATCH 35/47] Check all contacts that we haven't contacted in a month

---
 include/discover_poco.php | 7 ++++---
 include/socgraph.php      | 6 +++---
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/include/discover_poco.php b/include/discover_poco.php
index 687698b44f..e93b449b98 100644
--- a/include/discover_poco.php
+++ b/include/discover_poco.php
@@ -83,9 +83,10 @@ function discover_poco_run(&$argv, &$argc){
 
 function discover_users() {
 	logger("Discover users", LOGGER_DEBUG);
-	// To-Do: Maybe we should check old contact as well.
+
 	$users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url` FROM `gcontact`
-			WHERE `last_contact` = '0000-00-00 00:00:00' AND `last_failure` = '0000-00-00 00:00:00' AND
+			WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
+				`last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
 				`network` IN ('%s', '%s', '%s') ORDER BY rand()",
 			dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS));
 
@@ -104,7 +105,7 @@ function discover_users() {
 
 			if (poco_check_server($server_url, $gcontacts[0]["network"])) {
 				logger('Check user '.$user["url"]);
-				poco_last_updated($user["url"]);
+				poco_last_updated($user["url"], true);
 
 				if (++$checked > 100)
 					return;
diff --git a/include/socgraph.php b/include/socgraph.php
index 04374f3e54..5671891df5 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -425,7 +425,7 @@ function poco_detect_server($profile) {
 	return $server_url;
 }
 
-function poco_last_updated($profile) {
+function poco_last_updated($profile, $force = false) {
 
 	$gcontacts = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s'",
 			dbesc(normalise_link($profile)));
@@ -451,7 +451,7 @@ function poco_last_updated($profile) {
 
 		if ($server) {
 			$noscraperet = z_fetch_url($server[0]["noscrape"]."/".$gcontacts[0]["nick"]);
-			 if ($noscraperet["success"]) {
+			 if ($noscraperet["success"] AND ($noscraperet["body"] = "")) {
 				$noscrape = json_decode($noscraperet["body"], true);
 
 				if (($noscrape["name"] != "") AND ($noscrape["name"] != $gcontacts[0]["name"]))
@@ -514,7 +514,7 @@ function poco_last_updated($profile) {
 	}
 
 	// If we only can poll the feed, then we only do this once a while
-	if (!poco_do_update($gcontacts[0]["created"], $gcontacts[0]["updated"], $gcontacts[0]["last_failure"],  $gcontacts[0]["last_contact"]))
+	if (!$force AND !poco_do_update($gcontacts[0]["created"], $gcontacts[0]["updated"], $gcontacts[0]["last_failure"],  $gcontacts[0]["last_contact"]))
 		return $gcontacts[0]["updated"];
 
 	$data = probe_url($profile);

From ec161d798f426e53e3c1b963624ce6852c4ddd2a Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Tue, 28 Jul 2015 08:25:30 +0200
Subject: [PATCH 36/47] The discovery should check all profiles.

---
 include/discover_poco.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/include/discover_poco.php b/include/discover_poco.php
index e93b449b98..47c0cc2cd0 100644
--- a/include/discover_poco.php
+++ b/include/discover_poco.php
@@ -96,7 +96,7 @@ function discover_users() {
 	$checked = 0;
 
 	foreach ($users AS $user) {
-		if (poco_do_update($user["created"], $user["updated"], $user["last_failure"], $user["last_contact"])) {
+		//if (poco_do_update($user["created"], $user["updated"], $user["last_failure"], $user["last_contact"])) {
 
 			if ($user[0]["server_url"] != "")
                 		$server_url = $user[0]["server_url"];
@@ -112,7 +112,7 @@ function discover_users() {
 			} else
 				q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'",
 					dbesc(datetime_convert()), dbesc(normalise_link($user["url"])));
-		}
+		//}
 	}
 }
 

From 9f073f2fd58df9dee38991efbf4e44a66c0fc03a Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Tue, 28 Jul 2015 15:28:58 +0200
Subject: [PATCH 37/47] Some correction stuff for old entries in the global
 contacts

---
 include/discover_poco.php |  5 +++--
 include/socgraph.php      | 30 ++++++++++++++++++++++++++++--
 2 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/include/discover_poco.php b/include/discover_poco.php
index 47c0cc2cd0..d4564ae4a6 100644
--- a/include/discover_poco.php
+++ b/include/discover_poco.php
@@ -87,8 +87,9 @@ function discover_users() {
 	$users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url` FROM `gcontact`
 			WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
 				`last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND
-				`network` IN ('%s', '%s', '%s') ORDER BY rand()",
-			dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS));
+				`network` IN ('%s', '%s', '%s', '%s', '') ORDER BY rand()",
+			dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA),
+			dbesc(NETWORK_OSTATUS), dbesc(NETWORK_FEED));
 
 	if (!$users)
 		return;
diff --git a/include/socgraph.php b/include/socgraph.php
index 5671891df5..5e7cded9c2 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -176,7 +176,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	if (!isset($urlparts["scheme"]))
 		return $gcid;
 
-	if (in_array($urlparts["host"], array("facebook.com", "twitter.com", "www.twitter-rss.com",
+	if (in_array($urlparts["host"], array("www.facebook.com", "facebook.com", "twitter.com",
 						"identi.ca", "alpha.app.net")))
 		return $gcid;
 
@@ -434,15 +434,41 @@ function poco_last_updated($profile, $force = false) {
 		q("UPDATE `gcontact` SET `created` = '%s' WHERE `nurl` = '%s'",
 			dbesc(datetime_convert()), dbesc(normalise_link($profile)));
 
+	$urlparts = parse_url($profile);
+	if (!isset($urlparts["scheme"]))
+		return;
+
+	if (in_array($urlparts["host"], array("www.facebook.com", "facebook.com", "twitter.com",
+						"identi.ca", "alpha.app.net"))) {
+		q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'",
+			dbesc(NETWORK_PHANTOM), dbesc(normalise_link($profile)));
+		return;
+	}
+
 	if ($gcontacts[0]["server_url"] != "")
 		$server_url = $gcontacts[0]["server_url"];
 	else
 		$server_url = poco_detect_server($profile);
 
-	if ($server_url != "")
+	if ($server_url != "") {
 		if (!poco_check_server($server_url, $gcontacts[0]["network"]))
 			return false;
 
+		q("UPDATE `gcontact` SET `server_url` = '%s' WHERE `nurl` = '%s'",
+			dbesc($server_url), dbesc(normalise_link($profile)));
+	}
+
+	if (in_array($gcontacts[0]["network"], array("", NETWORK_FEED))) {
+		$server = q("SELECT `network` FROM `gserver` WHERE `nurl` = '%s' AND `network` != ''",
+			dbesc(normalise_link($server_url)));
+
+		if ($server)
+			q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'",
+				dbesc($server[0]["network"]), dbesc(normalise_link($profile)));
+		else
+			return;
+	}
+
 	// noscrape is really fast so we don't cache the call.
 	if (($gcontacts[0]["server_url"] != "") AND ($gcontacts[0]["nick"] != "")) {
 

From 678de014ad2f2d1fc0915f83998eba0e325f8720 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Tue, 28 Jul 2015 18:26:30 +0200
Subject: [PATCH 38/47] Some code beautification.

---
 include/discover_poco.php | 26 ++++++++++++--------------
 include/socgraph.php      | 39 ++++++++++++---------------------------
 2 files changed, 24 insertions(+), 41 deletions(-)

diff --git a/include/discover_poco.php b/include/discover_poco.php
index d4564ae4a6..a9eaf1cf67 100644
--- a/include/discover_poco.php
+++ b/include/discover_poco.php
@@ -97,23 +97,21 @@ function discover_users() {
 	$checked = 0;
 
 	foreach ($users AS $user) {
-		//if (poco_do_update($user["created"], $user["updated"], $user["last_failure"], $user["last_contact"])) {
 
-			if ($user[0]["server_url"] != "")
-                		$server_url = $user[0]["server_url"];
-        		else
-                		$server_url = poco_detect_server($user["url"]);
+		if ($user[0]["server_url"] != "")
+			$server_url = $user[0]["server_url"];
+		else
+			$server_url = poco_detect_server($user["url"]);
 
-			if (poco_check_server($server_url, $gcontacts[0]["network"])) {
-				logger('Check user '.$user["url"]);
-				poco_last_updated($user["url"], true);
+		if (poco_check_server($server_url, $gcontacts[0]["network"])) {
+			logger('Check user '.$user["url"]);
+			poco_last_updated($user["url"], true);
 
-				if (++$checked > 100)
-					return;
-			} else
-				q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'",
-					dbesc(datetime_convert()), dbesc(normalise_link($user["url"])));
-		//}
+			if (++$checked > 100)
+				return;
+		} else
+			q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'",
+				dbesc(datetime_convert()), dbesc(normalise_link($user["url"])));
 	}
 }
 
diff --git a/include/socgraph.php b/include/socgraph.php
index 5e7cded9c2..979f1cb367 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -7,6 +7,8 @@ require_once("include/html2bbcode.php");
 /*
  To-Do:
  - Move GNU Social URL schemata (http://server.tld/user/number) to http://server.tld/username
+ - Fetch profile data from profile page for Redmatrix users
+ - Detect if it is a forum
 */
 
 /*
@@ -219,17 +221,10 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 			$updated = $x[0]["updated"];
 
 		$created = $x[0]["created"];
-		$last_contact = $x[0]["last_contact"];
-		$last_failure = $x[0]["last_failure"];
 		$server_url = $x[0]["server_url"];
 		$nick = $x[0]["nick"];
-
-		if ($updated > $last_contact)
-			$last_contact = $updated;
 	} else {
 		$created = "0000-00-00 00:00:00";
-		$last_contact = "0000-00-00 00:00:00";
-		$last_failure = "0000-00-00 00:00:00";
 		$server_url = "";
 
 		$urlparts = parse_url($profile_url);
@@ -265,19 +260,6 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 
 	poco_check_server($server_url, $network);
 
-	// Fetch last update manually if it is enabled in the system
-	//if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00")
-	//	AND poco_do_update($created, $updated, $last_failure, $last_contact)
-	//	AND poco_reachable($profile_url, $server_url, $network)) {
-	//	$last_updated = poco_last_updated($profile_url);
-	//	if ($last_updated) {
-	//		$updated = $last_updated;
-	//		$last_contact = datetime_convert();
-	//		logger("Last updated for profile ".$profile_url.": ".$updated, LOGGER_DEBUG);
-	//	} else
-	//		$last_failure = datetime_convert();
-	//}
-
 	if(count($x)) {
 		$gcid = $x[0]['id'];
 
@@ -298,7 +280,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 
 		if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo || $x[0]['updated'] < $updated) {
 			q("UPDATE `gcontact` SET `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s', `server_url` = '%s',
-				`updated` = '%s', `last_contact` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d
+				`updated` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d
 				WHERE (`generation` >= %d OR `generation` = 0) AND `nurl` = '%s'",
 				dbesc($name),
 				dbesc($network),
@@ -307,7 +289,6 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 				dbesc($profile_url),
 				dbesc($server_url),
 				dbesc($updated),
-				dbesc($last_contact),
 				dbesc($location),
 				dbesc($about),
 				dbesc($keywords),
@@ -318,8 +299,8 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 			);
 		}
 	} else {
-		q("INSERT INTO `gcontact` (`name`, `nick`, `network`, `url`, `nurl`, `photo`, `connect`, `server_url`, `created`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`)
-			VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)",
+		q("INSERT INTO `gcontact` (`name`, `nick`, `network`, `url`, `nurl`, `photo`, `connect`, `server_url`, `created`, `updated`, `location`, `about`, `keywords`, `gender`, `generation`)
+			VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)",
 			dbesc($name),
 			dbesc($nick),
 			dbesc($network),
@@ -330,8 +311,6 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 			dbesc($server_url),
 			dbesc(datetime_convert()),
 			dbesc($updated),
-			dbesc($last_contact),
-			dbesc($last_failure),
 			dbesc($location),
 			dbesc($about),
 			dbesc($keywords),
@@ -451,8 +430,14 @@ function poco_last_updated($profile, $force = false) {
 		$server_url = poco_detect_server($profile);
 
 	if ($server_url != "") {
-		if (!poco_check_server($server_url, $gcontacts[0]["network"]))
+		if (!poco_check_server($server_url, $gcontacts[0]["network"], $force)) {
+
+			if ($force)
+				q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'",
+					dbesc(datetime_convert()), dbesc(normalise_link($profile)));
+
 			return false;
+		}
 
 		q("UPDATE `gcontact` SET `server_url` = '%s' WHERE `nurl` = '%s'",
 			dbesc($server_url), dbesc(normalise_link($profile)));

From b7c9b942f14e0b621b57f626d25f20b34e080b96 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Wed, 29 Jul 2015 00:21:08 +0200
Subject: [PATCH 39/47] Set the network name for some centralized networks.

---
 include/socgraph.php | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/include/socgraph.php b/include/socgraph.php
index 979f1cb367..f0ed7ec232 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -419,8 +419,14 @@ function poco_last_updated($profile, $force = false) {
 
 	if (in_array($urlparts["host"], array("www.facebook.com", "facebook.com", "twitter.com",
 						"identi.ca", "alpha.app.net"))) {
+		$networks = array("www.facebook.com" => NETWORK_FACEBOOK,
+				"facebook.com" => NETWORK_FACEBOOK,
+				"twitter.com" => NETWORK_TWITTER,
+				"identi.ca" => NETWORK_PUMPIO,
+				"alpha.app.net" => NETWORK_APPNET);
+
 		q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'",
-			dbesc(NETWORK_PHANTOM), dbesc(normalise_link($profile)));
+			dbesc($networks[$urlparts["host"]]), dbesc(normalise_link($profile)));
 		return;
 	}
 

From 098ccd09fa0cc0257ecbb59598d97246251b6d92 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Wed, 29 Jul 2015 06:31:18 +0200
Subject: [PATCH 40/47] Moved the detection for centralized networks.

---
 include/discover_poco.php | 36 ++++++++++++++++++++++++++++--------
 include/socgraph.php      | 17 -----------------
 2 files changed, 28 insertions(+), 25 deletions(-)

diff --git a/include/discover_poco.php b/include/discover_poco.php
index a9eaf1cf67..a4cf5db54e 100644
--- a/include/discover_poco.php
+++ b/include/discover_poco.php
@@ -13,10 +13,10 @@ function discover_poco_run(&$argv, &$argc){
 
 	if(is_null($db)) {
 	    @include(".htconfig.php");
-    	require_once("include/dba.php");
+	require_once("include/dba.php");
 	    $db = new dba($db_host, $db_user, $db_pass, $db_data);
-    	unset($db_host, $db_user, $db_pass, $db_data);
-  	};
+	unset($db_host, $db_user, $db_pass, $db_data);
+	};
 
 	require_once('include/session.php');
 	require_once('include/datetime.php');
@@ -53,12 +53,12 @@ function discover_poco_run(&$argv, &$argc){
 		if($pidfile->is_already_running()) {
 			logger("discover_poco: Already running");
 			if ($pidfile->running_time() > 19*60) {
-                                $pidfile->kill();
-                                logger("discover_poco: killed stale process");
+				$pidfile->kill();
+				logger("discover_poco: killed stale process");
 				// Calling a new instance
 				if ($mode == 0)
 					proc_run('php','include/discover_poco.php');
-                        }
+			}
 			exit;
 		}
 	}
@@ -98,8 +98,28 @@ function discover_users() {
 
 	foreach ($users AS $user) {
 
-		if ($user[0]["server_url"] != "")
-			$server_url = $user[0]["server_url"];
+		$urlparts = parse_url($user["url"]);
+		if (!isset($urlparts["scheme"])) {
+			q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'",
+				dbesc(NETWORK_PHANTOM), dbesc(normalise_link($user["url"])));
+			continue;
+		 }
+
+		if (in_array($urlparts["host"], array("www.facebook.com", "facebook.com", "twitter.com",
+							"identi.ca", "alpha.app.net"))) {
+			$networks = array("www.facebook.com" => NETWORK_FACEBOOK,
+					"facebook.com" => NETWORK_FACEBOOK,
+					"twitter.com" => NETWORK_TWITTER,
+					"identi.ca" => NETWORK_PUMPIO,
+					"alpha.app.net" => NETWORK_APPNET);
+
+			q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'",
+				dbesc($networks[$urlparts["host"]]), dbesc(normalise_link($user["url"])));
+			continue;
+		}
+
+		if ($user["server_url"] != "")
+			$server_url = $user["server_url"];
 		else
 			$server_url = poco_detect_server($user["url"]);
 
diff --git a/include/socgraph.php b/include/socgraph.php
index f0ed7ec232..618a9bf3ea 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -413,23 +413,6 @@ function poco_last_updated($profile, $force = false) {
 		q("UPDATE `gcontact` SET `created` = '%s' WHERE `nurl` = '%s'",
 			dbesc(datetime_convert()), dbesc(normalise_link($profile)));
 
-	$urlparts = parse_url($profile);
-	if (!isset($urlparts["scheme"]))
-		return;
-
-	if (in_array($urlparts["host"], array("www.facebook.com", "facebook.com", "twitter.com",
-						"identi.ca", "alpha.app.net"))) {
-		$networks = array("www.facebook.com" => NETWORK_FACEBOOK,
-				"facebook.com" => NETWORK_FACEBOOK,
-				"twitter.com" => NETWORK_TWITTER,
-				"identi.ca" => NETWORK_PUMPIO,
-				"alpha.app.net" => NETWORK_APPNET);
-
-		q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'",
-			dbesc($networks[$urlparts["host"]]), dbesc(normalise_link($profile)));
-		return;
-	}
-
 	if ($gcontacts[0]["server_url"] != "")
 		$server_url = $gcontacts[0]["server_url"];
 	else

From 016f9cb8c35ce5157fb9ad2c23a67f1204c0ac21 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Wed, 29 Jul 2015 07:32:00 +0200
Subject: [PATCH 41/47] Only suggest friendica contacts that where recently
 active

---
 include/Contact.php  | 7 ++++++-
 include/socgraph.php | 2 +-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/include/Contact.php b/include/Contact.php
index 103dbe9add..1215f40e75 100644
--- a/include/Contact.php
+++ b/include/Contact.php
@@ -267,7 +267,12 @@ function contact_photo_menu($contact) {
 
 
 function random_profile() {
-	$r = q("select url from gcontact where url like '%%://%%/profile/%%' order by rand() limit 1");
+	$r = q("SELECT `url` FROM `gcontact` WHERE `network` = '%s'
+				AND `last_contact` >= `last_failure`
+				AND `updated` > UTC_TIMESTAMP - INTERVAL 1 MONTH
+			ORDER BY rand() LIMIT 1",
+		dbesc(NETWORK_DFRN));
+
 	if(count($r))
 		return dirname($r[0]['url']);
 	return '';
diff --git a/include/socgraph.php b/include/socgraph.php
index 618a9bf3ea..872b730e05 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -1211,7 +1211,7 @@ function poco_discover($complete = false) {
 	$last_update = date("c", time() - (60 * 60 * 6)); // 24
 	$last_update = date("c", time() - (60 * 60 * 24)); // 24
 
-	$r = q("SELECT `poco`, `nurl`, `url`, `network` FROM `gserver` WHERE `last_contact` > `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update));
+	$r = q("SELECT `poco`, `nurl`, `url`, `network` FROM `gserver` WHERE `last_contact` >= `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update));
 	if ($r)
 		foreach ($r AS $server) {
 

From 4cffa18c62b8955a05e5788d001f06c064001651 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Wed, 29 Jul 2015 07:40:23 +0200
Subject: [PATCH 42/47] Added index

---
 database.sql | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/database.sql b/database.sql
index 67bc718221..91d151a104 100644
--- a/database.sql
+++ b/database.sql
@@ -317,7 +317,8 @@ CREATE TABLE IF NOT EXISTS `gcontact` (
 	`network` varchar(255) NOT NULL DEFAULT '',
 	`generation` tinyint(3) NOT NULL DEFAULT 0,
 	`server_url` varchar(255) NOT NULL DEFAULT '',
-	 INDEX `nurl` (`nurl`)
+	 INDEX `nurl` (`nurl`),
+	 INDEX `updated` (`updated`)
 ) DEFAULT CHARSET=utf8;
 
 --

From d5a48c46212a6c5014278a5390e175e6ca2f80a5 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Wed, 29 Jul 2015 07:57:28 +0200
Subject: [PATCH 43/47] Code beautification

---
 include/onepoll.php | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/include/onepoll.php b/include/onepoll.php
index 464a50fb9b..90d018524d 100644
--- a/include/onepoll.php
+++ b/include/onepoll.php
@@ -19,10 +19,10 @@ function onepoll_run(&$argv, &$argc){
 
 	if(is_null($db)) {
 	    @include(".htconfig.php");
-    	require_once("include/dba.php");
+	require_once("include/dba.php");
 	    $db = new dba($db_host, $db_user, $db_pass, $db_data);
-    	unset($db_host, $db_user, $db_pass, $db_data);
-  	};
+	unset($db_host, $db_user, $db_pass, $db_data);
+	};
 
 
 	require_once('include/session.php');

From 6ab304b5642c8d15a4c32cc4c84c45fa1c1a0110 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Thu, 30 Jul 2015 16:30:18 +0200
Subject: [PATCH 44/47] Bugfix: Fetching data from noscrape had several issues

---
 boot.php                |  2 +-
 include/dbstructure.php |  1 +
 include/socgraph.php    | 18 ++++++++++++++----
 update.php              |  2 +-
 4 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/boot.php b/boot.php
index ec803a1573..2a11501033 100644
--- a/boot.php
+++ b/boot.php
@@ -19,7 +19,7 @@ define ( 'FRIENDICA_PLATFORM',     'Friendica');
 define ( 'FRIENDICA_CODENAME',     'Lily of the valley');
 define ( 'FRIENDICA_VERSION',      '3.4.1' );
 define ( 'DFRN_PROTOCOL_VERSION',  '2.23'    );
-define ( 'DB_UPDATE_VERSION',      1186      );
+define ( 'DB_UPDATE_VERSION',      1187      );
 define ( 'EOL',                    "<br />\r\n"     );
 define ( 'ATOM_TIME',              'Y-m-d\TH:i:s\Z' );
 
diff --git a/include/dbstructure.php b/include/dbstructure.php
index e14ce64d81..2370b01bec 100644
--- a/include/dbstructure.php
+++ b/include/dbstructure.php
@@ -639,6 +639,7 @@ function db_definition() {
 					"about" => array("type" => "text", "not null" => "1"),
 					"keywords" => array("type" => "text", "not null" => "1"),
 					"gender" => array("type" => "varchar(32)", "not null" => "1", "default" => ""),
+					"community" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"),
 					"network" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"generation" => array("type" => "tinyint(3)", "not null" => "1", "default" => "0"),
 					"server_url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
diff --git a/include/socgraph.php b/include/socgraph.php
index 872b730e05..d114ef09cc 100644
--- a/include/socgraph.php
+++ b/include/socgraph.php
@@ -451,12 +451,14 @@ function poco_last_updated($profile, $force = false) {
 
 		if ($server) {
 			$noscraperet = z_fetch_url($server[0]["noscrape"]."/".$gcontacts[0]["nick"]);
-			 if ($noscraperet["success"] AND ($noscraperet["body"] = "")) {
+
+			 if ($noscraperet["success"] AND ($noscraperet["body"] != "")) {
+;
 				$noscrape = json_decode($noscraperet["body"], true);
 
-				if (($noscrape["name"] != "") AND ($noscrape["name"] != $gcontacts[0]["name"]))
+				if (($noscrape["fn"] != "") AND ($noscrape["fn"] != $gcontacts[0]["name"]))
 					q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'",
-						dbesc($noscrape["name"]), dbesc(normalise_link($profile)));
+						dbesc($noscrape["fn"]), dbesc(normalise_link($profile)));
 
 				if (($noscrape["photo"] != "") AND ($noscrape["photo"] != $gcontacts[0]["photo"]))
 					q("UPDATE `gcontact` SET `photo` = '%s' WHERE `nurl` = '%s'",
@@ -470,10 +472,18 @@ function poco_last_updated($profile, $force = false) {
 					q("UPDATE `gcontact` SET `gender` = '%s' WHERE `nurl` = '%s'",
 						dbesc($noscrape["gender"]), dbesc(normalise_link($profile)));
 
-				if (($noscrape["about"] != "") AND ($noscrape["about"] != $gcontacts[0]["name"]))
+				if (($noscrape["pdesc"] != "") AND ($noscrape["pdesc"] != $gcontacts[0]["about"]))
+					q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'",
+						dbesc($noscrape["pdesc"]), dbesc(normalise_link($profile)));
+
+				if (($noscrape["about"] != "") AND ($noscrape["about"] != $gcontacts[0]["about"]))
 					q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'",
 						dbesc($noscrape["about"]), dbesc(normalise_link($profile)));
 
+				if (isset($noscrape["comm"]) AND ($noscrape["comm"] != $gcontacts[0]["community"]))
+					q("UPDATE `gcontact` SET `community` = %d WHERE `nurl` = '%s'",
+						intval($noscrape["comm"]), dbesc(normalise_link($profile)));
+
 				if (isset($noscrape["tags"]))
 					$keywords = implode(" ", $noscrape["tags"]);
 				else
diff --git a/update.php b/update.php
index 4f762661a5..a2c8674028 100644
--- a/update.php
+++ b/update.php
@@ -1,6 +1,6 @@
 <?php
 
-define( 'UPDATE_VERSION' , 1186 );
+define( 'UPDATE_VERSION' , 1187 );
 
 /**
  *

From 0a998f99dfada0f201cdaad638447ea2a35b6e7a Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Thu, 30 Jul 2015 23:27:50 +0200
Subject: [PATCH 45/47] Added community search mode

---
 mod/dirfind.php | 16 ++++++++++++++--
 mod/search.php  |  4 ++++
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/mod/dirfind.php b/mod/dirfind.php
index 45fc93b7e8..d5cfcbcab0 100644
--- a/mod/dirfind.php
+++ b/mod/dirfind.php
@@ -16,6 +16,8 @@ function dirfind_init(&$a) {
 
 function dirfind_content(&$a) {
 
+	$community = false;
+
 	$local = get_config('system','poco_local_search');
 
 	$search = notags(trim($_REQUEST['search']));
@@ -23,6 +25,11 @@ function dirfind_content(&$a) {
 	if(strpos($search,'@') === 0)
 		$search = substr($search,1);
 
+	if(strpos($search,'!') === 0) {
+		$search = substr($search,1);
+		$community = true;
+	}
+
 	$o = '';
 
 	$o .= replace_macros(get_markup_template("section_title.tpl"),array(
@@ -33,12 +40,17 @@ function dirfind_content(&$a) {
 
 		if ($local) {
 
+			if ($community)
+				$extra_sql = " AND `community`";
+			else
+				$extra_sql = "";
+
 			$perpage = 80;
 			$startrec = (($a->pager['page']) * $perpage) - $perpage;
 
 			$count = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `network` IN ('%s', '%s', '%s') AND
 					(`url` REGEXP '%s' OR `name` REGEXP '%s' OR `location` REGEXP '%s' OR
-						`about` REGEXP '%s' OR `keywords` REGEXP '%s')",
+						`about` REGEXP '%s' OR `keywords` REGEXP '%s')".$extra_sql,
 					dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA),
 					dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)),
 					dbesc(escape_tags($search)), dbesc(escape_tags($search)));
@@ -49,7 +61,7 @@ function dirfind_content(&$a) {
 					WHERE `gcontact`.`network` IN ('%s', '%s', '%s') AND
 					((`gcontact`.`last_contact` >= `gcontact`.`last_failure`) OR (`gcontact`.`updated` >= `gcontact`.`last_failure`)) AND
 					(`gcontact`.`url` REGEXP '%s' OR `gcontact`.`name` REGEXP '%s' OR `gcontact`.`location` REGEXP '%s' OR
-						`gcontact`.`about` REGEXP '%s' OR `gcontact`.`keywords` REGEXP '%s')
+						`gcontact`.`about` REGEXP '%s' OR `gcontact`.`keywords` REGEXP '%s') $extra_sql
 						GROUP BY `gcontact`.`nurl`
 						ORDER BY `gcontact`.`updated` DESC LIMIT %d, %d",
 					intval(local_user()), dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA),
diff --git a/mod/search.php b/mod/search.php
index 339efc92db..a01984d244 100644
--- a/mod/search.php
+++ b/mod/search.php
@@ -120,6 +120,10 @@ function search_content(&$a) {
 		require_once('mod/dirfind.php');
 		return dirfind_content($a);
 	}
+	if(strpos($search,'!') === 0) {
+		require_once('mod/dirfind.php');
+		return dirfind_content($a);
+	}
 
 	if(! $search)
 		return $o;

From ebae45babc026827a7ed5f05379ca9630fa9beba Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sun, 2 Aug 2015 10:05:50 +0200
Subject: [PATCH 46/47] Clarified the settings.

---
 mod/admin.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mod/admin.php b/mod/admin.php
index cdfd3b5a68..d421154b17 100644
--- a/mod/admin.php
+++ b/mod/admin.php
@@ -710,8 +710,8 @@ function admin_page_site(&$a) {
 		'$maxloadavg'		=> array('maxloadavg', t("Maximum Load Average"), ((intval(get_config('system','maxloadavg')) > 0)?get_config('system','maxloadavg'):50), t("Maximum system load before delivery and poll processes are deferred - default 50.")),
 		'$maxloadavg_frontend'	=> array('maxloadavg_frontend', t("Maximum Load Average (Frontend)"), ((intval(get_config('system','maxloadavg_frontend')) > 0)?get_config('system','maxloadavg_frontend'):50), t("Maximum system load before the frontend quits service - default 50.")),
 
-		'$poco_completion'	=> array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful when communicating with Redmatrix and friendica servers before 3.3)")),
-		'$poco_discovery'	=> array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available."), $poco_discovery_choices),
+		'$poco_completion'	=> array('poco_completion', t("Periodical check of global contacts"), get_config('system','poco_completion'), t("If enabled, the global contacts are checked periodically for missing or outdated data and the vitality of the contacts and servers.")),
+		'$poco_discovery'	=> array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available. The recommened setting is 'Users, Global Contacts'"), $poco_discovery_choices),
 		'$poco_discovery_since'	=> array('poco_discovery_since', t("Timeframe for fetching global contacts"), (string) intval(get_config('system','poco_discovery_since')), t("When the discovery is activated, this value defines the timeframe for the global contacts that are fetched from other servers."), $poco_discovery_since_choices),
 		'$poco_local_search'	=> array('poco_local_search', t("Search the local directory"), get_config('system','poco_local_search'), t("Search the local directory instead of the global directory. When searching locally, every search will be executed on the global directory in the background. This improves the search results when the search is repeated.")),
 

From 2f147451f729167d103ccfe0669206476bd1e097 Mon Sep 17 00:00:00 2001
From: Michael Vogel <icarus@dabo.de>
Date: Sun, 2 Aug 2015 10:08:08 +0200
Subject: [PATCH 47/47] Just some more improved setting descriptions.

---
 mod/admin.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mod/admin.php b/mod/admin.php
index d421154b17..d725c45811 100644
--- a/mod/admin.php
+++ b/mod/admin.php
@@ -711,8 +711,8 @@ function admin_page_site(&$a) {
 		'$maxloadavg_frontend'	=> array('maxloadavg_frontend', t("Maximum Load Average (Frontend)"), ((intval(get_config('system','maxloadavg_frontend')) > 0)?get_config('system','maxloadavg_frontend'):50), t("Maximum system load before the frontend quits service - default 50.")),
 
 		'$poco_completion'	=> array('poco_completion', t("Periodical check of global contacts"), get_config('system','poco_completion'), t("If enabled, the global contacts are checked periodically for missing or outdated data and the vitality of the contacts and servers.")),
-		'$poco_discovery'	=> array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available. The recommened setting is 'Users, Global Contacts'"), $poco_discovery_choices),
-		'$poco_discovery_since'	=> array('poco_discovery_since', t("Timeframe for fetching global contacts"), (string) intval(get_config('system','poco_discovery_since')), t("When the discovery is activated, this value defines the timeframe for the global contacts that are fetched from other servers."), $poco_discovery_since_choices),
+		'$poco_discovery'	=> array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available. The fallback increases the server load, so the recommened setting is 'Users, Global Contacts'."), $poco_discovery_choices),
+		'$poco_discovery_since'	=> array('poco_discovery_since', t("Timeframe for fetching global contacts"), (string) intval(get_config('system','poco_discovery_since')), t("When the discovery is activated, this value defines the timeframe for the activity of the global contacts that are fetched from other servers."), $poco_discovery_since_choices),
 		'$poco_local_search'	=> array('poco_local_search', t("Search the local directory"), get_config('system','poco_local_search'), t("Search the local directory instead of the global directory. When searching locally, every search will be executed on the global directory in the background. This improves the search results when the search is repeated.")),
 
 		'$use_fulltext_engine'	=> array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")),