New field "created" in gcontacts. New option for timeframe when polling global contacts

include/dbstructure.php

@@ -626,10 +626,12 @@ function db_definition() {
 			"fields" => array(
 					"id" => array("type" => "int(10) unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"),
 					"name" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"nick" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"nurl" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"photo" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"connect" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
 					"updated" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
 					"last_contact" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
 					"last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
@@ -699,6 +701,7 @@ function db_definition() {
 					"noscrape" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
 					"network" => array("type" => "varchar(32)", "not null" => "1", "default" => ""),
 					"platform" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
 					"last_poco_query" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
 					"last_contact" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
 					"last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),

include/onepoll.php

@@ -110,7 +110,7 @@ function onepoll_run(&$argv, &$argc){
 	// - Check why we don't poll the Diaspora feed at the moment (some guid problem in the items?)
 	// - Check whether this is possible with Redmatrix
 	if ($contact["network"] == NETWORK_DIASPORA) {
-		if (poco_do_update($contact["last-item"], $contact["success_update"], $contact["failure_update"])) {
+		if (poco_do_update($contact["created"], $contact["last-item"], $contact["failure_update"], $contact["success_update"])) {
 			$last_updated = poco_last_updated($contact["url"]);
 			$updated = datetime_convert();
 			if ($last_updated) {

include/socgraph.php

@@ -205,19 +205,26 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 		if ($updated == "0000-00-00 00:00:00")
 			$updated = $x[0]["updated"];
 
+		$created = $x[0]["created"];
 		$last_contact = $x[0]["last_contact"];
 		$last_failure = $x[0]["last_failure"];
 		$server_url = $x[0]["server_url"];
+		$nick = $x[0]["nick"];
 	} else {
+		$created = "0000-00-00 00:00:00";
 		$last_contact = "0000-00-00 00:00:00";
 		$last_failure = "0000-00-00 00:00:00";
 		$server_url = "";
+
+		$urlparts = parse_url($profile_url);
+		$nick = end(explode("/", $urlparts["path"]));
 	}
 
 	if (($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == "")) {
 		$data = probe_url($profile_url);
 		$network = $data["network"];
 		$name = $data["name"];
+		$nick = $data["nick"];
 		$profile_url = $data["url"];
 		$profile_photo = $data["photo"];
 		$server_url = $data["baseurl"];
@@ -241,7 +248,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	poco_check_server($server_url, $network);
 
 	// Only fetch last update manually if it wasn't provided and enabled in the system
-	if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) {
+	if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($created, $updated, $last_failure, $last_contact)) {
 		$last_updated = poco_last_updated($profile_url);
 		if ($last_updated) {
 			$updated = $last_updated;
@@ -290,15 +297,17 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 			);
 		}
 	} else {
-		q("INSERT INTO `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `server_url`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`)
+		q("INSERT INTO `gcontact` (`name`, `nick`, `network`, `url`, `nurl`, `photo`, `connect`, `server_url`, `created`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`)
-			VALUES ('%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)",
+			VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)",
 			dbesc($name),
+			dbesc($nick),
 			dbesc($network),
 			dbesc($profile_url),
 			dbesc(normalise_link($profile_url)),
 			dbesc($profile_photo),
 			dbesc($connect_url),
 			dbesc($server_url),
+			dbesc(datetime_convert()),
 			dbesc($updated),
 			dbesc($last_contact),
 			dbesc($last_failure),
@@ -357,13 +366,79 @@ function poco_last_updated($profile) {
 	$gcontacts = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s'",
 			dbesc(normalise_link($profile)));
 
-	if ($gcontacts[0]["server_url"] != "") {
+	if ($gcontacts[0]["created"] == "0000-00-00 00:00:00")
-		$servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s' AND `last_contact` < `last_failure`", dbesc(normalise_link($gcontacts[0]["server_url"])));
+		q("UPDATE `gcontact` SET `created` = '%s' WHERE `nurl` = '%s'",
+			dbesc(datetime_convert()), dbesc(normalise_link($profile)));
 
-		if ($servers)
+	if (($gcontacts[0]["server_url"] != "") AND ($gcontacts[0]["network"] != ""))
+		if (!poco_check_server($gcontacts[0]["server_url"], $gcontacts[0]["network"]))
 			return false;
+
+	// noscrape is really fast so we don't cache the call.
+	if (($gcontacts[0]["server_url"] != "") AND ($gcontacts[0]["nick"] != "")) {
+
+		//  Use noscrape if possible
+		$server = q("SELECT `noscrape` FROM `gserver` WHERE `nurl` = '%s' AND `noscrape` != ''", dbesc(normalise_link($gcontacts[0]["server_url"])));
+
+		if ($server) {
+			$noscraperet = z_fetch_url($server[0]["noscrape"]."/".$gcontacts[0]["nick"]);
+			 if ($noscraperet["success"]) {
+				$noscrape = json_decode($noscraperet["body"], true);
+
+				if (($noscrape["name"] != "") AND ($noscrape["name"] != $gcontacts[0]["name"]))
+					q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'",
+						dbesc($noscrape["name"]), dbesc(normalise_link($profile)));
+
+				if (($noscrape["photo"] != "") AND ($noscrape["photo"] != $gcontacts[0]["photo"]))
+					q("UPDATE `gcontact` SET `photo` = '%s' WHERE `nurl` = '%s'",
+						dbesc($noscrape["photo"]), dbesc(normalise_link($profile)));
+
+				if (($noscrape["updated"] != "") AND ($noscrape["updated"] != $gcontacts[0]["updated"]))
+					q("UPDATE `gcontact` SET `updated` = '%s' WHERE `nurl` = '%s'",
+						dbesc($noscrape["updated"]), dbesc(normalise_link($profile)));
+
+				if (($noscrape["gender"] != "") AND ($noscrape["gender"] != $gcontacts[0]["gender"]))
+					q("UPDATE `gcontact` SET `gender` = '%s' WHERE `nurl` = '%s'",
+						dbesc($noscrape["gender"]), dbesc(normalise_link($profile)));
+
+				if (($noscrape["about"] != "") AND ($noscrape["about"] != $gcontacts[0]["name"]))
+					q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'",
+						dbesc($noscrape["about"]), dbesc(normalise_link($profile)));
+
+				$keywords = implode(" ", $noscrape["tags"]);
+				if (($keywords != "") AND ($keywords != $gcontacts[0]["keywords"]))
+					q("UPDATE `gcontact` SET `keywords` = '%s' WHERE `nurl` = '%s'",
+						dbesc($keywords), dbesc(normalise_link($profile)));
+
+				$location = $noscrape["locality"];
+
+				if ($noscrape["region"] != "") {
+					if ($location != "")
+						$location .= ", ";
+
+					$location .= $noscrape["region"];
+				}
+
+				if ($noscrape["country-name"] != "") {
+					if ($location != "")
+						$location .= ", ";
+
+					$location .= $noscrape["country-name"];
+				}
+
+				if (($location != "") AND ($location != $gcontacts[0]["location"]))
+					q("UPDATE `gcontact` SET `location` = '%s' WHERE `nurl` = '%s'",
+						dbesc($location), dbesc(normalise_link($profile)));
+
+				return $noscrape["updated"];
+			}
+		}
 	}
 
+	// If we only can poll the feed, then we only do this once a while
+	if (!poco_do_update($gcontacts[0]["created"], $gcontacts[0]["updated"], $gcontacts[0]["last_failure"],  $gcontacts[0]["last_contact"]))
+/		return $gcontacts[0]["updated"];
+
 	$data = probe_url($profile);
 
 	if (($data["poll"] == "") OR ($data["network"] == NETWORK_FEED)) {
@@ -376,6 +451,10 @@ function poco_last_updated($profile) {
 		q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'",
 			dbesc($data["name"]), dbesc(normalise_link($profile)));
 
+	if (($data["nick"] != "") AND ($data["nick"] != $gcontacts[0]["nick"]))
+		q("UPDATE `gcontact` SET `nick` = '%s' WHERE `nurl` = '%s'",
+			dbesc($data["nick"]), dbesc(normalise_link($profile)));
+
 	if (($data["addr"] != "") AND ($data["addr"] != $gcontacts[0]["connect"]))
 		q("UPDATE `gcontact` SET `connect` = '%s' WHERE `nurl` = '%s'",
 			dbesc($data["addr"]), dbesc(normalise_link($profile)));
@@ -388,11 +467,6 @@ function poco_last_updated($profile) {
 		q("UPDATE `gcontact` SET `server_url` = '%s' WHERE `nurl` = '%s'",
 			dbesc($data["baseurl"]), dbesc(normalise_link($profile)));
 
-	if ($data["baseurl"] != "")
-		poco_check_server($data["baseurl"], $data["network"]);
-
-	// To-Do: Use noscrape
-
 	$feedret = z_fetch_url($data["poll"]);
 
 	if (!$feedret["success"]) {
@@ -437,7 +511,7 @@ function poco_last_updated($profile) {
 	return($last_updated);
 }
 
-function poco_do_update($updated, $last_contact, $last_failure) {
+function poco_do_update($created, $updated, $last_failure,  $last_contact) {
 	$now = strtotime(datetime_convert());
 
 	if ($updated > $last_contact)
@@ -446,6 +520,11 @@ function poco_do_update($updated, $last_contact, $last_failure) {
 		$contact_time = strtotime($last_contact);
 
 	$failure_time = strtotime($last_failure);
+	$created_time = strtotime($created);
+
+	// If there is no "created" time then use the current time
+	if ($created_time <= 0)
+		$created_time = $now;
 
 	// If the last contact was less than 24 hours then don't update
 	if (($now - $contact_time) < (60 * 60 * 24))
@@ -459,12 +538,12 @@ function poco_do_update($updated, $last_contact, $last_failure) {
 	if ((($now - $contact_time) < (60 * 60 * 24 * 7)) AND ($contact_time > $failure_time))
 		return false;
 
-	// If the last contact time was more than a week ago, then only try once a week
+	// If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week
-	if ((($now - $contact_time) > (60 * 60 * 24 * 7)) AND (($now - $failure_time) < (60 * 60 * 24 * 7)))
+	if ((($now - $contact_time) > (60 * 60 * 24 * 7)) AND (($now - $created_time) > (60 * 60 * 24 * 7)) AND (($now - $failure_time) < (60 * 60 * 24 * 7)))
 		return false;
 
-	// If the last contact time was more than a month ago, then only try once a month - but only if there ever was a contact time
+	// If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month
-	if (($contact_time > 0) AND (($now - $contact_time) > (60 * 60 * 24 * 30)) AND (($now - $failure_time) < (60 * 60 * 24 * 30)))
+	if ((($now - $contact_time) > (60 * 60 * 24 * 30)) AND (($now - $created_time) > (60 * 60 * 24 * 30)) AND (($now - $failure_time) < (60 * 60 * 24 * 30)))
 		return false;
 
 	return true;
@@ -486,6 +565,11 @@ function poco_check_server($server_url, $network = "") {
 
 	$servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url)));
 	if ($servers) {
+
+		if ($servers[0]["created"] == "0000-00-00 00:00:00")
+			q("UPDATE `gserver` SET `created` = '%s' WHERE `nurl` = '%s'",
+				dbesc(datetime_convert()), dbesc(normalise_link($server_url)));
+
 		$poco = $servers[0]["poco"];
 		$noscrape = $servers[0]["noscrape"];
 
@@ -500,7 +584,7 @@ function poco_check_server($server_url, $network = "") {
 		$info = $servers[0]["info"];
 		$register_policy = $servers[0]["register_policy"];
 
-		if (!poco_do_update("", $last_contact, $last_failure))
+		if (!poco_do_update($servers[0]["created"], "", $last_failure, $last_contact))
 			return ($last_contact >= $last_failure);
 	} else {
 		$poco = "";
@@ -566,7 +650,7 @@ function poco_check_server($server_url, $network = "") {
 				if (isset($data->site->hubzilla)) {
 					$platform = $data->site->hubzilla->PLATFORM_NAME;
 					$version = $data->site->hubzilla->RED_VERSION;
-					$network = NETWORK_DFRN;
+					$network = NETWORK_DIASPORA;
 				}
 				if (isset($data->site->redmatrix)) {
 					if (isset($data->site->redmatrix->PLATFORM_NAME))
@@ -685,7 +769,7 @@ function poco_check_server($server_url, $network = "") {
 			dbesc(normalise_link($server_url))
 		);
 	else
-		q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `last_contact`, `last_failure`)
+		q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `created`, `last_contact`, `last_failure`)
 					VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s')",
 				dbesc($server_url),
 				dbesc(normalise_link($server_url)),
@@ -697,6 +781,7 @@ function poco_check_server($server_url, $network = "") {
 				dbesc($noscrape),
 				dbesc($network),
 				dbesc($platform),
+				dbesc(datetime_convert()),
 				dbesc($last_contact),
 				dbesc($last_failure),
 				dbesc(datetime_convert())
@@ -1028,8 +1113,14 @@ function poco_discover($complete = false) {
 
 				if (get_config('system','poco_discovery') > 1) {
 
+					$timeframe = get_config('system','poco_discovery_since');
+					if ($timeframe == 0)
+						$timeframe = 30;
+
+					$updatedSince = date("Y-m-d H:i:s", time() - $timeframe * 86400);
+
 					// Fetch all global contacts from the other server (Not working with Redmatrix and Friendica versions before 3.3)
-					$url = $server["poco"]."/@global?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
+					$url = $server["poco"]."/@global?updatedSince=".$updatedSince."&fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
 
 					$retdata = z_fetch_url($url);
 					if ($retdata["success"]) {

mod/admin.php

@@ -360,6 +360,7 @@ function admin_page_site_post(&$a){
 	$maxloadavg_frontend	=	((x($_POST,'maxloadavg_frontend'))	? intval(trim($_POST['maxloadavg_frontend']))	: 50);
 	$poco_completion	=	((x($_POST,'poco_completion'))		? intval(trim($_POST['poco_completion']))	: false);
 	$poco_discovery		=	((x($_POST,'poco_discovery'))		? intval(trim($_POST['poco_discovery']))	: 0);
+	$poco_discovery_since	=	((x($_POST,'poco_discovery_since'))	? intval(trim($_POST['poco_discovery_since']))	: 30);
 	$poco_local_search	=	((x($_POST,'poco_local_search'))	? intval(trim($_POST['poco_local_search']))	: false);
 	$dfrn_only		=	((x($_POST,'dfrn_only'))		? True						: False);
 	$ostatus_disabled	=	!((x($_POST,'ostatus_disabled'))	? True  					: False);
@@ -432,6 +433,7 @@ function admin_page_site_post(&$a){
 	set_config('system','maxloadavg_frontend',$maxloadavg_frontend);
 	set_config('system','poco_completion',$poco_completion);
 	set_config('system','poco_discovery',$poco_discovery);
+	set_config('system','poco_discovery_since',$poco_discovery_since);
 	set_config('system','poco_local_search',$poco_local_search);
 	set_config('config','sitename',$sitename);
 	set_config('config','hostname',$hostname);
@@ -595,6 +597,13 @@ function admin_page_site(&$a) {
 			"3" => t("Users, Global Contacts/fallback"),
 			);
 
+		$poco_discovery_since_choices = array(
+			"30" => t("One month"),
+			"91" => t("Three months"),
+			"182" => t("Half a year"),
+			"365" => t("One year"),
+			);
+
 		/* get user names to make the install a personal install of X */
 		$user_names = array();
 		$user_names['---'] = t('Multi user instance');
@@ -703,6 +712,7 @@ function admin_page_site(&$a) {
 
 		'$poco_completion'	=> array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful when communicating with Redmatrix and friendica servers before 3.3)")),
 		'$poco_discovery'	=> array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available."), $poco_discovery_choices),
+		'$poco_discovery_since'	=> array('poco_discovery_since', t("Timeframe for fetching global contacts"), (string) intval(get_config('system','poco_discovery_since')), t("When the discovery is activated, this value defines the timeframe for the global contacts that are fetched from other servers."), $poco_discovery_since_choices),
 		'$poco_local_search'	=> array('poco_local_search', t("Search the local directory"), get_config('system','poco_local_search'), t("Search the local directory instead of the global directory. When searching locally, every search will be executed on the global directory in the background. This improves the search results when the search is repeated.")),
 
 		'$use_fulltext_engine'	=> array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")),

view/templates/admin_site.tpl

@@ -118,6 +118,7 @@
 	<h3>{{$portable_contacts}}</h3>
 	{{include file="field_checkbox.tpl" field=$poco_completion}}
 	{{include file="field_select.tpl" field=$poco_discovery}}
+	{{include file="field_select.tpl" field=$poco_discovery_since}}
 	{{include file="field_checkbox.tpl" field=$poco_local_search}}
 	<div class="submit"><input type="submit" name="page_site" value="{{$submit|escape:'html'}}" /></div>