From 54a5a80b2425fe575643530b00c28578f1610448 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 17 Jul 2015 01:08:28 +0200 Subject: [PATCH 01/47] Poco: New fields for last contact and last failure of a contact. --- boot.php | 2 +- include/dbstructure.php | 2 + include/plaintext.php | 4 +- include/socgraph.php | 129 ++++++++++++++++++++++++++++++++++++---- mod/parse_url.php | 15 +++++ update.php | 2 +- 6 files changed, 139 insertions(+), 15 deletions(-) diff --git a/boot.php b/boot.php index 42498a6cc..ec803a157 100644 --- a/boot.php +++ b/boot.php @@ -19,7 +19,7 @@ define ( 'FRIENDICA_PLATFORM', 'Friendica'); define ( 'FRIENDICA_CODENAME', 'Lily of the valley'); define ( 'FRIENDICA_VERSION', '3.4.1' ); define ( 'DFRN_PROTOCOL_VERSION', '2.23' ); -define ( 'DB_UPDATE_VERSION', 1185 ); +define ( 'DB_UPDATE_VERSION', 1186 ); define ( 'EOL', "
\r\n" ); define ( 'ATOM_TIME', 'Y-m-d\TH:i:s\Z' ); diff --git a/include/dbstructure.php b/include/dbstructure.php index 0f81ee624..8688903ba 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -630,6 +630,8 @@ function db_definition() { "photo" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "connect" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "updated" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), + "last_contact" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), + "last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), "location" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "about" => array("type" => "text", "not null" => "1"), "keywords" => array("type" => "text", "not null" => "1"), diff --git a/include/plaintext.php b/include/plaintext.php index c8cdfa57d..4f435fc6a 100644 --- a/include/plaintext.php +++ b/include/plaintext.php @@ -52,7 +52,7 @@ function get_attached_data($body) { if (preg_match_all("(\[url=([$URLSearchString]*)\]\s*\[img\]([$URLSearchString]*)\[\/img\]\s*\[\/url\])ism", $body, $pictures, PREG_SET_ORDER)) { if (count($pictures) == 1) { // Checking, if the link goes to a picture - $data = parseurl_getsiteinfo($pictures[0][1], true); + $data = parseurl_getsiteinfo_cached($pictures[0][1], true); if ($data["type"] == "photo") { $post["type"] = "photo"; if (isset($data["images"][0])) @@ -95,7 +95,7 @@ function get_attached_data($body) { } } elseif (isset($post["url"]) AND ($post["type"] == "video")) { require_once("mod/parse_url.php"); - $data = parseurl_getsiteinfo($post["url"], true); + $data = parseurl_getsiteinfo_cached($post["url"], true); if (isset($data["images"][0])) $post["image"] = $data["images"][0]["src"]; diff --git a/include/socgraph.php b/include/socgraph.php index 1e1b6076f..1e8bd97aa 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -195,8 +195,20 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca $x = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1", dbesc(normalise_link($profile_url)) ); - if(count($x) AND ($network == "") AND ($x[0]["network"] != NETWORK_STATUSNET)) - $network = $x[0]["network"]; + + if (count($x)) { + if (($network == "") AND ($x[0]["network"] != NETWORK_STATUSNET)) + $network = $x[0]["network"]; + + if ($updated = "0000-00-00 00:00:00") + $updated = $x[0]["updated"]; + + $last_contact = $x[0]["last_contact"]; + $last_failure = $x[0]["last_failure"]; + } else { + $last_contact = "0000-00-00 00:00:00"; + $last_failure = "0000-00-00 00:00:00"; + } if (($network == "") OR ($name == "") OR ($profile_photo == "")) { require_once("include/Scrape.php"); @@ -223,6 +235,23 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca logger("profile-check generation: ".$generation." Network: ".$network." URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG); + if (poco_do_update($updated, $last_contact, $last_failure)) { + $last_updated = poco_last_updated($profile_url); + if ($last_updated) { + $updated = $last_updated; + $last_contact = datetime_convert(); + logger("Last updated for profile ".$profile_url.": ".$updated, LOGGER_DEBUG); + + if (count($x)) + q("UPDATE `gcontact` SET `last_contact` = '%s' WHERE `nurl` = '%s'", dbesc($last_contact), dbesc(normalise_link($profile_url))); + } else { + $last_failure = datetime_convert(); + + if (count($x)) + q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", dbesc($last_failure), dbesc(normalise_link($profile_url))); + } + } + if(count($x)) { $gcid = $x[0]['id']; @@ -241,6 +270,9 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca if (($generation == 0) AND ($x[0]['generation'] > 0)) $generation = $x[0]['generation']; + if ($last_contact < $x[0]['last_contact']) + $last_contact = $x[0]['last_contact']; + if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo || $x[0]['updated'] < $updated) { q("UPDATE `gcontact` SET `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s', `updated` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d @@ -261,7 +293,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca ); } } else { - q("INSERT INTO `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `updated`, `location`, `about`, `keywords`, `gender`, `generation`) + q("INSERT INTO `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`) VALUES ('%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s', '%s', %d)", dbesc($name), dbesc($network), @@ -270,6 +302,8 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca dbesc($profile_photo), dbesc($connect_url), dbesc($updated), + dbesc($last_contact), + dbesc($last_failure), dbesc($location), dbesc($about), dbesc($keywords), @@ -320,6 +354,79 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca return $gcid; } +function poco_last_updated($profile) { + $data = probe_url($profile); + + if (($data["poll"] == "") OR ($data["network"] == NETWORK_FEED)) + return false; + + $feedret = z_fetch_url($data["poll"]); + + if (!$feedret["success"]) + return false; + + $doc = new DOMDocument(); + @$doc->loadXML($feedret["body"]); + + $xpath = new DomXPath($doc); + $xpath->registerNamespace('atom', "http://www.w3.org/2005/Atom"); + + $entries = $xpath->query('/atom:feed/atom:entry'); + + $last_updated = ""; + + foreach ($entries AS $entry) { + $published = $xpath->query('atom:published/text()', $entry)->item(0)->nodeValue; + $updated = $xpath->query('atom:updated/text()', $entry)->item(0)->nodeValue; + + if ($last_updated < $published) + $last_updated = $published; + + if ($last_updated < $updated) + $last_updated = $updated; + } + + // Maybe there aren't any entries. Then check if it is a valid feed + if ($last_updated == "") + if ($xpath->query('/atom:feed')->length > 0) + $last_updated = "0000-00-00 00:00:00"; + + return($last_updated); +} + +function poco_do_update($updated, $last_contact, $last_failure) { + $now = strtotime(datetime_convert()); + + if ($updated > $last_contact) + $contact_time = strtotime($updated); + else + $contact_time = strtotime($last_contact); + + $failure_time = strtotime($last_failure); + + // If the last contact was less than 24 hours then don't update + if (($now - $contact_time) < (60 * 60 * 24)) + return false; + + // If the last failure was less than 24 hours then don't update + if (($now - $failure_time) < (60 * 60 * 24)) + return false; + + // If the last contact was less than a week ago and the last failure is older than a week then don't update + if ((($now - $contact_time) < (60 * 60 * 24 * 7)) AND ($contact_time > $failure_time)) + return false; + + // If the last contact time was more than a week ago, then only try once a week + if (($now - $contact_time) > (60 * 60 * 24 * 7) AND ($now - $failure_time) < (60 * 60 * 24 * 7)) + return false; + + // If the last contact time was more than a month ago, then only try once a month + if (($now - $contact_time) > (60 * 60 * 24 * 30) AND ($now - $failure_time) < (60 * 60 * 24 * 30)) + return false; + + return true; +} + function poco_contact_from_body($body, $created, $cid, $uid) { preg_replace_callback("/\[share(.*?)\].*?\[\/share\]/ism", function ($match) use ($created, $cid, $uid){ @@ -328,20 +435,20 @@ function poco_contact_from_body($body, $created, $cid, $uid) { } function sub_poco_from_share($share, $created, $cid, $uid) { - $profile = ""; - preg_match("/profile='(.*?)'/ism", $share[1], $matches); - if ($matches[1] != "") - $profile = $matches[1]; + $profile = ""; + preg_match("/profile='(.*?)'/ism", $share[1], $matches); + if ($matches[1] != "") + $profile = $matches[1]; - preg_match('/profile="(.*?)"/ism', $share[1], $matches); - if ($matches[1] != "") - $profile = $matches[1]; + preg_match('/profile="(.*?)"/ism', $share[1], $matches); + if ($matches[1] != "") + $profile = $matches[1]; if ($profile == "") return; logger("prepare poco_check for profile ".$profile, LOGGER_DEBUG); - poco_check($profile, "", "", "", "", "", "", "", "", $created, 3, $cid, $uid); + poco_check($profile, "", "", "", "", "", "", "", "", $created, 3, $cid, $uid); } function poco_store($item) { diff --git a/mod/parse_url.php b/mod/parse_url.php index 71d767675..50c6a15b0 100644 --- a/mod/parse_url.php +++ b/mod/parse_url.php @@ -50,6 +50,21 @@ function completeurl($url, $scheme) { return($complete); } +function parseurl_getsiteinfo_cached($url, $no_guessing = false, $do_oembed = true) { + + $data = Cache::get("parse_url:".$no_guessing.":".$do_oembed.":".$url); + if (!is_null($data)) { + $data = unserialize($data); + return $data; + } + + $data = parseurl_getsiteinfo($url, $no_guessing, $do_oembed); + + Cache::set("parse_url:".$no_guessing.":".$do_oembed.":".$url,serialize($data)); + + return $data; +} + function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $count = 1) { require_once("include/network.php"); diff --git a/update.php b/update.php index c182eb590..4f762661a 100644 --- a/update.php +++ b/update.php @@ -1,6 +1,6 @@ Date: Fri, 17 Jul 2015 08:38:20 +0200 Subject: [PATCH 02/47] Only suggest contacts that haven't failed recently --- include/socgraph.php | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/include/socgraph.php b/include/socgraph.php index 1e8bd97aa..6d14df613 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -2,6 +2,13 @@ require_once('include/datetime.php'); +/* + To-Do: + - noscrape for updating contact fields and "last updated" + - use /poco/@global for discovering contacts from other servers + - Make search for last activity optional +*/ + /* * poco_load * @@ -270,9 +277,6 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca if (($generation == 0) AND ($x[0]['generation'] > 0)) $generation = $x[0]['generation']; - if ($last_contact < $x[0]['last_contact']) - $last_contact = $x[0]['last_contact']; - if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo || $x[0]['updated'] < $updated) { q("UPDATE `gcontact` SET `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s', `updated` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d @@ -360,6 +364,8 @@ function poco_last_updated($profile) { if (($data["poll"] == "") OR ($data["network"] == NETWORK_FEED)) return false; + // To-Do: Use noscrape + $feedret = z_fetch_url($data["poll"]); if (!$feedret["success"]) @@ -649,6 +655,7 @@ function suggestion_query($uid, $start = 0, $limit = 80) { and not gcontact.name in ( select name from contact where uid = %d ) and not gcontact.id in ( select gcid from gcign where uid = %d ) AND `gcontact`.`updated` != '0000-00-00 00:00:00' + AND `gcontact`.`last_contact` >= `gcontact`.`last_failure` AND `gcontact`.`network` IN (%s) group by glink.gcid order by gcontact.updated desc,total desc limit %d, %d ", intval($uid), From 09d71abbfc507791124140ae6b21e92ffb050a5c Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 17 Jul 2015 08:58:42 +0200 Subject: [PATCH 03/47] poco: Only fetch last update manually if it wasn't provided --- include/socgraph.php | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/include/socgraph.php b/include/socgraph.php index 6d14df613..3ff934f72 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -174,6 +174,8 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca if ($profile_url == "") return $gcid; + $orig_updated = $updated; + // Don't store the statusnet connector as network // We can't simply set this to NETWORK_OSTATUS since the connector could have fetched posts from friendica as well if ($network == NETWORK_STATUSNET) @@ -207,7 +209,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca if (($network == "") AND ($x[0]["network"] != NETWORK_STATUSNET)) $network = $x[0]["network"]; - if ($updated = "0000-00-00 00:00:00") + if ($updated == "0000-00-00 00:00:00") $updated = $x[0]["updated"]; $last_contact = $x[0]["last_contact"]; @@ -242,7 +244,8 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca logger("profile-check generation: ".$generation." Network: ".$network." URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG); - if (poco_do_update($updated, $last_contact, $last_failure)) { + // Only fetch last update manually if it wasn't provided + if (($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) { $last_updated = poco_last_updated($profile_url); if ($last_updated) { $updated = $last_updated; From 639f94f8b90ce23420be40d9f183e57190a4ecf4 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 17 Jul 2015 23:05:50 +0200 Subject: [PATCH 04/47] Onepoll: mark the last request failure for this contact --- include/dbstructure.php | 1 + include/onepoll.php | 117 +++++++++++++++++++++++++++++----------- include/socgraph.php | 5 +- 3 files changed, 89 insertions(+), 34 deletions(-) diff --git a/include/dbstructure.php b/include/dbstructure.php index 8688903ba..98e225bf8 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -451,6 +451,7 @@ function db_definition() { "hub-verify" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "last-update" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "success_update" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), + "failure_update" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "name-date" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "uri-date" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "avatar-date" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), diff --git a/include/onepoll.php b/include/onepoll.php index 9723fadd5..73668126a 100644 --- a/include/onepoll.php +++ b/include/onepoll.php @@ -79,28 +79,69 @@ function onepoll_run(&$argv, &$argc){ $contacts = q("SELECT `contact`.* FROM `contact` WHERE ( `rel` = %d OR `rel` = %d ) AND `poll` != '' - AND NOT `network` IN ( '%s', '%s', '%s' ) + AND NOT `network` IN ( '%s', '%s' ) AND `contact`.`id` = %d AND `self` = 0 AND `contact`.`blocked` = 0 AND `contact`.`readonly` = 0 AND `contact`.`archive` = 0 LIMIT 1", intval(CONTACT_IS_SHARING), intval(CONTACT_IS_FRIEND), - dbesc(NETWORK_DIASPORA), dbesc(NETWORK_FACEBOOK), dbesc(NETWORK_PUMPIO), intval($contact_id) ); - if(! count($contacts)) { - // Maybe it is a Redmatrix account. Then we can fetch their contacts via poco - $contacts = q("SELECT `id`, `poco` FROM `contact` WHERE `id` = %d AND `poco` != ''", intval($contact_id)); - if ($contacts) - poco_load($contacts[0]['id'],$importer_uid,0,$contacts[0]['poco']); + if(! count($contacts)) return; - } $contact = $contacts[0]; + // load current friends if possible. + if (($contact['poco'] != "") AND ($contact['success_update'] > $contact['failure_update'])) { + $r = q("SELECT count(*) as total from glink + where `cid` = %d and updated > UTC_TIMESTAMP() - INTERVAL 1 DAY", + intval($contact['id']) + ); + if (count($r)) + if (!$r[0]['total']) + poco_load($contact['id'],$importer_uid,0,$contact['poco']); + } + + // To-Do: + // - Check why we don't poll the Diaspora feed at the moment (some guid problem in the items?) + // - Check whether this is possible with Redmatrix + if ($contact["network"] == NETWORK_DIASPORA) { + if (poco_do_update($contact["last-item"], $contact["success_update"], $contact["failure_update"])) { + $last_updated = poco_last_updated($contact["url"]); + $updated = datetime_convert(); + if ($last_updated) { + q("UPDATE `contact` SET `last-item` = '%s', `last-update` = '%s', `success_update` = '%s' WHERE `id` = %d", + dbesc($last_updated), + dbesc($updated), + dbesc($updated), + intval($contact['id']) + ); + + q("UPDATE `gcontact` SET `updated` = '%s', `last_contact` = '%s' WHERE `nurl` = '%s'", + dbesc($last_updated), + dbesc($updated), + dbesc($contact['nurl']) + ); + } else { + q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d", + dbesc($updated), + dbesc($updated), + intval($contact['id']) + ); + + q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc($updated), + dbesc($contact['nurl']) + ); + } + } + return; + } + $xml = false; $t = $contact['last-update']; @@ -177,7 +218,8 @@ function onepoll_run(&$argv, &$argc){ mark_for_death($contact); // set the last-update so we don't keep polling - $r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d", + $r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d", + dbesc(datetime_convert()), dbesc(datetime_convert()), intval($contact['id']) ); @@ -190,7 +232,8 @@ function onepoll_run(&$argv, &$argc){ mark_for_death($contact); - $r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d", + $r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d", + dbesc(datetime_convert()), dbesc(datetime_convert()), intval($contact['id']) ); @@ -207,7 +250,8 @@ function onepoll_run(&$argv, &$argc){ // set the last-update so we don't keep polling - $r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d", + $r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d", + dbesc(datetime_convert()), dbesc(datetime_convert()), intval($contact['id']) ); @@ -554,14 +598,14 @@ function onepoll_run(&$argv, &$argc){ logger('poller: received xml : ' . $xml, LOGGER_DATA); if(! strstr($xml,'<')) { logger('poller: post_handshake: response from ' . $url . ' did not contain XML.'); - $r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d", + $r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d", + dbesc(datetime_convert()), dbesc(datetime_convert()), intval($contact['id']) ); return; } - consume_feed($xml,$importer,$contact,$hub,1,1); @@ -588,29 +632,38 @@ function onepoll_run(&$argv, &$argc){ } } } - } - $updated = datetime_convert(); + $updated = datetime_convert(); - $r = q("UPDATE `contact` SET `last-update` = '%s', `success_update` = '%s' WHERE `id` = %d", - dbesc($updated), - dbesc($updated), - intval($contact['id']) - ); - - - // load current friends if possible. - - if($contact['poco']) { - $r = q("SELECT count(*) as total from glink - where `cid` = %d and updated > UTC_TIMESTAMP() - INTERVAL 1 DAY", + $r = q("UPDATE `contact` SET `last-update` = '%s', `success_update` = '%s' WHERE `id` = %d", + dbesc($updated), + dbesc($updated), + intval($contact['id']) + ); + + q("UPDATE `gcontact` SET `last_contact` = '%s' WHERE `nurl` = '%s'", + dbesc($updated), + dbesc($contact['nurl']) + ); + + } elseif (in_array($contact["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, NETWORK_FEED))) { + $updated = datetime_convert(); + + $r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d", + dbesc($updated), + dbesc($updated), + intval($contact['id']) + ); + + q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc($updated), + dbesc($contact['nurl']) + ); + } else { + $r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d", + dbesc($updated), intval($contact['id']) ); - } - if(count($r)) { - if(! $r[0]['total']) { - poco_load($contact['id'],$importer_uid,0,$contact['poco']); - } } return; diff --git a/include/socgraph.php b/include/socgraph.php index 3ff934f72..c676157ea 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -1,12 +1,15 @@ Date: Sat, 18 Jul 2015 00:48:42 +0200 Subject: [PATCH 05/47] Poco: Option to activate and deactivate detection of last activity. --- include/socgraph.php | 7 ++----- mod/admin.php | 5 +++++ mod/poco.php | 10 ++++++---- view/templates/admin_site.tpl | 11 +++++++---- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/include/socgraph.php b/include/socgraph.php index c676157ea..5c9b1d98a 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -7,9 +7,6 @@ require_once("include/Scrape.php"); To-Do: - noscrape for updating contact fields and "last updated" - use /poco/@global for discovering contacts from other servers - - Make search for last activity optional - - only export contacts via poco where update is higher than failure - - check your own contacts in some way as well */ /* @@ -245,8 +242,8 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca logger("profile-check generation: ".$generation." Network: ".$network." URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG); - // Only fetch last update manually if it wasn't provided - if (($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) { + // Only fetch last update manually if it wasn't provided and enabled in the system + if (get_config('system','ld_discover_activity') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) { $last_updated = poco_last_updated($profile_url); if ($last_updated) { $updated = $last_updated; diff --git a/mod/admin.php b/mod/admin.php index 28a7a91e4..214ce62ea 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -358,6 +358,7 @@ function admin_page_site_post(&$a){ $poll_interval = ((x($_POST,'poll_interval')) ? intval(trim($_POST['poll_interval'])) : 0); $maxloadavg = ((x($_POST,'maxloadavg')) ? intval(trim($_POST['maxloadavg'])) : 50); $maxloadavg_frontend = ((x($_POST,'maxloadavg_frontend')) ? intval(trim($_POST['maxloadavg_frontend'])) : 50); + $ld_discover_activity = ((x($_POST,'ld_discover_activity')) ? intval(trim($_POST['ld_discover_activity'])) : false); $dfrn_only = ((x($_POST,'dfrn_only')) ? True : False); $ostatus_disabled = !((x($_POST,'ostatus_disabled')) ? True : False); $ostatus_poll_interval = ((x($_POST,'ostatus_poll_interval')) ? intval(trim($_POST['ostatus_poll_interval'])) : 0); @@ -427,6 +428,7 @@ function admin_page_site_post(&$a){ set_config('system','poll_interval',$poll_interval); set_config('system','maxloadavg',$maxloadavg); set_config('system','maxloadavg_frontend',$maxloadavg_frontend); + set_config('system','ld_discover_activity',$ld_discover_activity); set_config('config','sitename',$sitename); set_config('config','hostname',$hostname); set_config('config','sender_email', $sender_email); @@ -630,6 +632,7 @@ function admin_page_site(&$a) { '$upload' => t('File upload'), '$corporate' => t('Policies'), '$advanced' => t('Advanced'), + '$local_directory' => t('Local Directory (Portable Contacts)'), '$performance' => t('Performance'), '$relocate'=> t('Relocate - WARNING: advanced function. Could make this server unreachable.'), '$baseurl' => $a->get_baseurl(true), @@ -687,6 +690,8 @@ function admin_page_site(&$a) { '$maxloadavg' => array('maxloadavg', t("Maximum Load Average"), ((intval(get_config('system','maxloadavg')) > 0)?get_config('system','maxloadavg'):50), t("Maximum system load before delivery and poll processes are deferred - default 50.")), '$maxloadavg_frontend' => array('maxloadavg_frontend', t("Maximum Load Average (Frontend)"), ((intval(get_config('system','maxloadavg_frontend')) > 0)?get_config('system','maxloadavg_frontend'):50), t("Maximum system load before the frontend quits service - default 50.")), + '$ld_discover_activity' => array('ld_discover_activity', t("Discover last activity"), get_config('system','ld_discover_activity'), t("Update the last activity when this isn't provided via the 'portable contacts' functionality. (Useful for poco exchange with Redmatrix and friendica servers before 3.3)")), + '$use_fulltext_engine' => array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")), '$suppress_language' => array('suppress_language', t("Suppress Language"), get_config('system','suppress_language'), t("Suppress language information in meta information about a posting.")), '$suppress_tags' => array('suppress_tags', t("Suppress Tags"), get_config('system','suppress_tags'), t("Suppress showing a list of hashtags at the end of the posting.")), diff --git a/mod/poco.php b/mod/poco.php index 89f2f879a..0b62e93e3 100644 --- a/mod/poco.php +++ b/mod/poco.php @@ -61,12 +61,13 @@ function poco_init(&$a) { $update_limit = date("Y-m-d H:i:s",strtotime($_GET['updatedSince'])); if ($global) { - $r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND `network` IN ('%s')", + $r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND `last_contact` >= `last_failure` AND `network` IN ('%s')", dbesc($update_limit), dbesc(NETWORK_DFRN) ); } elseif($system_mode) { $r = q("SELECT count(*) AS `total` FROM `contact` WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') + AND `success_update` >= `failure_update` AND `uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) ", dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), @@ -75,7 +76,7 @@ function poco_init(&$a) { ); } else { $r = q("SELECT count(*) AS `total` FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0 - AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra", + AND `success_update` >= `failure_update` AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra", intval($user['uid']), dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), @@ -95,7 +96,7 @@ function poco_init(&$a) { if ($global) { - $r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') LIMIT %d, %d", + $r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') AND `last_contact` >= `last_failure` LIMIT %d, %d", dbesc($update_limit), dbesc(NETWORK_DFRN), intval($startIndex), @@ -106,6 +107,7 @@ function poco_init(&$a) { `profile`.`address` AS `paddress`, `profile`.`region` AS `pregion`, `profile`.`postal-code` AS `ppostalcode`, `profile`.`country-name` AS `pcountry` FROM `contact` INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid` WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') AND `profile`.`is-default` + AND `contact`.`success_update` >= `contact`.`failure_update` AND `contact`.`uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) LIMIT %d, %d", dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), @@ -116,7 +118,7 @@ function poco_init(&$a) { ); } else { $r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0 - AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra LIMIT %d, %d", + AND `success_update` >= `failure_update` AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra LIMIT %d, %d", intval($user['uid']), dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), diff --git a/view/templates/admin_site.tpl b/view/templates/admin_site.tpl index 34c590b2e..5809d5ac6 100644 --- a/view/templates/admin_site.tpl +++ b/view/templates/admin_site.tpl @@ -59,8 +59,6 @@ {{include file="field_checkbox.tpl" field=$old_share}} {{include file="field_checkbox.tpl" field=$hide_help}} {{include file="field_select.tpl" field=$singleuser}} - -

{{$registration}}

@@ -70,13 +68,13 @@ {{include file="field_checkbox.tpl" field=$no_multi_reg}} {{include file="field_checkbox.tpl" field=$no_openid}} {{include file="field_checkbox.tpl" field=$no_regfullname}} -

{{$upload}}

{{include file="field_input.tpl" field=$maximagesize}} {{include file="field_input.tpl" field=$maximagelength}} {{include file="field_input.tpl" field=$jpegimagequality}} +

{{$corporate}}

{{include file="field_input.tpl" field=$allowed_sites}} @@ -99,7 +97,7 @@

{{$advanced}}

- {{include file="field_select.tpl" field=$rino}} + {{include file="field_select.tpl" field=$rino}} {{include file="field_checkbox.tpl" field=$no_utf}} {{include file="field_checkbox.tpl" field=$verifyssl}} {{include file="field_input.tpl" field=$proxy}} @@ -115,6 +113,11 @@ {{include file="field_input.tpl" field=$basepath}} {{include file="field_checkbox.tpl" field=$suppress_language}} {{include file="field_checkbox.tpl" field=$suppress_tags}} +
+ +

{{$local_directory}}

+ {{include file="field_checkbox.tpl" field=$ld_discover_activity}} +

{{$performance}}

{{include file="field_checkbox.tpl" field=$use_fulltext_engine}} From 38d7b5e32641cccfbdcfb9ed11c2ece0243f7698 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 18 Jul 2015 20:15:21 +0200 Subject: [PATCH 06/47] New table "gserver" for server data of the global contacts --- database.sql | 27 ++++- include/dbstructure.php | 21 ++++ include/socgraph.php | 251 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 293 insertions(+), 6 deletions(-) diff --git a/database.sql b/database.sql index b65dee657..ce5f051cd 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ --- Friendica 3.4.0 (Lily of the valley) --- DB_UPDATE_VERSION 1185 +-- Friendica 3.4.1 (Lily of the valley) +-- DB_UPDATE_VERSION 1186 -- ------------------------------------------ @@ -136,6 +136,7 @@ CREATE TABLE IF NOT EXISTS `contact` ( `hub-verify` varchar(255) NOT NULL DEFAULT '', `last-update` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `success_update` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', + `failure_update` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `name-date` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `uri-date` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `avatar-date` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', @@ -305,12 +306,15 @@ CREATE TABLE IF NOT EXISTS `gcontact` ( `photo` varchar(255) NOT NULL DEFAULT '', `connect` varchar(255) NOT NULL DEFAULT '', `updated` datetime DEFAULT '0000-00-00 00:00:00', + `last_contact` datetime DEFAULT '0000-00-00 00:00:00', + `last_failure` datetime DEFAULT '0000-00-00 00:00:00', `location` varchar(255) NOT NULL DEFAULT '', `about` text NOT NULL, `keywords` text NOT NULL, `gender` varchar(32) NOT NULL DEFAULT '', `network` varchar(255) NOT NULL DEFAULT '', `generation` tinyint(3) NOT NULL DEFAULT 0, + `server_url` varchar(255) NOT NULL DEFAULT '', INDEX `nurl` (`nurl`) ) DEFAULT CHARSET=utf8; @@ -352,6 +356,25 @@ CREATE TABLE IF NOT EXISTS `group_member` ( INDEX `uid_gid_contactid` (`uid`,`gid`,`contact-id`) ) DEFAULT CHARSET=utf8; +-- +-- TABLE gserver +-- +CREATE TABLE IF NOT EXISTS `gserver` ( + `url` varchar(255) NOT NULL DEFAULT '', + `nurl` varchar(255) NOT NULL DEFAULT '', + `version` varchar(255) NOT NULL DEFAULT '', + `site_name` varchar(255) NOT NULL DEFAULT '', + `info` text NOT NULL, + `register_policy` tinyint(1) NOT NULL DEFAULT 0, + `poco` varchar(255) NOT NULL DEFAULT '', + `noscrape` varchar(255) NOT NULL DEFAULT '', + `network` varchar(32) NOT NULL DEFAULT '', + `platform` varchar(255) NOT NULL DEFAULT '', + `last_poco_query` datetime DEFAULT '0000-00-00 00:00:00', + `last_contact` datetime DEFAULT '0000-00-00 00:00:00', + `last_failure` datetime DEFAULT '0000-00-00 00:00:00' +) DEFAULT CHARSET=utf8; + -- -- TABLE guid -- diff --git a/include/dbstructure.php b/include/dbstructure.php index 98e225bf8..9d0856c25 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -639,6 +639,7 @@ function db_definition() { "gender" => array("type" => "varchar(32)", "not null" => "1", "default" => ""), "network" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "generation" => array("type" => "tinyint(3)", "not null" => "1", "default" => "0"), + "server_url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), ), "indexes" => array( "PRIMARY" => array("id"), @@ -686,6 +687,26 @@ function db_definition() { "uid_gid_contactid" => array("uid","gid","contact-id"), ) ); + $database["gserver"] = array( + "fields" => array( + "url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "nurl" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "version" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "site_name" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "info" => array("type" => "text", "not null" => "1"), + "register_policy" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"), + "poco" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "noscrape" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "network" => array("type" => "varchar(32)", "not null" => "1", "default" => ""), + "platform" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "last_poco_query" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), + "last_contact" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), + "last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), + ), + "indexes" => array( + "PRIMARY" => array("nurl"), + ) + ); $database["guid"] = array( "fields" => array( "id" => array("type" => "int(10) unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"), diff --git a/include/socgraph.php b/include/socgraph.php index 5c9b1d98a..a6899e952 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -214,17 +214,20 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca $last_contact = $x[0]["last_contact"]; $last_failure = $x[0]["last_failure"]; + $server_url = $x[0]["server_url"]; } else { $last_contact = "0000-00-00 00:00:00"; $last_failure = "0000-00-00 00:00:00"; + $server_url = ""; } - if (($network == "") OR ($name == "") OR ($profile_photo == "")) { + if (($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == "")) { $data = probe_url($profile_url); $network = $data["network"]; $name = $data["name"]; $profile_url = $data["url"]; $profile_photo = $data["photo"]; + $server_url = $data["baseurl"]; } if (count($x) AND ($x[0]["network"] == "") AND ($network != "")) { @@ -260,6 +263,14 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca } } + poco_check_server($server_url, $network); + + // Test - remove before flight + if ($last_contact > $last_failure) + q("UPDATE `gserver` SET `last_contact` = '%s' WHERE `nurl` = '%s'", dbesc($last_contact), dbesc(normalise_link($server_url))); + else + q("UPDATE `gserver` SET `last_failure` = '%s' WHERE `nurl` = '%s'", dbesc($last_failure), dbesc(normalise_link($server_url))); + if(count($x)) { $gcid = $x[0]['id']; @@ -279,7 +290,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca $generation = $x[0]['generation']; if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo || $x[0]['updated'] < $updated) { - q("UPDATE `gcontact` SET `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s', + q("UPDATE `gcontact` SET `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s', `server_url` = '%s', `updated` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d WHERE (`generation` >= %d OR `generation` = 0) AND `nurl` = '%s'", dbesc($name), @@ -287,6 +298,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca dbesc($profile_photo), dbesc($connect_url), dbesc($profile_url), + dbesc($server_url), dbesc($updated), dbesc($location), dbesc($about), @@ -298,14 +310,15 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca ); } } else { - q("INSERT INTO `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`) - VALUES ('%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s', '%s', %d)", + q("INSERT INTO `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `server_url`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`) + VALUES ('%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)", dbesc($name), dbesc($network), dbesc($profile_url), dbesc(normalise_link($profile_url)), dbesc($profile_photo), dbesc($connect_url), + dbesc($server_url), dbesc($updated), dbesc($last_contact), dbesc($last_failure), @@ -434,6 +447,236 @@ function poco_do_update($updated, $last_contact, $last_failure) { return true; } +function poco_to_boolean($val) { + if (($val == "true") OR ($val == 1)) + return(true); + if (($val == "false") OR ($val == 0)) + return(false); + + return ($val); +} + +function poco_check_server($server_url, $network = "") { + + if ($server_url == "") + return; + + $servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url))); + if ($servers) { + $poco = $servers[0]["poco"]; + $noscrape = $servers[0]["noscrape"]; + + if ($network == "") + $network = $servers[0]["network"]; + + $last_contact = $servers[0]["last_contact"]; + $last_failure = $servers[0]["last_failure"]; + $version = $servers[0]["version"]; + $platform = $servers[0]["platform"]; + $site_name = $servers[0]["site_name"]; + $info = $servers[0]["info"]; + $register_policy = $servers[0]["register_policy"]; + + // Only check the server once a week + if (strtotime(datetime_convert()) < (strtotime($last_contact) + (60 * 60 * 24 * 7))) + return; + + if (strtotime(datetime_convert()) < (strtotime($last_failure) + (60 * 60 * 24 * 7))) + return; + } else { + $poco = ""; + $noscrape = ""; + $version = ""; + $platform = ""; + $site_name = ""; + $info = ""; + $register_policy = -1; + + $last_contact = "0000-00-00 00:00:00"; + $last_failure = "0000-00-00 00:00:00"; + } + + $failure = false; + $orig_last_failure = $last_failure; + + // Check if the page is accessible via SSL. + $server_url = str_replace("http://", "https://", $server_url); + $serverret = z_fetch_url($server_url."/.well-known/host-meta"); + + // Maybe the page is unencrypted only? + $xmlobj = @simplexml_load_string($serverret["body"],'SimpleXMLElement',0, "http://docs.oasis-open.org/ns/xri/xrd-1.0"); + if (!$serverret["success"] OR ($serverret["body"] == "") OR (@sizeof($xmlobj) == 0) OR !is_object($xmlobj)) { + $server_url = str_replace("https://", "http://", $server_url); + $serverret = z_fetch_url($server_url."/.well-known/host-meta"); + + $xmlobj = @simplexml_load_string($serverret["body"],'SimpleXMLElement',0, "http://docs.oasis-open.org/ns/xri/xrd-1.0"); + } + + if (!$serverret["success"] OR ($serverret["body"] == "") OR (sizeof($xmlobj) == 0) OR !is_object($xmlobj)) { + $last_failure = datetime_convert(); + $failure = true; + } elseif ($network == NETWORK_DIASPORA) + $last_contact = datetime_convert(); + + if (!$failure) { + // Test for Statusnet + // Will also return data for Friendica and GNU Social - but it will be overwritten later + // The "not implemented" is a special treatment for really, really old Friendica versions + $serverret = z_fetch_url($server_url."/api/statusnet/version.json"); + if ($serverret["success"] AND ($serverret["body"] != '{"error":"not implemented"}') AND ($serverret["body"] != '') AND (strlen($serverret["body"]) < 250)) { + $platform = "StatusNet"; + $version = trim($serverret["body"], '"'); + $network = NETWORK_OSTATUS; + } + + // Test for GNU Social + $serverret = z_fetch_url($server_url."/api/gnusocial/version.json"); + if ($serverret["success"] AND ($serverret["body"] != '{"error":"not implemented"}') AND ($serverret["body"] != '') AND (strlen($serverret["body"]) < 250)) { + $platform = "GNU Social"; + $version = trim($serverret["body"], '"'); + $network = NETWORK_OSTATUS; + } + + $serverret = z_fetch_url($server_url."/api/statusnet/config.json"); + if ($serverret["success"]) { + $data = json_decode($serverret["body"]); + + if (isset($data->site->server)) { + $last_contact = datetime_convert(); + + if (isset($data->site->redmatrix)) { + if (isset($data->site->redmatrix->PLATFORM_NAME)) + $platform = $data->site->redmatrix->PLATFORM_NAME; + elseif (isset($data->site->redmatrix->RED_PLATFORM)) + $platform = $data->site->redmatrix->RED_PLATFORM; + + $version = $data->site->redmatrix->RED_VERSION; + $network = NETWORK_DIASPORA; + } + if (isset($data->site->friendica)) { + $platform = $data->site->friendica->FRIENDICA_PLATFORM; + $version = $data->site->friendica->FRIENDICA_VERSION; + $network = NETWORK_DFRN; + } + + $site_name = $data->site->name; + + $data->site->closed = poco_to_boolean($data->site->closed); + $data->site->private = poco_to_boolean($data->site->private); + $data->site->inviteonly = poco_to_boolean($data->site->inviteonly); + + if (!$data->site->closed AND !$data->site->private and $data->site->inviteonly) + $register_policy = REGISTER_APPROVE; + elseif (!$data->site->closed AND !$data->site->private) + $register_policy = REGISTER_OPEN; + else + $register_policy = REGISTER_CLOSED; + } + } + } + + // Query statistics.json. Optional package for Diaspora, Friendica and Redmatrix + if (!$failure) { + $serverret = z_fetch_url($server_url."/statistics.json"); + if ($serverret["success"]) { + $data = json_decode($serverret["body"]); + if ($version == "") + $version = $data->version; + + $site_name = $data->name; + + if (isset($data->network) AND ($platform == "")) + $platform = $data->network; + + if ($data->registrations_open) + $register_policy = REGISTER_OPEN; + else + $register_policy = REGISTER_CLOSED; + + if (isset($data->version)) + $last_contact = datetime_convert(); + } + } + + // Check for noscrape + // Friendica servers could be detected as OStatus servers + if (!$failure AND in_array($network, array(NETWORK_DFRN, NETWORK_OSTATUS))) { + $serverret = z_fetch_url($server_url."/friendica/json"); + + if ($serverret["success"]) { + $data = json_decode($serverret["body"]); + + if (isset($data->version)) { + $last_contact = datetime_convert(); + $network = NETWORK_DFRN; + + $noscrape = $data->no_scrape_url; + $version = $data->version; + $site_name = $data->site_name; + $info = $data->info; + $register_policy_str = $data->register_policy; + $platform = $data->platform; + + switch ($register_policy_str) { + case "REGISTER_CLOSED": + $register_policy = REGISTER_CLOSED; + break; + case "REGISTER_APPROVE": + $register_policy = REGISTER_APPROVE; + break; + case "REGISTER_OPEN": + $register_policy = REGISTER_OPEN; + break; + } + } + } + } + + // Look for poco + if (!$failure) { + $serverret = z_fetch_url($server_url."/poco"); + if ($serverret["success"]) { + $data = json_decode($serverret["body"]); + if (isset($data->totalResults)) { + $poco = $server_url."/poco"; + $last_contact = datetime_convert(); + } + } + } + + if ($servers) + q("UPDATE `gserver` SET `url` = '%s', `version` = '%s', `site_name` = '%s', `info` = '%s', `register_policy` = %d, `poco` = '%s', `noscrape` = '%s', + `network` = '%s', `platform` = '%s', `last_contact` = '%s', `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc($server_url), + dbesc($version), + dbesc($site_name), + dbesc($info), + intval($register_policy), + dbesc($poco), + dbesc($noscrape), + dbesc($network), + dbesc($platform), + dbesc($last_contact), + dbesc($last_failure), + dbesc(normalise_link($server_url)) + ); + else + q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `last_contact`) + VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s')", + dbesc($server_url), + dbesc(normalise_link($server_url)), + dbesc($version), + dbesc($site_name), + dbesc($info), + intval($register_policy), + dbesc($poco), + dbesc($noscrape), + dbesc($network), + dbesc($platform), + dbesc(datetime_convert()) + ); +} + function poco_contact_from_body($body, $created, $cid, $uid) { preg_replace_callback("/\[share(.*?)\].*?\[\/share\]/ism", function ($match) use ($created, $cid, $uid){ From cf3214c9040d84102cfe30943ee42211305412d3 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 18 Jul 2015 22:26:06 +0200 Subject: [PATCH 07/47] Query other servers periodically for their known contacts --- include/discover_poco.php | 71 +++++++++++++++++++++ include/poller.php | 4 ++ include/socgraph.php | 115 ++++++++++++++++++++++++++++------ mod/admin.php | 13 ++-- view/templates/admin_site.tpl | 5 +- 5 files changed, 183 insertions(+), 25 deletions(-) create mode 100644 include/discover_poco.php diff --git a/include/discover_poco.php b/include/discover_poco.php new file mode 100644 index 000000000..5a493ab96 --- /dev/null +++ b/include/discover_poco.php @@ -0,0 +1,71 @@ + $maxsysload) { + logger('system: load ' . $load[0] . ' too high. discover_poco deferred to next scheduled run.'); + return; + } + } + + $lockpath = get_lockpath(); + if ($lockpath != '') { + $pidfile = new pidfile($lockpath, 'discover_poco'); + if($pidfile->is_already_running()) { + logger("discover_poco: Already running"); + if ($pidfile->running_time() > 19*60) { + $pidfile->kill(); + logger("discover_poco: killed stale process"); + // Calling a new instance + proc_run('php','include/discover_poco.php'); + } + exit; + } + } + + $a->set_baseurl(get_config('system','url')); + + load_hooks(); + + logger('start'); + + if (get_config('system','poco_discovery')) + poco_discover(); + + logger('end'); + + return; +} + +if (array_search(__file__,get_included_files())===0){ + discover_poco_run($_SERVER["argv"],$_SERVER["argc"]); + killme(); +} diff --git a/include/poller.php b/include/poller.php index d971d4f00..44ac94daa 100644 --- a/include/poller.php +++ b/include/poller.php @@ -82,6 +82,10 @@ function poller_run(&$argv, &$argc){ proc_run('php',"include/dsprphotoq.php"); + // run the process to discover global contacts in the background + + proc_run('php',"include/discover_poco.php"); + // expire any expired accounts q("UPDATE user SET `account_expired` = 1 where `account_expired` = 0 diff --git a/include/socgraph.php b/include/socgraph.php index a6899e952..1d0dfd6f7 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -2,12 +2,7 @@ require_once('include/datetime.php'); require_once("include/Scrape.php"); - -/* - To-Do: - - noscrape for updating contact fields and "last updated" - - use /poco/@global for discovering contacts from other servers -*/ +require_once("include/html2bbcode.php"); /* * poco_load @@ -28,8 +23,6 @@ require_once("include/Scrape.php"); function poco_load($cid,$uid = 0,$zcid = 0,$url = null) { - require_once("include/html2bbcode.php"); - $a = get_app(); if($cid) { @@ -246,7 +239,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca logger("profile-check generation: ".$generation." Network: ".$network." URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG); // Only fetch last update manually if it wasn't provided and enabled in the system - if (get_config('system','ld_discover_activity') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) { + if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) { $last_updated = poco_last_updated($profile_url); if ($last_updated) { $updated = $last_updated; @@ -266,10 +259,10 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca poco_check_server($server_url, $network); // Test - remove before flight - if ($last_contact > $last_failure) - q("UPDATE `gserver` SET `last_contact` = '%s' WHERE `nurl` = '%s'", dbesc($last_contact), dbesc(normalise_link($server_url))); - else - q("UPDATE `gserver` SET `last_failure` = '%s' WHERE `nurl` = '%s'", dbesc($last_failure), dbesc(normalise_link($server_url))); + //if ($last_contact > $last_failure) + // q("UPDATE `gserver` SET `last_contact` = '%s' WHERE `nurl` = '%s'", dbesc($last_contact), dbesc(normalise_link($server_url))); + //else + // q("UPDATE `gserver` SET `last_failure` = '%s' WHERE `nurl` = '%s'", dbesc($last_failure), dbesc(normalise_link($server_url))); if(count($x)) { $gcid = $x[0]['id']; @@ -448,12 +441,12 @@ function poco_do_update($updated, $last_contact, $last_failure) { } function poco_to_boolean($val) { - if (($val == "true") OR ($val == 1)) - return(true); - if (($val == "false") OR ($val == 0)) - return(false); + if (($val == "true") OR ($val == 1)) + return(true); + if (($val == "false") OR ($val == 0)) + return(false); - return ($val); + return ($val); } function poco_check_server($server_url, $network = "") { @@ -976,3 +969,89 @@ function update_suggestions() { } } } + +function poco_discover() { + + $last_update = date("c", time() - (60 * 60 * 24)); + + $r = q("SELECT `poco`, `nurl` FROM `gserver` WHERE `last_contact` > `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update)); + if ($r) + foreach ($r AS $server) { + $url = $server["poco"]."/@global?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; + + $retdata = z_fetch_url($url); + if ($retdata["success"]) { + poco_discover_server(json_decode($retdata["body"])); + q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"])); + break; + } else + q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"])); + + } +} + +function poco_discover_server($data) { + + foreach ($data->entry AS $entry) { + $profile_url = ''; + $profile_photo = ''; + $connect_url = ''; + $name = ''; + $network = ''; + $updated = '0000-00-00 00:00:00'; + $location = ''; + $about = ''; + $keywords = ''; + $gender = ''; + $generation = 0; + + $name = $entry->displayName; + + if(isset($entry->urls)) { + foreach($entry->urls as $url) { + if($url->type == 'profile') { + $profile_url = $url->value; + continue; + } + if($url->type == 'webfinger') { + $connect_url = str_replace('acct:' , '', $url->value); + continue; + } + } + } + if(isset($entry->photos)) { + foreach($entry->photos as $photo) { + if($photo->type == 'profile') { + $profile_photo = $photo->value; + continue; + } + } + } + + if(isset($entry->updated)) + $updated = date("Y-m-d H:i:s", strtotime($entry->updated)); + + if(isset($entry->network)) + $network = $entry->network; + + if(isset($entry->currentLocation)) + $location = $entry->currentLocation; + + if(isset($entry->aboutMe)) + $about = html2bbcode($entry->aboutMe); + + if(isset($entry->gender)) + $gender = $entry->gender; + + if(isset($entry->generation) AND ($entry->generation > 0)) + $generation = ++$entry->generation; + + if(isset($entry->tags)) + foreach($entry->tags as $tag) + $keywords = implode(", ", $tag); + + if ($generation > 0) + poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $generation); + } +} +?> diff --git a/mod/admin.php b/mod/admin.php index 214ce62ea..393fe1614 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -358,7 +358,8 @@ function admin_page_site_post(&$a){ $poll_interval = ((x($_POST,'poll_interval')) ? intval(trim($_POST['poll_interval'])) : 0); $maxloadavg = ((x($_POST,'maxloadavg')) ? intval(trim($_POST['maxloadavg'])) : 50); $maxloadavg_frontend = ((x($_POST,'maxloadavg_frontend')) ? intval(trim($_POST['maxloadavg_frontend'])) : 50); - $ld_discover_activity = ((x($_POST,'ld_discover_activity')) ? intval(trim($_POST['ld_discover_activity'])) : false); + $poco_completion = ((x($_POST,'poco_completion')) ? intval(trim($_POST['poco_completion'])) : false); + $poco_discovery = ((x($_POST,'poco_discovery')) ? intval(trim($_POST['poco_discovery'])) : false); $dfrn_only = ((x($_POST,'dfrn_only')) ? True : False); $ostatus_disabled = !((x($_POST,'ostatus_disabled')) ? True : False); $ostatus_poll_interval = ((x($_POST,'ostatus_poll_interval')) ? intval(trim($_POST['ostatus_poll_interval'])) : 0); @@ -428,7 +429,8 @@ function admin_page_site_post(&$a){ set_config('system','poll_interval',$poll_interval); set_config('system','maxloadavg',$maxloadavg); set_config('system','maxloadavg_frontend',$maxloadavg_frontend); - set_config('system','ld_discover_activity',$ld_discover_activity); + set_config('system','poco_completion',$poco_completion); + set_config('system','poco_discovery',$poco_discovery); set_config('config','sitename',$sitename); set_config('config','hostname',$hostname); set_config('config','sender_email', $sender_email); @@ -436,7 +438,7 @@ function admin_page_site_post(&$a){ set_config('system','suppress_tags',$suppress_tags); set_config('system','shortcut_icon',$shortcut_icon); set_config('system','touch_icon',$touch_icon); - + if ($banner==""){ // don't know why, but del_config doesn't work... q("DELETE FROM `config` WHERE `cat` = '%s' AND `k` = '%s' LIMIT 1", @@ -632,7 +634,7 @@ function admin_page_site(&$a) { '$upload' => t('File upload'), '$corporate' => t('Policies'), '$advanced' => t('Advanced'), - '$local_directory' => t('Local Directory (Portable Contacts)'), + '$portable_contacts' => t('Portable Contact Directory'), '$performance' => t('Performance'), '$relocate'=> t('Relocate - WARNING: advanced function. Could make this server unreachable.'), '$baseurl' => $a->get_baseurl(true), @@ -690,7 +692,8 @@ function admin_page_site(&$a) { '$maxloadavg' => array('maxloadavg', t("Maximum Load Average"), ((intval(get_config('system','maxloadavg')) > 0)?get_config('system','maxloadavg'):50), t("Maximum system load before delivery and poll processes are deferred - default 50.")), '$maxloadavg_frontend' => array('maxloadavg_frontend', t("Maximum Load Average (Frontend)"), ((intval(get_config('system','maxloadavg_frontend')) > 0)?get_config('system','maxloadavg_frontend'):50), t("Maximum system load before the frontend quits service - default 50.")), - '$ld_discover_activity' => array('ld_discover_activity', t("Discover last activity"), get_config('system','ld_discover_activity'), t("Update the last activity when this isn't provided via the 'portable contacts' functionality. (Useful for poco exchange with Redmatrix and friendica servers before 3.3)")), + '$poco_completion' => array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful for poco exchange with Redmatrix and friendica servers before 3.3)")), + '$poco_discovery' => array('poco_discovery', t("Discover contacts from other servers"), get_config('system','poco_discovery'), t("Periodically query other friendica servers for their recent contacts.")), '$use_fulltext_engine' => array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")), '$suppress_language' => array('suppress_language', t("Suppress Language"), get_config('system','suppress_language'), t("Suppress language information in meta information about a posting.")), diff --git a/view/templates/admin_site.tpl b/view/templates/admin_site.tpl index 5809d5ac6..22d785f6f 100644 --- a/view/templates/admin_site.tpl +++ b/view/templates/admin_site.tpl @@ -115,8 +115,9 @@ {{include file="field_checkbox.tpl" field=$suppress_tags}}
-

{{$local_directory}}

- {{include file="field_checkbox.tpl" field=$ld_discover_activity}} +

{{$portable_contacts}}

+ {{include file="field_checkbox.tpl" field=$poco_completion}} + {{include file="field_checkbox.tpl" field=$poco_discovery}}

{{$performance}}

From e47c65eb1357aeb4d0471914336574e02194c979 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 18 Jul 2015 23:33:54 +0200 Subject: [PATCH 08/47] Query the other servers for their users. --- include/socgraph.php | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/include/socgraph.php b/include/socgraph.php index 1d0dfd6f7..99d72d41c 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -977,20 +977,27 @@ function poco_discover() { $r = q("SELECT `poco`, `nurl` FROM `gserver` WHERE `last_contact` > `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update)); if ($r) foreach ($r AS $server) { - $url = $server["poco"]."/@global?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; + // Fetch all users from the other server + $url = $server["poco"]."/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; $retdata = z_fetch_url($url); if ($retdata["success"]) { - poco_discover_server(json_decode($retdata["body"])); + poco_discover_server(json_decode($retdata["body"]), 2); + + // Fetch all global contacts from the other server (Not working with Redmatrix and Friendica versions before 3.3) + $url = $server["poco"]."/@global?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; + + $retdata = z_fetch_url($url); + if ($retdata["success"]) + poco_discover_server(json_decode($retdata["body"])); + q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"])); break; - } else - q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"])); - + } } } -function poco_discover_server($data) { +function poco_discover_server($data, $default_generation = 0) { foreach ($data->entry AS $entry) { $profile_url = ''; @@ -1003,7 +1010,7 @@ function poco_discover_server($data) { $about = ''; $keywords = ''; $gender = ''; - $generation = 0; + $generation = $default_generation; $name = $entry->displayName; From 2a5fb7b175f487b277254a76a484687f7e491e05 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 19 Jul 2015 09:41:36 +0200 Subject: [PATCH 09/47] Option to discover contacts from remote users (fallback option) --- include/discover_poco.php | 2 +- include/socgraph.php | 60 +++++++++++++++++++++++++++++------ mod/admin.php | 15 ++++++--- view/templates/admin_site.tpl | 2 +- 4 files changed, 64 insertions(+), 15 deletions(-) diff --git a/include/discover_poco.php b/include/discover_poco.php index 5a493ab96..c69e27fef 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -57,7 +57,7 @@ function discover_poco_run(&$argv, &$argc){ logger('start'); - if (get_config('system','poco_discovery')) + if (get_config('system','poco_discovery') > 0) poco_discover(); logger('end'); diff --git a/include/socgraph.php b/include/socgraph.php index 99d72d41c..18c2c7c4c 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -970,7 +970,7 @@ function update_suggestions() { } } -function poco_discover() { +function poco_discover($complete = false) { $last_update = date("c", time() - (60 * 60 * 24)); @@ -980,25 +980,64 @@ function poco_discover() { // Fetch all users from the other server $url = $server["poco"]."/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; + logger("Fetch all users from the server ".$server["nurl"], LOGGER_DEBUG); + $retdata = z_fetch_url($url); if ($retdata["success"]) { - poco_discover_server(json_decode($retdata["body"]), 2); + $data = json_decode($retdata["body"]); + poco_discover_server($data, 2); - // Fetch all global contacts from the other server (Not working with Redmatrix and Friendica versions before 3.3) - $url = $server["poco"]."/@global?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; + if (get_config('system','poco_discovery') > 1) { - $retdata = z_fetch_url($url); - if ($retdata["success"]) - poco_discover_server(json_decode($retdata["body"])); + // Fetch all global contacts from the other server (Not working with Redmatrix and Friendica versions before 3.3) + $url = $server["poco"]."/@global?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; + + $retdata = z_fetch_url($url); + if ($retdata["success"]) { + logger("Fetch all global contacts from the server ".$server["nurl"], LOGGER_DEBUG); + poco_discover_server(json_decode($retdata["body"])); + } elseif (get_config('system','poco_discovery') > 2) { + logger("Fetch contacts from users of the server ".$server["nurl"], LOGGER_DEBUG); + poco_discover_server_users($data); + } + } q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"])); - break; + if (!$complete) + break; } } } +function poco_discover_server_users($data) { + foreach ($data->entry AS $entry) { + $username = ""; + if (isset($entry->urls)) { + foreach($entry->urls as $url) + if($url->type == 'profile') { + $profile_url = $url->value; + $urlparts = parse_url($profile_url); + $username = end(explode("/", $urlparts["path"])); + } + } + if ($username != "") { + logger("Fetch contacts for the user ".$username." from the server ".$server["nurl"], LOGGER_DEBUG); + + // Fetch all contacts from a given user from the other server + $url = $server["poco"]."/".$username."/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; + + $retdata = z_fetch_url($url); + if ($retdata["success"]) + poco_discover_server(json_decode($retdata["body"]), 3); + } + } +} + function poco_discover_server($data, $default_generation = 0) { + if (!isset($data->entry) OR !count($data->entry)) + return; + foreach ($data->entry AS $entry) { $profile_url = ''; $profile_photo = ''; @@ -1057,8 +1096,11 @@ function poco_discover_server($data, $default_generation = 0) { foreach($entry->tags as $tag) $keywords = implode(", ", $tag); - if ($generation > 0) + if ($generation > 0) { + logger("Store profile ".$profile_url, LOGGER_DEBUG); poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $generation); + logger("Done for profile ".$profile_url, LOGGER_DEBUG); + } } } ?> diff --git a/mod/admin.php b/mod/admin.php index 393fe1614..8c3e2441b 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -359,7 +359,7 @@ function admin_page_site_post(&$a){ $maxloadavg = ((x($_POST,'maxloadavg')) ? intval(trim($_POST['maxloadavg'])) : 50); $maxloadavg_frontend = ((x($_POST,'maxloadavg_frontend')) ? intval(trim($_POST['maxloadavg_frontend'])) : 50); $poco_completion = ((x($_POST,'poco_completion')) ? intval(trim($_POST['poco_completion'])) : false); - $poco_discovery = ((x($_POST,'poco_discovery')) ? intval(trim($_POST['poco_discovery'])) : false); + $poco_discovery = ((x($_POST,'poco_discovery')) ? intval(trim($_POST['poco_discovery'])) : 0); $dfrn_only = ((x($_POST,'dfrn_only')) ? True : False); $ostatus_disabled = !((x($_POST,'ostatus_disabled')) ? True : False); $ostatus_poll_interval = ((x($_POST,'ostatus_poll_interval')) ? intval(trim($_POST['ostatus_poll_interval'])) : 0); @@ -382,8 +382,8 @@ function admin_page_site_post(&$a){ $old_pager = ((x($_POST,'old_pager')) ? True : False); $only_tag_search = ((x($_POST,'only_tag_search')) ? True : False); $rino = ((x($_POST,'rino')) ? intval($_POST['rino']) : 0); - - + + if($ssl_policy != intval(get_config('system','ssl_policy'))) { if($ssl_policy == SSL_POLICY_FULL) { q("update `contact` set @@ -586,6 +586,13 @@ function admin_page_site(&$a) { "1440" => t("Daily") ); + $poco_discovery_choices = array( + "0" => t("Disabled"), + "1" => t("Users"), + "2" => t("Users, Global Contacts"), + "3" => t("Users, Global Contacts/fallback"), + ); + /* get user names to make the install a personal install of X */ $user_names = array(); $user_names['---'] = t('Multi user instance'); @@ -693,7 +700,7 @@ function admin_page_site(&$a) { '$maxloadavg_frontend' => array('maxloadavg_frontend', t("Maximum Load Average (Frontend)"), ((intval(get_config('system','maxloadavg_frontend')) > 0)?get_config('system','maxloadavg_frontend'):50), t("Maximum system load before the frontend quits service - default 50.")), '$poco_completion' => array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful for poco exchange with Redmatrix and friendica servers before 3.3)")), - '$poco_discovery' => array('poco_discovery', t("Discover contacts from other servers"), get_config('system','poco_discovery'), t("Periodically query other friendica servers for their recent contacts.")), + '$poco_discovery' => array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for profiles. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available."), $poco_discovery_choices), '$use_fulltext_engine' => array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")), '$suppress_language' => array('suppress_language', t("Suppress Language"), get_config('system','suppress_language'), t("Suppress language information in meta information about a posting.")), diff --git a/view/templates/admin_site.tpl b/view/templates/admin_site.tpl index 22d785f6f..f0461e1b8 100644 --- a/view/templates/admin_site.tpl +++ b/view/templates/admin_site.tpl @@ -117,7 +117,7 @@

{{$portable_contacts}}

{{include file="field_checkbox.tpl" field=$poco_completion}} - {{include file="field_checkbox.tpl" field=$poco_discovery}} + {{include file="field_select.tpl" field=$poco_discovery}}

{{$performance}}

From 93e93f58bd703dc4dbe254e1e6ca80fc41739971 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 19 Jul 2015 10:40:33 +0200 Subject: [PATCH 10/47] Query Redmatrix contacts for their contacts as well. --- include/socgraph.php | 9 +++++++-- mod/admin.php | 6 +++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/include/socgraph.php b/include/socgraph.php index 18c2c7c4c..c97a0f15c 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -939,6 +939,7 @@ function update_suggestions() { $done = array(); + // To-Do: Check if it is really neccessary to poll the own server poco_load(0,0,0,$a->get_baseurl() . '/poco'); $done[] = $a->get_baseurl() . '/poco'; @@ -949,6 +950,9 @@ function update_suggestions() { $j = json_decode($x); if($j->entries) { foreach($j->entries as $entry) { + + poco_check_server($entry->url); + $url = $entry->url . '/poco'; if(! in_array($url,$done)) poco_load(0,0,0,$entry->url . '/poco'); @@ -957,8 +961,9 @@ function update_suggestions() { } } - $r = q("select distinct(poco) as poco from contact where network = '%s'", - dbesc(NETWORK_DFRN) + // Query your contacts from Friendica and Redmatrix/Hubzilla for their contacts + $r = q("SELECT DISTINCT(`poco`) AS `poco` FROM `contact` WHERE `network` IN ('%s', '%s')", + dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA) ); if(count($r)) { diff --git a/mod/admin.php b/mod/admin.php index 8c3e2441b..a079bdabf 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -641,7 +641,7 @@ function admin_page_site(&$a) { '$upload' => t('File upload'), '$corporate' => t('Policies'), '$advanced' => t('Advanced'), - '$portable_contacts' => t('Portable Contact Directory'), + '$portable_contacts' => t('Auto Discovered Contact Directory'), '$performance' => t('Performance'), '$relocate'=> t('Relocate - WARNING: advanced function. Could make this server unreachable.'), '$baseurl' => $a->get_baseurl(true), @@ -699,8 +699,8 @@ function admin_page_site(&$a) { '$maxloadavg' => array('maxloadavg', t("Maximum Load Average"), ((intval(get_config('system','maxloadavg')) > 0)?get_config('system','maxloadavg'):50), t("Maximum system load before delivery and poll processes are deferred - default 50.")), '$maxloadavg_frontend' => array('maxloadavg_frontend', t("Maximum Load Average (Frontend)"), ((intval(get_config('system','maxloadavg_frontend')) > 0)?get_config('system','maxloadavg_frontend'):50), t("Maximum system load before the frontend quits service - default 50.")), - '$poco_completion' => array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful for poco exchange with Redmatrix and friendica servers before 3.3)")), - '$poco_discovery' => array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for profiles. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available."), $poco_discovery_choices), + '$poco_completion' => array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful when communicating with Redmatrix and friendica servers before 3.3)")), + '$poco_discovery' => array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available."), $poco_discovery_choices), '$use_fulltext_engine' => array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")), '$suppress_language' => array('suppress_language', t("Suppress Language"), get_config('system','suppress_language'), t("Suppress language information in meta information about a posting.")), From 0bede65a6b26826158703a0a6a164c2e0dac9851 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 19 Jul 2015 13:23:01 +0200 Subject: [PATCH 11/47] Use local search --- mod/dirfind.php | 65 ++++++++++++++++++++++++++++------- view/templates/match.tpl | 2 +- view/templates/peoplefind.tpl | 2 +- 3 files changed, 54 insertions(+), 15 deletions(-) diff --git a/mod/dirfind.php b/mod/dirfind.php index 0c6d79480..c49d37e48 100644 --- a/mod/dirfind.php +++ b/mod/dirfind.php @@ -16,11 +16,13 @@ function dirfind_init(&$a) { function dirfind_content(&$a) { + $local = true; + $search = notags(trim($_REQUEST['search'])); if(strpos($search,'@') === 0) $search = substr($search,1); - + $o = ''; $o .= replace_macros(get_markup_template("section_title.tpl"),array( @@ -29,16 +31,53 @@ function dirfind_content(&$a) { if($search) { - $p = (($a->pager['page'] != 1) ? '&p=' . $a->pager['page'] : ''); - - if(strlen(get_config('system','directory_submit_url'))) - $x = fetch_url('http://dir.friendica.com/lsearch?f=' . $p . '&search=' . urlencode($search)); + if ($local) { -//TODO fallback local search if global dir not available. -// else -// $x = post_url($a->get_baseurl() . '/lsearch', $params); + $perpage = 80; + $startrec = (($a->pager['page']) * $perpage) - $perpage; - $j = json_decode($x); + $count = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `network` IN ('%s', '%s', '%s') AND + (`url` REGEXP '%s' OR `name` REGEXP '%s' OR `location` REGEXP '%s' OR + `about` REGEXP '%s' OR `keywords` REGEXP '%s')", + dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), + dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)), + dbesc(escape_tags($search)), dbesc(escape_tags($search))); + + $results = q("SELECT `url`, `name`, `photo`, `keywords` FROM `gcontact`WHERE `network` IN ('%s', '%s', '%s') AND + (`url` REGEXP '%s' OR `name` REGEXP '%s' OR `location` REGEXP '%s' OR `about` REGEXP '%s' OR + `keywords` REGEXP '%s') ORDER BY `name` ASC LIMIT %d, %d", + dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), + dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)), + dbesc(escape_tags($search)), dbesc(escape_tags($search)), + intval($startrec), intval($perpage)); + + $j = new stdClass(); + $j->total = $count[0]["total"]; + $j->items_page = $perpage; + $j->page = $a->pager['page']; + foreach ($results AS $result) { + if ($result["name"] == "") { + $urlparts = parse_url($result["url"]); + $result["name"] = end(explode("/", $urlparts["path"])); + } + + $objresult = new stdClass(); + $objresult->name = $result["name"]; + $objresult->url = $result["url"]; + $objresult->photo = $result["photo"]; + $objresult->tags = $result["keywords"]; + + $j->results[] = $objresult; + } + } else { + + $p = (($a->pager['page'] != 1) ? '&p=' . $a->pager['page'] : ''); + + if(strlen(get_config('system','directory_submit_url'))) + $x = fetch_url('http://dir.friendica.com/lsearch?f=' . $p . '&search=' . urlencode($search)); + + $j = json_decode($x); + } if($j->total) { $a->set_pager_total($j->total); @@ -46,21 +85,21 @@ function dirfind_content(&$a) { } if(count($j->results)) { - + $tpl = get_markup_template('match.tpl'); foreach($j->results as $jj) { - + $o .= replace_macros($tpl,array( '$url' => zrl($jj->url), '$name' => $jj->name, - '$photo' => $jj->photo, + '$photo' => proxy_url($jj->photo), '$tags' => $jj->tags )); } } else { info( t('No matches') . EOL); - } + } } diff --git a/view/templates/match.tpl b/view/templates/match.tpl index f4bbc3c1b..32f046e6a 100644 --- a/view/templates/match.tpl +++ b/view/templates/match.tpl @@ -2,7 +2,7 @@
- {{$name}} + {{$name}}
diff --git a/view/templates/peoplefind.tpl b/view/templates/peoplefind.tpl index de8cd011b..45c4c0b62 100644 --- a/view/templates/peoplefind.tpl +++ b/view/templates/peoplefind.tpl @@ -2,7 +2,7 @@

{{$findpeople}}

{{$desc}}
-
+
From f3c83fb51479c1aca19fd2528dc6d68d1f160739 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 19 Jul 2015 18:02:24 +0200 Subject: [PATCH 12/47] Automatically updating the searched contacts while searching it. --- include/discover_poco.php | 68 ++++++++++++++++++++++++++++++++--- mod/admin.php | 3 ++ mod/dirfind.php | 33 +++++++++++++---- view/templates/admin_site.tpl | 1 + 4 files changed, 94 insertions(+), 11 deletions(-) diff --git a/include/discover_poco.php b/include/discover_poco.php index c69e27fef..504d97287 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -36,9 +36,18 @@ function discover_poco_run(&$argv, &$argc){ } } + if(($argc > 2) && ($argv[1] == "dirsearch")) { + $search = urldecode($argv[2]); + $searchmode = 1; + } elseif ($argc == 1) { + $search = ""; + $searchmode = 0; + } else + die("Unknown or missing parameter ".$argv[1]."\n"); + $lockpath = get_lockpath(); if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'discover_poco'); + $pidfile = new pidfile($lockpath, 'discover_poco'.urlencode($search)); if($pidfile->is_already_running()) { logger("discover_poco: Already running"); if ($pidfile->running_time() > 19*60) { @@ -55,16 +64,67 @@ function discover_poco_run(&$argv, &$argc){ load_hooks(); - logger('start'); + logger('start '.$search); - if (get_config('system','poco_discovery') > 0) + if (($search != "") and get_config('system','poco_local_search')) + discover_directory($search); + elseif (($search == "") and get_config('system','poco_discovery') > 0) poco_discover(); - logger('end'); + logger('end '.$search); return; } +function discover_directory($search) { + + $data = Cache::get("dirsearch:".$search); + if (!is_null($data)){ + // Only search for the same item every 24 hours + if (time() < $data + (60 * 60 * 24)) { + logger("Already searched for ".$search." in the last 24 hours", LOGGER_DEBUG); + return; + } + } + + $x = fetch_url("http://dir.friendica.com/lsearch?p=1&n=500&search=".urlencode($search)); + $j = json_decode($x); + + if(count($j->results)) + foreach($j->results as $jj) { + // Check if the contact already exists + $exists = q("SELECT `id`, `last_contact`, `last_failure` FROM `gcontact` WHERE `nurl` = '%s'", normalise_link($jj->url)); + if ($exists) { + logger("Profile ".$jj->url." already exists (".$search.")", LOGGER_DEBUG); + + if ($exists[0]["last_contact"] < $exists[0]["last_failure"]) + continue; + + $last_updated = poco_last_updated($jj->url); + $last_contact = datetime_convert(); + + if ($last_updated) { + logger("Mark profile ".$jj->url." as accessible (".$search.")", LOGGER_DEBUG); + q("UPDATE `gcontact` SET `updated` = '%s', `last_contact` = '%s' WHERE `nurl` = '%s'", + dbesc($last_updated), dbesc($last_contact), dbesc(normalise_link($jj->url))); + } else { + logger("Mark profile ".$jj->url." as unaccessible (".$search.")", LOGGER_DEBUG); + q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc($last_contact), dbesc(normalise_link($jj->url))); + } + continue; + } + + logger("Check if profile ".$jj->url." is reachable (".$search.")", LOGGER_DEBUG); + $data = probe_url($jj->url); + if ($data["network"] == NETWORK_DFRN) { + logger("Add profile ".$jj->url." to local directory (".$search.")", LOGGER_DEBUG); + poco_check($data["url"], $data["name"], $data["network"], $data["photo"], "", "", "", $jj->tags, $data["addr"], "", 0); + } + } + Cache::set("dirsearch:".$search, time()); +} + if (array_search(__file__,get_included_files())===0){ discover_poco_run($_SERVER["argv"],$_SERVER["argc"]); killme(); diff --git a/mod/admin.php b/mod/admin.php index a079bdabf..ea0196db8 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -360,6 +360,7 @@ function admin_page_site_post(&$a){ $maxloadavg_frontend = ((x($_POST,'maxloadavg_frontend')) ? intval(trim($_POST['maxloadavg_frontend'])) : 50); $poco_completion = ((x($_POST,'poco_completion')) ? intval(trim($_POST['poco_completion'])) : false); $poco_discovery = ((x($_POST,'poco_discovery')) ? intval(trim($_POST['poco_discovery'])) : 0); + $poco_local_search = ((x($_POST,'poco_local_search')) ? intval(trim($_POST['poco_local_search'])) : false); $dfrn_only = ((x($_POST,'dfrn_only')) ? True : False); $ostatus_disabled = !((x($_POST,'ostatus_disabled')) ? True : False); $ostatus_poll_interval = ((x($_POST,'ostatus_poll_interval')) ? intval(trim($_POST['ostatus_poll_interval'])) : 0); @@ -431,6 +432,7 @@ function admin_page_site_post(&$a){ set_config('system','maxloadavg_frontend',$maxloadavg_frontend); set_config('system','poco_completion',$poco_completion); set_config('system','poco_discovery',$poco_discovery); + set_config('system','poco_local_search',$poco_local_search); set_config('config','sitename',$sitename); set_config('config','hostname',$hostname); set_config('config','sender_email', $sender_email); @@ -701,6 +703,7 @@ function admin_page_site(&$a) { '$poco_completion' => array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful when communicating with Redmatrix and friendica servers before 3.3)")), '$poco_discovery' => array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available."), $poco_discovery_choices), + '$poco_local_search' => array('poco_local_search', t("Search the local directory"), get_config('system','poco_local_search'), t("Search the local directory instead of the global directory. When searching locally, every search will be executed on the global directory in the background. This improves the search results when the search is repeated.")), '$use_fulltext_engine' => array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")), '$suppress_language' => array('suppress_language', t("Suppress Language"), get_config('system','suppress_language'), t("Suppress language information in meta information about a posting.")), diff --git a/mod/dirfind.php b/mod/dirfind.php index c49d37e48..996d2a464 100644 --- a/mod/dirfind.php +++ b/mod/dirfind.php @@ -16,7 +16,7 @@ function dirfind_init(&$a) { function dirfind_content(&$a) { - $local = true; + $local = get_config('system','poco_local_search'); $search = notags(trim($_REQUEST['search'])); @@ -43,14 +43,18 @@ function dirfind_content(&$a) { dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search))); - $results = q("SELECT `url`, `name`, `photo`, `keywords` FROM `gcontact`WHERE `network` IN ('%s', '%s', '%s') AND - (`url` REGEXP '%s' OR `name` REGEXP '%s' OR `location` REGEXP '%s' OR `about` REGEXP '%s' OR - `keywords` REGEXP '%s') ORDER BY `name` ASC LIMIT %d, %d", - dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), + $results = q("SELECT `contact`.`id` AS `cid`, `gcontact`.`url`, `gcontact`.`name`, `gcontact`.`photo`, `gcontact`.`keywords` + FROM `gcontact` + LEFT JOIN `contact` ON `contact`.`nurl` = `gcontact`.`nurl` AND `contact`.`uid` = %d + WHERE `gcontact`.`network` IN ('%s', '%s', '%s') AND `gcontact`.`last_contact` >= `gcontact`.`last_failure` AND + (`gcontact`.`url` REGEXP '%s' OR `gcontact`.`name` REGEXP '%s' OR `gcontact`.`location` REGEXP '%s' OR + `gcontact`.`about` REGEXP '%s' OR `gcontact`.`keywords` REGEXP '%s') + GROUP BY `gcontact`.`nurl` + ORDER BY `gcontact`.`updated` DESC LIMIT %d, %d", + intval(local_user()), dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)), intval($startrec), intval($perpage)); - $j = new stdClass(); $j->total = $count[0]["total"]; $j->items_page = $perpage; @@ -62,6 +66,7 @@ function dirfind_content(&$a) { } $objresult = new stdClass(); + $objresult->cid = $result["cid"]; $objresult->name = $result["name"]; $objresult->url = $result["url"]; $objresult->photo = $result["photo"]; @@ -69,6 +74,9 @@ function dirfind_content(&$a) { $j->results[] = $objresult; } + + // Add found profiles from the global directory to the local directory + proc_run('php','include/discover_poco.php', "dirsearch", urlencode($search)); } else { $p = (($a->pager['page'] != 1) ? '&p=' . $a->pager['page'] : ''); @@ -89,11 +97,22 @@ function dirfind_content(&$a) { $tpl = get_markup_template('match.tpl'); foreach($j->results as $jj) { + // If We already know this contact then don't show the "connect" button + if ($jj->cid > 0) { + $connlnk = ""; + $conntxt = ""; + } else { + $connlnk = $a->get_baseurl().'/follow/?url='.(($jj->connect) ? $jj->connect : $jj->url); + $conntxt = t('Connect'); + } + $o .= replace_macros($tpl,array( '$url' => zrl($jj->url), '$name' => $jj->name, '$photo' => proxy_url($jj->photo), - '$tags' => $jj->tags + '$tags' => $jj->tags, + '$conntxt' => $conntxt, + '$connlnk' => $connlnk, )); } } diff --git a/view/templates/admin_site.tpl b/view/templates/admin_site.tpl index f0461e1b8..2f448d1b9 100644 --- a/view/templates/admin_site.tpl +++ b/view/templates/admin_site.tpl @@ -118,6 +118,7 @@

{{$portable_contacts}}

{{include file="field_checkbox.tpl" field=$poco_completion}} {{include file="field_select.tpl" field=$poco_discovery}} + {{include file="field_checkbox.tpl" field=$poco_local_search}}

{{$performance}}

From 3622f1a335998c8052be78d83b1e18708a509bbe Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 19 Jul 2015 21:29:24 +0200 Subject: [PATCH 13/47] Do some caching --- include/discover_poco.php | 25 +++++++------ include/onepoll.php | 11 ------ include/socgraph.php | 79 +++++++++++++++++++++++++-------------- 3 files changed, 64 insertions(+), 51 deletions(-) diff --git a/include/discover_poco.php b/include/discover_poco.php index 504d97287..74ed83b47 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -100,21 +100,22 @@ function discover_directory($search) { if ($exists[0]["last_contact"] < $exists[0]["last_failure"]) continue; - $last_updated = poco_last_updated($jj->url); - $last_contact = datetime_convert(); - - if ($last_updated) { - logger("Mark profile ".$jj->url." as accessible (".$search.")", LOGGER_DEBUG); - q("UPDATE `gcontact` SET `updated` = '%s', `last_contact` = '%s' WHERE `nurl` = '%s'", - dbesc($last_updated), dbesc($last_contact), dbesc(normalise_link($jj->url))); - } else { - logger("Mark profile ".$jj->url." as unaccessible (".$search.")", LOGGER_DEBUG); - q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", - dbesc($last_contact), dbesc(normalise_link($jj->url))); - } + // Update the contact + poco_last_updated($jj->url); continue; } + // Harcoded paths aren't so good. But in this case it is okay. + // First: We only will get Friendica contacts (which always are using this url schema) + // Second: There will be no further problems if we are doing a mistake + $server_url = preg_replace("=(https?://)(.*)/profile/(.*)=ism", "$1$2", $jj->url); + if ($server_url != $jj->url) + if (!poco_check_server($server_url)) { + logger("Friendica server ".$server_url." doesn't answer.", LOGGER_DEBUG); + continue; + } + logger("Friendica server ".$server_url." seems to be okay.", LOGGER_DEBUG); + logger("Check if profile ".$jj->url." is reachable (".$search.")", LOGGER_DEBUG); $data = probe_url($jj->url); if ($data["network"] == NETWORK_DFRN) { diff --git a/include/onepoll.php b/include/onepoll.php index 73668126a..c0c2f1a34 100644 --- a/include/onepoll.php +++ b/include/onepoll.php @@ -120,23 +120,12 @@ function onepoll_run(&$argv, &$argc){ dbesc($updated), intval($contact['id']) ); - - q("UPDATE `gcontact` SET `updated` = '%s', `last_contact` = '%s' WHERE `nurl` = '%s'", - dbesc($last_updated), - dbesc($updated), - dbesc($contact['nurl']) - ); } else { q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d", dbesc($updated), dbesc($updated), intval($contact['id']) ); - - q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", - dbesc($updated), - dbesc($contact['nurl']) - ); } } return; diff --git a/include/socgraph.php b/include/socgraph.php index c97a0f15c..9ccae5040 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -238,6 +238,8 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca logger("profile-check generation: ".$generation." Network: ".$network." URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG); + poco_check_server($server_url, $network); + // Only fetch last update manually if it wasn't provided and enabled in the system if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) { $last_updated = poco_last_updated($profile_url); @@ -245,25 +247,10 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca $updated = $last_updated; $last_contact = datetime_convert(); logger("Last updated for profile ".$profile_url.": ".$updated, LOGGER_DEBUG); - - if (count($x)) - q("UPDATE `gcontact` SET `last_contact` = '%s' WHERE `nurl` = '%s'", dbesc($last_contact), dbesc(normalise_link($profile_url))); - } else { + } else $last_failure = datetime_convert(); - - if (count($x)) - q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", dbesc($last_failure), dbesc(normalise_link($profile_url))); - } } - poco_check_server($server_url, $network); - - // Test - remove before flight - //if ($last_contact > $last_failure) - // q("UPDATE `gserver` SET `last_contact` = '%s' WHERE `nurl` = '%s'", dbesc($last_contact), dbesc(normalise_link($server_url))); - //else - // q("UPDATE `gserver` SET `last_failure` = '%s' WHERE `nurl` = '%s'", dbesc($last_failure), dbesc(normalise_link($server_url))); - if(count($x)) { $gcid = $x[0]['id']; @@ -366,17 +353,53 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca } function poco_last_updated($profile) { + + $gcontacts = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s'", + dbesc(normalise_link($profile))); + + if ($gcontacts[0]["server_url"] != "") { + $servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s' AND `last_contact` < `last_failure`", dbesc(normalise_link($gcontacts[0]["server_url"]))); + + if ($servers) + return false; + } + $data = probe_url($profile); - if (($data["poll"] == "") OR ($data["network"] == NETWORK_FEED)) + if (($data["poll"] == "") OR ($data["network"] == NETWORK_FEED)) { + q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($profile))); return false; + } + + if (($data["name"] != "") AND ($data["name"] != $gcontacts[0]["name"])) + q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'", + dbesc($data["name"]), dbesc(normalise_link($profile))); + + if (($data["addr"] != "") AND ($data["addr"] != $gcontacts[0]["connect"])) + q("UPDATE `gcontact` SET `connect` = '%s' WHERE `nurl` = '%s'", + dbesc($data["addr"]), dbesc(normalise_link($profile))); + + if (($data["photo"] != "") AND ($data["photo"] != $gcontacts[0]["photo"])) + q("UPDATE `gcontact` SET `photo` = '%s' WHERE `nurl` = '%s'", + dbesc($data["photo"]), dbesc(normalise_link($profile))); + + if (($data["baseurl"] != "") AND ($data["baseurl"] != $gcontacts[0]["server_url"])) + q("UPDATE `gcontact` SET `server_url` = '%s' WHERE `nurl` = '%s'", + dbesc($data["baseurl"]), dbesc(normalise_link($profile))); + + if ($data["baseurl"] != "") + poco_check_server($data["baseurl"], $data["network"]); // To-Do: Use noscrape $feedret = z_fetch_url($data["poll"]); - if (!$feedret["success"]) + if (!$feedret["success"]) { + q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($profile))); return false; + } $doc = new DOMDocument(); @$doc->loadXML($feedret["body"]); @@ -404,6 +427,9 @@ function poco_last_updated($profile) { if ($xpath->query('/atom:feed')->length > 0) $last_updated = "0000-00-00 00:00:00"; + q("UPDATE `gcontact` SET `updated` = '%s', `last_contact` = '%s' WHERE `nurl` = '%s'", + dbesc($last_updated), dbesc(datetime_convert()), dbesc(normalise_link($profile))); + return($last_updated); } @@ -430,11 +456,11 @@ function poco_do_update($updated, $last_contact, $last_failure) { return false; // If the last contact time was more than a week ago, then only try once a week - if (($now - $contact_time) > (60 * 60 * 24 * 7) AND ($now - $failure_time) < (60 * 60 * 24 * 7)) + if ((($now - $contact_time) > (60 * 60 * 24 * 7)) AND (($now - $failure_time) < (60 * 60 * 24 * 7))) return false; - // If the last contact time was more than a month ago, then only try once a month - if (($now - $contact_time) > (60 * 60 * 24 * 30) AND ($now - $failure_time) < (60 * 60 * 24 * 30)) + // If the last contact time was more than a month ago, then only try once a month - but only if there ever was a contact time + if (($contact_time > 0) AND (($now - $contact_time) > (60 * 60 * 24 * 30)) AND (($now - $failure_time) < (60 * 60 * 24 * 30))) return false; return true; @@ -452,7 +478,7 @@ function poco_to_boolean($val) { function poco_check_server($server_url, $network = "") { if ($server_url == "") - return; + return false; $servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url))); if ($servers) { @@ -470,12 +496,8 @@ function poco_check_server($server_url, $network = "") { $info = $servers[0]["info"]; $register_policy = $servers[0]["register_policy"]; - // Only check the server once a week - if (strtotime(datetime_convert()) < (strtotime($last_contact) + (60 * 60 * 24 * 7))) - return; - - if (strtotime(datetime_convert()) < (strtotime($last_failure) + (60 * 60 * 24 * 7))) - return; + if (!poco_do_update("", $last_contact, $last_failure)) + return ($last_contact >= $last_failure); } else { $poco = ""; $noscrape = ""; @@ -668,6 +690,7 @@ function poco_check_server($server_url, $network = "") { dbesc($platform), dbesc(datetime_convert()) ); + return $failure; } function poco_contact_from_body($body, $created, $cid, $uid) { From a4aec06e5b7064433bff87e115ad7b63107c6d0a Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 20 Jul 2015 00:14:14 +0200 Subject: [PATCH 14/47] Support for detecting "Hubzilla" (and some bugfixing in the server detection) --- include/socgraph.php | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/include/socgraph.php b/include/socgraph.php index 9ccae5040..064fc182b 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -430,6 +430,10 @@ function poco_last_updated($profile) { q("UPDATE `gcontact` SET `updated` = '%s', `last_contact` = '%s' WHERE `nurl` = '%s'", dbesc($last_updated), dbesc(datetime_convert()), dbesc(normalise_link($profile))); + if (($gcontacts[0]["generation"] == 0)) + q("UPDATE `gcontact` SET `generation` = 9 WHERE `nurl` = '%s'", + dbesc(normalise_link($profile))); + return($last_updated); } @@ -559,6 +563,11 @@ function poco_check_server($server_url, $network = "") { if (isset($data->site->server)) { $last_contact = datetime_convert(); + if (isset($data->site->hubzilla)) { + $platform = $data->site->hubzilla->PLATFORM_NAME; + $version = $data->site->hubzilla->RED_VERSION; + $network = NETWORK_DFRN; + } if (isset($data->site->redmatrix)) { if (isset($data->site->redmatrix->PLATFORM_NAME)) $platform = $data->site->redmatrix->PLATFORM_NAME; @@ -676,8 +685,8 @@ function poco_check_server($server_url, $network = "") { dbesc(normalise_link($server_url)) ); else - q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `last_contact`) - VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s')", + q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `last_contact`, `last_failure`) + VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s')", dbesc($server_url), dbesc(normalise_link($server_url)), dbesc($version), @@ -688,9 +697,11 @@ function poco_check_server($server_url, $network = "") { dbesc($noscrape), dbesc($network), dbesc($platform), + dbesc($last_contact), + dbesc($last_failure), dbesc(datetime_convert()) ); - return $failure; + return !$failure; } function poco_contact_from_body($body, $created, $cid, $uid) { From 9717fa82fe02413bf007cff4698538121d71b611 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Tue, 21 Jul 2015 00:05:44 +0200 Subject: [PATCH 15/47] New field "created" in gcontacts. New option for timeframe when polling global contacts --- include/dbstructure.php | 3 + include/onepoll.php | 2 +- include/socgraph.php | 131 ++++++++++++++++++++++++++++------ mod/admin.php | 10 +++ view/templates/admin_site.tpl | 1 + 5 files changed, 126 insertions(+), 21 deletions(-) diff --git a/include/dbstructure.php b/include/dbstructure.php index 9d0856c25..c079efeb5 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -626,10 +626,12 @@ function db_definition() { "fields" => array( "id" => array("type" => "int(10) unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"), "name" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "nick" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "nurl" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "photo" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "connect" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "updated" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), "last_contact" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), "last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), @@ -699,6 +701,7 @@ function db_definition() { "noscrape" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "network" => array("type" => "varchar(32)", "not null" => "1", "default" => ""), "platform" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "last_poco_query" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), "last_contact" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), "last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), diff --git a/include/onepoll.php b/include/onepoll.php index c0c2f1a34..464a50fb9 100644 --- a/include/onepoll.php +++ b/include/onepoll.php @@ -110,7 +110,7 @@ function onepoll_run(&$argv, &$argc){ // - Check why we don't poll the Diaspora feed at the moment (some guid problem in the items?) // - Check whether this is possible with Redmatrix if ($contact["network"] == NETWORK_DIASPORA) { - if (poco_do_update($contact["last-item"], $contact["success_update"], $contact["failure_update"])) { + if (poco_do_update($contact["created"], $contact["last-item"], $contact["failure_update"], $contact["success_update"])) { $last_updated = poco_last_updated($contact["url"]); $updated = datetime_convert(); if ($last_updated) { diff --git a/include/socgraph.php b/include/socgraph.php index 064fc182b..45cf3a650 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -205,19 +205,26 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca if ($updated == "0000-00-00 00:00:00") $updated = $x[0]["updated"]; + $created = $x[0]["created"]; $last_contact = $x[0]["last_contact"]; $last_failure = $x[0]["last_failure"]; $server_url = $x[0]["server_url"]; + $nick = $x[0]["nick"]; } else { + $created = "0000-00-00 00:00:00"; $last_contact = "0000-00-00 00:00:00"; $last_failure = "0000-00-00 00:00:00"; $server_url = ""; + + $urlparts = parse_url($profile_url); + $nick = end(explode("/", $urlparts["path"])); } if (($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == "")) { $data = probe_url($profile_url); $network = $data["network"]; $name = $data["name"]; + $nick = $data["nick"]; $profile_url = $data["url"]; $profile_photo = $data["photo"]; $server_url = $data["baseurl"]; @@ -241,7 +248,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca poco_check_server($server_url, $network); // Only fetch last update manually if it wasn't provided and enabled in the system - if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) { + if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($created, $updated, $last_failure, $last_contact)) { $last_updated = poco_last_updated($profile_url); if ($last_updated) { $updated = $last_updated; @@ -290,15 +297,17 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca ); } } else { - q("INSERT INTO `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `server_url`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`) - VALUES ('%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)", + q("INSERT INTO `gcontact` (`name`, `nick`, `network`, `url`, `nurl`, `photo`, `connect`, `server_url`, `created`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`) + VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)", dbesc($name), + dbesc($nick), dbesc($network), dbesc($profile_url), dbesc(normalise_link($profile_url)), dbesc($profile_photo), dbesc($connect_url), dbesc($server_url), + dbesc(datetime_convert()), dbesc($updated), dbesc($last_contact), dbesc($last_failure), @@ -357,13 +366,79 @@ function poco_last_updated($profile) { $gcontacts = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s'", dbesc(normalise_link($profile))); - if ($gcontacts[0]["server_url"] != "") { - $servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s' AND `last_contact` < `last_failure`", dbesc(normalise_link($gcontacts[0]["server_url"]))); + if ($gcontacts[0]["created"] == "0000-00-00 00:00:00") + q("UPDATE `gcontact` SET `created` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($profile))); - if ($servers) + if (($gcontacts[0]["server_url"] != "") AND ($gcontacts[0]["network"] != "")) + if (!poco_check_server($gcontacts[0]["server_url"], $gcontacts[0]["network"])) return false; + + // noscrape is really fast so we don't cache the call. + if (($gcontacts[0]["server_url"] != "") AND ($gcontacts[0]["nick"] != "")) { + + // Use noscrape if possible + $server = q("SELECT `noscrape` FROM `gserver` WHERE `nurl` = '%s' AND `noscrape` != ''", dbesc(normalise_link($gcontacts[0]["server_url"]))); + + if ($server) { + $noscraperet = z_fetch_url($server[0]["noscrape"]."/".$gcontacts[0]["nick"]); + if ($noscraperet["success"]) { + $noscrape = json_decode($noscraperet["body"], true); + + if (($noscrape["name"] != "") AND ($noscrape["name"] != $gcontacts[0]["name"])) + q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'", + dbesc($noscrape["name"]), dbesc(normalise_link($profile))); + + if (($noscrape["photo"] != "") AND ($noscrape["photo"] != $gcontacts[0]["photo"])) + q("UPDATE `gcontact` SET `photo` = '%s' WHERE `nurl` = '%s'", + dbesc($noscrape["photo"]), dbesc(normalise_link($profile))); + + if (($noscrape["updated"] != "") AND ($noscrape["updated"] != $gcontacts[0]["updated"])) + q("UPDATE `gcontact` SET `updated` = '%s' WHERE `nurl` = '%s'", + dbesc($noscrape["updated"]), dbesc(normalise_link($profile))); + + if (($noscrape["gender"] != "") AND ($noscrape["gender"] != $gcontacts[0]["gender"])) + q("UPDATE `gcontact` SET `gender` = '%s' WHERE `nurl` = '%s'", + dbesc($noscrape["gender"]), dbesc(normalise_link($profile))); + + if (($noscrape["about"] != "") AND ($noscrape["about"] != $gcontacts[0]["name"])) + q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'", + dbesc($noscrape["about"]), dbesc(normalise_link($profile))); + + $keywords = implode(" ", $noscrape["tags"]); + if (($keywords != "") AND ($keywords != $gcontacts[0]["keywords"])) + q("UPDATE `gcontact` SET `keywords` = '%s' WHERE `nurl` = '%s'", + dbesc($keywords), dbesc(normalise_link($profile))); + + $location = $noscrape["locality"]; + + if ($noscrape["region"] != "") { + if ($location != "") + $location .= ", "; + + $location .= $noscrape["region"]; + } + + if ($noscrape["country-name"] != "") { + if ($location != "") + $location .= ", "; + + $location .= $noscrape["country-name"]; + } + + if (($location != "") AND ($location != $gcontacts[0]["location"])) + q("UPDATE `gcontact` SET `location` = '%s' WHERE `nurl` = '%s'", + dbesc($location), dbesc(normalise_link($profile))); + + return $noscrape["updated"]; + } + } } + // If we only can poll the feed, then we only do this once a while + if (!poco_do_update($gcontacts[0]["created"], $gcontacts[0]["updated"], $gcontacts[0]["last_failure"], $gcontacts[0]["last_contact"])) +/ return $gcontacts[0]["updated"]; + $data = probe_url($profile); if (($data["poll"] == "") OR ($data["network"] == NETWORK_FEED)) { @@ -376,6 +451,10 @@ function poco_last_updated($profile) { q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'", dbesc($data["name"]), dbesc(normalise_link($profile))); + if (($data["nick"] != "") AND ($data["nick"] != $gcontacts[0]["nick"])) + q("UPDATE `gcontact` SET `nick` = '%s' WHERE `nurl` = '%s'", + dbesc($data["nick"]), dbesc(normalise_link($profile))); + if (($data["addr"] != "") AND ($data["addr"] != $gcontacts[0]["connect"])) q("UPDATE `gcontact` SET `connect` = '%s' WHERE `nurl` = '%s'", dbesc($data["addr"]), dbesc(normalise_link($profile))); @@ -388,11 +467,6 @@ function poco_last_updated($profile) { q("UPDATE `gcontact` SET `server_url` = '%s' WHERE `nurl` = '%s'", dbesc($data["baseurl"]), dbesc(normalise_link($profile))); - if ($data["baseurl"] != "") - poco_check_server($data["baseurl"], $data["network"]); - - // To-Do: Use noscrape - $feedret = z_fetch_url($data["poll"]); if (!$feedret["success"]) { @@ -437,7 +511,7 @@ function poco_last_updated($profile) { return($last_updated); } -function poco_do_update($updated, $last_contact, $last_failure) { +function poco_do_update($created, $updated, $last_failure, $last_contact) { $now = strtotime(datetime_convert()); if ($updated > $last_contact) @@ -446,6 +520,11 @@ function poco_do_update($updated, $last_contact, $last_failure) { $contact_time = strtotime($last_contact); $failure_time = strtotime($last_failure); + $created_time = strtotime($created); + + // If there is no "created" time then use the current time + if ($created_time <= 0) + $created_time = $now; // If the last contact was less than 24 hours then don't update if (($now - $contact_time) < (60 * 60 * 24)) @@ -459,12 +538,12 @@ function poco_do_update($updated, $last_contact, $last_failure) { if ((($now - $contact_time) < (60 * 60 * 24 * 7)) AND ($contact_time > $failure_time)) return false; - // If the last contact time was more than a week ago, then only try once a week - if ((($now - $contact_time) > (60 * 60 * 24 * 7)) AND (($now - $failure_time) < (60 * 60 * 24 * 7))) + // If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week + if ((($now - $contact_time) > (60 * 60 * 24 * 7)) AND (($now - $created_time) > (60 * 60 * 24 * 7)) AND (($now - $failure_time) < (60 * 60 * 24 * 7))) return false; - // If the last contact time was more than a month ago, then only try once a month - but only if there ever was a contact time - if (($contact_time > 0) AND (($now - $contact_time) > (60 * 60 * 24 * 30)) AND (($now - $failure_time) < (60 * 60 * 24 * 30))) + // If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month + if ((($now - $contact_time) > (60 * 60 * 24 * 30)) AND (($now - $created_time) > (60 * 60 * 24 * 30)) AND (($now - $failure_time) < (60 * 60 * 24 * 30))) return false; return true; @@ -486,6 +565,11 @@ function poco_check_server($server_url, $network = "") { $servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url))); if ($servers) { + + if ($servers[0]["created"] == "0000-00-00 00:00:00") + q("UPDATE `gserver` SET `created` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($server_url))); + $poco = $servers[0]["poco"]; $noscrape = $servers[0]["noscrape"]; @@ -500,7 +584,7 @@ function poco_check_server($server_url, $network = "") { $info = $servers[0]["info"]; $register_policy = $servers[0]["register_policy"]; - if (!poco_do_update("", $last_contact, $last_failure)) + if (!poco_do_update($servers[0]["created"], "", $last_failure, $last_contact)) return ($last_contact >= $last_failure); } else { $poco = ""; @@ -566,7 +650,7 @@ function poco_check_server($server_url, $network = "") { if (isset($data->site->hubzilla)) { $platform = $data->site->hubzilla->PLATFORM_NAME; $version = $data->site->hubzilla->RED_VERSION; - $network = NETWORK_DFRN; + $network = NETWORK_DIASPORA; } if (isset($data->site->redmatrix)) { if (isset($data->site->redmatrix->PLATFORM_NAME)) @@ -685,7 +769,7 @@ function poco_check_server($server_url, $network = "") { dbesc(normalise_link($server_url)) ); else - q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `last_contact`, `last_failure`) + q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `created`, `last_contact`, `last_failure`) VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s')", dbesc($server_url), dbesc(normalise_link($server_url)), @@ -697,6 +781,7 @@ function poco_check_server($server_url, $network = "") { dbesc($noscrape), dbesc($network), dbesc($platform), + dbesc(datetime_convert()), dbesc($last_contact), dbesc($last_failure), dbesc(datetime_convert()) @@ -1028,8 +1113,14 @@ function poco_discover($complete = false) { if (get_config('system','poco_discovery') > 1) { + $timeframe = get_config('system','poco_discovery_since'); + if ($timeframe == 0) + $timeframe = 30; + + $updatedSince = date("Y-m-d H:i:s", time() - $timeframe * 86400); + // Fetch all global contacts from the other server (Not working with Redmatrix and Friendica versions before 3.3) - $url = $server["poco"]."/@global?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; + $url = $server["poco"]."/@global?updatedSince=".$updatedSince."&fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; $retdata = z_fetch_url($url); if ($retdata["success"]) { diff --git a/mod/admin.php b/mod/admin.php index ea0196db8..cdfd3b5a6 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -360,6 +360,7 @@ function admin_page_site_post(&$a){ $maxloadavg_frontend = ((x($_POST,'maxloadavg_frontend')) ? intval(trim($_POST['maxloadavg_frontend'])) : 50); $poco_completion = ((x($_POST,'poco_completion')) ? intval(trim($_POST['poco_completion'])) : false); $poco_discovery = ((x($_POST,'poco_discovery')) ? intval(trim($_POST['poco_discovery'])) : 0); + $poco_discovery_since = ((x($_POST,'poco_discovery_since')) ? intval(trim($_POST['poco_discovery_since'])) : 30); $poco_local_search = ((x($_POST,'poco_local_search')) ? intval(trim($_POST['poco_local_search'])) : false); $dfrn_only = ((x($_POST,'dfrn_only')) ? True : False); $ostatus_disabled = !((x($_POST,'ostatus_disabled')) ? True : False); @@ -432,6 +433,7 @@ function admin_page_site_post(&$a){ set_config('system','maxloadavg_frontend',$maxloadavg_frontend); set_config('system','poco_completion',$poco_completion); set_config('system','poco_discovery',$poco_discovery); + set_config('system','poco_discovery_since',$poco_discovery_since); set_config('system','poco_local_search',$poco_local_search); set_config('config','sitename',$sitename); set_config('config','hostname',$hostname); @@ -595,6 +597,13 @@ function admin_page_site(&$a) { "3" => t("Users, Global Contacts/fallback"), ); + $poco_discovery_since_choices = array( + "30" => t("One month"), + "91" => t("Three months"), + "182" => t("Half a year"), + "365" => t("One year"), + ); + /* get user names to make the install a personal install of X */ $user_names = array(); $user_names['---'] = t('Multi user instance'); @@ -703,6 +712,7 @@ function admin_page_site(&$a) { '$poco_completion' => array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful when communicating with Redmatrix and friendica servers before 3.3)")), '$poco_discovery' => array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available."), $poco_discovery_choices), + '$poco_discovery_since' => array('poco_discovery_since', t("Timeframe for fetching global contacts"), (string) intval(get_config('system','poco_discovery_since')), t("When the discovery is activated, this value defines the timeframe for the global contacts that are fetched from other servers."), $poco_discovery_since_choices), '$poco_local_search' => array('poco_local_search', t("Search the local directory"), get_config('system','poco_local_search'), t("Search the local directory instead of the global directory. When searching locally, every search will be executed on the global directory in the background. This improves the search results when the search is repeated.")), '$use_fulltext_engine' => array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")), diff --git a/view/templates/admin_site.tpl b/view/templates/admin_site.tpl index 2f448d1b9..9b3c75a0e 100644 --- a/view/templates/admin_site.tpl +++ b/view/templates/admin_site.tpl @@ -118,6 +118,7 @@

{{$portable_contacts}}

{{include file="field_checkbox.tpl" field=$poco_completion}} {{include file="field_select.tpl" field=$poco_discovery}} + {{include file="field_select.tpl" field=$poco_discovery_since}} {{include file="field_checkbox.tpl" field=$poco_local_search}}
From 749a646480f6352af3b9a912811fdc182c0b7c4e Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Wed, 22 Jul 2015 11:51:37 +0200 Subject: [PATCH 16/47] Improved scraping and poco discovery. --- include/Scrape.php | 74 +++++++++++++-------------- include/socgraph.php | 118 +++++++++++++++++++++++++++++++++++++------ 2 files changed, 137 insertions(+), 55 deletions(-) diff --git a/include/Scrape.php b/include/Scrape.php index 90fb1a9e3..83c099769 100644 --- a/include/Scrape.php +++ b/include/Scrape.php @@ -249,7 +249,7 @@ function scrape_feed($url) { $ret['feed_atom'] = $url; return $ret; } - if(stristr($line,'application/rss+xml') || stristr($s,'loadHTML($s); + $xpath = new DomXPath($doc); + + $base = $xpath->query("//base"); + foreach ($base as $node) { + $attr = array(); + + if ($node->attributes->length) + foreach ($node->attributes as $attribute) + $attr[$attribute->name] = $attribute->value; + + if ($attr["href"] != "") + $basename = $attr["href"] ; } - if(! $dom) { - logger('scrape_feed: failed to parse.'); - return $ret; + $list = $xpath->query("//link"); + foreach ($list as $node) { + $attr = array(); + + if ($node->attributes->length) + foreach ($node->attributes as $attribute) + $attr[$attribute->name] = $attribute->value; + + if (($attr["rel"] == "alternate") AND ($attr["type"] == "application/atom+xml")) + $ret["feed_atom"] = $attr["href"]; + + if (($attr["rel"] == "alternate") AND ($attr["type"] == "application/rss+xml")) + $ret["feed_rss"] = $attr["href"]; } - - $head = $dom->getElementsByTagName('base'); - if($head) { - foreach($head as $head0) { - $basename = $head0->getAttribute('href'); - break; - } - } - if(! $basename) - $basename = implode('/', array_slice(explode('/',$url),0,3)) . '/'; - - $items = $dom->getElementsByTagName('link'); - - // get Atom/RSS link elements, take the first one of either. - - if($items) { - foreach($items as $item) { - $x = $item->getAttribute('rel'); - if(($x === 'alternate') && ($item->getAttribute('type') === 'application/atom+xml')) { - if(! x($ret,'feed_atom')) - $ret['feed_atom'] = $item->getAttribute('href'); - } - if(($x === 'alternate') && ($item->getAttribute('type') === 'application/rss+xml')) { - if(! x($ret,'feed_rss')) - $ret['feed_rss'] = $item->getAttribute('href'); - } - } - } - - // Drupal and perhaps others only provide relative URL's. Turn them into absolute. + // Drupal and perhaps others only provide relative URLs. Turn them into absolute. if(x($ret,'feed_atom') && (! strstr($ret['feed_atom'],'://'))) $ret['feed_atom'] = $basename . $ret['feed_atom']; @@ -509,6 +502,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) { } if($mode == PROBE_NORMAL) { + if(strlen($zot)) { $s = fetch_url($zot); if($s) { @@ -528,6 +522,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) { } } + if(strlen($dfrn)) { $ret = scrape_dfrn(($hcard) ? $hcard : $dfrn); if(is_array($ret) && x($ret,'dfrn-request')) { @@ -634,6 +629,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) { if($check_feed) { $feedret = scrape_feed(($poll) ? $poll : $url); + logger('probe_url: scrape_feed ' . (($poll)? $poll : $url) . ' returns: ' . print_r($feedret,true), LOGGER_DATA); if(count($feedret) && ($feedret['feed_atom'] || $feedret['feed_rss'])) { $poll = ((x($feedret,'feed_atom')) ? unamp($feedret['feed_atom']) : unamp($feedret['feed_rss'])); @@ -653,7 +649,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) { logger('probe_url: scrape_feed: headers: ' . $a->get_curl_headers(), LOGGER_DATA); // Don't try and parse an empty string - $feed->set_raw_data(($xml) ? $xml : ''); + $feed->set_raw_data(($xml) ? $xml : ''); $feed->init(); if($feed->error()) diff --git a/include/socgraph.php b/include/socgraph.php index 45cf3a650..9348cb67b 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -4,6 +4,11 @@ require_once('include/datetime.php'); require_once("include/Scrape.php"); require_once("include/html2bbcode.php"); +/* + To-Do: + - Move GNU Social URL schemata (http://server.tld/user/number) to http://server.tld/username +*/ + /* * poco_load * @@ -167,6 +172,14 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca if ($profile_url == "") return $gcid; + $urlparts = parse_url($profile_url); + if (!isset($urlparts["scheme"])) + return $gcid; + + if (in_array($urlparts["host"], array("facebook.com", "twitter.com", "www.twitter-rss.com", + "identi.ca", "alpha.app.net"))) + return $gcid; + $orig_updated = $updated; // Don't store the statusnet connector as network @@ -220,7 +233,8 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca $nick = end(explode("/", $urlparts["path"])); } - if (($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == "")) { + if ((($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == "")) + AND poco_reachable($profile_url, $server_url, $network)) { $data = probe_url($profile_url); $network = $data["network"]; $name = $data["name"]; @@ -248,7 +262,9 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca poco_check_server($server_url, $network); // Only fetch last update manually if it wasn't provided and enabled in the system - if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($created, $updated, $last_failure, $last_contact)) { + if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") + AND poco_do_update($created, $updated, $last_failure, $last_contact) + AND poco_reachable($profile_url, $server_url, $network)) { $last_updated = poco_last_updated($profile_url); if ($last_updated) { $updated = $last_updated; @@ -361,6 +377,49 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca return $gcid; } +function poco_reachable($profile, $server = "", $network = "") { + + if ($server == "") + $server = poco_detect_server($profile); + + if ($server == "") + return true; + + return poco_check_server($server, $network); +} + +function poco_detect_server($profile) { + + // Try to detect the server path based upon some known standard paths + $server_url = ""; + + if ($server_url == "") { + $friendica = preg_replace("=(https?://)(.*)/profile/(.*)=ism", "$1$2", $profile); + if ($friendica != $profile) { + $server_url = $friendica; + $network = NETWORK_DFRN; + } + } + + if ($server_url == "") { + $diaspora = preg_replace("=(https?://)(.*)/u/(.*)=ism", "$1$2", $profile); + if ($diaspora != $profile) { + $server_url = $diaspora; + $network = NETWORK_DIASPORA; + } + } + + if ($server_url == "") { + $red = preg_replace("=(https?://)(.*)/channel/(.*)=ism", "$1$2", $profile); + if ($red != $profile) { + $server_url = $red; + $network = NETWORK_DIASPORA; + } + } + + return $server_url; +} + function poco_last_updated($profile) { $gcontacts = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s'", @@ -437,7 +496,7 @@ function poco_last_updated($profile) { // If we only can poll the feed, then we only do this once a while if (!poco_do_update($gcontacts[0]["created"], $gcontacts[0]["updated"], $gcontacts[0]["last_failure"], $gcontacts[0]["last_contact"])) -/ return $gcontacts[0]["updated"]; + return $gcontacts[0]["updated"]; $data = probe_url($profile); @@ -558,7 +617,7 @@ function poco_to_boolean($val) { return ($val); } -function poco_check_server($server_url, $network = "") { +function poco_check_server($server_url, $network = "", $force = false) { if ($server_url == "") return false; @@ -584,8 +643,10 @@ function poco_check_server($server_url, $network = "") { $info = $servers[0]["info"]; $register_policy = $servers[0]["register_policy"]; - if (!poco_do_update($servers[0]["created"], "", $last_failure, $last_contact)) + if (!$force AND !poco_do_update($servers[0]["created"], "", $last_failure, $last_contact)) { + logger("Use cached data for server ".$server_url, LOGGER_DEBUG); return ($last_contact >= $last_failure); + } } else { $poco = ""; $noscrape = ""; @@ -598,6 +659,7 @@ function poco_check_server($server_url, $network = "") { $last_contact = "0000-00-00 00:00:00"; $last_failure = "0000-00-00 00:00:00"; } + logger("Server ".$server_url." is unknown. Start discovery.", LOGGER_DEBUG); $failure = false; $orig_last_failure = $last_failure; @@ -752,6 +814,9 @@ function poco_check_server($server_url, $network = "") { } } + // Check again if the server exists + $servers = q("SELECT `nurl` FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url))); + if ($servers) q("UPDATE `gserver` SET `url` = '%s', `version` = '%s', `site_name` = '%s', `info` = '%s', `register_policy` = %d, `poco` = '%s', `noscrape` = '%s', `network` = '%s', `platform` = '%s', `last_contact` = '%s', `last_failure` = '%s' WHERE `nurl` = '%s'", @@ -770,7 +835,7 @@ function poco_check_server($server_url, $network = "") { ); else q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `created`, `last_contact`, `last_failure`) - VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s')", + VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s')", dbesc($server_url), dbesc(normalise_link($server_url)), dbesc($version), @@ -786,6 +851,9 @@ function poco_check_server($server_url, $network = "") { dbesc($last_failure), dbesc(datetime_convert()) ); + + logger("End discovery for server ".$server_url, LOGGER_DEBUG); + return !$failure; } @@ -1096,11 +1164,17 @@ function update_suggestions() { function poco_discover($complete = false) { - $last_update = date("c", time() - (60 * 60 * 24)); + $no_of_queries = 5; - $r = q("SELECT `poco`, `nurl` FROM `gserver` WHERE `last_contact` > `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update)); + $last_update = date("c", time() - (60 * 60 * 6)); // 24 + + $r = q("SELECT `poco`, `nurl`, `url`, `network` FROM `gserver` WHERE `last_contact` > `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update)); if ($r) foreach ($r AS $server) { + + if (!poco_check_server($server["url"], $server["network"])) + continue; + // Fetch all users from the other server $url = $server["poco"]."/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; @@ -1109,6 +1183,7 @@ function poco_discover($complete = false) { $retdata = z_fetch_url($url); if ($retdata["success"]) { $data = json_decode($retdata["body"]); + poco_discover_server($data, 2); if (get_config('system','poco_discovery') > 1) { @@ -1122,24 +1197,29 @@ function poco_discover($complete = false) { // Fetch all global contacts from the other server (Not working with Redmatrix and Friendica versions before 3.3) $url = $server["poco"]."/@global?updatedSince=".$updatedSince."&fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; + $success = false; + $retdata = z_fetch_url($url); if ($retdata["success"]) { logger("Fetch all global contacts from the server ".$server["nurl"], LOGGER_DEBUG); - poco_discover_server(json_decode($retdata["body"])); - } elseif (get_config('system','poco_discovery') > 2) { + $success = poco_discover_server(json_decode($retdata["body"])); + } + + if (!$success AND (get_config('system','poco_discovery') > 2)) { logger("Fetch contacts from users of the server ".$server["nurl"], LOGGER_DEBUG); - poco_discover_server_users($data); + poco_discover_server_users($data, $server); } } q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"])); - if (!$complete) + if (!$complete AND (--$no_of_queries == 0)) break; - } + } else // If the server hadn't replied correctly, then force a sanity check + poco_check_server($server["url"], $server["network"], true); } } -function poco_discover_server_users($data) { +function poco_discover_server_users($data, $server) { foreach ($data->entry AS $entry) { $username = ""; if (isset($entry->urls)) { @@ -1166,7 +1246,9 @@ function poco_discover_server_users($data) { function poco_discover_server($data, $default_generation = 0) { if (!isset($data->entry) OR !count($data->entry)) - return; + return false; + + $success = false; foreach ($data->entry AS $entry) { $profile_url = ''; @@ -1195,6 +1277,7 @@ function poco_discover_server($data, $default_generation = 0) { } } } + if(isset($entry->photos)) { foreach($entry->photos as $photo) { if($photo->type == 'profile') { @@ -1227,10 +1310,13 @@ function poco_discover_server($data, $default_generation = 0) { $keywords = implode(", ", $tag); if ($generation > 0) { + $success = true; + logger("Store profile ".$profile_url, LOGGER_DEBUG); - poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $generation); + poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $generation, 0, 0, 0); logger("Done for profile ".$profile_url, LOGGER_DEBUG); } } + return $success; } ?> From 56a43dcbdf1cfb6d45acc1eb6f96c6183467acb4 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Thu, 23 Jul 2015 00:32:41 +0200 Subject: [PATCH 17/47] New Diaspora detection --- include/socgraph.php | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/include/socgraph.php b/include/socgraph.php index 9348cb67b..f72fe5f6f 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -464,7 +464,11 @@ function poco_last_updated($profile) { q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'", dbesc($noscrape["about"]), dbesc(normalise_link($profile))); - $keywords = implode(" ", $noscrape["tags"]); + if (isset($noscrape["tags"])) + $keywords = implode(" ", $noscrape["tags"]); + else + $keywords = ""; + if (($keywords != "") AND ($keywords != $gcontacts[0]["keywords"])) q("UPDATE `gcontact` SET `keywords` = '%s' WHERE `nurl` = '%s'", dbesc($keywords), dbesc(normalise_link($profile))); @@ -683,6 +687,23 @@ function poco_check_server($server_url, $network = "", $force = false) { } elseif ($network == NETWORK_DIASPORA) $last_contact = datetime_convert(); + if (!$failure) { + // Test for Diaspora + $serverret = z_fetch_url($server_url); + + $lines = explode("\n",$serverret["header"]); + if(count($lines)) + foreach($lines as $line) { + $line = trim($line); + if(stristr($line,'X-Diaspora-Version:')) { + $platform = "Diaspora"; + $version = trim(str_replace("X-Diaspora-Version:", "", $line)); + $version = trim(str_replace("x-diaspora-version:", "", $version)); + $network = NETWORK_DIASPORA; + } + } + } + if (!$failure) { // Test for Statusnet // Will also return data for Friendica and GNU Social - but it will be overwritten later @@ -758,6 +779,9 @@ function poco_check_server($server_url, $network = "", $force = false) { if (isset($data->network) AND ($platform == "")) $platform = $data->network; + if ($platform == "Diaspora") + $network = NETWORK_DIASPORA; + if ($data->registrations_open) $register_policy = REGISTER_OPEN; else @@ -773,6 +797,9 @@ function poco_check_server($server_url, $network = "", $force = false) { if (!$failure AND in_array($network, array(NETWORK_DFRN, NETWORK_OSTATUS))) { $serverret = z_fetch_url($server_url."/friendica/json"); + if (!$serverret["success"]) + $serverret = z_fetch_url($server_url."/friendika/json"); + if ($serverret["success"]) { $data = json_decode($serverret["body"]); @@ -1220,6 +1247,10 @@ function poco_discover($complete = false) { } function poco_discover_server_users($data, $server) { + + if (!isset($data->entry)) + return; + foreach ($data->entry AS $entry) { $username = ""; if (isset($entry->urls)) { From bb5186ae277c500614fab54df3255c869af9ec53 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Thu, 23 Jul 2015 08:35:45 +0200 Subject: [PATCH 18/47] Use an ID as a primary field. --- include/dbstructure.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/dbstructure.php b/include/dbstructure.php index c079efeb5..7d6d015a3 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -691,6 +691,7 @@ function db_definition() { ); $database["gserver"] = array( "fields" => array( + "id" => array("type" => "int(10) unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"), "url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "nurl" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "version" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), @@ -707,7 +708,8 @@ function db_definition() { "last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), ), "indexes" => array( - "PRIMARY" => array("nurl"), + "PRIMARY" => array("id"), + "nurl" => array("nurl"), ) ); $database["guid"] = array( From 5989b5dd68d380f22df25dc324f75b504ccf5760 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Thu, 23 Jul 2015 08:37:10 +0200 Subject: [PATCH 19/47] Adopted changes to database.sql --- database.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/database.sql b/database.sql index ce5f051cd..67bc71822 100644 --- a/database.sql +++ b/database.sql @@ -301,10 +301,12 @@ CREATE TABLE IF NOT EXISTS `gcign` ( CREATE TABLE IF NOT EXISTS `gcontact` ( `id` int(10) unsigned NOT NULL auto_increment PRIMARY KEY, `name` varchar(255) NOT NULL DEFAULT '', + `nick` varchar(255) NOT NULL DEFAULT '', `url` varchar(255) NOT NULL DEFAULT '', `nurl` varchar(255) NOT NULL DEFAULT '', `photo` varchar(255) NOT NULL DEFAULT '', `connect` varchar(255) NOT NULL DEFAULT '', + `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `updated` datetime DEFAULT '0000-00-00 00:00:00', `last_contact` datetime DEFAULT '0000-00-00 00:00:00', `last_failure` datetime DEFAULT '0000-00-00 00:00:00', @@ -360,6 +362,7 @@ CREATE TABLE IF NOT EXISTS `group_member` ( -- TABLE gserver -- CREATE TABLE IF NOT EXISTS `gserver` ( + `id` int(10) unsigned NOT NULL auto_increment PRIMARY KEY, `url` varchar(255) NOT NULL DEFAULT '', `nurl` varchar(255) NOT NULL DEFAULT '', `version` varchar(255) NOT NULL DEFAULT '', @@ -370,9 +373,11 @@ CREATE TABLE IF NOT EXISTS `gserver` ( `noscrape` varchar(255) NOT NULL DEFAULT '', `network` varchar(32) NOT NULL DEFAULT '', `platform` varchar(255) NOT NULL DEFAULT '', + `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `last_poco_query` datetime DEFAULT '0000-00-00 00:00:00', `last_contact` datetime DEFAULT '0000-00-00 00:00:00', - `last_failure` datetime DEFAULT '0000-00-00 00:00:00' + `last_failure` datetime DEFAULT '0000-00-00 00:00:00', + INDEX `nurl` (`nurl`) ) DEFAULT CHARSET=utf8; -- From 201d95a8558506ede370e3f4d1986428fd6826bb Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Thu, 23 Jul 2015 10:28:40 +0200 Subject: [PATCH 20/47] Set contact as reachable when there was data from noscrape --- include/socgraph.php | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/include/socgraph.php b/include/socgraph.php index f72fe5f6f..272fcea99 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -429,8 +429,13 @@ function poco_last_updated($profile) { q("UPDATE `gcontact` SET `created` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc(normalise_link($profile))); - if (($gcontacts[0]["server_url"] != "") AND ($gcontacts[0]["network"] != "")) - if (!poco_check_server($gcontacts[0]["server_url"], $gcontacts[0]["network"])) + if ($gcontacts[0]["server_url"] != "") + $server_url = $gcontacts[0]["server_url"]; + else + $server_url = poco_detect_server($profile); + + if ($server_url != "") + if (!poco_check_server($pserver_url, $gcontacts[0]["network"])) return false; // noscrape is really fast so we don't cache the call. @@ -493,6 +498,11 @@ function poco_last_updated($profile) { q("UPDATE `gcontact` SET `location` = '%s' WHERE `nurl` = '%s'", dbesc($location), dbesc(normalise_link($profile))); + // If we got data from noscrape then mark the contact as reachable + if (is_array($noscrape) AND count($noscrape)) + q("UPDATE `gcontact` SET `last_contact` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($profile))); + return $noscrape["updated"]; } } From cbd2fc1572028a4b7e6753d4abebb46995328145 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Thu, 23 Jul 2015 20:57:08 +0200 Subject: [PATCH 21/47] "last_contact" is now set automatically if it is lower than the update value. --- include/socgraph.php | 6 +++++- mod/poco.php | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/socgraph.php b/include/socgraph.php index 272fcea99..b9028724a 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -223,6 +223,9 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca $last_failure = $x[0]["last_failure"]; $server_url = $x[0]["server_url"]; $nick = $x[0]["nick"]; + + if ($updated > $last_contact) + $last_contact = $updated; } else { $created = "0000-00-00 00:00:00"; $last_contact = "0000-00-00 00:00:00"; @@ -294,7 +297,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo || $x[0]['updated'] < $updated) { q("UPDATE `gcontact` SET `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s', `server_url` = '%s', - `updated` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d + `updated` = '%s', `last_contact` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d WHERE (`generation` >= %d OR `generation` = 0) AND `nurl` = '%s'", dbesc($name), dbesc($network), @@ -303,6 +306,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca dbesc($profile_url), dbesc($server_url), dbesc($updated), + dbesc($last_contact), dbesc($location), dbesc($about), dbesc($keywords), diff --git a/mod/poco.php b/mod/poco.php index 0b62e93e3..250add179 100644 --- a/mod/poco.php +++ b/mod/poco.php @@ -61,7 +61,7 @@ function poco_init(&$a) { $update_limit = date("Y-m-d H:i:s",strtotime($_GET['updatedSince'])); if ($global) { - $r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND `last_contact` >= `last_failure` AND `network` IN ('%s')", + $r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND ((`last_contact` >= `last_failure`) OR (`updated` > `last_failure`)) AND `network` IN ('%s')", dbesc($update_limit), dbesc(NETWORK_DFRN) ); From 880f486044a519e4dd22c7e7f52ebf6e5f50b723 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Thu, 23 Jul 2015 22:21:22 +0200 Subject: [PATCH 22/47] Check not only for the last contact but also for the last update for the sanity check. --- include/discover_poco.php | 5 +++-- mod/dirfind.php | 3 ++- mod/poco.php | 6 +++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/include/discover_poco.php b/include/discover_poco.php index 74ed83b47..7dbafcafd 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -93,11 +93,12 @@ function discover_directory($search) { if(count($j->results)) foreach($j->results as $jj) { // Check if the contact already exists - $exists = q("SELECT `id`, `last_contact`, `last_failure` FROM `gcontact` WHERE `nurl` = '%s'", normalise_link($jj->url)); + $exists = q("SELECT `id`, `last_contact`, `last_failure`, `updated` FROM `gcontact` WHERE `nurl` = '%s'", normalise_link($jj->url)); if ($exists) { logger("Profile ".$jj->url." already exists (".$search.")", LOGGER_DEBUG); - if ($exists[0]["last_contact"] < $exists[0]["last_failure"]) + if (($exists[0]["last_contact"] < $exists[0]["last_failure"]) AND + ($exists[0]["updated"] < $exists[0]["last_failure"])) continue; // Update the contact diff --git a/mod/dirfind.php b/mod/dirfind.php index 996d2a464..45fc93b7e 100644 --- a/mod/dirfind.php +++ b/mod/dirfind.php @@ -46,7 +46,8 @@ function dirfind_content(&$a) { $results = q("SELECT `contact`.`id` AS `cid`, `gcontact`.`url`, `gcontact`.`name`, `gcontact`.`photo`, `gcontact`.`keywords` FROM `gcontact` LEFT JOIN `contact` ON `contact`.`nurl` = `gcontact`.`nurl` AND `contact`.`uid` = %d - WHERE `gcontact`.`network` IN ('%s', '%s', '%s') AND `gcontact`.`last_contact` >= `gcontact`.`last_failure` AND + WHERE `gcontact`.`network` IN ('%s', '%s', '%s') AND + ((`gcontact`.`last_contact` >= `gcontact`.`last_failure`) OR (`gcontact`.`updated` >= `gcontact`.`last_failure`)) AND (`gcontact`.`url` REGEXP '%s' OR `gcontact`.`name` REGEXP '%s' OR `gcontact`.`location` REGEXP '%s' OR `gcontact`.`about` REGEXP '%s' OR `gcontact`.`keywords` REGEXP '%s') GROUP BY `gcontact`.`nurl` diff --git a/mod/poco.php b/mod/poco.php index 250add179..0b08e30bf 100644 --- a/mod/poco.php +++ b/mod/poco.php @@ -61,7 +61,7 @@ function poco_init(&$a) { $update_limit = date("Y-m-d H:i:s",strtotime($_GET['updatedSince'])); if ($global) { - $r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND ((`last_contact` >= `last_failure`) OR (`updated` > `last_failure`)) AND `network` IN ('%s')", + $r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND ((`last_contact` >= `last_failure`) OR (`updated` >= `last_failure`)) AND `network` IN ('%s')", dbesc($update_limit), dbesc(NETWORK_DFRN) ); @@ -96,7 +96,7 @@ function poco_init(&$a) { if ($global) { - $r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') AND `last_contact` >= `last_failure` LIMIT %d, %d", + $r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') AND ((`last_contact` >= `last_failure`) OR (`updated` > `last_failure`)) LIMIT %d, %d", dbesc($update_limit), dbesc(NETWORK_DFRN), intval($startIndex), @@ -107,7 +107,7 @@ function poco_init(&$a) { `profile`.`address` AS `paddress`, `profile`.`region` AS `pregion`, `profile`.`postal-code` AS `ppostalcode`, `profile`.`country-name` AS `pcountry` FROM `contact` INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid` WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') AND `profile`.`is-default` - AND `contact`.`success_update` >= `contact`.`failure_update` + AND ((`contact`.`success_update` >= `contact`.`failure_update`) OR (`contact`.`last-item` >= `contact`.`failure_update`)) AND `contact`.`uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) LIMIT %d, %d", dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), From 39d2720e7b9d7674cbdf4c76bc579f3819691528 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Thu, 23 Jul 2015 23:27:15 +0200 Subject: [PATCH 23/47] Update the contact when activated --- include/socgraph.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/socgraph.php b/include/socgraph.php index b9028724a..7d4cca18a 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -264,8 +264,9 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca poco_check_server($server_url, $network); - // Only fetch last update manually if it wasn't provided and enabled in the system - if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") + // Fetch last update manually if it is enabled in the system + // AND ($orig_updated == "0000-00-00 00:00:00") + if (get_config('system','poco_completion') AND poco_do_update($created, $updated, $last_failure, $last_contact) AND poco_reachable($profile_url, $server_url, $network)) { $last_updated = poco_last_updated($profile_url); From e3e9de2e74b623a6843ee684a7978d7aa51e34b6 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Fri, 24 Jul 2015 07:23:57 +0200 Subject: [PATCH 24/47] Export all federation contacts through poco --- include/dbstructure.php | 1 + include/socgraph.php | 4 ++-- mod/poco.php | 12 +++++++++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/include/dbstructure.php b/include/dbstructure.php index 7d6d015a3..e14ce64d8 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -646,6 +646,7 @@ function db_definition() { "indexes" => array( "PRIMARY" => array("id"), "nurl" => array("nurl"), + "updated" => array("updated"), ) ); $database["glink"] = array( diff --git a/include/socgraph.php b/include/socgraph.php index 7d4cca18a..1f3536a9f 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -613,8 +613,8 @@ function poco_do_update($created, $updated, $last_failure, $last_contact) { return false; // If the last contact was less than a week ago and the last failure is older than a week then don't update - if ((($now - $contact_time) < (60 * 60 * 24 * 7)) AND ($contact_time > $failure_time)) - return false; + //if ((($now - $contact_time) < (60 * 60 * 24 * 7)) AND ($contact_time > $failure_time)) + // return false; // If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week if ((($now - $contact_time) > (60 * 60 * 24 * 7)) AND (($now - $created_time) > (60 * 60 * 24 * 7)) AND (($now - $failure_time) < (60 * 60 * 24 * 7))) diff --git a/mod/poco.php b/mod/poco.php index 0b08e30bf..53c7f5b0b 100644 --- a/mod/poco.php +++ b/mod/poco.php @@ -61,9 +61,12 @@ function poco_init(&$a) { $update_limit = date("Y-m-d H:i:s",strtotime($_GET['updatedSince'])); if ($global) { - $r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND ((`last_contact` >= `last_failure`) OR (`updated` >= `last_failure`)) AND `network` IN ('%s')", + //$r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND ((`last_contact` >= `last_failure`) OR (`updated` >= `last_failure`)) AND `network` IN ('%s')", + $r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND `updated` >= `last_failure` AND `network` IN ('%s', '%s', '%s')", dbesc($update_limit), - dbesc(NETWORK_DFRN) + dbesc(NETWORK_DFRN), + dbesc(NETWORK_DIASPORA), + dbesc(NETWORK_OSTATUS) ); } elseif($system_mode) { $r = q("SELECT count(*) AS `total` FROM `contact` WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') @@ -96,9 +99,12 @@ function poco_init(&$a) { if ($global) { - $r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') AND ((`last_contact` >= `last_failure`) OR (`updated` > `last_failure`)) LIMIT %d, %d", + //$r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') AND ((`last_contact` >= `last_failure`) OR (`updated` > `last_failure`)) LIMIT %d, %d", + $r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s', '%1', '%1') and `updated` > `last_failure` LIMIT %d, %d", dbesc($update_limit), dbesc(NETWORK_DFRN), + dbesc(NETWORK_DIASPORA), + dbesc(NETWORK_OSTATUS), intval($startIndex), intval($itemsPerPage) ); From 68e318e3162e992acbc31fe4d2efeff26ae6d3d5 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 25 Jul 2015 10:50:00 +0200 Subject: [PATCH 25/47] Don't export contacts with empty network --- mod/poco.php | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mod/poco.php b/mod/poco.php index 53c7f5b0b..d180650a4 100644 --- a/mod/poco.php +++ b/mod/poco.php @@ -69,8 +69,8 @@ function poco_init(&$a) { dbesc(NETWORK_OSTATUS) ); } elseif($system_mode) { - $r = q("SELECT count(*) AS `total` FROM `contact` WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') - AND `success_update` >= `failure_update` + $r = q("SELECT count(*) AS `total` FROM `contact` WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s') + AND (`success_update` >= `failure_update` OR `last-item` >= `failure_update`) AND `uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) ", dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), @@ -79,7 +79,8 @@ function poco_init(&$a) { ); } else { $r = q("SELECT count(*) AS `total` FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0 - AND `success_update` >= `failure_update` AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra", + AND (`success_update` >= `failure_update` OR `last-item` >= `failure_update`) + AND `network` IN ('%s', '%s', '%s', '%s') $sql_extra", intval($user['uid']), dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), @@ -112,7 +113,7 @@ function poco_init(&$a) { $r = q("SELECT `contact`.*, `profile`.`about` AS `pabout`, `profile`.`locality` AS `plocation`, `profile`.`pub_keywords`, `profile`.`gender` AS `pgender`, `profile`.`address` AS `paddress`, `profile`.`region` AS `pregion`, `profile`.`postal-code` AS `ppostalcode`, `profile`.`country-name` AS `pcountry` FROM `contact` INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid` - WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') AND `profile`.`is-default` + WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s') AND `profile`.`is-default` AND ((`contact`.`success_update` >= `contact`.`failure_update`) OR (`contact`.`last-item` >= `contact`.`failure_update`)) AND `contact`.`uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) LIMIT %d, %d", dbesc(NETWORK_DFRN), @@ -124,7 +125,8 @@ function poco_init(&$a) { ); } else { $r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0 - AND `success_update` >= `failure_update` AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra LIMIT %d, %d", + AND (`success_update` >= `failure_update` OR `last-item` >= `failure_update`) + AND `network` IN ('%s', '%s', '%s', '%s') $sql_extra LIMIT %d, %d", intval($user['uid']), dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), From dcfcd6f906e9b7a10d4e733ff53879149c05e80a Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 25 Jul 2015 12:05:27 +0200 Subject: [PATCH 26/47] Poco: added caching --- mod/poco.php | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/mod/poco.php b/mod/poco.php index d180650a4..f84fc964d 100644 --- a/mod/poco.php +++ b/mod/poco.php @@ -98,10 +98,11 @@ function poco_init(&$a) { $startIndex = 0; $itemsPerPage = ((x($_GET,'count') && intval($_GET['count'])) ? intval($_GET['count']) : $totalResults); - if ($global) { + logger("Start global query", LOGGER_DEBUG); //$r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') AND ((`last_contact` >= `last_failure`) OR (`updated` > `last_failure`)) LIMIT %d, %d", - $r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s', '%1', '%1') and `updated` > `last_failure` LIMIT %d, %d", + $r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s', '%s', '%s') AND `updated` > `last_failure` + ORDER BY `updated` DESC LIMIT %d, %d", dbesc($update_limit), dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), @@ -110,6 +111,7 @@ function poco_init(&$a) { intval($itemsPerPage) ); } elseif($system_mode) { + logger("Start system mode query", LOGGER_DEBUG); $r = q("SELECT `contact`.*, `profile`.`about` AS `pabout`, `profile`.`locality` AS `plocation`, `profile`.`pub_keywords`, `profile`.`gender` AS `pgender`, `profile`.`address` AS `paddress`, `profile`.`region` AS `pregion`, `profile`.`postal-code` AS `ppostalcode`, `profile`.`country-name` AS `pcountry` FROM `contact` INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid` @@ -124,6 +126,7 @@ function poco_init(&$a) { intval($itemsPerPage) ); } else { + logger("Start query for user ".$user['nickname'], LOGGER_DEBUG); $r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0 AND (`success_update` >= `failure_update` OR `last-item` >= `failure_update`) AND `network` IN ('%s', '%s', '%s', '%s') $sql_extra LIMIT %d, %d", @@ -136,6 +139,7 @@ function poco_init(&$a) { intval($itemsPerPage) ); } + logger("Query done", LOGGER_DEBUG); $ret = array(); if(x($_GET,'sorted')) @@ -216,13 +220,19 @@ function poco_init(&$a) { if (($rr['keywords'] == "") AND isset($rr['pub_keywords'])) $rr['keywords'] = $rr['pub_keywords']; + $about = Cache::get("about:".$rr['updated'].":".$rr['nurl']); + if (is_null($about)) { + $about = bbcode($rr['about'], false, false); + Cache::set("about:".$rr['updated'].":".$rr['nurl'],$about); + } + $entry = array(); if($fields_ret['id']) $entry['id'] = (int)$rr['id']; if($fields_ret['displayName']) $entry['displayName'] = $rr['name']; if($fields_ret['aboutMe']) - $entry['aboutMe'] = bbcode($rr['about'], false, false); + $entry['aboutMe'] = $about; if($fields_ret['currentLocation']) $entry['currentLocation'] = $rr['location']; if($fields_ret['gender']) @@ -305,6 +315,8 @@ function poco_init(&$a) { else http_status_exit(500); + logger("End of poco", LOGGER_DEBUG); + if($format === 'xml') { header('Content-type: text/xml'); echo replace_macros(get_markup_template('poco_xml.tpl'),array_xmlify(array('$response' => $ret))); From 5f524006dd058dd913510bfb6604aefaf1f27c9b Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 25 Jul 2015 14:51:56 +0200 Subject: [PATCH 27/47] Poco: Only fetch the latest post if it wasn't provided. --- include/socgraph.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/socgraph.php b/include/socgraph.php index 1f3536a9f..6111e0779 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -265,8 +265,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca poco_check_server($server_url, $network); // Fetch last update manually if it is enabled in the system - // AND ($orig_updated == "0000-00-00 00:00:00") - if (get_config('system','poco_completion') + if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($created, $updated, $last_failure, $last_contact) AND poco_reachable($profile_url, $server_url, $network)) { $last_updated = poco_last_updated($profile_url); From 072d7f6b3c9ee2b3a01f91044dfa95771320a633 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 25 Jul 2015 19:59:27 +0200 Subject: [PATCH 28/47] Do a recheck when the server isn't available. --- include/socgraph.php | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/include/socgraph.php b/include/socgraph.php index 6111e0779..a84d2d7be 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -239,6 +239,18 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca if ((($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == "")) AND poco_reachable($profile_url, $server_url, $network)) { $data = probe_url($profile_url); + + // If the system doesn't seem to react, recheck the server + if ($data["network"] == NETWORK_FEED) { + logger("Recheck the server for profile ".$profile_url, LOGGER_DEBUG); + if ($server_url == "") + $url_check = poco_detect_server($profile_url); + else + $url_check = $server_url; + + poco_check_server($url_check, $network, true); + } + $network = $data["network"]; $name = $data["name"]; $nick = $data["nick"]; @@ -1208,6 +1220,7 @@ function poco_discover($complete = false) { $no_of_queries = 5; $last_update = date("c", time() - (60 * 60 * 6)); // 24 + $last_update = date("c", time() - (60 * 60 * 24)); // 24 $r = q("SELECT `poco`, `nurl`, `url`, `network` FROM `gserver` WHERE `last_contact` > `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update)); if ($r) From fc9c6e6bfd1cf31959d79077e701b086c5d8c713 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 25 Jul 2015 20:10:42 +0200 Subject: [PATCH 29/47] Poco: always do a forced check for availability of the server --- include/socgraph.php | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/include/socgraph.php b/include/socgraph.php index a84d2d7be..0c78cfe23 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -237,20 +237,9 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca } if ((($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == "")) - AND poco_reachable($profile_url, $server_url, $network)) { + AND poco_reachable($profile_url, $server_url, $network, true)) { $data = probe_url($profile_url); - // If the system doesn't seem to react, recheck the server - if ($data["network"] == NETWORK_FEED) { - logger("Recheck the server for profile ".$profile_url, LOGGER_DEBUG); - if ($server_url == "") - $url_check = poco_detect_server($profile_url); - else - $url_check = $server_url; - - poco_check_server($url_check, $network, true); - } - $network = $data["network"]; $name = $data["name"]; $nick = $data["nick"]; @@ -393,7 +382,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca return $gcid; } -function poco_reachable($profile, $server = "", $network = "") { +function poco_reachable($profile, $server = "", $network = "", $force = false) { if ($server == "") $server = poco_detect_server($profile); @@ -401,7 +390,7 @@ function poco_reachable($profile, $server = "", $network = "") { if ($server == "") return true; - return poco_check_server($server, $network); + return poco_check_server($server, $network, $force); } function poco_detect_server($profile) { From 166519fc4db8eec57ce17c4ab7b094d2252b65e1 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 26 Jul 2015 14:41:34 +0200 Subject: [PATCH 30/47] Move the contact discovery into the background --- include/discover_poco.php | 32 ++++++++++++++++++++++++++------ include/poller.php | 4 ++++ include/socgraph.php | 22 +++++++++++----------- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/include/discover_poco.php b/include/discover_poco.php index 7dbafcafd..68f483dab 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -38,23 +38,26 @@ function discover_poco_run(&$argv, &$argc){ if(($argc > 2) && ($argv[1] == "dirsearch")) { $search = urldecode($argv[2]); - $searchmode = 1; + $mode = 1; + } elseif(($argc == 2) && ($argv[1] == "checkcontact")) { + $mode = 2; } elseif ($argc == 1) { $search = ""; - $searchmode = 0; + $mode = 0; } else die("Unknown or missing parameter ".$argv[1]."\n"); $lockpath = get_lockpath(); if ($lockpath != '') { - $pidfile = new pidfile($lockpath, 'discover_poco'.urlencode($search)); + $pidfile = new pidfile($lockpath, 'discover_poco'.$mode.urlencode($search)); if($pidfile->is_already_running()) { logger("discover_poco: Already running"); if ($pidfile->running_time() > 19*60) { $pidfile->kill(); logger("discover_poco: killed stale process"); // Calling a new instance - proc_run('php','include/discover_poco.php'); + if ($mode == 0) + proc_run('php','include/discover_poco.php'); } exit; } @@ -66,9 +69,11 @@ function discover_poco_run(&$argv, &$argc){ logger('start '.$search); - if (($search != "") and get_config('system','poco_local_search')) + if (($mode == 2) AND get_config('system','poco_completion')) + discover_users(); + elseif (($mode == 1) AND ($search != "") and get_config('system','poco_local_search')) discover_directory($search); - elseif (($search == "") and get_config('system','poco_discovery') > 0) + elseif (($mode == 0) AND ($search == "") and (get_config('system','poco_discovery') > 0)) poco_discover(); logger('end '.$search); @@ -76,6 +81,21 @@ function discover_poco_run(&$argv, &$argc){ return; } +function discover_users() { + $users = q("SELECT `url` FROM `gcontact` WHERE `updated` = '0000-00-00 00:00:00' AND + `last_contact` = '0000-00-00 00:00:00' AND `last_failure` = '0000-00-00 00:00:00' AND + `network` IN ('%s', '%s', '%s') ORDER BY rand() LIMIT 50", + dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); + + if (!$users) + return; + + foreach ($users AS $user) { + logger('Check user '.$user["url"]); + poco_last_updated($user["url"]); + } +} + function discover_directory($search) { $data = Cache::get("dirsearch:".$search); diff --git a/include/poller.php b/include/poller.php index 44ac94daa..7cc1a2852 100644 --- a/include/poller.php +++ b/include/poller.php @@ -86,6 +86,10 @@ function poller_run(&$argv, &$argc){ proc_run('php',"include/discover_poco.php"); + // run the process to update locally stored global contacts in the background + + proc_run('php',"include/discover_poco.php", "checkcontact"); + // expire any expired accounts q("UPDATE user SET `account_expired` = 1 where `account_expired` = 0 diff --git a/include/socgraph.php b/include/socgraph.php index 0c78cfe23..53763626d 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -266,17 +266,17 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca poco_check_server($server_url, $network); // Fetch last update manually if it is enabled in the system - if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") - AND poco_do_update($created, $updated, $last_failure, $last_contact) - AND poco_reachable($profile_url, $server_url, $network)) { - $last_updated = poco_last_updated($profile_url); - if ($last_updated) { - $updated = $last_updated; - $last_contact = datetime_convert(); - logger("Last updated for profile ".$profile_url.": ".$updated, LOGGER_DEBUG); - } else - $last_failure = datetime_convert(); - } + //if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") + // AND poco_do_update($created, $updated, $last_failure, $last_contact) + // AND poco_reachable($profile_url, $server_url, $network)) { + // $last_updated = poco_last_updated($profile_url); + // if ($last_updated) { + // $updated = $last_updated; + // $last_contact = datetime_convert(); + // logger("Last updated for profile ".$profile_url.": ".$updated, LOGGER_DEBUG); + // } else + // $last_failure = datetime_convert(); + //} if(count($x)) { $gcid = $x[0]['id']; From 7136a39f9e4dd7adce5adb3bc13ec3ccd4c3483d Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 26 Jul 2015 16:37:05 +0200 Subject: [PATCH 31/47] Discovery for unchecked contacts --- include/discover_poco.php | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/include/discover_poco.php b/include/discover_poco.php index 68f483dab..bc5ee5d32 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -82,17 +82,21 @@ function discover_poco_run(&$argv, &$argc){ } function discover_users() { - $users = q("SELECT `url` FROM `gcontact` WHERE `updated` = '0000-00-00 00:00:00' AND - `last_contact` = '0000-00-00 00:00:00' AND `last_failure` = '0000-00-00 00:00:00' AND - `network` IN ('%s', '%s', '%s') ORDER BY rand() LIMIT 50", + // To-Do: Maybe we should check old contact as well. + $users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact` FROM `gcontact` + WHERE `updated` = '0000-00-00 00:00:00' AND `last_contact` = '0000-00-00 00:00:00' AND + `last_failure` = '0000-00-00 00:00:00' AND `network` IN ('%s', '%s', '%s') + ORDER BY rand() LIMIT 100", dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); if (!$users) return; foreach ($users AS $user) { - logger('Check user '.$user["url"]); - poco_last_updated($user["url"]); + if (poco_do_update($user["created"], $user["updated"], $user["last_failure"], $user["last_contact"])) { + logger('Check user '.$user["url"]); + poco_last_updated($user["url"]); + } } } From e5bf7bae7b2e2d670744e4b5ec6fa82b28ca59e2 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 26 Jul 2015 16:52:37 +0200 Subject: [PATCH 32/47] Just some improvements ... --- include/discover_poco.php | 21 +++++++++++++++++---- include/socgraph.php | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/include/discover_poco.php b/include/discover_poco.php index bc5ee5d32..481bedcf9 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -83,19 +83,32 @@ function discover_poco_run(&$argv, &$argc){ function discover_users() { // To-Do: Maybe we should check old contact as well. - $users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact` FROM `gcontact` + $users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url` FROM `gcontact` WHERE `updated` = '0000-00-00 00:00:00' AND `last_contact` = '0000-00-00 00:00:00' AND `last_failure` = '0000-00-00 00:00:00' AND `network` IN ('%s', '%s', '%s') - ORDER BY rand() LIMIT 100", + ORDER BY rand()", dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); if (!$users) return; + $checked = 0; + foreach ($users AS $user) { if (poco_do_update($user["created"], $user["updated"], $user["last_failure"], $user["last_contact"])) { - logger('Check user '.$user["url"]); - poco_last_updated($user["url"]); + + if ($user[0]["server_url"] != "") + $server_url = $user[0]["server_url"]; + else + $server_url = poco_detect_server($user["url"]); + + if (poco_check_server($server_url, $gcontacts[0]["network"])) { + logger('Check user '.$user["url"]); + poco_last_updated($user["url"]); + + if (++$checked > 100) + return; + } } } } diff --git a/include/socgraph.php b/include/socgraph.php index 53763626d..04374f3e5 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -440,7 +440,7 @@ function poco_last_updated($profile) { $server_url = poco_detect_server($profile); if ($server_url != "") - if (!poco_check_server($pserver_url, $gcontacts[0]["network"])) + if (!poco_check_server($server_url, $gcontacts[0]["network"])) return false; // noscrape is really fast so we don't cache the call. From a0011ba58e7503a90e643268b63db173e9b1a268 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 26 Jul 2015 17:14:32 +0200 Subject: [PATCH 33/47] Discover users who never are contacted. --- include/discover_poco.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/discover_poco.php b/include/discover_poco.php index 481bedcf9..bd3483307 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -82,11 +82,11 @@ function discover_poco_run(&$argv, &$argc){ } function discover_users() { + logger("Discover users", LOGGER_DEBUG); // To-Do: Maybe we should check old contact as well. $users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url` FROM `gcontact` - WHERE `updated` = '0000-00-00 00:00:00' AND `last_contact` = '0000-00-00 00:00:00' AND - `last_failure` = '0000-00-00 00:00:00' AND `network` IN ('%s', '%s', '%s') - ORDER BY rand()", + WHERE `last_contact` = '0000-00-00 00:00:00' AND `last_failure` = '0000-00-00 00:00:00' AND + `network` IN ('%s', '%s', '%s') ORDER BY rand()", dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); if (!$users) From e0955535902b13982f58c063d47ab00a9d4d1db9 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 27 Jul 2015 07:16:06 +0200 Subject: [PATCH 34/47] Mark contacts from not reachable servers as unreachable --- include/discover_poco.php | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/discover_poco.php b/include/discover_poco.php index bd3483307..687698b44 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -108,7 +108,9 @@ function discover_users() { if (++$checked > 100) return; - } + } else + q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($user["url"]))); } } } From 4b2c2c480231cdccc40fea0e31aaac5811507ae7 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Mon, 27 Jul 2015 08:14:04 +0200 Subject: [PATCH 35/47] Check all contacts that we haven't contacted in a month --- include/discover_poco.php | 7 ++++--- include/socgraph.php | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/include/discover_poco.php b/include/discover_poco.php index 687698b44..e93b449b9 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -83,9 +83,10 @@ function discover_poco_run(&$argv, &$argc){ function discover_users() { logger("Discover users", LOGGER_DEBUG); - // To-Do: Maybe we should check old contact as well. + $users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url` FROM `gcontact` - WHERE `last_contact` = '0000-00-00 00:00:00' AND `last_failure` = '0000-00-00 00:00:00' AND + WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND + `last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND `network` IN ('%s', '%s', '%s') ORDER BY rand()", dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); @@ -104,7 +105,7 @@ function discover_users() { if (poco_check_server($server_url, $gcontacts[0]["network"])) { logger('Check user '.$user["url"]); - poco_last_updated($user["url"]); + poco_last_updated($user["url"], true); if (++$checked > 100) return; diff --git a/include/socgraph.php b/include/socgraph.php index 04374f3e5..5671891df 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -425,7 +425,7 @@ function poco_detect_server($profile) { return $server_url; } -function poco_last_updated($profile) { +function poco_last_updated($profile, $force = false) { $gcontacts = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s'", dbesc(normalise_link($profile))); @@ -451,7 +451,7 @@ function poco_last_updated($profile) { if ($server) { $noscraperet = z_fetch_url($server[0]["noscrape"]."/".$gcontacts[0]["nick"]); - if ($noscraperet["success"]) { + if ($noscraperet["success"] AND ($noscraperet["body"] = "")) { $noscrape = json_decode($noscraperet["body"], true); if (($noscrape["name"] != "") AND ($noscrape["name"] != $gcontacts[0]["name"])) @@ -514,7 +514,7 @@ function poco_last_updated($profile) { } // If we only can poll the feed, then we only do this once a while - if (!poco_do_update($gcontacts[0]["created"], $gcontacts[0]["updated"], $gcontacts[0]["last_failure"], $gcontacts[0]["last_contact"])) + if (!$force AND !poco_do_update($gcontacts[0]["created"], $gcontacts[0]["updated"], $gcontacts[0]["last_failure"], $gcontacts[0]["last_contact"])) return $gcontacts[0]["updated"]; $data = probe_url($profile); From ec161d798f426e53e3c1b963624ce6852c4ddd2a Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Tue, 28 Jul 2015 08:25:30 +0200 Subject: [PATCH 36/47] The discovery should check all profiles. --- include/discover_poco.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/discover_poco.php b/include/discover_poco.php index e93b449b9..47c0cc2cd 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -96,7 +96,7 @@ function discover_users() { $checked = 0; foreach ($users AS $user) { - if (poco_do_update($user["created"], $user["updated"], $user["last_failure"], $user["last_contact"])) { + //if (poco_do_update($user["created"], $user["updated"], $user["last_failure"], $user["last_contact"])) { if ($user[0]["server_url"] != "") $server_url = $user[0]["server_url"]; @@ -112,7 +112,7 @@ function discover_users() { } else q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc(normalise_link($user["url"]))); - } + //} } } From 9f073f2fd58df9dee38991efbf4e44a66c0fc03a Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Tue, 28 Jul 2015 15:28:58 +0200 Subject: [PATCH 37/47] Some correction stuff for old entries in the global contacts --- include/discover_poco.php | 5 +++-- include/socgraph.php | 30 ++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/include/discover_poco.php b/include/discover_poco.php index 47c0cc2cd..d4564ae4a 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -87,8 +87,9 @@ function discover_users() { $users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url` FROM `gcontact` WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND `last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND - `network` IN ('%s', '%s', '%s') ORDER BY rand()", - dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_OSTATUS)); + `network` IN ('%s', '%s', '%s', '%s', '') ORDER BY rand()", + dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), + dbesc(NETWORK_OSTATUS), dbesc(NETWORK_FEED)); if (!$users) return; diff --git a/include/socgraph.php b/include/socgraph.php index 5671891df..5e7cded9c 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -176,7 +176,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca if (!isset($urlparts["scheme"])) return $gcid; - if (in_array($urlparts["host"], array("facebook.com", "twitter.com", "www.twitter-rss.com", + if (in_array($urlparts["host"], array("www.facebook.com", "facebook.com", "twitter.com", "identi.ca", "alpha.app.net"))) return $gcid; @@ -434,15 +434,41 @@ function poco_last_updated($profile, $force = false) { q("UPDATE `gcontact` SET `created` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc(normalise_link($profile))); + $urlparts = parse_url($profile); + if (!isset($urlparts["scheme"])) + return; + + if (in_array($urlparts["host"], array("www.facebook.com", "facebook.com", "twitter.com", + "identi.ca", "alpha.app.net"))) { + q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'", + dbesc(NETWORK_PHANTOM), dbesc(normalise_link($profile))); + return; + } + if ($gcontacts[0]["server_url"] != "") $server_url = $gcontacts[0]["server_url"]; else $server_url = poco_detect_server($profile); - if ($server_url != "") + if ($server_url != "") { if (!poco_check_server($server_url, $gcontacts[0]["network"])) return false; + q("UPDATE `gcontact` SET `server_url` = '%s' WHERE `nurl` = '%s'", + dbesc($server_url), dbesc(normalise_link($profile))); + } + + if (in_array($gcontacts[0]["network"], array("", NETWORK_FEED))) { + $server = q("SELECT `network` FROM `gserver` WHERE `nurl` = '%s' AND `network` != ''", + dbesc(normalise_link($server_url))); + + if ($server) + q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'", + dbesc($server[0]["network"]), dbesc(normalise_link($profile))); + else + return; + } + // noscrape is really fast so we don't cache the call. if (($gcontacts[0]["server_url"] != "") AND ($gcontacts[0]["nick"] != "")) { From 678de014ad2f2d1fc0915f83998eba0e325f8720 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Tue, 28 Jul 2015 18:26:30 +0200 Subject: [PATCH 38/47] Some code beautification. --- include/discover_poco.php | 26 ++++++++++++-------------- include/socgraph.php | 39 ++++++++++++--------------------------- 2 files changed, 24 insertions(+), 41 deletions(-) diff --git a/include/discover_poco.php b/include/discover_poco.php index d4564ae4a..a9eaf1cf6 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -97,23 +97,21 @@ function discover_users() { $checked = 0; foreach ($users AS $user) { - //if (poco_do_update($user["created"], $user["updated"], $user["last_failure"], $user["last_contact"])) { - if ($user[0]["server_url"] != "") - $server_url = $user[0]["server_url"]; - else - $server_url = poco_detect_server($user["url"]); + if ($user[0]["server_url"] != "") + $server_url = $user[0]["server_url"]; + else + $server_url = poco_detect_server($user["url"]); - if (poco_check_server($server_url, $gcontacts[0]["network"])) { - logger('Check user '.$user["url"]); - poco_last_updated($user["url"], true); + if (poco_check_server($server_url, $gcontacts[0]["network"])) { + logger('Check user '.$user["url"]); + poco_last_updated($user["url"], true); - if (++$checked > 100) - return; - } else - q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", - dbesc(datetime_convert()), dbesc(normalise_link($user["url"]))); - //} + if (++$checked > 100) + return; + } else + q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($user["url"]))); } } diff --git a/include/socgraph.php b/include/socgraph.php index 5e7cded9c..979f1cb36 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -7,6 +7,8 @@ require_once("include/html2bbcode.php"); /* To-Do: - Move GNU Social URL schemata (http://server.tld/user/number) to http://server.tld/username + - Fetch profile data from profile page for Redmatrix users + - Detect if it is a forum */ /* @@ -219,17 +221,10 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca $updated = $x[0]["updated"]; $created = $x[0]["created"]; - $last_contact = $x[0]["last_contact"]; - $last_failure = $x[0]["last_failure"]; $server_url = $x[0]["server_url"]; $nick = $x[0]["nick"]; - - if ($updated > $last_contact) - $last_contact = $updated; } else { $created = "0000-00-00 00:00:00"; - $last_contact = "0000-00-00 00:00:00"; - $last_failure = "0000-00-00 00:00:00"; $server_url = ""; $urlparts = parse_url($profile_url); @@ -265,19 +260,6 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca poco_check_server($server_url, $network); - // Fetch last update manually if it is enabled in the system - //if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") - // AND poco_do_update($created, $updated, $last_failure, $last_contact) - // AND poco_reachable($profile_url, $server_url, $network)) { - // $last_updated = poco_last_updated($profile_url); - // if ($last_updated) { - // $updated = $last_updated; - // $last_contact = datetime_convert(); - // logger("Last updated for profile ".$profile_url.": ".$updated, LOGGER_DEBUG); - // } else - // $last_failure = datetime_convert(); - //} - if(count($x)) { $gcid = $x[0]['id']; @@ -298,7 +280,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo || $x[0]['updated'] < $updated) { q("UPDATE `gcontact` SET `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s', `server_url` = '%s', - `updated` = '%s', `last_contact` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d + `updated` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d WHERE (`generation` >= %d OR `generation` = 0) AND `nurl` = '%s'", dbesc($name), dbesc($network), @@ -307,7 +289,6 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca dbesc($profile_url), dbesc($server_url), dbesc($updated), - dbesc($last_contact), dbesc($location), dbesc($about), dbesc($keywords), @@ -318,8 +299,8 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca ); } } else { - q("INSERT INTO `gcontact` (`name`, `nick`, `network`, `url`, `nurl`, `photo`, `connect`, `server_url`, `created`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`) - VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)", + q("INSERT INTO `gcontact` (`name`, `nick`, `network`, `url`, `nurl`, `photo`, `connect`, `server_url`, `created`, `updated`, `location`, `about`, `keywords`, `gender`, `generation`) + VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)", dbesc($name), dbesc($nick), dbesc($network), @@ -330,8 +311,6 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca dbesc($server_url), dbesc(datetime_convert()), dbesc($updated), - dbesc($last_contact), - dbesc($last_failure), dbesc($location), dbesc($about), dbesc($keywords), @@ -451,8 +430,14 @@ function poco_last_updated($profile, $force = false) { $server_url = poco_detect_server($profile); if ($server_url != "") { - if (!poco_check_server($server_url, $gcontacts[0]["network"])) + if (!poco_check_server($server_url, $gcontacts[0]["network"], $force)) { + + if ($force) + q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($profile))); + return false; + } q("UPDATE `gcontact` SET `server_url` = '%s' WHERE `nurl` = '%s'", dbesc($server_url), dbesc(normalise_link($profile))); From b7c9b942f14e0b621b57f626d25f20b34e080b96 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Wed, 29 Jul 2015 00:21:08 +0200 Subject: [PATCH 39/47] Set the network name for some centralized networks. --- include/socgraph.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/include/socgraph.php b/include/socgraph.php index 979f1cb36..f0ed7ec23 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -419,8 +419,14 @@ function poco_last_updated($profile, $force = false) { if (in_array($urlparts["host"], array("www.facebook.com", "facebook.com", "twitter.com", "identi.ca", "alpha.app.net"))) { + $networks = array("www.facebook.com" => NETWORK_FACEBOOK, + "facebook.com" => NETWORK_FACEBOOK, + "twitter.com" => NETWORK_TWITTER, + "identi.ca" => NETWORK_PUMPIO, + "alpha.app.net" => NETWORK_APPNET); + q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'", - dbesc(NETWORK_PHANTOM), dbesc(normalise_link($profile))); + dbesc($networks[$urlparts["host"]]), dbesc(normalise_link($profile))); return; } From 098ccd09fa0cc0257ecbb59598d97246251b6d92 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Wed, 29 Jul 2015 06:31:18 +0200 Subject: [PATCH 40/47] Moved the detection for centralized networks. --- include/discover_poco.php | 36 ++++++++++++++++++++++++++++-------- include/socgraph.php | 17 ----------------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/include/discover_poco.php b/include/discover_poco.php index a9eaf1cf6..a4cf5db54 100644 --- a/include/discover_poco.php +++ b/include/discover_poco.php @@ -13,10 +13,10 @@ function discover_poco_run(&$argv, &$argc){ if(is_null($db)) { @include(".htconfig.php"); - require_once("include/dba.php"); + require_once("include/dba.php"); $db = new dba($db_host, $db_user, $db_pass, $db_data); - unset($db_host, $db_user, $db_pass, $db_data); - }; + unset($db_host, $db_user, $db_pass, $db_data); + }; require_once('include/session.php'); require_once('include/datetime.php'); @@ -53,12 +53,12 @@ function discover_poco_run(&$argv, &$argc){ if($pidfile->is_already_running()) { logger("discover_poco: Already running"); if ($pidfile->running_time() > 19*60) { - $pidfile->kill(); - logger("discover_poco: killed stale process"); + $pidfile->kill(); + logger("discover_poco: killed stale process"); // Calling a new instance if ($mode == 0) proc_run('php','include/discover_poco.php'); - } + } exit; } } @@ -98,8 +98,28 @@ function discover_users() { foreach ($users AS $user) { - if ($user[0]["server_url"] != "") - $server_url = $user[0]["server_url"]; + $urlparts = parse_url($user["url"]); + if (!isset($urlparts["scheme"])) { + q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'", + dbesc(NETWORK_PHANTOM), dbesc(normalise_link($user["url"]))); + continue; + } + + if (in_array($urlparts["host"], array("www.facebook.com", "facebook.com", "twitter.com", + "identi.ca", "alpha.app.net"))) { + $networks = array("www.facebook.com" => NETWORK_FACEBOOK, + "facebook.com" => NETWORK_FACEBOOK, + "twitter.com" => NETWORK_TWITTER, + "identi.ca" => NETWORK_PUMPIO, + "alpha.app.net" => NETWORK_APPNET); + + q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'", + dbesc($networks[$urlparts["host"]]), dbesc(normalise_link($user["url"]))); + continue; + } + + if ($user["server_url"] != "") + $server_url = $user["server_url"]; else $server_url = poco_detect_server($user["url"]); diff --git a/include/socgraph.php b/include/socgraph.php index f0ed7ec23..618a9bf3e 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -413,23 +413,6 @@ function poco_last_updated($profile, $force = false) { q("UPDATE `gcontact` SET `created` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc(normalise_link($profile))); - $urlparts = parse_url($profile); - if (!isset($urlparts["scheme"])) - return; - - if (in_array($urlparts["host"], array("www.facebook.com", "facebook.com", "twitter.com", - "identi.ca", "alpha.app.net"))) { - $networks = array("www.facebook.com" => NETWORK_FACEBOOK, - "facebook.com" => NETWORK_FACEBOOK, - "twitter.com" => NETWORK_TWITTER, - "identi.ca" => NETWORK_PUMPIO, - "alpha.app.net" => NETWORK_APPNET); - - q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'", - dbesc($networks[$urlparts["host"]]), dbesc(normalise_link($profile))); - return; - } - if ($gcontacts[0]["server_url"] != "") $server_url = $gcontacts[0]["server_url"]; else From 016f9cb8c35ce5157fb9ad2c23a67f1204c0ac21 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Wed, 29 Jul 2015 07:32:00 +0200 Subject: [PATCH 41/47] Only suggest friendica contacts that where recently active --- include/Contact.php | 7 ++++++- include/socgraph.php | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/include/Contact.php b/include/Contact.php index 103dbe9ad..1215f40e7 100644 --- a/include/Contact.php +++ b/include/Contact.php @@ -267,7 +267,12 @@ function contact_photo_menu($contact) { function random_profile() { - $r = q("select url from gcontact where url like '%%://%%/profile/%%' order by rand() limit 1"); + $r = q("SELECT `url` FROM `gcontact` WHERE `network` = '%s' + AND `last_contact` >= `last_failure` + AND `updated` > UTC_TIMESTAMP - INTERVAL 1 MONTH + ORDER BY rand() LIMIT 1", + dbesc(NETWORK_DFRN)); + if(count($r)) return dirname($r[0]['url']); return ''; diff --git a/include/socgraph.php b/include/socgraph.php index 618a9bf3e..872b730e0 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -1211,7 +1211,7 @@ function poco_discover($complete = false) { $last_update = date("c", time() - (60 * 60 * 6)); // 24 $last_update = date("c", time() - (60 * 60 * 24)); // 24 - $r = q("SELECT `poco`, `nurl`, `url`, `network` FROM `gserver` WHERE `last_contact` > `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update)); + $r = q("SELECT `poco`, `nurl`, `url`, `network` FROM `gserver` WHERE `last_contact` >= `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update)); if ($r) foreach ($r AS $server) { From 4cffa18c62b8955a05e5788d001f06c064001651 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Wed, 29 Jul 2015 07:40:23 +0200 Subject: [PATCH 42/47] Added index --- database.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/database.sql b/database.sql index 67bc71822..91d151a10 100644 --- a/database.sql +++ b/database.sql @@ -317,7 +317,8 @@ CREATE TABLE IF NOT EXISTS `gcontact` ( `network` varchar(255) NOT NULL DEFAULT '', `generation` tinyint(3) NOT NULL DEFAULT 0, `server_url` varchar(255) NOT NULL DEFAULT '', - INDEX `nurl` (`nurl`) + INDEX `nurl` (`nurl`), + INDEX `updated` (`updated`) ) DEFAULT CHARSET=utf8; -- From d5a48c46212a6c5014278a5390e175e6ca2f80a5 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Wed, 29 Jul 2015 07:57:28 +0200 Subject: [PATCH 43/47] Code beautification --- include/onepoll.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/onepoll.php b/include/onepoll.php index 464a50fb9..90d018524 100644 --- a/include/onepoll.php +++ b/include/onepoll.php @@ -19,10 +19,10 @@ function onepoll_run(&$argv, &$argc){ if(is_null($db)) { @include(".htconfig.php"); - require_once("include/dba.php"); + require_once("include/dba.php"); $db = new dba($db_host, $db_user, $db_pass, $db_data); - unset($db_host, $db_user, $db_pass, $db_data); - }; + unset($db_host, $db_user, $db_pass, $db_data); + }; require_once('include/session.php'); From 6ab304b5642c8d15a4c32cc4c84c45fa1c1a0110 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Thu, 30 Jul 2015 16:30:18 +0200 Subject: [PATCH 44/47] Bugfix: Fetching data from noscrape had several issues --- boot.php | 2 +- include/dbstructure.php | 1 + include/socgraph.php | 18 ++++++++++++++---- update.php | 2 +- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/boot.php b/boot.php index ec803a157..2a1150103 100644 --- a/boot.php +++ b/boot.php @@ -19,7 +19,7 @@ define ( 'FRIENDICA_PLATFORM', 'Friendica'); define ( 'FRIENDICA_CODENAME', 'Lily of the valley'); define ( 'FRIENDICA_VERSION', '3.4.1' ); define ( 'DFRN_PROTOCOL_VERSION', '2.23' ); -define ( 'DB_UPDATE_VERSION', 1186 ); +define ( 'DB_UPDATE_VERSION', 1187 ); define ( 'EOL', "
\r\n" ); define ( 'ATOM_TIME', 'Y-m-d\TH:i:s\Z' ); diff --git a/include/dbstructure.php b/include/dbstructure.php index e14ce64d8..2370b01be 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -639,6 +639,7 @@ function db_definition() { "about" => array("type" => "text", "not null" => "1"), "keywords" => array("type" => "text", "not null" => "1"), "gender" => array("type" => "varchar(32)", "not null" => "1", "default" => ""), + "community" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"), "network" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "generation" => array("type" => "tinyint(3)", "not null" => "1", "default" => "0"), "server_url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), diff --git a/include/socgraph.php b/include/socgraph.php index 872b730e0..d114ef09c 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -451,12 +451,14 @@ function poco_last_updated($profile, $force = false) { if ($server) { $noscraperet = z_fetch_url($server[0]["noscrape"]."/".$gcontacts[0]["nick"]); - if ($noscraperet["success"] AND ($noscraperet["body"] = "")) { + + if ($noscraperet["success"] AND ($noscraperet["body"] != "")) { +; $noscrape = json_decode($noscraperet["body"], true); - if (($noscrape["name"] != "") AND ($noscrape["name"] != $gcontacts[0]["name"])) + if (($noscrape["fn"] != "") AND ($noscrape["fn"] != $gcontacts[0]["name"])) q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'", - dbesc($noscrape["name"]), dbesc(normalise_link($profile))); + dbesc($noscrape["fn"]), dbesc(normalise_link($profile))); if (($noscrape["photo"] != "") AND ($noscrape["photo"] != $gcontacts[0]["photo"])) q("UPDATE `gcontact` SET `photo` = '%s' WHERE `nurl` = '%s'", @@ -470,10 +472,18 @@ function poco_last_updated($profile, $force = false) { q("UPDATE `gcontact` SET `gender` = '%s' WHERE `nurl` = '%s'", dbesc($noscrape["gender"]), dbesc(normalise_link($profile))); - if (($noscrape["about"] != "") AND ($noscrape["about"] != $gcontacts[0]["name"])) + if (($noscrape["pdesc"] != "") AND ($noscrape["pdesc"] != $gcontacts[0]["about"])) + q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'", + dbesc($noscrape["pdesc"]), dbesc(normalise_link($profile))); + + if (($noscrape["about"] != "") AND ($noscrape["about"] != $gcontacts[0]["about"])) q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'", dbesc($noscrape["about"]), dbesc(normalise_link($profile))); + if (isset($noscrape["comm"]) AND ($noscrape["comm"] != $gcontacts[0]["community"])) + q("UPDATE `gcontact` SET `community` = %d WHERE `nurl` = '%s'", + intval($noscrape["comm"]), dbesc(normalise_link($profile))); + if (isset($noscrape["tags"])) $keywords = implode(" ", $noscrape["tags"]); else diff --git a/update.php b/update.php index 4f762661a..a2c867402 100644 --- a/update.php +++ b/update.php @@ -1,6 +1,6 @@ Date: Thu, 30 Jul 2015 23:27:50 +0200 Subject: [PATCH 45/47] Added community search mode --- mod/dirfind.php | 16 ++++++++++++++-- mod/search.php | 4 ++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/mod/dirfind.php b/mod/dirfind.php index 45fc93b7e..d5cfcbcab 100644 --- a/mod/dirfind.php +++ b/mod/dirfind.php @@ -16,6 +16,8 @@ function dirfind_init(&$a) { function dirfind_content(&$a) { + $community = false; + $local = get_config('system','poco_local_search'); $search = notags(trim($_REQUEST['search'])); @@ -23,6 +25,11 @@ function dirfind_content(&$a) { if(strpos($search,'@') === 0) $search = substr($search,1); + if(strpos($search,'!') === 0) { + $search = substr($search,1); + $community = true; + } + $o = ''; $o .= replace_macros(get_markup_template("section_title.tpl"),array( @@ -33,12 +40,17 @@ function dirfind_content(&$a) { if ($local) { + if ($community) + $extra_sql = " AND `community`"; + else + $extra_sql = ""; + $perpage = 80; $startrec = (($a->pager['page']) * $perpage) - $perpage; $count = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `network` IN ('%s', '%s', '%s') AND (`url` REGEXP '%s' OR `name` REGEXP '%s' OR `location` REGEXP '%s' OR - `about` REGEXP '%s' OR `keywords` REGEXP '%s')", + `about` REGEXP '%s' OR `keywords` REGEXP '%s')".$extra_sql, dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search))); @@ -49,7 +61,7 @@ function dirfind_content(&$a) { WHERE `gcontact`.`network` IN ('%s', '%s', '%s') AND ((`gcontact`.`last_contact` >= `gcontact`.`last_failure`) OR (`gcontact`.`updated` >= `gcontact`.`last_failure`)) AND (`gcontact`.`url` REGEXP '%s' OR `gcontact`.`name` REGEXP '%s' OR `gcontact`.`location` REGEXP '%s' OR - `gcontact`.`about` REGEXP '%s' OR `gcontact`.`keywords` REGEXP '%s') + `gcontact`.`about` REGEXP '%s' OR `gcontact`.`keywords` REGEXP '%s') $extra_sql GROUP BY `gcontact`.`nurl` ORDER BY `gcontact`.`updated` DESC LIMIT %d, %d", intval(local_user()), dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), diff --git a/mod/search.php b/mod/search.php index 339efc92d..a01984d24 100644 --- a/mod/search.php +++ b/mod/search.php @@ -120,6 +120,10 @@ function search_content(&$a) { require_once('mod/dirfind.php'); return dirfind_content($a); } + if(strpos($search,'!') === 0) { + require_once('mod/dirfind.php'); + return dirfind_content($a); + } if(! $search) return $o; From ebae45babc026827a7ed5f05379ca9630fa9beba Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 2 Aug 2015 10:05:50 +0200 Subject: [PATCH 46/47] Clarified the settings. --- mod/admin.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mod/admin.php b/mod/admin.php index cdfd3b5a6..d421154b1 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -710,8 +710,8 @@ function admin_page_site(&$a) { '$maxloadavg' => array('maxloadavg', t("Maximum Load Average"), ((intval(get_config('system','maxloadavg')) > 0)?get_config('system','maxloadavg'):50), t("Maximum system load before delivery and poll processes are deferred - default 50.")), '$maxloadavg_frontend' => array('maxloadavg_frontend', t("Maximum Load Average (Frontend)"), ((intval(get_config('system','maxloadavg_frontend')) > 0)?get_config('system','maxloadavg_frontend'):50), t("Maximum system load before the frontend quits service - default 50.")), - '$poco_completion' => array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful when communicating with Redmatrix and friendica servers before 3.3)")), - '$poco_discovery' => array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available."), $poco_discovery_choices), + '$poco_completion' => array('poco_completion', t("Periodical check of global contacts"), get_config('system','poco_completion'), t("If enabled, the global contacts are checked periodically for missing or outdated data and the vitality of the contacts and servers.")), + '$poco_discovery' => array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available. The recommened setting is 'Users, Global Contacts'"), $poco_discovery_choices), '$poco_discovery_since' => array('poco_discovery_since', t("Timeframe for fetching global contacts"), (string) intval(get_config('system','poco_discovery_since')), t("When the discovery is activated, this value defines the timeframe for the global contacts that are fetched from other servers."), $poco_discovery_since_choices), '$poco_local_search' => array('poco_local_search', t("Search the local directory"), get_config('system','poco_local_search'), t("Search the local directory instead of the global directory. When searching locally, every search will be executed on the global directory in the background. This improves the search results when the search is repeated.")), From 2f147451f729167d103ccfe0669206476bd1e097 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sun, 2 Aug 2015 10:08:08 +0200 Subject: [PATCH 47/47] Just some more improved setting descriptions. --- mod/admin.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mod/admin.php b/mod/admin.php index d421154b1..d725c4581 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -711,8 +711,8 @@ function admin_page_site(&$a) { '$maxloadavg_frontend' => array('maxloadavg_frontend', t("Maximum Load Average (Frontend)"), ((intval(get_config('system','maxloadavg_frontend')) > 0)?get_config('system','maxloadavg_frontend'):50), t("Maximum system load before the frontend quits service - default 50.")), '$poco_completion' => array('poco_completion', t("Periodical check of global contacts"), get_config('system','poco_completion'), t("If enabled, the global contacts are checked periodically for missing or outdated data and the vitality of the contacts and servers.")), - '$poco_discovery' => array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available. The recommened setting is 'Users, Global Contacts'"), $poco_discovery_choices), - '$poco_discovery_since' => array('poco_discovery_since', t("Timeframe for fetching global contacts"), (string) intval(get_config('system','poco_discovery_since')), t("When the discovery is activated, this value defines the timeframe for the global contacts that are fetched from other servers."), $poco_discovery_since_choices), + '$poco_discovery' => array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available. The fallback increases the server load, so the recommened setting is 'Users, Global Contacts'."), $poco_discovery_choices), + '$poco_discovery_since' => array('poco_discovery_since', t("Timeframe for fetching global contacts"), (string) intval(get_config('system','poco_discovery_since')), t("When the discovery is activated, this value defines the timeframe for the activity of the global contacts that are fetched from other servers."), $poco_discovery_since_choices), '$poco_local_search' => array('poco_local_search', t("Search the local directory"), get_config('system','poco_local_search'), t("Search the local directory instead of the global directory. When searching locally, every search will be executed on the global directory in the background. This improves the search results when the search is repeated.")), '$use_fulltext_engine' => array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")),