diff --git a/boot.php b/boot.php index 42498a6cc6..2a11501033 100644 --- a/boot.php +++ b/boot.php @@ -19,7 +19,7 @@ define ( 'FRIENDICA_PLATFORM', 'Friendica'); define ( 'FRIENDICA_CODENAME', 'Lily of the valley'); define ( 'FRIENDICA_VERSION', '3.4.1' ); define ( 'DFRN_PROTOCOL_VERSION', '2.23' ); -define ( 'DB_UPDATE_VERSION', 1185 ); +define ( 'DB_UPDATE_VERSION', 1187 ); define ( 'EOL', "
\r\n" ); define ( 'ATOM_TIME', 'Y-m-d\TH:i:s\Z' ); diff --git a/database.sql b/database.sql index b65dee657b..91d151a104 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ --- Friendica 3.4.0 (Lily of the valley) --- DB_UPDATE_VERSION 1185 +-- Friendica 3.4.1 (Lily of the valley) +-- DB_UPDATE_VERSION 1186 -- ------------------------------------------ @@ -136,6 +136,7 @@ CREATE TABLE IF NOT EXISTS `contact` ( `hub-verify` varchar(255) NOT NULL DEFAULT '', `last-update` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `success_update` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', + `failure_update` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `name-date` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `uri-date` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `avatar-date` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', @@ -300,18 +301,24 @@ CREATE TABLE IF NOT EXISTS `gcign` ( CREATE TABLE IF NOT EXISTS `gcontact` ( `id` int(10) unsigned NOT NULL auto_increment PRIMARY KEY, `name` varchar(255) NOT NULL DEFAULT '', + `nick` varchar(255) NOT NULL DEFAULT '', `url` varchar(255) NOT NULL DEFAULT '', `nurl` varchar(255) NOT NULL DEFAULT '', `photo` varchar(255) NOT NULL DEFAULT '', `connect` varchar(255) NOT NULL DEFAULT '', + `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', `updated` datetime DEFAULT '0000-00-00 00:00:00', + `last_contact` datetime DEFAULT '0000-00-00 00:00:00', + `last_failure` datetime DEFAULT '0000-00-00 00:00:00', `location` varchar(255) NOT NULL DEFAULT '', `about` text NOT NULL, `keywords` text NOT NULL, `gender` varchar(32) NOT NULL DEFAULT '', `network` varchar(255) NOT NULL DEFAULT '', `generation` tinyint(3) NOT NULL DEFAULT 0, - INDEX `nurl` (`nurl`) + `server_url` varchar(255) NOT NULL DEFAULT '', + INDEX `nurl` (`nurl`), + INDEX `updated` (`updated`) ) DEFAULT CHARSET=utf8; -- @@ -352,6 +359,28 @@ CREATE TABLE IF NOT EXISTS `group_member` ( INDEX `uid_gid_contactid` (`uid`,`gid`,`contact-id`) ) DEFAULT CHARSET=utf8; +-- +-- TABLE gserver +-- +CREATE TABLE IF NOT EXISTS `gserver` ( + `id` int(10) unsigned NOT NULL auto_increment PRIMARY KEY, + `url` varchar(255) NOT NULL DEFAULT '', + `nurl` varchar(255) NOT NULL DEFAULT '', + `version` varchar(255) NOT NULL DEFAULT '', + `site_name` varchar(255) NOT NULL DEFAULT '', + `info` text NOT NULL, + `register_policy` tinyint(1) NOT NULL DEFAULT 0, + `poco` varchar(255) NOT NULL DEFAULT '', + `noscrape` varchar(255) NOT NULL DEFAULT '', + `network` varchar(32) NOT NULL DEFAULT '', + `platform` varchar(255) NOT NULL DEFAULT '', + `created` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', + `last_poco_query` datetime DEFAULT '0000-00-00 00:00:00', + `last_contact` datetime DEFAULT '0000-00-00 00:00:00', + `last_failure` datetime DEFAULT '0000-00-00 00:00:00', + INDEX `nurl` (`nurl`) +) DEFAULT CHARSET=utf8; + -- -- TABLE guid -- diff --git a/include/Contact.php b/include/Contact.php index 103dbe9add..1215f40e75 100644 --- a/include/Contact.php +++ b/include/Contact.php @@ -267,7 +267,12 @@ function contact_photo_menu($contact) { function random_profile() { - $r = q("select url from gcontact where url like '%%://%%/profile/%%' order by rand() limit 1"); + $r = q("SELECT `url` FROM `gcontact` WHERE `network` = '%s' + AND `last_contact` >= `last_failure` + AND `updated` > UTC_TIMESTAMP - INTERVAL 1 MONTH + ORDER BY rand() LIMIT 1", + dbesc(NETWORK_DFRN)); + if(count($r)) return dirname($r[0]['url']); return ''; diff --git a/include/Scrape.php b/include/Scrape.php index 90fb1a9e3d..83c099769d 100644 --- a/include/Scrape.php +++ b/include/Scrape.php @@ -249,7 +249,7 @@ function scrape_feed($url) { $ret['feed_atom'] = $url; return $ret; } - if(stristr($line,'application/rss+xml') || stristr($s,'loadHTML($s); + $xpath = new DomXPath($doc); + + $base = $xpath->query("//base"); + foreach ($base as $node) { + $attr = array(); + + if ($node->attributes->length) + foreach ($node->attributes as $attribute) + $attr[$attribute->name] = $attribute->value; + + if ($attr["href"] != "") + $basename = $attr["href"] ; } - if(! $dom) { - logger('scrape_feed: failed to parse.'); - return $ret; + $list = $xpath->query("//link"); + foreach ($list as $node) { + $attr = array(); + + if ($node->attributes->length) + foreach ($node->attributes as $attribute) + $attr[$attribute->name] = $attribute->value; + + if (($attr["rel"] == "alternate") AND ($attr["type"] == "application/atom+xml")) + $ret["feed_atom"] = $attr["href"]; + + if (($attr["rel"] == "alternate") AND ($attr["type"] == "application/rss+xml")) + $ret["feed_rss"] = $attr["href"]; } - - $head = $dom->getElementsByTagName('base'); - if($head) { - foreach($head as $head0) { - $basename = $head0->getAttribute('href'); - break; - } - } - if(! $basename) - $basename = implode('/', array_slice(explode('/',$url),0,3)) . '/'; - - $items = $dom->getElementsByTagName('link'); - - // get Atom/RSS link elements, take the first one of either. - - if($items) { - foreach($items as $item) { - $x = $item->getAttribute('rel'); - if(($x === 'alternate') && ($item->getAttribute('type') === 'application/atom+xml')) { - if(! x($ret,'feed_atom')) - $ret['feed_atom'] = $item->getAttribute('href'); - } - if(($x === 'alternate') && ($item->getAttribute('type') === 'application/rss+xml')) { - if(! x($ret,'feed_rss')) - $ret['feed_rss'] = $item->getAttribute('href'); - } - } - } - - // Drupal and perhaps others only provide relative URL's. Turn them into absolute. + // Drupal and perhaps others only provide relative URLs. Turn them into absolute. if(x($ret,'feed_atom') && (! strstr($ret['feed_atom'],'://'))) $ret['feed_atom'] = $basename . $ret['feed_atom']; @@ -509,6 +502,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) { } if($mode == PROBE_NORMAL) { + if(strlen($zot)) { $s = fetch_url($zot); if($s) { @@ -528,6 +522,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) { } } + if(strlen($dfrn)) { $ret = scrape_dfrn(($hcard) ? $hcard : $dfrn); if(is_array($ret) && x($ret,'dfrn-request')) { @@ -634,6 +629,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) { if($check_feed) { $feedret = scrape_feed(($poll) ? $poll : $url); + logger('probe_url: scrape_feed ' . (($poll)? $poll : $url) . ' returns: ' . print_r($feedret,true), LOGGER_DATA); if(count($feedret) && ($feedret['feed_atom'] || $feedret['feed_rss'])) { $poll = ((x($feedret,'feed_atom')) ? unamp($feedret['feed_atom']) : unamp($feedret['feed_rss'])); @@ -653,7 +649,7 @@ function probe_url($url, $mode = PROBE_NORMAL, $level = 1) { logger('probe_url: scrape_feed: headers: ' . $a->get_curl_headers(), LOGGER_DATA); // Don't try and parse an empty string - $feed->set_raw_data(($xml) ? $xml : ''); + $feed->set_raw_data(($xml) ? $xml : ''); $feed->init(); if($feed->error()) diff --git a/include/dbstructure.php b/include/dbstructure.php index 0f81ee6249..2370b01bec 100644 --- a/include/dbstructure.php +++ b/include/dbstructure.php @@ -451,6 +451,7 @@ function db_definition() { "hub-verify" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "last-update" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "success_update" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), + "failure_update" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "name-date" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "uri-date" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "avatar-date" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), @@ -625,21 +626,28 @@ function db_definition() { "fields" => array( "id" => array("type" => "int(10) unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"), "name" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "nick" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "nurl" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "photo" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "connect" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), "updated" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), + "last_contact" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), + "last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), "location" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "about" => array("type" => "text", "not null" => "1"), "keywords" => array("type" => "text", "not null" => "1"), "gender" => array("type" => "varchar(32)", "not null" => "1", "default" => ""), + "community" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"), "network" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), "generation" => array("type" => "tinyint(3)", "not null" => "1", "default" => "0"), + "server_url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), ), "indexes" => array( "PRIMARY" => array("id"), "nurl" => array("nurl"), + "updated" => array("updated"), ) ); $database["glink"] = array( @@ -683,6 +691,29 @@ function db_definition() { "uid_gid_contactid" => array("uid","gid","contact-id"), ) ); + $database["gserver"] = array( + "fields" => array( + "id" => array("type" => "int(10) unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"), + "url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "nurl" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "version" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "site_name" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "info" => array("type" => "text", "not null" => "1"), + "register_policy" => array("type" => "tinyint(1)", "not null" => "1", "default" => "0"), + "poco" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "noscrape" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "network" => array("type" => "varchar(32)", "not null" => "1", "default" => ""), + "platform" => array("type" => "varchar(255)", "not null" => "1", "default" => ""), + "created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"), + "last_poco_query" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), + "last_contact" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), + "last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"), + ), + "indexes" => array( + "PRIMARY" => array("id"), + "nurl" => array("nurl"), + ) + ); $database["guid"] = array( "fields" => array( "id" => array("type" => "int(10) unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"), diff --git a/include/discover_poco.php b/include/discover_poco.php new file mode 100644 index 0000000000..a4cf5db54e --- /dev/null +++ b/include/discover_poco.php @@ -0,0 +1,192 @@ + $maxsysload) { + logger('system: load ' . $load[0] . ' too high. discover_poco deferred to next scheduled run.'); + return; + } + } + + if(($argc > 2) && ($argv[1] == "dirsearch")) { + $search = urldecode($argv[2]); + $mode = 1; + } elseif(($argc == 2) && ($argv[1] == "checkcontact")) { + $mode = 2; + } elseif ($argc == 1) { + $search = ""; + $mode = 0; + } else + die("Unknown or missing parameter ".$argv[1]."\n"); + + $lockpath = get_lockpath(); + if ($lockpath != '') { + $pidfile = new pidfile($lockpath, 'discover_poco'.$mode.urlencode($search)); + if($pidfile->is_already_running()) { + logger("discover_poco: Already running"); + if ($pidfile->running_time() > 19*60) { + $pidfile->kill(); + logger("discover_poco: killed stale process"); + // Calling a new instance + if ($mode == 0) + proc_run('php','include/discover_poco.php'); + } + exit; + } + } + + $a->set_baseurl(get_config('system','url')); + + load_hooks(); + + logger('start '.$search); + + if (($mode == 2) AND get_config('system','poco_completion')) + discover_users(); + elseif (($mode == 1) AND ($search != "") and get_config('system','poco_local_search')) + discover_directory($search); + elseif (($mode == 0) AND ($search == "") and (get_config('system','poco_discovery') > 0)) + poco_discover(); + + logger('end '.$search); + + return; +} + +function discover_users() { + logger("Discover users", LOGGER_DEBUG); + + $users = q("SELECT `url`, `created`, `updated`, `last_failure`, `last_contact`, `server_url` FROM `gcontact` + WHERE `last_contact` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND + `last_failure` < UTC_TIMESTAMP - INTERVAL 1 MONTH AND + `network` IN ('%s', '%s', '%s', '%s', '') ORDER BY rand()", + dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), + dbesc(NETWORK_OSTATUS), dbesc(NETWORK_FEED)); + + if (!$users) + return; + + $checked = 0; + + foreach ($users AS $user) { + + $urlparts = parse_url($user["url"]); + if (!isset($urlparts["scheme"])) { + q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'", + dbesc(NETWORK_PHANTOM), dbesc(normalise_link($user["url"]))); + continue; + } + + if (in_array($urlparts["host"], array("www.facebook.com", "facebook.com", "twitter.com", + "identi.ca", "alpha.app.net"))) { + $networks = array("www.facebook.com" => NETWORK_FACEBOOK, + "facebook.com" => NETWORK_FACEBOOK, + "twitter.com" => NETWORK_TWITTER, + "identi.ca" => NETWORK_PUMPIO, + "alpha.app.net" => NETWORK_APPNET); + + q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'", + dbesc($networks[$urlparts["host"]]), dbesc(normalise_link($user["url"]))); + continue; + } + + if ($user["server_url"] != "") + $server_url = $user["server_url"]; + else + $server_url = poco_detect_server($user["url"]); + + if (poco_check_server($server_url, $gcontacts[0]["network"])) { + logger('Check user '.$user["url"]); + poco_last_updated($user["url"], true); + + if (++$checked > 100) + return; + } else + q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($user["url"]))); + } +} + +function discover_directory($search) { + + $data = Cache::get("dirsearch:".$search); + if (!is_null($data)){ + // Only search for the same item every 24 hours + if (time() < $data + (60 * 60 * 24)) { + logger("Already searched for ".$search." in the last 24 hours", LOGGER_DEBUG); + return; + } + } + + $x = fetch_url("http://dir.friendica.com/lsearch?p=1&n=500&search=".urlencode($search)); + $j = json_decode($x); + + if(count($j->results)) + foreach($j->results as $jj) { + // Check if the contact already exists + $exists = q("SELECT `id`, `last_contact`, `last_failure`, `updated` FROM `gcontact` WHERE `nurl` = '%s'", normalise_link($jj->url)); + if ($exists) { + logger("Profile ".$jj->url." already exists (".$search.")", LOGGER_DEBUG); + + if (($exists[0]["last_contact"] < $exists[0]["last_failure"]) AND + ($exists[0]["updated"] < $exists[0]["last_failure"])) + continue; + + // Update the contact + poco_last_updated($jj->url); + continue; + } + + // Harcoded paths aren't so good. But in this case it is okay. + // First: We only will get Friendica contacts (which always are using this url schema) + // Second: There will be no further problems if we are doing a mistake + $server_url = preg_replace("=(https?://)(.*)/profile/(.*)=ism", "$1$2", $jj->url); + if ($server_url != $jj->url) + if (!poco_check_server($server_url)) { + logger("Friendica server ".$server_url." doesn't answer.", LOGGER_DEBUG); + continue; + } + logger("Friendica server ".$server_url." seems to be okay.", LOGGER_DEBUG); + + logger("Check if profile ".$jj->url." is reachable (".$search.")", LOGGER_DEBUG); + $data = probe_url($jj->url); + if ($data["network"] == NETWORK_DFRN) { + logger("Add profile ".$jj->url." to local directory (".$search.")", LOGGER_DEBUG); + poco_check($data["url"], $data["name"], $data["network"], $data["photo"], "", "", "", $jj->tags, $data["addr"], "", 0); + } + } + Cache::set("dirsearch:".$search, time()); +} + +if (array_search(__file__,get_included_files())===0){ + discover_poco_run($_SERVER["argv"],$_SERVER["argc"]); + killme(); +} diff --git a/include/onepoll.php b/include/onepoll.php index 9723fadd51..90d018524d 100644 --- a/include/onepoll.php +++ b/include/onepoll.php @@ -19,10 +19,10 @@ function onepoll_run(&$argv, &$argc){ if(is_null($db)) { @include(".htconfig.php"); - require_once("include/dba.php"); + require_once("include/dba.php"); $db = new dba($db_host, $db_user, $db_pass, $db_data); - unset($db_host, $db_user, $db_pass, $db_data); - }; + unset($db_host, $db_user, $db_pass, $db_data); + }; require_once('include/session.php'); @@ -79,28 +79,58 @@ function onepoll_run(&$argv, &$argc){ $contacts = q("SELECT `contact`.* FROM `contact` WHERE ( `rel` = %d OR `rel` = %d ) AND `poll` != '' - AND NOT `network` IN ( '%s', '%s', '%s' ) + AND NOT `network` IN ( '%s', '%s' ) AND `contact`.`id` = %d AND `self` = 0 AND `contact`.`blocked` = 0 AND `contact`.`readonly` = 0 AND `contact`.`archive` = 0 LIMIT 1", intval(CONTACT_IS_SHARING), intval(CONTACT_IS_FRIEND), - dbesc(NETWORK_DIASPORA), dbesc(NETWORK_FACEBOOK), dbesc(NETWORK_PUMPIO), intval($contact_id) ); - if(! count($contacts)) { - // Maybe it is a Redmatrix account. Then we can fetch their contacts via poco - $contacts = q("SELECT `id`, `poco` FROM `contact` WHERE `id` = %d AND `poco` != ''", intval($contact_id)); - if ($contacts) - poco_load($contacts[0]['id'],$importer_uid,0,$contacts[0]['poco']); + if(! count($contacts)) return; - } $contact = $contacts[0]; + // load current friends if possible. + if (($contact['poco'] != "") AND ($contact['success_update'] > $contact['failure_update'])) { + $r = q("SELECT count(*) as total from glink + where `cid` = %d and updated > UTC_TIMESTAMP() - INTERVAL 1 DAY", + intval($contact['id']) + ); + if (count($r)) + if (!$r[0]['total']) + poco_load($contact['id'],$importer_uid,0,$contact['poco']); + } + + // To-Do: + // - Check why we don't poll the Diaspora feed at the moment (some guid problem in the items?) + // - Check whether this is possible with Redmatrix + if ($contact["network"] == NETWORK_DIASPORA) { + if (poco_do_update($contact["created"], $contact["last-item"], $contact["failure_update"], $contact["success_update"])) { + $last_updated = poco_last_updated($contact["url"]); + $updated = datetime_convert(); + if ($last_updated) { + q("UPDATE `contact` SET `last-item` = '%s', `last-update` = '%s', `success_update` = '%s' WHERE `id` = %d", + dbesc($last_updated), + dbesc($updated), + dbesc($updated), + intval($contact['id']) + ); + } else { + q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d", + dbesc($updated), + dbesc($updated), + intval($contact['id']) + ); + } + } + return; + } + $xml = false; $t = $contact['last-update']; @@ -177,7 +207,8 @@ function onepoll_run(&$argv, &$argc){ mark_for_death($contact); // set the last-update so we don't keep polling - $r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d", + $r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d", + dbesc(datetime_convert()), dbesc(datetime_convert()), intval($contact['id']) ); @@ -190,7 +221,8 @@ function onepoll_run(&$argv, &$argc){ mark_for_death($contact); - $r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d", + $r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d", + dbesc(datetime_convert()), dbesc(datetime_convert()), intval($contact['id']) ); @@ -207,7 +239,8 @@ function onepoll_run(&$argv, &$argc){ // set the last-update so we don't keep polling - $r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d", + $r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d", + dbesc(datetime_convert()), dbesc(datetime_convert()), intval($contact['id']) ); @@ -554,14 +587,14 @@ function onepoll_run(&$argv, &$argc){ logger('poller: received xml : ' . $xml, LOGGER_DATA); if(! strstr($xml,'<')) { logger('poller: post_handshake: response from ' . $url . ' did not contain XML.'); - $r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d", + $r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d", + dbesc(datetime_convert()), dbesc(datetime_convert()), intval($contact['id']) ); return; } - consume_feed($xml,$importer,$contact,$hub,1,1); @@ -588,29 +621,38 @@ function onepoll_run(&$argv, &$argc){ } } } - } - $updated = datetime_convert(); + $updated = datetime_convert(); - $r = q("UPDATE `contact` SET `last-update` = '%s', `success_update` = '%s' WHERE `id` = %d", - dbesc($updated), - dbesc($updated), - intval($contact['id']) - ); - - - // load current friends if possible. - - if($contact['poco']) { - $r = q("SELECT count(*) as total from glink - where `cid` = %d and updated > UTC_TIMESTAMP() - INTERVAL 1 DAY", + $r = q("UPDATE `contact` SET `last-update` = '%s', `success_update` = '%s' WHERE `id` = %d", + dbesc($updated), + dbesc($updated), + intval($contact['id']) + ); + + q("UPDATE `gcontact` SET `last_contact` = '%s' WHERE `nurl` = '%s'", + dbesc($updated), + dbesc($contact['nurl']) + ); + + } elseif (in_array($contact["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, NETWORK_FEED))) { + $updated = datetime_convert(); + + $r = q("UPDATE `contact` SET `last-update` = '%s', `failure_update` = '%s' WHERE `id` = %d", + dbesc($updated), + dbesc($updated), + intval($contact['id']) + ); + + q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc($updated), + dbesc($contact['nurl']) + ); + } else { + $r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d", + dbesc($updated), intval($contact['id']) ); - } - if(count($r)) { - if(! $r[0]['total']) { - poco_load($contact['id'],$importer_uid,0,$contact['poco']); - } } return; diff --git a/include/plaintext.php b/include/plaintext.php index c8cdfa57df..4f435fc6a2 100644 --- a/include/plaintext.php +++ b/include/plaintext.php @@ -52,7 +52,7 @@ function get_attached_data($body) { if (preg_match_all("(\[url=([$URLSearchString]*)\]\s*\[img\]([$URLSearchString]*)\[\/img\]\s*\[\/url\])ism", $body, $pictures, PREG_SET_ORDER)) { if (count($pictures) == 1) { // Checking, if the link goes to a picture - $data = parseurl_getsiteinfo($pictures[0][1], true); + $data = parseurl_getsiteinfo_cached($pictures[0][1], true); if ($data["type"] == "photo") { $post["type"] = "photo"; if (isset($data["images"][0])) @@ -95,7 +95,7 @@ function get_attached_data($body) { } } elseif (isset($post["url"]) AND ($post["type"] == "video")) { require_once("mod/parse_url.php"); - $data = parseurl_getsiteinfo($post["url"], true); + $data = parseurl_getsiteinfo_cached($post["url"], true); if (isset($data["images"][0])) $post["image"] = $data["images"][0]["src"]; diff --git a/include/poller.php b/include/poller.php index d971d4f004..7cc1a28527 100644 --- a/include/poller.php +++ b/include/poller.php @@ -82,6 +82,14 @@ function poller_run(&$argv, &$argc){ proc_run('php',"include/dsprphotoq.php"); + // run the process to discover global contacts in the background + + proc_run('php',"include/discover_poco.php"); + + // run the process to update locally stored global contacts in the background + + proc_run('php',"include/discover_poco.php", "checkcontact"); + // expire any expired accounts q("UPDATE user SET `account_expired` = 1 where `account_expired` = 0 diff --git a/include/socgraph.php b/include/socgraph.php index 1e1b6076f2..d114ef09cc 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -1,6 +1,15 @@ = %d OR `generation` = 0) AND `nurl` = '%s'", dbesc($name), @@ -250,6 +287,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca dbesc($profile_photo), dbesc($connect_url), dbesc($profile_url), + dbesc($server_url), dbesc($updated), dbesc($location), dbesc($about), @@ -261,14 +299,17 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca ); } } else { - q("INSERT INTO `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `updated`, `location`, `about`, `keywords`, `gender`, `generation`) - VALUES ('%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s', '%s', %d)", + q("INSERT INTO `gcontact` (`name`, `nick`, `network`, `url`, `nurl`, `photo`, `connect`, `server_url`, `created`, `updated`, `location`, `about`, `keywords`, `gender`, `generation`) + VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)", dbesc($name), + dbesc($nick), dbesc($network), dbesc($profile_url), dbesc(normalise_link($profile_url)), dbesc($profile_photo), dbesc($connect_url), + dbesc($server_url), + dbesc(datetime_convert()), dbesc($updated), dbesc($location), dbesc($about), @@ -320,6 +361,554 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca return $gcid; } +function poco_reachable($profile, $server = "", $network = "", $force = false) { + + if ($server == "") + $server = poco_detect_server($profile); + + if ($server == "") + return true; + + return poco_check_server($server, $network, $force); +} + +function poco_detect_server($profile) { + + // Try to detect the server path based upon some known standard paths + $server_url = ""; + + if ($server_url == "") { + $friendica = preg_replace("=(https?://)(.*)/profile/(.*)=ism", "$1$2", $profile); + if ($friendica != $profile) { + $server_url = $friendica; + $network = NETWORK_DFRN; + } + } + + if ($server_url == "") { + $diaspora = preg_replace("=(https?://)(.*)/u/(.*)=ism", "$1$2", $profile); + if ($diaspora != $profile) { + $server_url = $diaspora; + $network = NETWORK_DIASPORA; + } + } + + if ($server_url == "") { + $red = preg_replace("=(https?://)(.*)/channel/(.*)=ism", "$1$2", $profile); + if ($red != $profile) { + $server_url = $red; + $network = NETWORK_DIASPORA; + } + } + + return $server_url; +} + +function poco_last_updated($profile, $force = false) { + + $gcontacts = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s'", + dbesc(normalise_link($profile))); + + if ($gcontacts[0]["created"] == "0000-00-00 00:00:00") + q("UPDATE `gcontact` SET `created` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($profile))); + + if ($gcontacts[0]["server_url"] != "") + $server_url = $gcontacts[0]["server_url"]; + else + $server_url = poco_detect_server($profile); + + if ($server_url != "") { + if (!poco_check_server($server_url, $gcontacts[0]["network"], $force)) { + + if ($force) + q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($profile))); + + return false; + } + + q("UPDATE `gcontact` SET `server_url` = '%s' WHERE `nurl` = '%s'", + dbesc($server_url), dbesc(normalise_link($profile))); + } + + if (in_array($gcontacts[0]["network"], array("", NETWORK_FEED))) { + $server = q("SELECT `network` FROM `gserver` WHERE `nurl` = '%s' AND `network` != ''", + dbesc(normalise_link($server_url))); + + if ($server) + q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'", + dbesc($server[0]["network"]), dbesc(normalise_link($profile))); + else + return; + } + + // noscrape is really fast so we don't cache the call. + if (($gcontacts[0]["server_url"] != "") AND ($gcontacts[0]["nick"] != "")) { + + // Use noscrape if possible + $server = q("SELECT `noscrape` FROM `gserver` WHERE `nurl` = '%s' AND `noscrape` != ''", dbesc(normalise_link($gcontacts[0]["server_url"]))); + + if ($server) { + $noscraperet = z_fetch_url($server[0]["noscrape"]."/".$gcontacts[0]["nick"]); + + if ($noscraperet["success"] AND ($noscraperet["body"] != "")) { +; + $noscrape = json_decode($noscraperet["body"], true); + + if (($noscrape["fn"] != "") AND ($noscrape["fn"] != $gcontacts[0]["name"])) + q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'", + dbesc($noscrape["fn"]), dbesc(normalise_link($profile))); + + if (($noscrape["photo"] != "") AND ($noscrape["photo"] != $gcontacts[0]["photo"])) + q("UPDATE `gcontact` SET `photo` = '%s' WHERE `nurl` = '%s'", + dbesc($noscrape["photo"]), dbesc(normalise_link($profile))); + + if (($noscrape["updated"] != "") AND ($noscrape["updated"] != $gcontacts[0]["updated"])) + q("UPDATE `gcontact` SET `updated` = '%s' WHERE `nurl` = '%s'", + dbesc($noscrape["updated"]), dbesc(normalise_link($profile))); + + if (($noscrape["gender"] != "") AND ($noscrape["gender"] != $gcontacts[0]["gender"])) + q("UPDATE `gcontact` SET `gender` = '%s' WHERE `nurl` = '%s'", + dbesc($noscrape["gender"]), dbesc(normalise_link($profile))); + + if (($noscrape["pdesc"] != "") AND ($noscrape["pdesc"] != $gcontacts[0]["about"])) + q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'", + dbesc($noscrape["pdesc"]), dbesc(normalise_link($profile))); + + if (($noscrape["about"] != "") AND ($noscrape["about"] != $gcontacts[0]["about"])) + q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'", + dbesc($noscrape["about"]), dbesc(normalise_link($profile))); + + if (isset($noscrape["comm"]) AND ($noscrape["comm"] != $gcontacts[0]["community"])) + q("UPDATE `gcontact` SET `community` = %d WHERE `nurl` = '%s'", + intval($noscrape["comm"]), dbesc(normalise_link($profile))); + + if (isset($noscrape["tags"])) + $keywords = implode(" ", $noscrape["tags"]); + else + $keywords = ""; + + if (($keywords != "") AND ($keywords != $gcontacts[0]["keywords"])) + q("UPDATE `gcontact` SET `keywords` = '%s' WHERE `nurl` = '%s'", + dbesc($keywords), dbesc(normalise_link($profile))); + + $location = $noscrape["locality"]; + + if ($noscrape["region"] != "") { + if ($location != "") + $location .= ", "; + + $location .= $noscrape["region"]; + } + + if ($noscrape["country-name"] != "") { + if ($location != "") + $location .= ", "; + + $location .= $noscrape["country-name"]; + } + + if (($location != "") AND ($location != $gcontacts[0]["location"])) + q("UPDATE `gcontact` SET `location` = '%s' WHERE `nurl` = '%s'", + dbesc($location), dbesc(normalise_link($profile))); + + // If we got data from noscrape then mark the contact as reachable + if (is_array($noscrape) AND count($noscrape)) + q("UPDATE `gcontact` SET `last_contact` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($profile))); + + return $noscrape["updated"]; + } + } + } + + // If we only can poll the feed, then we only do this once a while + if (!$force AND !poco_do_update($gcontacts[0]["created"], $gcontacts[0]["updated"], $gcontacts[0]["last_failure"], $gcontacts[0]["last_contact"])) + return $gcontacts[0]["updated"]; + + $data = probe_url($profile); + + if (($data["poll"] == "") OR ($data["network"] == NETWORK_FEED)) { + q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($profile))); + return false; + } + + if (($data["name"] != "") AND ($data["name"] != $gcontacts[0]["name"])) + q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'", + dbesc($data["name"]), dbesc(normalise_link($profile))); + + if (($data["nick"] != "") AND ($data["nick"] != $gcontacts[0]["nick"])) + q("UPDATE `gcontact` SET `nick` = '%s' WHERE `nurl` = '%s'", + dbesc($data["nick"]), dbesc(normalise_link($profile))); + + if (($data["addr"] != "") AND ($data["addr"] != $gcontacts[0]["connect"])) + q("UPDATE `gcontact` SET `connect` = '%s' WHERE `nurl` = '%s'", + dbesc($data["addr"]), dbesc(normalise_link($profile))); + + if (($data["photo"] != "") AND ($data["photo"] != $gcontacts[0]["photo"])) + q("UPDATE `gcontact` SET `photo` = '%s' WHERE `nurl` = '%s'", + dbesc($data["photo"]), dbesc(normalise_link($profile))); + + if (($data["baseurl"] != "") AND ($data["baseurl"] != $gcontacts[0]["server_url"])) + q("UPDATE `gcontact` SET `server_url` = '%s' WHERE `nurl` = '%s'", + dbesc($data["baseurl"]), dbesc(normalise_link($profile))); + + $feedret = z_fetch_url($data["poll"]); + + if (!$feedret["success"]) { + q("UPDATE `gcontact` SET `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($profile))); + return false; + } + + $doc = new DOMDocument(); + @$doc->loadXML($feedret["body"]); + + $xpath = new DomXPath($doc); + $xpath->registerNamespace('atom', "http://www.w3.org/2005/Atom"); + + $entries = $xpath->query('/atom:feed/atom:entry'); + + $last_updated = ""; + + foreach ($entries AS $entry) { + $published = $xpath->query('atom:published/text()', $entry)->item(0)->nodeValue; + $updated = $xpath->query('atom:updated/text()', $entry)->item(0)->nodeValue; + + if ($last_updated < $published) + $last_updated = $published; + + if ($last_updated < $updated) + $last_updated = $updated; + } + + // Maybe there aren't any entries. Then check if it is a valid feed + if ($last_updated == "") + if ($xpath->query('/atom:feed')->length > 0) + $last_updated = "0000-00-00 00:00:00"; + + q("UPDATE `gcontact` SET `updated` = '%s', `last_contact` = '%s' WHERE `nurl` = '%s'", + dbesc($last_updated), dbesc(datetime_convert()), dbesc(normalise_link($profile))); + + if (($gcontacts[0]["generation"] == 0)) + q("UPDATE `gcontact` SET `generation` = 9 WHERE `nurl` = '%s'", + dbesc(normalise_link($profile))); + + return($last_updated); +} + +function poco_do_update($created, $updated, $last_failure, $last_contact) { + $now = strtotime(datetime_convert()); + + if ($updated > $last_contact) + $contact_time = strtotime($updated); + else + $contact_time = strtotime($last_contact); + + $failure_time = strtotime($last_failure); + $created_time = strtotime($created); + + // If there is no "created" time then use the current time + if ($created_time <= 0) + $created_time = $now; + + // If the last contact was less than 24 hours then don't update + if (($now - $contact_time) < (60 * 60 * 24)) + return false; + + // If the last failure was less than 24 hours then don't update + if (($now - $failure_time) < (60 * 60 * 24)) + return false; + + // If the last contact was less than a week ago and the last failure is older than a week then don't update + //if ((($now - $contact_time) < (60 * 60 * 24 * 7)) AND ($contact_time > $failure_time)) + // return false; + + // If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week + if ((($now - $contact_time) > (60 * 60 * 24 * 7)) AND (($now - $created_time) > (60 * 60 * 24 * 7)) AND (($now - $failure_time) < (60 * 60 * 24 * 7))) + return false; + + // If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month + if ((($now - $contact_time) > (60 * 60 * 24 * 30)) AND (($now - $created_time) > (60 * 60 * 24 * 30)) AND (($now - $failure_time) < (60 * 60 * 24 * 30))) + return false; + + return true; +} + +function poco_to_boolean($val) { + if (($val == "true") OR ($val == 1)) + return(true); + if (($val == "false") OR ($val == 0)) + return(false); + + return ($val); +} + +function poco_check_server($server_url, $network = "", $force = false) { + + if ($server_url == "") + return false; + + $servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url))); + if ($servers) { + + if ($servers[0]["created"] == "0000-00-00 00:00:00") + q("UPDATE `gserver` SET `created` = '%s' WHERE `nurl` = '%s'", + dbesc(datetime_convert()), dbesc(normalise_link($server_url))); + + $poco = $servers[0]["poco"]; + $noscrape = $servers[0]["noscrape"]; + + if ($network == "") + $network = $servers[0]["network"]; + + $last_contact = $servers[0]["last_contact"]; + $last_failure = $servers[0]["last_failure"]; + $version = $servers[0]["version"]; + $platform = $servers[0]["platform"]; + $site_name = $servers[0]["site_name"]; + $info = $servers[0]["info"]; + $register_policy = $servers[0]["register_policy"]; + + if (!$force AND !poco_do_update($servers[0]["created"], "", $last_failure, $last_contact)) { + logger("Use cached data for server ".$server_url, LOGGER_DEBUG); + return ($last_contact >= $last_failure); + } + } else { + $poco = ""; + $noscrape = ""; + $version = ""; + $platform = ""; + $site_name = ""; + $info = ""; + $register_policy = -1; + + $last_contact = "0000-00-00 00:00:00"; + $last_failure = "0000-00-00 00:00:00"; + } + logger("Server ".$server_url." is unknown. Start discovery.", LOGGER_DEBUG); + + $failure = false; + $orig_last_failure = $last_failure; + + // Check if the page is accessible via SSL. + $server_url = str_replace("http://", "https://", $server_url); + $serverret = z_fetch_url($server_url."/.well-known/host-meta"); + + // Maybe the page is unencrypted only? + $xmlobj = @simplexml_load_string($serverret["body"],'SimpleXMLElement',0, "http://docs.oasis-open.org/ns/xri/xrd-1.0"); + if (!$serverret["success"] OR ($serverret["body"] == "") OR (@sizeof($xmlobj) == 0) OR !is_object($xmlobj)) { + $server_url = str_replace("https://", "http://", $server_url); + $serverret = z_fetch_url($server_url."/.well-known/host-meta"); + + $xmlobj = @simplexml_load_string($serverret["body"],'SimpleXMLElement',0, "http://docs.oasis-open.org/ns/xri/xrd-1.0"); + } + + if (!$serverret["success"] OR ($serverret["body"] == "") OR (sizeof($xmlobj) == 0) OR !is_object($xmlobj)) { + $last_failure = datetime_convert(); + $failure = true; + } elseif ($network == NETWORK_DIASPORA) + $last_contact = datetime_convert(); + + if (!$failure) { + // Test for Diaspora + $serverret = z_fetch_url($server_url); + + $lines = explode("\n",$serverret["header"]); + if(count($lines)) + foreach($lines as $line) { + $line = trim($line); + if(stristr($line,'X-Diaspora-Version:')) { + $platform = "Diaspora"; + $version = trim(str_replace("X-Diaspora-Version:", "", $line)); + $version = trim(str_replace("x-diaspora-version:", "", $version)); + $network = NETWORK_DIASPORA; + } + } + } + + if (!$failure) { + // Test for Statusnet + // Will also return data for Friendica and GNU Social - but it will be overwritten later + // The "not implemented" is a special treatment for really, really old Friendica versions + $serverret = z_fetch_url($server_url."/api/statusnet/version.json"); + if ($serverret["success"] AND ($serverret["body"] != '{"error":"not implemented"}') AND ($serverret["body"] != '') AND (strlen($serverret["body"]) < 250)) { + $platform = "StatusNet"; + $version = trim($serverret["body"], '"'); + $network = NETWORK_OSTATUS; + } + + // Test for GNU Social + $serverret = z_fetch_url($server_url."/api/gnusocial/version.json"); + if ($serverret["success"] AND ($serverret["body"] != '{"error":"not implemented"}') AND ($serverret["body"] != '') AND (strlen($serverret["body"]) < 250)) { + $platform = "GNU Social"; + $version = trim($serverret["body"], '"'); + $network = NETWORK_OSTATUS; + } + + $serverret = z_fetch_url($server_url."/api/statusnet/config.json"); + if ($serverret["success"]) { + $data = json_decode($serverret["body"]); + + if (isset($data->site->server)) { + $last_contact = datetime_convert(); + + if (isset($data->site->hubzilla)) { + $platform = $data->site->hubzilla->PLATFORM_NAME; + $version = $data->site->hubzilla->RED_VERSION; + $network = NETWORK_DIASPORA; + } + if (isset($data->site->redmatrix)) { + if (isset($data->site->redmatrix->PLATFORM_NAME)) + $platform = $data->site->redmatrix->PLATFORM_NAME; + elseif (isset($data->site->redmatrix->RED_PLATFORM)) + $platform = $data->site->redmatrix->RED_PLATFORM; + + $version = $data->site->redmatrix->RED_VERSION; + $network = NETWORK_DIASPORA; + } + if (isset($data->site->friendica)) { + $platform = $data->site->friendica->FRIENDICA_PLATFORM; + $version = $data->site->friendica->FRIENDICA_VERSION; + $network = NETWORK_DFRN; + } + + $site_name = $data->site->name; + + $data->site->closed = poco_to_boolean($data->site->closed); + $data->site->private = poco_to_boolean($data->site->private); + $data->site->inviteonly = poco_to_boolean($data->site->inviteonly); + + if (!$data->site->closed AND !$data->site->private and $data->site->inviteonly) + $register_policy = REGISTER_APPROVE; + elseif (!$data->site->closed AND !$data->site->private) + $register_policy = REGISTER_OPEN; + else + $register_policy = REGISTER_CLOSED; + } + } + } + + // Query statistics.json. Optional package for Diaspora, Friendica and Redmatrix + if (!$failure) { + $serverret = z_fetch_url($server_url."/statistics.json"); + if ($serverret["success"]) { + $data = json_decode($serverret["body"]); + if ($version == "") + $version = $data->version; + + $site_name = $data->name; + + if (isset($data->network) AND ($platform == "")) + $platform = $data->network; + + if ($platform == "Diaspora") + $network = NETWORK_DIASPORA; + + if ($data->registrations_open) + $register_policy = REGISTER_OPEN; + else + $register_policy = REGISTER_CLOSED; + + if (isset($data->version)) + $last_contact = datetime_convert(); + } + } + + // Check for noscrape + // Friendica servers could be detected as OStatus servers + if (!$failure AND in_array($network, array(NETWORK_DFRN, NETWORK_OSTATUS))) { + $serverret = z_fetch_url($server_url."/friendica/json"); + + if (!$serverret["success"]) + $serverret = z_fetch_url($server_url."/friendika/json"); + + if ($serverret["success"]) { + $data = json_decode($serverret["body"]); + + if (isset($data->version)) { + $last_contact = datetime_convert(); + $network = NETWORK_DFRN; + + $noscrape = $data->no_scrape_url; + $version = $data->version; + $site_name = $data->site_name; + $info = $data->info; + $register_policy_str = $data->register_policy; + $platform = $data->platform; + + switch ($register_policy_str) { + case "REGISTER_CLOSED": + $register_policy = REGISTER_CLOSED; + break; + case "REGISTER_APPROVE": + $register_policy = REGISTER_APPROVE; + break; + case "REGISTER_OPEN": + $register_policy = REGISTER_OPEN; + break; + } + } + } + } + + // Look for poco + if (!$failure) { + $serverret = z_fetch_url($server_url."/poco"); + if ($serverret["success"]) { + $data = json_decode($serverret["body"]); + if (isset($data->totalResults)) { + $poco = $server_url."/poco"; + $last_contact = datetime_convert(); + } + } + } + + // Check again if the server exists + $servers = q("SELECT `nurl` FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url))); + + if ($servers) + q("UPDATE `gserver` SET `url` = '%s', `version` = '%s', `site_name` = '%s', `info` = '%s', `register_policy` = %d, `poco` = '%s', `noscrape` = '%s', + `network` = '%s', `platform` = '%s', `last_contact` = '%s', `last_failure` = '%s' WHERE `nurl` = '%s'", + dbesc($server_url), + dbesc($version), + dbesc($site_name), + dbesc($info), + intval($register_policy), + dbesc($poco), + dbesc($noscrape), + dbesc($network), + dbesc($platform), + dbesc($last_contact), + dbesc($last_failure), + dbesc(normalise_link($server_url)) + ); + else + q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `created`, `last_contact`, `last_failure`) + VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s', '%s')", + dbesc($server_url), + dbesc(normalise_link($server_url)), + dbesc($version), + dbesc($site_name), + dbesc($info), + intval($register_policy), + dbesc($poco), + dbesc($noscrape), + dbesc($network), + dbesc($platform), + dbesc(datetime_convert()), + dbesc($last_contact), + dbesc($last_failure), + dbesc(datetime_convert()) + ); + + logger("End discovery for server ".$server_url, LOGGER_DEBUG); + + return !$failure; +} + function poco_contact_from_body($body, $created, $cid, $uid) { preg_replace_callback("/\[share(.*?)\].*?\[\/share\]/ism", function ($match) use ($created, $cid, $uid){ @@ -328,20 +917,20 @@ function poco_contact_from_body($body, $created, $cid, $uid) { } function sub_poco_from_share($share, $created, $cid, $uid) { - $profile = ""; - preg_match("/profile='(.*?)'/ism", $share[1], $matches); - if ($matches[1] != "") - $profile = $matches[1]; + $profile = ""; + preg_match("/profile='(.*?)'/ism", $share[1], $matches); + if ($matches[1] != "") + $profile = $matches[1]; - preg_match('/profile="(.*?)"/ism', $share[1], $matches); - if ($matches[1] != "") - $profile = $matches[1]; + preg_match('/profile="(.*?)"/ism', $share[1], $matches); + if ($matches[1] != "") + $profile = $matches[1]; if ($profile == "") return; logger("prepare poco_check for profile ".$profile, LOGGER_DEBUG); - poco_check($profile, "", "", "", "", "", "", "", "", $created, 3, $cid, $uid); + poco_check($profile, "", "", "", "", "", "", "", "", $created, 3, $cid, $uid); } function poco_store($item) { @@ -542,6 +1131,7 @@ function suggestion_query($uid, $start = 0, $limit = 80) { and not gcontact.name in ( select name from contact where uid = %d ) and not gcontact.id in ( select gcid from gcign where uid = %d ) AND `gcontact`.`updated` != '0000-00-00 00:00:00' + AND `gcontact`.`last_contact` >= `gcontact`.`last_failure` AND `gcontact`.`network` IN (%s) group by glink.gcid order by gcontact.updated desc,total desc limit %d, %d ", intval($uid), @@ -588,6 +1178,7 @@ function update_suggestions() { $done = array(); + // To-Do: Check if it is really neccessary to poll the own server poco_load(0,0,0,$a->get_baseurl() . '/poco'); $done[] = $a->get_baseurl() . '/poco'; @@ -598,6 +1189,9 @@ function update_suggestions() { $j = json_decode($x); if($j->entries) { foreach($j->entries as $entry) { + + poco_check_server($entry->url); + $url = $entry->url . '/poco'; if(! in_array($url,$done)) poco_load(0,0,0,$entry->url . '/poco'); @@ -606,8 +1200,9 @@ function update_suggestions() { } } - $r = q("select distinct(poco) as poco from contact where network = '%s'", - dbesc(NETWORK_DFRN) + // Query your contacts from Friendica and Redmatrix/Hubzilla for their contacts + $r = q("SELECT DISTINCT(`poco`) AS `poco` FROM `contact` WHERE `network` IN ('%s', '%s')", + dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA) ); if(count($r)) { @@ -618,3 +1213,167 @@ function update_suggestions() { } } } + +function poco_discover($complete = false) { + + $no_of_queries = 5; + + $last_update = date("c", time() - (60 * 60 * 6)); // 24 + $last_update = date("c", time() - (60 * 60 * 24)); // 24 + + $r = q("SELECT `poco`, `nurl`, `url`, `network` FROM `gserver` WHERE `last_contact` >= `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update)); + if ($r) + foreach ($r AS $server) { + + if (!poco_check_server($server["url"], $server["network"])) + continue; + + // Fetch all users from the other server + $url = $server["poco"]."/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; + + logger("Fetch all users from the server ".$server["nurl"], LOGGER_DEBUG); + + $retdata = z_fetch_url($url); + if ($retdata["success"]) { + $data = json_decode($retdata["body"]); + + poco_discover_server($data, 2); + + if (get_config('system','poco_discovery') > 1) { + + $timeframe = get_config('system','poco_discovery_since'); + if ($timeframe == 0) + $timeframe = 30; + + $updatedSince = date("Y-m-d H:i:s", time() - $timeframe * 86400); + + // Fetch all global contacts from the other server (Not working with Redmatrix and Friendica versions before 3.3) + $url = $server["poco"]."/@global?updatedSince=".$updatedSince."&fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; + + $success = false; + + $retdata = z_fetch_url($url); + if ($retdata["success"]) { + logger("Fetch all global contacts from the server ".$server["nurl"], LOGGER_DEBUG); + $success = poco_discover_server(json_decode($retdata["body"])); + } + + if (!$success AND (get_config('system','poco_discovery') > 2)) { + logger("Fetch contacts from users of the server ".$server["nurl"], LOGGER_DEBUG); + poco_discover_server_users($data, $server); + } + } + + q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"])); + if (!$complete AND (--$no_of_queries == 0)) + break; + } else // If the server hadn't replied correctly, then force a sanity check + poco_check_server($server["url"], $server["network"], true); + } +} + +function poco_discover_server_users($data, $server) { + + if (!isset($data->entry)) + return; + + foreach ($data->entry AS $entry) { + $username = ""; + if (isset($entry->urls)) { + foreach($entry->urls as $url) + if($url->type == 'profile') { + $profile_url = $url->value; + $urlparts = parse_url($profile_url); + $username = end(explode("/", $urlparts["path"])); + } + } + if ($username != "") { + logger("Fetch contacts for the user ".$username." from the server ".$server["nurl"], LOGGER_DEBUG); + + // Fetch all contacts from a given user from the other server + $url = $server["poco"]."/".$username."/?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; + + $retdata = z_fetch_url($url); + if ($retdata["success"]) + poco_discover_server(json_decode($retdata["body"]), 3); + } + } +} + +function poco_discover_server($data, $default_generation = 0) { + + if (!isset($data->entry) OR !count($data->entry)) + return false; + + $success = false; + + foreach ($data->entry AS $entry) { + $profile_url = ''; + $profile_photo = ''; + $connect_url = ''; + $name = ''; + $network = ''; + $updated = '0000-00-00 00:00:00'; + $location = ''; + $about = ''; + $keywords = ''; + $gender = ''; + $generation = $default_generation; + + $name = $entry->displayName; + + if(isset($entry->urls)) { + foreach($entry->urls as $url) { + if($url->type == 'profile') { + $profile_url = $url->value; + continue; + } + if($url->type == 'webfinger') { + $connect_url = str_replace('acct:' , '', $url->value); + continue; + } + } + } + + if(isset($entry->photos)) { + foreach($entry->photos as $photo) { + if($photo->type == 'profile') { + $profile_photo = $photo->value; + continue; + } + } + } + + if(isset($entry->updated)) + $updated = date("Y-m-d H:i:s", strtotime($entry->updated)); + + if(isset($entry->network)) + $network = $entry->network; + + if(isset($entry->currentLocation)) + $location = $entry->currentLocation; + + if(isset($entry->aboutMe)) + $about = html2bbcode($entry->aboutMe); + + if(isset($entry->gender)) + $gender = $entry->gender; + + if(isset($entry->generation) AND ($entry->generation > 0)) + $generation = ++$entry->generation; + + if(isset($entry->tags)) + foreach($entry->tags as $tag) + $keywords = implode(", ", $tag); + + if ($generation > 0) { + $success = true; + + logger("Store profile ".$profile_url, LOGGER_DEBUG); + poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $generation, 0, 0, 0); + logger("Done for profile ".$profile_url, LOGGER_DEBUG); + } + } + return $success; +} +?> diff --git a/mod/admin.php b/mod/admin.php index 28a7a91e4b..d725c45811 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -358,6 +358,10 @@ function admin_page_site_post(&$a){ $poll_interval = ((x($_POST,'poll_interval')) ? intval(trim($_POST['poll_interval'])) : 0); $maxloadavg = ((x($_POST,'maxloadavg')) ? intval(trim($_POST['maxloadavg'])) : 50); $maxloadavg_frontend = ((x($_POST,'maxloadavg_frontend')) ? intval(trim($_POST['maxloadavg_frontend'])) : 50); + $poco_completion = ((x($_POST,'poco_completion')) ? intval(trim($_POST['poco_completion'])) : false); + $poco_discovery = ((x($_POST,'poco_discovery')) ? intval(trim($_POST['poco_discovery'])) : 0); + $poco_discovery_since = ((x($_POST,'poco_discovery_since')) ? intval(trim($_POST['poco_discovery_since'])) : 30); + $poco_local_search = ((x($_POST,'poco_local_search')) ? intval(trim($_POST['poco_local_search'])) : false); $dfrn_only = ((x($_POST,'dfrn_only')) ? True : False); $ostatus_disabled = !((x($_POST,'ostatus_disabled')) ? True : False); $ostatus_poll_interval = ((x($_POST,'ostatus_poll_interval')) ? intval(trim($_POST['ostatus_poll_interval'])) : 0); @@ -380,8 +384,8 @@ function admin_page_site_post(&$a){ $old_pager = ((x($_POST,'old_pager')) ? True : False); $only_tag_search = ((x($_POST,'only_tag_search')) ? True : False); $rino = ((x($_POST,'rino')) ? intval($_POST['rino']) : 0); - - + + if($ssl_policy != intval(get_config('system','ssl_policy'))) { if($ssl_policy == SSL_POLICY_FULL) { q("update `contact` set @@ -427,6 +431,10 @@ function admin_page_site_post(&$a){ set_config('system','poll_interval',$poll_interval); set_config('system','maxloadavg',$maxloadavg); set_config('system','maxloadavg_frontend',$maxloadavg_frontend); + set_config('system','poco_completion',$poco_completion); + set_config('system','poco_discovery',$poco_discovery); + set_config('system','poco_discovery_since',$poco_discovery_since); + set_config('system','poco_local_search',$poco_local_search); set_config('config','sitename',$sitename); set_config('config','hostname',$hostname); set_config('config','sender_email', $sender_email); @@ -434,7 +442,7 @@ function admin_page_site_post(&$a){ set_config('system','suppress_tags',$suppress_tags); set_config('system','shortcut_icon',$shortcut_icon); set_config('system','touch_icon',$touch_icon); - + if ($banner==""){ // don't know why, but del_config doesn't work... q("DELETE FROM `config` WHERE `cat` = '%s' AND `k` = '%s' LIMIT 1", @@ -582,6 +590,20 @@ function admin_page_site(&$a) { "1440" => t("Daily") ); + $poco_discovery_choices = array( + "0" => t("Disabled"), + "1" => t("Users"), + "2" => t("Users, Global Contacts"), + "3" => t("Users, Global Contacts/fallback"), + ); + + $poco_discovery_since_choices = array( + "30" => t("One month"), + "91" => t("Three months"), + "182" => t("Half a year"), + "365" => t("One year"), + ); + /* get user names to make the install a personal install of X */ $user_names = array(); $user_names['---'] = t('Multi user instance'); @@ -630,6 +652,7 @@ function admin_page_site(&$a) { '$upload' => t('File upload'), '$corporate' => t('Policies'), '$advanced' => t('Advanced'), + '$portable_contacts' => t('Auto Discovered Contact Directory'), '$performance' => t('Performance'), '$relocate'=> t('Relocate - WARNING: advanced function. Could make this server unreachable.'), '$baseurl' => $a->get_baseurl(true), @@ -687,6 +710,11 @@ function admin_page_site(&$a) { '$maxloadavg' => array('maxloadavg', t("Maximum Load Average"), ((intval(get_config('system','maxloadavg')) > 0)?get_config('system','maxloadavg'):50), t("Maximum system load before delivery and poll processes are deferred - default 50.")), '$maxloadavg_frontend' => array('maxloadavg_frontend', t("Maximum Load Average (Frontend)"), ((intval(get_config('system','maxloadavg_frontend')) > 0)?get_config('system','maxloadavg_frontend'):50), t("Maximum system load before the frontend quits service - default 50.")), + '$poco_completion' => array('poco_completion', t("Periodical check of global contacts"), get_config('system','poco_completion'), t("If enabled, the global contacts are checked periodically for missing or outdated data and the vitality of the contacts and servers.")), + '$poco_discovery' => array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available. The fallback increases the server load, so the recommened setting is 'Users, Global Contacts'."), $poco_discovery_choices), + '$poco_discovery_since' => array('poco_discovery_since', t("Timeframe for fetching global contacts"), (string) intval(get_config('system','poco_discovery_since')), t("When the discovery is activated, this value defines the timeframe for the activity of the global contacts that are fetched from other servers."), $poco_discovery_since_choices), + '$poco_local_search' => array('poco_local_search', t("Search the local directory"), get_config('system','poco_local_search'), t("Search the local directory instead of the global directory. When searching locally, every search will be executed on the global directory in the background. This improves the search results when the search is repeated.")), + '$use_fulltext_engine' => array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")), '$suppress_language' => array('suppress_language', t("Suppress Language"), get_config('system','suppress_language'), t("Suppress language information in meta information about a posting.")), '$suppress_tags' => array('suppress_tags', t("Suppress Tags"), get_config('system','suppress_tags'), t("Suppress showing a list of hashtags at the end of the posting.")), diff --git a/mod/dirfind.php b/mod/dirfind.php index 0c6d79480f..d5cfcbcab0 100644 --- a/mod/dirfind.php +++ b/mod/dirfind.php @@ -16,11 +16,20 @@ function dirfind_init(&$a) { function dirfind_content(&$a) { + $community = false; + + $local = get_config('system','poco_local_search'); + $search = notags(trim($_REQUEST['search'])); if(strpos($search,'@') === 0) $search = substr($search,1); - + + if(strpos($search,'!') === 0) { + $search = substr($search,1); + $community = true; + } + $o = ''; $o .= replace_macros(get_markup_template("section_title.tpl"),array( @@ -29,16 +38,67 @@ function dirfind_content(&$a) { if($search) { - $p = (($a->pager['page'] != 1) ? '&p=' . $a->pager['page'] : ''); - - if(strlen(get_config('system','directory_submit_url'))) - $x = fetch_url('http://dir.friendica.com/lsearch?f=' . $p . '&search=' . urlencode($search)); + if ($local) { -//TODO fallback local search if global dir not available. -// else -// $x = post_url($a->get_baseurl() . '/lsearch', $params); + if ($community) + $extra_sql = " AND `community`"; + else + $extra_sql = ""; - $j = json_decode($x); + $perpage = 80; + $startrec = (($a->pager['page']) * $perpage) - $perpage; + + $count = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `network` IN ('%s', '%s', '%s') AND + (`url` REGEXP '%s' OR `name` REGEXP '%s' OR `location` REGEXP '%s' OR + `about` REGEXP '%s' OR `keywords` REGEXP '%s')".$extra_sql, + dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), + dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)), + dbesc(escape_tags($search)), dbesc(escape_tags($search))); + + $results = q("SELECT `contact`.`id` AS `cid`, `gcontact`.`url`, `gcontact`.`name`, `gcontact`.`photo`, `gcontact`.`keywords` + FROM `gcontact` + LEFT JOIN `contact` ON `contact`.`nurl` = `gcontact`.`nurl` AND `contact`.`uid` = %d + WHERE `gcontact`.`network` IN ('%s', '%s', '%s') AND + ((`gcontact`.`last_contact` >= `gcontact`.`last_failure`) OR (`gcontact`.`updated` >= `gcontact`.`last_failure`)) AND + (`gcontact`.`url` REGEXP '%s' OR `gcontact`.`name` REGEXP '%s' OR `gcontact`.`location` REGEXP '%s' OR + `gcontact`.`about` REGEXP '%s' OR `gcontact`.`keywords` REGEXP '%s') $extra_sql + GROUP BY `gcontact`.`nurl` + ORDER BY `gcontact`.`updated` DESC LIMIT %d, %d", + intval(local_user()), dbesc(NETWORK_DFRN), dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), + dbesc(escape_tags($search)), dbesc(escape_tags($search)), dbesc(escape_tags($search)), + dbesc(escape_tags($search)), dbesc(escape_tags($search)), + intval($startrec), intval($perpage)); + $j = new stdClass(); + $j->total = $count[0]["total"]; + $j->items_page = $perpage; + $j->page = $a->pager['page']; + foreach ($results AS $result) { + if ($result["name"] == "") { + $urlparts = parse_url($result["url"]); + $result["name"] = end(explode("/", $urlparts["path"])); + } + + $objresult = new stdClass(); + $objresult->cid = $result["cid"]; + $objresult->name = $result["name"]; + $objresult->url = $result["url"]; + $objresult->photo = $result["photo"]; + $objresult->tags = $result["keywords"]; + + $j->results[] = $objresult; + } + + // Add found profiles from the global directory to the local directory + proc_run('php','include/discover_poco.php', "dirsearch", urlencode($search)); + } else { + + $p = (($a->pager['page'] != 1) ? '&p=' . $a->pager['page'] : ''); + + if(strlen(get_config('system','directory_submit_url'))) + $x = fetch_url('http://dir.friendica.com/lsearch?f=' . $p . '&search=' . urlencode($search)); + + $j = json_decode($x); + } if($j->total) { $a->set_pager_total($j->total); @@ -46,21 +106,32 @@ function dirfind_content(&$a) { } if(count($j->results)) { - + $tpl = get_markup_template('match.tpl'); foreach($j->results as $jj) { - + + // If We already know this contact then don't show the "connect" button + if ($jj->cid > 0) { + $connlnk = ""; + $conntxt = ""; + } else { + $connlnk = $a->get_baseurl().'/follow/?url='.(($jj->connect) ? $jj->connect : $jj->url); + $conntxt = t('Connect'); + } + $o .= replace_macros($tpl,array( '$url' => zrl($jj->url), '$name' => $jj->name, - '$photo' => $jj->photo, - '$tags' => $jj->tags + '$photo' => proxy_url($jj->photo), + '$tags' => $jj->tags, + '$conntxt' => $conntxt, + '$connlnk' => $connlnk, )); } } else { info( t('No matches') . EOL); - } + } } diff --git a/mod/parse_url.php b/mod/parse_url.php index 71d767675f..50c6a15b0e 100644 --- a/mod/parse_url.php +++ b/mod/parse_url.php @@ -50,6 +50,21 @@ function completeurl($url, $scheme) { return($complete); } +function parseurl_getsiteinfo_cached($url, $no_guessing = false, $do_oembed = true) { + + $data = Cache::get("parse_url:".$no_guessing.":".$do_oembed.":".$url); + if (!is_null($data)) { + $data = unserialize($data); + return $data; + } + + $data = parseurl_getsiteinfo($url, $no_guessing, $do_oembed); + + Cache::set("parse_url:".$no_guessing.":".$do_oembed.":".$url,serialize($data)); + + return $data; +} + function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $count = 1) { require_once("include/network.php"); diff --git a/mod/poco.php b/mod/poco.php index 89f2f879a0..f84fc964d9 100644 --- a/mod/poco.php +++ b/mod/poco.php @@ -61,12 +61,16 @@ function poco_init(&$a) { $update_limit = date("Y-m-d H:i:s",strtotime($_GET['updatedSince'])); if ($global) { - $r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND `network` IN ('%s')", + //$r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND ((`last_contact` >= `last_failure`) OR (`updated` >= `last_failure`)) AND `network` IN ('%s')", + $r = q("SELECT count(*) AS `total` FROM `gcontact` WHERE `updated` >= '%s' AND `updated` >= `last_failure` AND `network` IN ('%s', '%s', '%s')", dbesc($update_limit), - dbesc(NETWORK_DFRN) + dbesc(NETWORK_DFRN), + dbesc(NETWORK_DIASPORA), + dbesc(NETWORK_OSTATUS) ); } elseif($system_mode) { - $r = q("SELECT count(*) AS `total` FROM `contact` WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') + $r = q("SELECT count(*) AS `total` FROM `contact` WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s') + AND (`success_update` >= `failure_update` OR `last-item` >= `failure_update`) AND `uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) ", dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), @@ -75,7 +79,8 @@ function poco_init(&$a) { ); } else { $r = q("SELECT count(*) AS `total` FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0 - AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra", + AND (`success_update` >= `failure_update` OR `last-item` >= `failure_update`) + AND `network` IN ('%s', '%s', '%s', '%s') $sql_extra", intval($user['uid']), dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), @@ -93,19 +98,25 @@ function poco_init(&$a) { $startIndex = 0; $itemsPerPage = ((x($_GET,'count') && intval($_GET['count'])) ? intval($_GET['count']) : $totalResults); - if ($global) { - $r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') LIMIT %d, %d", + logger("Start global query", LOGGER_DEBUG); + //$r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s') AND ((`last_contact` >= `last_failure`) OR (`updated` > `last_failure`)) LIMIT %d, %d", + $r = q("SELECT * FROM `gcontact` WHERE `updated` > '%s' AND `network` IN ('%s', '%s', '%s') AND `updated` > `last_failure` + ORDER BY `updated` DESC LIMIT %d, %d", dbesc($update_limit), dbesc(NETWORK_DFRN), + dbesc(NETWORK_DIASPORA), + dbesc(NETWORK_OSTATUS), intval($startIndex), intval($itemsPerPage) ); } elseif($system_mode) { + logger("Start system mode query", LOGGER_DEBUG); $r = q("SELECT `contact`.*, `profile`.`about` AS `pabout`, `profile`.`locality` AS `plocation`, `profile`.`pub_keywords`, `profile`.`gender` AS `pgender`, `profile`.`address` AS `paddress`, `profile`.`region` AS `pregion`, `profile`.`postal-code` AS `ppostalcode`, `profile`.`country-name` AS `pcountry` FROM `contact` INNER JOIN `profile` ON `profile`.`uid` = `contact`.`uid` - WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s', '') AND `profile`.`is-default` + WHERE `self` = 1 AND `network` IN ('%s', '%s', '%s', '%s') AND `profile`.`is-default` + AND ((`contact`.`success_update` >= `contact`.`failure_update`) OR (`contact`.`last-item` >= `contact`.`failure_update`)) AND `contact`.`uid` IN (SELECT `uid` FROM `pconfig` WHERE `cat` = 'system' AND `k` = 'suggestme' AND `v` = 1) LIMIT %d, %d", dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), @@ -115,8 +126,10 @@ function poco_init(&$a) { intval($itemsPerPage) ); } else { + logger("Start query for user ".$user['nickname'], LOGGER_DEBUG); $r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `blocked` = 0 AND `pending` = 0 AND `hidden` = 0 AND `archive` = 0 - AND `network` IN ('%s', '%s', '%s', '%s', '') $sql_extra LIMIT %d, %d", + AND (`success_update` >= `failure_update` OR `last-item` >= `failure_update`) + AND `network` IN ('%s', '%s', '%s', '%s') $sql_extra LIMIT %d, %d", intval($user['uid']), dbesc(NETWORK_DFRN), dbesc(NETWORK_DIASPORA), @@ -126,6 +139,7 @@ function poco_init(&$a) { intval($itemsPerPage) ); } + logger("Query done", LOGGER_DEBUG); $ret = array(); if(x($_GET,'sorted')) @@ -206,13 +220,19 @@ function poco_init(&$a) { if (($rr['keywords'] == "") AND isset($rr['pub_keywords'])) $rr['keywords'] = $rr['pub_keywords']; + $about = Cache::get("about:".$rr['updated'].":".$rr['nurl']); + if (is_null($about)) { + $about = bbcode($rr['about'], false, false); + Cache::set("about:".$rr['updated'].":".$rr['nurl'],$about); + } + $entry = array(); if($fields_ret['id']) $entry['id'] = (int)$rr['id']; if($fields_ret['displayName']) $entry['displayName'] = $rr['name']; if($fields_ret['aboutMe']) - $entry['aboutMe'] = bbcode($rr['about'], false, false); + $entry['aboutMe'] = $about; if($fields_ret['currentLocation']) $entry['currentLocation'] = $rr['location']; if($fields_ret['gender']) @@ -295,6 +315,8 @@ function poco_init(&$a) { else http_status_exit(500); + logger("End of poco", LOGGER_DEBUG); + if($format === 'xml') { header('Content-type: text/xml'); echo replace_macros(get_markup_template('poco_xml.tpl'),array_xmlify(array('$response' => $ret))); diff --git a/mod/search.php b/mod/search.php index 339efc92db..a01984d244 100644 --- a/mod/search.php +++ b/mod/search.php @@ -120,6 +120,10 @@ function search_content(&$a) { require_once('mod/dirfind.php'); return dirfind_content($a); } + if(strpos($search,'!') === 0) { + require_once('mod/dirfind.php'); + return dirfind_content($a); + } if(! $search) return $o; diff --git a/update.php b/update.php index c182eb590e..a2c8674028 100644 --- a/update.php +++ b/update.php @@ -1,6 +1,6 @@

{{$registration}}

@@ -70,13 +68,13 @@ {{include file="field_checkbox.tpl" field=$no_multi_reg}} {{include file="field_checkbox.tpl" field=$no_openid}} {{include file="field_checkbox.tpl" field=$no_regfullname}} -

{{$upload}}

{{include file="field_input.tpl" field=$maximagesize}} {{include file="field_input.tpl" field=$maximagelength}} {{include file="field_input.tpl" field=$jpegimagequality}} +

{{$corporate}}

{{include file="field_input.tpl" field=$allowed_sites}} @@ -99,7 +97,7 @@

{{$advanced}}

- {{include file="field_select.tpl" field=$rino}} + {{include file="field_select.tpl" field=$rino}} {{include file="field_checkbox.tpl" field=$no_utf}} {{include file="field_checkbox.tpl" field=$verifyssl}} {{include file="field_input.tpl" field=$proxy}} @@ -115,6 +113,14 @@ {{include file="field_input.tpl" field=$basepath}} {{include file="field_checkbox.tpl" field=$suppress_language}} {{include file="field_checkbox.tpl" field=$suppress_tags}} +
+ +

{{$portable_contacts}}

+ {{include file="field_checkbox.tpl" field=$poco_completion}} + {{include file="field_select.tpl" field=$poco_discovery}} + {{include file="field_select.tpl" field=$poco_discovery_since}} + {{include file="field_checkbox.tpl" field=$poco_local_search}} +

{{$performance}}

{{include file="field_checkbox.tpl" field=$use_fulltext_engine}} diff --git a/view/templates/match.tpl b/view/templates/match.tpl index f4bbc3c1bb..32f046e6aa 100644 --- a/view/templates/match.tpl +++ b/view/templates/match.tpl @@ -2,7 +2,7 @@
- {{$name}} + {{$name}}
diff --git a/view/templates/peoplefind.tpl b/view/templates/peoplefind.tpl index de8cd011b4..45c4c0b622 100644 --- a/view/templates/peoplefind.tpl +++ b/view/templates/peoplefind.tpl @@ -2,7 +2,7 @@

{{$findpeople}}

{{$desc}}
-
+