From cf3214c9040d84102cfe30943ee42211305412d3 Mon Sep 17 00:00:00 2001 From: Michael Vogel Date: Sat, 18 Jul 2015 22:26:06 +0200 Subject: [PATCH] Query other servers periodically for their known contacts --- include/discover_poco.php | 71 +++++++++++++++++++++ include/poller.php | 4 ++ include/socgraph.php | 115 ++++++++++++++++++++++++++++------ mod/admin.php | 13 ++-- view/templates/admin_site.tpl | 5 +- 5 files changed, 183 insertions(+), 25 deletions(-) create mode 100644 include/discover_poco.php diff --git a/include/discover_poco.php b/include/discover_poco.php new file mode 100644 index 0000000000..5a493ab967 --- /dev/null +++ b/include/discover_poco.php @@ -0,0 +1,71 @@ + $maxsysload) { + logger('system: load ' . $load[0] . ' too high. discover_poco deferred to next scheduled run.'); + return; + } + } + + $lockpath = get_lockpath(); + if ($lockpath != '') { + $pidfile = new pidfile($lockpath, 'discover_poco'); + if($pidfile->is_already_running()) { + logger("discover_poco: Already running"); + if ($pidfile->running_time() > 19*60) { + $pidfile->kill(); + logger("discover_poco: killed stale process"); + // Calling a new instance + proc_run('php','include/discover_poco.php'); + } + exit; + } + } + + $a->set_baseurl(get_config('system','url')); + + load_hooks(); + + logger('start'); + + if (get_config('system','poco_discovery')) + poco_discover(); + + logger('end'); + + return; +} + +if (array_search(__file__,get_included_files())===0){ + discover_poco_run($_SERVER["argv"],$_SERVER["argc"]); + killme(); +} diff --git a/include/poller.php b/include/poller.php index d971d4f004..44ac94daa4 100644 --- a/include/poller.php +++ b/include/poller.php @@ -82,6 +82,10 @@ function poller_run(&$argv, &$argc){ proc_run('php',"include/dsprphotoq.php"); + // run the process to discover global contacts in the background + + proc_run('php',"include/discover_poco.php"); + // expire any expired accounts q("UPDATE user SET `account_expired` = 1 where `account_expired` = 0 diff --git a/include/socgraph.php b/include/socgraph.php index a6899e952c..1d0dfd6f79 100644 --- a/include/socgraph.php +++ b/include/socgraph.php @@ -2,12 +2,7 @@ require_once('include/datetime.php'); require_once("include/Scrape.php"); - -/* - To-Do: - - noscrape for updating contact fields and "last updated" - - use /poco/@global for discovering contacts from other servers -*/ +require_once("include/html2bbcode.php"); /* * poco_load @@ -28,8 +23,6 @@ require_once("include/Scrape.php"); function poco_load($cid,$uid = 0,$zcid = 0,$url = null) { - require_once("include/html2bbcode.php"); - $a = get_app(); if($cid) { @@ -246,7 +239,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca logger("profile-check generation: ".$generation." Network: ".$network." URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG); // Only fetch last update manually if it wasn't provided and enabled in the system - if (get_config('system','ld_discover_activity') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) { + if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) { $last_updated = poco_last_updated($profile_url); if ($last_updated) { $updated = $last_updated; @@ -266,10 +259,10 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca poco_check_server($server_url, $network); // Test - remove before flight - if ($last_contact > $last_failure) - q("UPDATE `gserver` SET `last_contact` = '%s' WHERE `nurl` = '%s'", dbesc($last_contact), dbesc(normalise_link($server_url))); - else - q("UPDATE `gserver` SET `last_failure` = '%s' WHERE `nurl` = '%s'", dbesc($last_failure), dbesc(normalise_link($server_url))); + //if ($last_contact > $last_failure) + // q("UPDATE `gserver` SET `last_contact` = '%s' WHERE `nurl` = '%s'", dbesc($last_contact), dbesc(normalise_link($server_url))); + //else + // q("UPDATE `gserver` SET `last_failure` = '%s' WHERE `nurl` = '%s'", dbesc($last_failure), dbesc(normalise_link($server_url))); if(count($x)) { $gcid = $x[0]['id']; @@ -448,12 +441,12 @@ function poco_do_update($updated, $last_contact, $last_failure) { } function poco_to_boolean($val) { - if (($val == "true") OR ($val == 1)) - return(true); - if (($val == "false") OR ($val == 0)) - return(false); + if (($val == "true") OR ($val == 1)) + return(true); + if (($val == "false") OR ($val == 0)) + return(false); - return ($val); + return ($val); } function poco_check_server($server_url, $network = "") { @@ -976,3 +969,89 @@ function update_suggestions() { } } } + +function poco_discover() { + + $last_update = date("c", time() - (60 * 60 * 24)); + + $r = q("SELECT `poco`, `nurl` FROM `gserver` WHERE `last_contact` > `last_failure` AND `poco` != '' AND `last_poco_query` < '%s' ORDER BY RAND()", dbesc($last_update)); + if ($r) + foreach ($r AS $server) { + $url = $server["poco"]."/@global?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation"; + + $retdata = z_fetch_url($url); + if ($retdata["success"]) { + poco_discover_server(json_decode($retdata["body"])); + q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"])); + break; + } else + q("UPDATE `gserver` SET `last_poco_query` = '%s' WHERE `nurl` = '%s'", dbesc(datetime_convert()), dbesc($server["nurl"])); + + } +} + +function poco_discover_server($data) { + + foreach ($data->entry AS $entry) { + $profile_url = ''; + $profile_photo = ''; + $connect_url = ''; + $name = ''; + $network = ''; + $updated = '0000-00-00 00:00:00'; + $location = ''; + $about = ''; + $keywords = ''; + $gender = ''; + $generation = 0; + + $name = $entry->displayName; + + if(isset($entry->urls)) { + foreach($entry->urls as $url) { + if($url->type == 'profile') { + $profile_url = $url->value; + continue; + } + if($url->type == 'webfinger') { + $connect_url = str_replace('acct:' , '', $url->value); + continue; + } + } + } + if(isset($entry->photos)) { + foreach($entry->photos as $photo) { + if($photo->type == 'profile') { + $profile_photo = $photo->value; + continue; + } + } + } + + if(isset($entry->updated)) + $updated = date("Y-m-d H:i:s", strtotime($entry->updated)); + + if(isset($entry->network)) + $network = $entry->network; + + if(isset($entry->currentLocation)) + $location = $entry->currentLocation; + + if(isset($entry->aboutMe)) + $about = html2bbcode($entry->aboutMe); + + if(isset($entry->gender)) + $gender = $entry->gender; + + if(isset($entry->generation) AND ($entry->generation > 0)) + $generation = ++$entry->generation; + + if(isset($entry->tags)) + foreach($entry->tags as $tag) + $keywords = implode(", ", $tag); + + if ($generation > 0) + poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $generation); + } +} +?> diff --git a/mod/admin.php b/mod/admin.php index 214ce62ea9..393fe16147 100644 --- a/mod/admin.php +++ b/mod/admin.php @@ -358,7 +358,8 @@ function admin_page_site_post(&$a){ $poll_interval = ((x($_POST,'poll_interval')) ? intval(trim($_POST['poll_interval'])) : 0); $maxloadavg = ((x($_POST,'maxloadavg')) ? intval(trim($_POST['maxloadavg'])) : 50); $maxloadavg_frontend = ((x($_POST,'maxloadavg_frontend')) ? intval(trim($_POST['maxloadavg_frontend'])) : 50); - $ld_discover_activity = ((x($_POST,'ld_discover_activity')) ? intval(trim($_POST['ld_discover_activity'])) : false); + $poco_completion = ((x($_POST,'poco_completion')) ? intval(trim($_POST['poco_completion'])) : false); + $poco_discovery = ((x($_POST,'poco_discovery')) ? intval(trim($_POST['poco_discovery'])) : false); $dfrn_only = ((x($_POST,'dfrn_only')) ? True : False); $ostatus_disabled = !((x($_POST,'ostatus_disabled')) ? True : False); $ostatus_poll_interval = ((x($_POST,'ostatus_poll_interval')) ? intval(trim($_POST['ostatus_poll_interval'])) : 0); @@ -428,7 +429,8 @@ function admin_page_site_post(&$a){ set_config('system','poll_interval',$poll_interval); set_config('system','maxloadavg',$maxloadavg); set_config('system','maxloadavg_frontend',$maxloadavg_frontend); - set_config('system','ld_discover_activity',$ld_discover_activity); + set_config('system','poco_completion',$poco_completion); + set_config('system','poco_discovery',$poco_discovery); set_config('config','sitename',$sitename); set_config('config','hostname',$hostname); set_config('config','sender_email', $sender_email); @@ -436,7 +438,7 @@ function admin_page_site_post(&$a){ set_config('system','suppress_tags',$suppress_tags); set_config('system','shortcut_icon',$shortcut_icon); set_config('system','touch_icon',$touch_icon); - + if ($banner==""){ // don't know why, but del_config doesn't work... q("DELETE FROM `config` WHERE `cat` = '%s' AND `k` = '%s' LIMIT 1", @@ -632,7 +634,7 @@ function admin_page_site(&$a) { '$upload' => t('File upload'), '$corporate' => t('Policies'), '$advanced' => t('Advanced'), - '$local_directory' => t('Local Directory (Portable Contacts)'), + '$portable_contacts' => t('Portable Contact Directory'), '$performance' => t('Performance'), '$relocate'=> t('Relocate - WARNING: advanced function. Could make this server unreachable.'), '$baseurl' => $a->get_baseurl(true), @@ -690,7 +692,8 @@ function admin_page_site(&$a) { '$maxloadavg' => array('maxloadavg', t("Maximum Load Average"), ((intval(get_config('system','maxloadavg')) > 0)?get_config('system','maxloadavg'):50), t("Maximum system load before delivery and poll processes are deferred - default 50.")), '$maxloadavg_frontend' => array('maxloadavg_frontend', t("Maximum Load Average (Frontend)"), ((intval(get_config('system','maxloadavg_frontend')) > 0)?get_config('system','maxloadavg_frontend'):50), t("Maximum system load before the frontend quits service - default 50.")), - '$ld_discover_activity' => array('ld_discover_activity', t("Discover last activity"), get_config('system','ld_discover_activity'), t("Update the last activity when this isn't provided via the 'portable contacts' functionality. (Useful for poco exchange with Redmatrix and friendica servers before 3.3)")), + '$poco_completion' => array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful for poco exchange with Redmatrix and friendica servers before 3.3)")), + '$poco_discovery' => array('poco_discovery', t("Discover contacts from other servers"), get_config('system','poco_discovery'), t("Periodically query other friendica servers for their recent contacts.")), '$use_fulltext_engine' => array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")), '$suppress_language' => array('suppress_language', t("Suppress Language"), get_config('system','suppress_language'), t("Suppress language information in meta information about a posting.")), diff --git a/view/templates/admin_site.tpl b/view/templates/admin_site.tpl index 5809d5ac6f..22d785f6ff 100644 --- a/view/templates/admin_site.tpl +++ b/view/templates/admin_site.tpl @@ -115,8 +115,9 @@ {{include file="field_checkbox.tpl" field=$suppress_tags}}
-

{{$local_directory}}

- {{include file="field_checkbox.tpl" field=$ld_discover_activity}} +

{{$portable_contacts}}

+ {{include file="field_checkbox.tpl" field=$poco_completion}} + {{include file="field_checkbox.tpl" field=$poco_discovery}}

{{$performance}}