friendica
/
friendica
mirror of https://github.com/friendica/friendica.git
4
6
Fork 3
Code Issues Releases 13 Wiki Activity
Browse Source

New field "created" in gcontacts. New option for timeframe when polling global contacts

pull/1791/head
Michael Vogel 6 years ago
parent
a4aec06e5b
commit
9717fa82fe
5 changed files with 126 additions and 21 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      include/dbstructure.php
  2. 2
      include/onepoll.php
  3. 131
      include/socgraph.php
  4. 10
      mod/admin.php
  5. 1
      view/templates/admin_site.tpl

3
include/dbstructure.php
View File

@ -626,10 +626,12 @@ function db_definition() {
"fields" => array(
"id" => array("type" => "int(10) unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"),
"name" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
"nick" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
"url" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
"nurl" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
"photo" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
"connect" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
"created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
"updated" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
"last_contact" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
"last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
@ -699,6 +701,7 @@ function db_definition() {
"noscrape" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
"network" => array("type" => "varchar(32)", "not null" => "1", "default" => ""),
"platform" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
"created" => array("type" => "datetime", "not null" => "1", "default" => "0000-00-00 00:00:00"),
"last_poco_query" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
"last_contact" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),
"last_failure" => array("type" => "datetime", "default" => "0000-00-00 00:00:00"),

2
include/onepoll.php
View File

@ -110,7 +110,7 @@ function onepoll_run(&$argv, &$argc){
// - Check why we don't poll the Diaspora feed at the moment (some guid problem in the items?)
// - Check whether this is possible with Redmatrix
if ($contact["network"] == NETWORK_DIASPORA) {
if (poco_do_update($contact["last-item"], $contact["success_update"], $contact["failure_update"])) {
if (poco_do_update($contact["created"], $contact["last-item"], $contact["failure_update"], $contact["success_update"])) {
$last_updated = poco_last_updated($contact["url"]);
$updated = datetime_convert();
if ($last_updated) {

131
include/socgraph.php
View File

@ -205,19 +205,26 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
if ($updated == "0000-00-00 00:00:00")
$updated = $x[0]["updated"];
$created = $x[0]["created"];
$last_contact = $x[0]["last_contact"];
$last_failure = $x[0]["last_failure"];
$server_url = $x[0]["server_url"];
$nick = $x[0]["nick"];
} else {
$created = "0000-00-00 00:00:00";
$last_contact = "0000-00-00 00:00:00";
$last_failure = "0000-00-00 00:00:00";
$server_url = "";
$urlparts = parse_url($profile_url);
$nick = end(explode("/", $urlparts["path"]));
}
if (($network == "") OR ($name == "") OR ($profile_photo == "") OR ($server_url == "")) {
$data = probe_url($profile_url);
$network = $data["network"];
$name = $data["name"];
$nick = $data["nick"];
$profile_url = $data["url"];
$profile_photo = $data["photo"];
$server_url = $data["baseurl"];
@ -241,7 +248,7 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
poco_check_server($server_url, $network);
// Only fetch last update manually if it wasn't provided and enabled in the system
if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($updated, $last_contact, $last_failure)) {
if (get_config('system','poco_completion') AND ($orig_updated == "0000-00-00 00:00:00") AND poco_do_update($created, $updated, $last_failure, $last_contact)) {
$last_updated = poco_last_updated($profile_url);
if ($last_updated) {
$updated = $last_updated;
@ -290,15 +297,17 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
);
}
} else {
q("INSERT INTO `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `server_url`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`)
VALUES ('%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)",
q("INSERT INTO `gcontact` (`name`, `nick`, `network`, `url`, `nurl`, `photo`, `connect`, `server_url`, `created`, `updated`, `last_contact`, `last_failure`, `location`, `about`, `keywords`, `gender`, `generation`)
VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)",
dbesc($name),
dbesc($nick),
dbesc($network),
dbesc($profile_url),
dbesc(normalise_link($profile_url)),
dbesc($profile_photo),
dbesc($connect_url),
dbesc($server_url),
dbesc(datetime_convert()),
dbesc($updated),
dbesc($last_contact),
dbesc($last_failure),
@ -357,13 +366,79 @@ function poco_last_updated($profile) {
$gcontacts = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s'",
dbesc(normalise_link($profile)));
if ($gcontacts[0]["server_url"] != "") {
$servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s' AND `last_contact` < `last_failure`", dbesc(normalise_link($gcontacts[0]["server_url"])));
if ($gcontacts[0]["created"] == "0000-00-00 00:00:00")
q("UPDATE `gcontact` SET `created` = '%s' WHERE `nurl` = '%s'",
dbesc(datetime_convert()), dbesc(normalise_link($profile)));
if ($servers)
if (($gcontacts[0]["server_url"] != "") AND ($gcontacts[0]["network"] != ""))
if (!poco_check_server($gcontacts[0]["server_url"], $gcontacts[0]["network"]))
return false;
// noscrape is really fast so we don't cache the call.
if (($gcontacts[0]["server_url"] != "") AND ($gcontacts[0]["nick"] != "")) {
// Use noscrape if possible
$server = q("SELECT `noscrape` FROM `gserver` WHERE `nurl` = '%s' AND `noscrape` != ''", dbesc(normalise_link($gcontacts[0]["server_url"])));
if ($server) {
$noscraperet = z_fetch_url($server[0]["noscrape"]."/".$gcontacts[0]["nick"]);
if ($noscraperet["success"]) {
$noscrape = json_decode($noscraperet["body"], true);
if (($noscrape["name"] != "") AND ($noscrape["name"] != $gcontacts[0]["name"]))
q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'",
dbesc($noscrape["name"]), dbesc(normalise_link($profile)));
if (($noscrape["photo"] != "") AND ($noscrape["photo"] != $gcontacts[0]["photo"]))
q("UPDATE `gcontact` SET `photo` = '%s' WHERE `nurl` = '%s'",
dbesc($noscrape["photo"]), dbesc(normalise_link($profile)));
if (($noscrape["updated"] != "") AND ($noscrape["updated"] != $gcontacts[0]["updated"]))
q("UPDATE `gcontact` SET `updated` = '%s' WHERE `nurl` = '%s'",
dbesc($noscrape["updated"]), dbesc(normalise_link($profile)));
if (($noscrape["gender"] != "") AND ($noscrape["gender"] != $gcontacts[0]["gender"]))
q("UPDATE `gcontact` SET `gender` = '%s' WHERE `nurl` = '%s'",
dbesc($noscrape["gender"]), dbesc(normalise_link($profile)));
if (($noscrape["about"] != "") AND ($noscrape["about"] != $gcontacts[0]["name"]))
q("UPDATE `gcontact` SET `about` = '%s' WHERE `nurl` = '%s'",
dbesc($noscrape["about"]), dbesc(normalise_link($profile)));
$keywords = implode(" ", $noscrape["tags"]);
if (($keywords != "") AND ($keywords != $gcontacts[0]["keywords"]))
q("UPDATE `gcontact` SET `keywords` = '%s' WHERE `nurl` = '%s'",
dbesc($keywords), dbesc(normalise_link($profile)));
$location = $noscrape["locality"];
if ($noscrape["region"] != "") {
if ($location != "")
$location .= ", ";
$location .= $noscrape["region"];
}
if ($noscrape["country-name"] != "") {
if ($location != "")
$location .= ", ";
$location .= $noscrape["country-name"];
}
if (($location != "") AND ($location != $gcontacts[0]["location"]))
q("UPDATE `gcontact` SET `location` = '%s' WHERE `nurl` = '%s'",
dbesc($location), dbesc(normalise_link($profile)));
return $noscrape["updated"];
}
}
}
// If we only can poll the feed, then we only do this once a while
if (!poco_do_update($gcontacts[0]["created"], $gcontacts[0]["updated"], $gcontacts[0]["last_failure"], $gcontacts[0]["last_contact"]))
/ return $gcontacts[0]["updated"];
$data = probe_url($profile);
if (($data["poll"] == "") OR ($data["network"] == NETWORK_FEED)) {
@ -376,6 +451,10 @@ function poco_last_updated($profile) {
q("UPDATE `gcontact` SET `name` = '%s' WHERE `nurl` = '%s'",
dbesc($data["name"]), dbesc(normalise_link($profile)));
if (($data["nick"] != "") AND ($data["nick"] != $gcontacts[0]["nick"]))
q("UPDATE `gcontact` SET `nick` = '%s' WHERE `nurl` = '%s'",
dbesc($data["nick"]), dbesc(normalise_link($profile)));
if (($data["addr"] != "") AND ($data["addr"] != $gcontacts[0]["connect"]))
q("UPDATE `gcontact` SET `connect` = '%s' WHERE `nurl` = '%s'",
dbesc($data["addr"]), dbesc(normalise_link($profile)));
@ -388,11 +467,6 @@ function poco_last_updated($profile) {
q("UPDATE `gcontact` SET `server_url` = '%s' WHERE `nurl` = '%s'",
dbesc($data["baseurl"]), dbesc(normalise_link($profile)));
if ($data["baseurl"] != "")
poco_check_server($data["baseurl"], $data["network"]);
// To-Do: Use noscrape
$feedret = z_fetch_url($data["poll"]);
if (!$feedret["success"]) {
@ -437,7 +511,7 @@ function poco_last_updated($profile) {
return($last_updated);
}
function poco_do_update($updated, $last_contact, $last_failure) {
function poco_do_update($created, $updated, $last_failure, $last_contact) {
$now = strtotime(datetime_convert());
if ($updated > $last_contact)
@ -446,6 +520,11 @@ function poco_do_update($updated, $last_contact, $last_failure) {
$contact_time = strtotime($last_contact);
$failure_time = strtotime($last_failure);
$created_time = strtotime($created);
// If there is no "created" time then use the current time
if ($created_time <= 0)
$created_time = $now;
// If the last contact was less than 24 hours then don't update
if (($now - $contact_time) < (60 * 60 * 24))
@ -459,12 +538,12 @@ function poco_do_update($updated, $last_contact, $last_failure) {
if ((($now - $contact_time) < (60 * 60 * 24 * 7)) AND ($contact_time > $failure_time))
return false;
// If the last contact time was more than a week ago, then only try once a week
if ((($now - $contact_time) > (60 * 60 * 24 * 7)) AND (($now - $failure_time) < (60 * 60 * 24 * 7)))
// If the last contact time was more than a week ago and the contact was created more than a week ago, then only try once a week
if ((($now - $contact_time) > (60 * 60 * 24 * 7)) AND (($now - $created_time) > (60 * 60 * 24 * 7)) AND (($now - $failure_time) < (60 * 60 * 24 * 7)))
return false;
// If the last contact time was more than a month ago, then only try once a month - but only if there ever was a contact time
if (($contact_time > 0) AND (($now - $contact_time) > (60 * 60 * 24 * 30)) AND (($now - $failure_time) < (60 * 60 * 24 * 30)))
// If the last contact time was more than a month ago and the contact was created more than a month ago, then only try once a month
if ((($now - $contact_time) > (60 * 60 * 24 * 30)) AND (($now - $created_time) > (60 * 60 * 24 * 30)) AND (($now - $failure_time) < (60 * 60 * 24 * 30)))
return false;
return true;
@ -486,6 +565,11 @@ function poco_check_server($server_url, $network = "") {
$servers = q("SELECT * FROM `gserver` WHERE `nurl` = '%s'", dbesc(normalise_link($server_url)));
if ($servers) {
if ($servers[0]["created"] == "0000-00-00 00:00:00")
q("UPDATE `gserver` SET `created` = '%s' WHERE `nurl` = '%s'",
dbesc(datetime_convert()), dbesc(normalise_link($server_url)));
$poco = $servers[0]["poco"];
$noscrape = $servers[0]["noscrape"];
@ -500,7 +584,7 @@ function poco_check_server($server_url, $network = "") {
$info = $servers[0]["info"];
$register_policy = $servers[0]["register_policy"];
if (!poco_do_update("", $last_contact, $last_failure))
if (!poco_do_update($servers[0]["created"], "", $last_failure, $last_contact))
return ($last_contact >= $last_failure);
} else {
$poco = "";
@ -566,7 +650,7 @@ function poco_check_server($server_url, $network = "") {
if (isset($data->site->hubzilla)) {
$platform = $data->site->hubzilla->PLATFORM_NAME;
$version = $data->site->hubzilla->RED_VERSION;
$network = NETWORK_DFRN;
$network = NETWORK_DIASPORA;
}
if (isset($data->site->redmatrix)) {
if (isset($data->site->redmatrix->PLATFORM_NAME))
@ -685,7 +769,7 @@ function poco_check_server($server_url, $network = "") {
dbesc(normalise_link($server_url))
);
else
q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `last_contact`, `last_failure`)
q("INSERT INTO `gserver` (`url`, `nurl`, `version`, `site_name`, `info`, `register_policy`, `poco`, `noscrape`, `network`, `platform`, `created`, `last_contact`, `last_failure`)
VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '%s', '%s')",
dbesc($server_url),
dbesc(normalise_link($server_url)),
@ -697,6 +781,7 @@ function poco_check_server($server_url, $network = "") {
dbesc($noscrape),
dbesc($network),
dbesc($platform),
dbesc(datetime_convert()),
dbesc($last_contact),
dbesc($last_failure),
dbesc(datetime_convert())
@ -1028,8 +1113,14 @@ function poco_discover($complete = false) {
if (get_config('system','poco_discovery') > 1) {
$timeframe = get_config('system','poco_discovery_since');
if ($timeframe == 0)
$timeframe = 30;
$updatedSince = date("Y-m-d H:i:s", time() - $timeframe * 86400);
// Fetch all global contacts from the other server (Not working with Redmatrix and Friendica versions before 3.3)
$url = $server["poco"]."/@global?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
$url = $server["poco"]."/@global?updatedSince=".$updatedSince."&fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation";
$retdata = z_fetch_url($url);
if ($retdata["success"]) {

10
mod/admin.php
View File

@ -360,6 +360,7 @@ function admin_page_site_post(&$a){
$maxloadavg_frontend = ((x($_POST,'maxloadavg_frontend')) ? intval(trim($_POST['maxloadavg_frontend'])) : 50);
$poco_completion = ((x($_POST,'poco_completion')) ? intval(trim($_POST['poco_completion'])) : false);
$poco_discovery = ((x($_POST,'poco_discovery')) ? intval(trim($_POST['poco_discovery'])) : 0);
$poco_discovery_since = ((x($_POST,'poco_discovery_since')) ? intval(trim($_POST['poco_discovery_since'])) : 30);
$poco_local_search = ((x($_POST,'poco_local_search')) ? intval(trim($_POST['poco_local_search'])) : false);
$dfrn_only = ((x($_POST,'dfrn_only')) ? True : False);
$ostatus_disabled = !((x($_POST,'ostatus_disabled')) ? True : False);
@ -432,6 +433,7 @@ function admin_page_site_post(&$a){
set_config('system','maxloadavg_frontend',$maxloadavg_frontend);
set_config('system','poco_completion',$poco_completion);
set_config('system','poco_discovery',$poco_discovery);
set_config('system','poco_discovery_since',$poco_discovery_since);
set_config('system','poco_local_search',$poco_local_search);
set_config('config','sitename',$sitename);
set_config('config','hostname',$hostname);
@ -595,6 +597,13 @@ function admin_page_site(&$a) {
"3" => t("Users, Global Contacts/fallback"),
);
$poco_discovery_since_choices = array(
"30" => t("One month"),
"91" => t("Three months"),
"182" => t("Half a year"),
"365" => t("One year"),
);
/* get user names to make the install a personal install of X */
$user_names = array();
$user_names['---'] = t('Multi user instance');
@ -703,6 +712,7 @@ function admin_page_site(&$a) {
'$poco_completion' => array('poco_completion', t("Completion of incoming contacts"), get_config('system','poco_completion'), t("Complete data of incomplete incoming contacts that are provided by the 'portable contacts' functionality. (Useful when communicating with Redmatrix and friendica servers before 3.3)")),
'$poco_discovery' => array('poco_discovery', t("Discover contacts from other servers"), (string) intval(get_config('system','poco_discovery')), t("Periodically query other servers for contacts. You can choose between 'users': the users on the remote system, 'Global Contacts': active contacts that are known on the system. The fallback is meant for Redmatrix servers and older friendica servers, where global contacts weren't available."), $poco_discovery_choices),
'$poco_discovery_since' => array('poco_discovery_since', t("Timeframe for fetching global contacts"), (string) intval(get_config('system','poco_discovery_since')), t("When the discovery is activated, this value defines the timeframe for the global contacts that are fetched from other servers."), $poco_discovery_since_choices),
'$poco_local_search' => array('poco_local_search', t("Search the local directory"), get_config('system','poco_local_search'), t("Search the local directory instead of the global directory. When searching locally, every search will be executed on the global directory in the background. This improves the search results when the search is repeated.")),
'$use_fulltext_engine' => array('use_fulltext_engine', t("Use MySQL full text engine"), get_config('system','use_fulltext_engine'), t("Activates the full text engine. Speeds up search - but can only search for four and more characters.")),

1
view/templates/admin_site.tpl
View File

@ -118,6 +118,7 @@
<h3>{{$portable_contacts}}</h3>
{{include file="field_checkbox.tpl" field=$poco_completion}}
{{include file="field_select.tpl" field=$poco_discovery}}
{{include file="field_select.tpl" field=$poco_discovery_since}}
{{include file="field_checkbox.tpl" field=$poco_local_search}}
<div class="submit"><input type="submit" name="page_site" value="{{$submit|escape:'html'}}" /></div>

Write Preview
Loading…
Cancel
Save