Merge remote-tracking branch 'upstream/develop' into 1601-dfrn

This commit is contained in:
Michael Vogel 2016-01-24 08:18:38 +01:00
commit 1471385157
5 changed files with 428 additions and 485 deletions

View file

@ -163,6 +163,9 @@ function cron_run(&$argv, &$argc){
// Repair missing Diaspora values in contacts
cron_repair_diaspora($a);
// Repair entries in the database
cron_repair_database();
$manual_id = 0;
$generation = 0;
$force = false;
@ -416,6 +419,24 @@ function cron_repair_diaspora(&$a) {
}
}
/**
* @brief Do some repairs in database entries
*
*/
function cron_repair_database() {
// Set the parent if it wasn't set. (Shouldn't happen - but does sometimes)
// This call is very "cheap" so we can do it at any time without a problem
q("UPDATE `item` INNER JOIN `item` AS `parent` ON `parent`.`uri` = `item`.`parent-uri` AND `parent`.`uid` = `item`.`uid` SET `item`.`parent` = `parent`.`id` WHERE `item`.`parent` = 0");
/// @todo
/// - remove duplicated contacts with uid=0 (We could do this at the place where the contacts are stored)
/// - remove thread entries without item
/// - remove sign entries without item
/// - remove children when parent got lost
/// - set contact-id in item when not present
}
if (array_search(__file__,get_included_files())===0){
cron_run($_SERVER["argv"],$_SERVER["argc"]);
killme();

View file

@ -33,7 +33,7 @@ function gprobe_run(&$argv, &$argc){
$url = hex2bin($argv[1]);
$r = q("select * from gcontact where nurl = '%s' limit 1",
$r = q("SELECT `id`, `url`, `network` FROM `gcontact` WHERE `nurl` = '%s' ORDER BY `id` LIMIT 1",
dbesc(normalise_link($url))
);
@ -58,21 +58,16 @@ function gprobe_run(&$argv, &$argc){
if (is_null($result))
Cache::set("gprobe:".$urlparts["host"],serialize($arr));
if(count($arr) && x($arr,'network') && $arr['network'] === NETWORK_DFRN) {
q("insert into `gcontact` (`name`,`url`,`nurl`,`photo`)
values ( '%s', '%s', '%s', '%s') ",
dbesc($arr['name']),
dbesc($arr['url']),
dbesc(normalise_link($arr['url'])),
dbesc($arr['photo'])
);
}
$r = q("select * from gcontact where nurl = '%s' limit 1",
if (!in_array($result["network"], array(NETWORK_FEED, NETWORK_PHANTOM)))
update_gcontact($arr);
$r = q("SELECT `id`, `url`, `network` FROM `gcontact` WHERE `nurl` = '%s' ORDER BY `id` LIMIT 1",
dbesc(normalise_link($url))
);
}
if(count($r))
poco_load(0,0,$r[0]['id'], str_replace('/profile/','/poco/',$r[0]['url']));
if ($r[0]["network"] == NETWORK_DFRN)
poco_load(0,0,$r[0]['id'], str_replace('/profile/','/poco/',$r[0]['url']));
logger("gprobe end for ".normalise_link($url), LOGGER_DEBUG);
return;

View file

@ -39,8 +39,10 @@ function poller_run(&$argv, &$argc){
}
// Checking the number of workers
if (poller_too_much_workers(1))
if (poller_too_much_workers(1)) {
poller_kill_stale_workers();
return;
}
if(($argc <= 1) OR ($argv[1] != "no_cron")) {
// Run the cron job that calls all other jobs
@ -50,16 +52,7 @@ function poller_run(&$argv, &$argc){
proc_run("php","include/cronhooks.php");
// Cleaning dead processes
$r = q("SELECT DISTINCT(`pid`) FROM `workerqueue` WHERE `executed` != '0000-00-00 00:00:00'");
foreach($r AS $pid)
if (!posix_kill($pid["pid"], 0))
q("UPDATE `workerqueue` SET `executed` = '0000-00-00 00:00:00', `pid` = 0 WHERE `pid` = %d",
intval($pid["pid"]));
else {
/// @TODO Kill long running processes
/// But: Update processes (like the database update) mustn't be killed
}
poller_kill_stale_workers();
} else
// Sleep four seconds before checking for running processes again to avoid having too many workers
sleep(4);
@ -124,6 +117,32 @@ function poller_run(&$argv, &$argc){
}
/**
* @brief fix the queue entry if the worker process died
*
*/
function poller_kill_stale_workers() {
$r = q("SELECT `pid`, `executed` FROM `workerqueue` WHERE `executed` != '0000-00-00 00:00:00'");
foreach($r AS $pid)
if (!posix_kill($pid["pid"], 0))
q("UPDATE `workerqueue` SET `executed` = '0000-00-00 00:00:00', `pid` = 0 WHERE `pid` = %d",
intval($pid["pid"]));
else {
// Kill long running processes
$duration = (time() - strtotime($pid["executed"])) / 60;
if ($duration > 180) {
logger("Worker process ".$pid["pid"]." took more than 3 hours. It will be killed now.");
posix_kill($pid["pid"], SIGTERM);
// Question: If a process is stale: Should we remove it or should we reschedule it?
// By now we rescheduling it. It's maybe not the wisest decision?
q("UPDATE `workerqueue` SET `executed` = '0000-00-00 00:00:00', `pid` = 0 WHERE `pid` = %d",
intval($pid["pid"]));
} else
logger("Worker process ".$pid["pid"]." now runs for ".round($duration)." minutes. That's okay.", LOGGER_DEBUG);
}
}
function poller_too_much_workers($stage) {
$queues = get_config("system", "worker_queues");

View file

@ -289,93 +289,25 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
poco_check_server($server_url, $network);
if(count($x)) {
$gcid = $x[0]['id'];
$gcontact = array("url" => $profile_url,
"addr" => $addr,
"alias" => $alias,
"name" => $name,
"network" => $network,
"photo" => $profile_photo,
"about" => $about,
"location" => $location,
"gender" => $gender,
"keywords" => $keywords,
"server_url" => $server_url,
"connect" => $connect_url,
"notify" => $notify,
"updated" => $updated,
"generation" => $generation);
if (($location == "") AND ($x[0]['location'] != ""))
$location = $x[0]['location'];
$gcid = update_gcontact($gcontact);
if (($about == "") AND ($x[0]['about'] != ""))
$about = $x[0]['about'];
if (($gender == "") AND ($x[0]['gender'] != ""))
$gender = $x[0]['gender'];
if (($keywords == "") AND ($x[0]['keywords'] != ""))
$keywords = $x[0]['keywords'];
if (($addr == "") AND ($x[0]['addr'] != ""))
$addr = $x[0]['addr'];
if (($alias == "") AND ($x[0]['alias'] != ""))
$alias = $x[0]['alias'];
if (($notify == "") AND ($x[0]['notify'] != ""))
$notify = $x[0]['notify'];
if (($generation == 0) AND ($x[0]['generation'] > 0))
$generation = $x[0]['generation'];
if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo || $x[0]['updated'] < $updated) {
q("UPDATE `gcontact` SET `name` = '%s', `addr` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s', `server_url` = '%s',
`updated` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d,
`alias` = '$s', `notify` = '%s'
WHERE (`generation` >= %d OR `generation` = 0) AND `nurl` = '%s'",
dbesc($name),
dbesc($addr),
dbesc($network),
dbesc($profile_photo),
dbesc($connect_url),
dbesc($profile_url),
dbesc($server_url),
dbesc($updated),
dbesc($location),
dbesc($about),
dbesc($keywords),
dbesc($gender),
dbesc($alias),
dbesc($notify),
intval($generation),
intval($generation),
dbesc(normalise_link($profile_url))
);
}
} else {
// Maybe another process had inserted the entry after the first check, so it again
$x = q("SELECT `id` FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1",
dbesc(normalise_link($profile_url))
);
if(!$x) {
q("INSERT INTO `gcontact` (`name`, `nick`, `addr`, `network`, `url`, `nurl`, `photo`, `connect`, `server_url`, `created`, `updated`, `location`, `about`, `keywords`, `gender`, `alias`, `notify`, `generation`)
VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d)",
dbesc($name),
dbesc($nick),
dbesc($addr),
dbesc($network),
dbesc($profile_url),
dbesc(normalise_link($profile_url)),
dbesc($profile_photo),
dbesc($connect_url),
dbesc($server_url),
dbesc(datetime_convert()),
dbesc($updated),
dbesc($location),
dbesc($about),
dbesc($keywords),
dbesc($gender),
dbesc($alias),
dbesc($notify),
intval($generation)
);
$x = q("SELECT `id` FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1",
dbesc(normalise_link($profile_url))
);
}
if(count($x))
$gcid = $x[0]['id'];
}
if(! $gcid)
if(!$gcid)
return $gcid;
$r = q("SELECT * FROM `glink` WHERE `cid` = %d AND `uid` = %d AND `gcid` = %d AND `zcid` = %d LIMIT 1",
@ -402,13 +334,6 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
);
}
// For unknown reasons there are sometimes duplicates
//q("DELETE FROM `gcontact` WHERE `nurl` = '%s' AND `id` != %d AND
// NOT EXISTS (SELECT `gcid` FROM `glink` WHERE `gcid` = `gcontact`.`id`)",
// dbesc(normalise_link($profile_url)),
// intval($gcid)
//);
return $gcid;
}
@ -1540,7 +1465,7 @@ function get_gcontact_id($contact) {
if ($contact["network"] == NETWORK_STATUSNET)
$contact["network"] = NETWORK_OSTATUS;
$r = q("SELECT `id` FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1",
$r = q("SELECT `id` FROM `gcontact` WHERE `nurl` = '%s' ORDER BY `id` LIMIT 2",
dbesc(normalise_link($contact["url"])));
if ($r)
@ -1562,13 +1487,18 @@ function get_gcontact_id($contact) {
intval($contact["generation"])
);
$r = q("SELECT `id` FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1",
$r = q("SELECT `id` FROM `gcontact` WHERE `nurl` = '%s' ORDER BY `id` LIMIT 2",
dbesc(normalise_link($contact["url"])));
if ($r)
$gcontact_id = $r[0]["id"];
}
if ((count($r) > 1) AND ($gcontact_id > 0) AND ($contact["url"] != ""))
q("DELETE FROM `gcontact` WHERE `nurl` = '%s' AND `id` != %d",
dbesc(normalise_link($contact["url"])),
intval($gcontact_id));
return $gcontact_id;
}
@ -1587,78 +1517,55 @@ function update_gcontact($contact) {
if (!$gcontact_id)
return false;
$r = q("SELECT `name`, `nick`, `photo`, `location`, `about`, `addr`, `generation`, `birthday`, `gender`, `keywords`, `hide`, `nsfw`, `network`, `alias`, `notify`, `url`
$r = q("SELECT `name`, `nick`, `photo`, `location`, `about`, `addr`, `generation`, `birthday`, `gender`, `keywords`,
`hide`, `nsfw`, `network`, `alias`, `notify`, `server_url`, `connect`, `updated`, `url`
FROM `gcontact` WHERE `id` = %d LIMIT 1",
intval($gcontact_id));
if ($contact["generation"] == 0)
$contact["generation"] = $r[0]["generation"];
// Get all field names
$fields = array();
foreach ($r[0] AS $field => $data)
$fields[$field] = $data;
if ($contact["photo"] == "")
$contact["photo"] = $r[0]["photo"];
unset($fields["url"]);
unset($fields["updated"]);
if ($contact["name"] == "")
$contact["name"] = $r[0]["name"];
if ($contact["nick"] == "")
$contact["nick"] = $r[0]["nick"];
if ($contact["addr"] == "")
$contact["addr"] = $r[0]["addr"];
if ($contact["location"] =="")
$contact["location"] = $r[0]["location"];
if ($contact["about"] =="")
$contact["about"] = $r[0]["about"];
if ($contact["birthday"] =="")
$contact["birthday"] = $r[0]["birthday"];
if ($contact["gender"] =="")
$contact["gender"] = $r[0]["gender"];
if ($contact["keywords"] =="")
$contact["keywords"] = $r[0]["keywords"];
if (!isset($contact["hide"]))
$contact["hide"] = $r[0]["hide"];
if (!isset($contact["nsfw"]))
$contact["nsfw"] = $r[0]["nsfw"];
if ($contact["network"] =="")
$contact["network"] = $r[0]["network"];
if ($contact["alias"] =="")
$contact["alias"] = $r[0]["alias"];
if ($contact["url"] =="")
$contact["url"] = $r[0]["url"];
if ($contact["notify"] =="")
$contact["notify"] = $r[0]["notify"];
// assign all unassigned fields from the database entry
foreach ($fields AS $field => $data)
if (!isset($contact[$field]))
$contact[$field] = $r[0][$field];
if ($contact["network"] == NETWORK_STATUSNET)
$contact["network"] = NETWORK_OSTATUS;
if (($contact["photo"] != $r[0]["photo"]) OR ($contact["name"] != $r[0]["name"]) OR ($contact["nick"] != $r[0]["nick"]) OR ($contact["addr"] != $r[0]["addr"]) OR
($contact["birthday"] != $r[0]["birthday"]) OR ($contact["gender"] != $r[0]["gender"]) OR ($contact["keywords"] != $r[0]["keywords"]) OR
($contact["hide"] != $r[0]["hide"]) OR ($contact["nsfw"] != $r[0]["nsfw"]) OR ($contact["network"] != $r[0]["network"]) OR
($contact["alias"] != $r[0]["alias"]) OR ($contact["notify"] != $r[0]["notify"]) OR ($contact["url"] != $r[0]["url"]) OR
($contact["location"] != $r[0]["location"]) OR ($contact["about"] != $r[0]["about"]) OR ($contact["generation"] < $r[0]["generation"])) {
if (!isset($contact["updated"]))
$contact["updated"] = datetime_convert();
// Check if any field changed
$update = false;
unset($fields["generation"]);
foreach ($fields AS $field => $data)
if ($contact[$field] != $r[0][$field])
$update = true;
if ($contact["generation"] < $r[0]["generation"])
$update = true;
if ($update) {
q("UPDATE `gcontact` SET `photo` = '%s', `name` = '%s', `nick` = '%s', `addr` = '%s', `network` = '%s',
`birthday` = '%s', `gender` = '%s', `keywords` = %d, `hide` = %d, `nsfw` = %d,
`alias` = '%s', `notify` = '%s', `url` = '%s',
`location` = '%s', `about` = '%s', `generation` = %d, `updated` = '%s'
`location` = '%s', `about` = '%s', `generation` = %d, `updated` = '%s',
`server_url` = '%s', `connect` = '%s'
WHERE `nurl` = '%s' AND (`generation` = 0 OR `generation` >= %d)",
dbesc($contact["photo"]), dbesc($contact["name"]), dbesc($contact["nick"]),
dbesc($contact["addr"]), dbesc($contact["network"]), dbesc($contact["birthday"]),
dbesc($contact["gender"]), dbesc($contact["keywords"]), intval($contact["hide"]),
intval($contact["nsfw"]), dbesc($contact["alias"]), dbesc($contact["notify"]),
dbesc($contact["url"]), dbesc($contact["location"]), dbesc($contact["about"]),
intval($contact["generation"]), dbesc(datetime_convert()),
intval($contact["generation"]), dbesc($contact["updated"]),
dbesc($contact["server_url"]), dbesc($contact["connect"]),
dbesc(normalise_link($contact["url"])), intval($contact["generation"]));
}

File diff suppressed because it is too large Load diff