MrPetovan/friendica
2
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #3461 from annando/1705-some-more-dba

Browse source 
DBA: poller / SQL errors / dbclean
This commit is contained in:
Hypolite Petovan 2017-05-13 00:22:01 -04:00 committed by GitHub
parent 2bbcbdc704 e884090efa
commit db0c3620c0
10 changed files with 72 additions and 102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
boot.php
View file

@ -1077,14 +1077,10 @@ function proc_run($cmd) {
array_shift($argv);
$parameters = json_encode($argv);
$found = q("SELECT `id` FROM `workerqueue` WHERE `parameter` = '%s'", dbesc($parameters));
$found = dba::select('workerqueue', array('id'), array('parameter' => $parameters), array('limit' => 1));
if (!dbm::is_result($found)) {
q("INSERT INTO `workerqueue` (`parameter`, `created`, `priority`)
VALUES ('%s', '%s', %d)",
dbesc($parameters),
dbesc(datetime_convert()),
intval($priority));
dba::insert('workerqueue', array('parameter' => $parameters, 'created' => datetime_convert(), 'priority' => $priority));
}
// Should we quit and wait for the poller to be called as a cronjob?

24
include/Contact.php
View file

@ -23,29 +23,7 @@ function user_remove($uid) {
$r[0]['nickname']
);
/// @todo Should be done in a background job since this likely will run into a time out
// don't delete yet, will be done later when contacts have deleted my stuff
// q("DELETE FROM `contact` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `gcign` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `group` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `group_member` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `intro` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `event` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `item` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `item_id` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `mail` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `mailacct` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `manage` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `notify` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `photo` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `attach` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `profile` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `profile_check` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `pconfig` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `search` WHERE `uid` = %d", intval($uid));
q("DELETE FROM `spam` WHERE `uid` = %d", intval($uid));
// don't delete yet, will be done later when contacts have deleted my stuff
// q("DELETE FROM `user` WHERE `uid` = %d", intval($uid));
// The user and related data will be deleted in "cron_expire_and_remove_users" (cronjobs.php)
q("UPDATE `user` SET `account_removed` = 1, `account_expires_on` = UTC_TIMESTAMP() WHERE `uid` = %d", intval($uid));
proc_run(PRIORITY_HIGH, "include/notifier.php", "removeme", $uid);

5
include/cronjobs.php
View file

@ -110,12 +110,11 @@ function cron_expire_and_remove_users() {
AND `account_expires_on` > '%s'
AND `account_expires_on` < UTC_TIMESTAMP()", dbesc(NULL_DATE));
// delete user and contact records for recently removed accounts
// delete user records for recently removed accounts
$r = q("SELECT * FROM `user` WHERE `account_removed` AND `account_expires_on` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
if (dbm::is_result($r)) {
foreach ($r as $user) {
q("DELETE FROM `contact` WHERE `uid` = %d", intval($user['uid']));
q("DELETE FROM `user` WHERE `uid` = %d", intval($user['uid']));
dba::delete('user', array('uid' => $user['uid']));
}
}
}

43
include/dbclean.php
View file

@ -18,13 +18,11 @@ function dbclean_run(&$argv, &$argc) {
}
if ($stage == 0) {
proc_run(PRIORITY_LOW, 'include/dbclean.php', 1);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 2);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 3);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 4);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 5);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 6);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 7);
for ($i = 1; $i <= 7; $i++) {
if (!Config::get('system', 'finished-dbclean-'.$i)) {
proc_run(PRIORITY_LOW, 'include/dbclean.php', $i);
}
}
} else {
remove_orphans($stage);
}
@ -54,9 +52,15 @@ function remove_orphans($stage = 0) {
}
} else {
logger("No global item orphans found");
}
dba::close($r);
logger("Done deleting ".$count." old global item entries from item table without user copy");
// We will eventually set this value when we found a good way to delete these items in another way.
// if ($count < $limit) {
// Config::set('system', 'finished-dbclean-1', true);
// }
} elseif ($stage == 2) {
logger("Deleting items without parents");
$r = dba::p("SELECT `id` FROM `item` WHERE NOT EXISTS (SELECT `id` FROM `item` AS `i` WHERE `item`.`parent` = `i`.`id`) LIMIT ".intval($limit));
@ -71,6 +75,10 @@ function remove_orphans($stage = 0) {
}
dba::close($r);
logger("Done deleting ".$count." items without parents");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-2', true);
}
} elseif ($stage == 3) {
logger("Deleting orphaned data from thread table");
$r = dba::p("SELECT `iid` FROM `thread` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`) LIMIT ".intval($limit));
@ -83,9 +91,12 @@ function remove_orphans($stage = 0) {
} else {
logger("No thread orphans found");
}
dba::close($r);
logger("Done deleting ".$count." orphaned data from thread table");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-3', true);
}
} elseif ($stage == 4) {
logger("Deleting orphaned data from notify table");
$r = dba::p("SELECT `iid` FROM `notify` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`) LIMIT ".intval($limit));
@ -100,6 +111,10 @@ function remove_orphans($stage = 0) {
}
dba::close($r);
logger("Done deleting ".$count." orphaned data from notify table");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-4', true);
}
} elseif ($stage == 5) {
logger("Deleting orphaned data from notify-threads table");
$r = dba::p("SELECT `id` FROM `notify-threads` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `notify-threads`.`master-parent-item`) LIMIT ".intval($limit));
@ -114,6 +129,10 @@ function remove_orphans($stage = 0) {
}
dba::close($r);
logger("Done deleting ".$count." orphaned data from notify-threads table");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-5', true);
}
} elseif ($stage == 6) {
logger("Deleting orphaned data from sign table");
$r = dba::p("SELECT `iid` FROM `sign` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`) LIMIT ".intval($limit));
@ -128,6 +147,10 @@ function remove_orphans($stage = 0) {
}
dba::close($r);
logger("Done deleting ".$count." orphaned data from sign table");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-6', true);
}
} elseif ($stage == 7) {
logger("Deleting orphaned data from term table");
$r = dba::p("SELECT `oid` FROM `term` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`) LIMIT ".intval($limit));
@ -142,6 +165,10 @@ function remove_orphans($stage = 0) {
}
dba::close($r);
logger("Done deleting ".$count." orphaned data from term table");
if ($count < $limit) {
Config::set('system', 'finished-dbclean-7', true);
}
}
// Call it again if not all entries were purged

27
include/items.php
View file

@ -501,6 +501,20 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
}
}
if ($notify) {
$guid_prefix = "";
} elseif ((trim($arr['guid']) == "") AND (trim($arr['plink']) != "")) {
$arr['guid'] = uri_to_guid($arr['plink']);
} elseif ((trim($arr['guid']) == "") AND (trim($arr['uri']) != "")) {
$arr['guid'] = uri_to_guid($arr['uri']);
} else {
$parsed = parse_url($arr["author-link"]);
$guid_prefix = hash("crc32", $parsed["host"]);
}
$arr['guid'] = ((x($arr, 'guid')) ? notags(trim($arr['guid'])) : get_guid(32, $guid_prefix));
$arr['uri'] = ((x($arr, 'uri')) ? notags(trim($arr['uri'])) : item_new_uri($a->get_hostname(), $uid, $arr['guid']));
// Store conversation data
$arr = store_conversation($arr);
@ -585,20 +599,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
item_add_language_opt($arr);
if ($notify) {
$guid_prefix = "";
} elseif ((trim($arr['guid']) == "") AND (trim($arr['plink']) != "")) {
$arr['guid'] = uri_to_guid($arr['plink']);
} elseif ((trim($arr['guid']) == "") AND (trim($arr['uri']) != "")) {
$arr['guid'] = uri_to_guid($arr['uri']);
} else {
$parsed = parse_url($arr["author-link"]);
$guid_prefix = hash("crc32", $parsed["host"]);
}
$arr['wall'] = ((x($arr, 'wall')) ? intval($arr['wall']) : 0);
$arr['guid'] = ((x($arr, 'guid')) ? notags(trim($arr['guid'])) : get_guid(32, $guid_prefix));
$arr['uri'] = ((x($arr, 'uri')) ? notags(trim($arr['uri'])) : item_new_uri($a->get_hostname(), $uid, $arr['guid']));
$arr['extid'] = ((x($arr, 'extid')) ? notags(trim($arr['extid'])) : '');
$arr['author-name'] = ((x($arr, 'author-name')) ? trim($arr['author-name']) : '');
$arr['author-link'] = ((x($arr, 'author-link')) ? notags(trim($arr['author-link'])) : '');

25
include/poller.php
View file

@ -121,12 +121,8 @@ function poller_execute($queue) {
return false;
}
$upd = q("UPDATE `workerqueue` SET `executed` = '%s', `pid` = %d WHERE `id` = %d AND `pid` = 0",
dbesc(datetime_convert()),
intval($mypid),
intval($queue["id"]));
if (!$upd) {
if (!dba::update('workerqueue', array('executed' => datetime_convert(), 'pid' => $mypid),
array('id' => $queue["id"], 'pid' => 0))) {
logger("Couldn't update queue entry ".$queue["id"]." - skip this execution", LOGGER_DEBUG);
dba::commit();
return true;
@ -156,7 +152,7 @@ function poller_execute($queue) {
if (!validate_include($include)) {
logger("Include file ".$argv[0]." is not valid!");
q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($queue["id"]));
dba::delete('workerqueue', array('id' => $queue["id"]));
return true;
}
@ -168,7 +164,7 @@ function poller_execute($queue) {
poller_exec_function($queue, $funcname, $argv);
q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($queue["id"]));
dba::delete('workerqueue', array('id' => $queue["id"]));
} else {
logger("Function ".$funcname." does not exist");
}
@ -376,8 +372,8 @@ function poller_kill_stale_workers() {
foreach ($r AS $pid) {
if (!posix_kill($pid["pid"], 0)) {
q("UPDATE `workerqueue` SET `executed` = '%s', `pid` = 0 WHERE `pid` = %d",
dbesc(NULL_DATE), intval($pid["pid"]));
dba::update('workerqueue', array('executed' => NULL_DATE, 'pid' => 0),
array('pid' => $pid["pid"]));
} else {
// Kill long running processes
@ -401,12 +397,9 @@ function poller_kill_stale_workers() {
// We killed the stale process.
// To avoid a blocking situation we reschedule the process at the beginning of the queue.
// Additionally we are lowering the priority.
q("UPDATE `workerqueue` SET `executed` = '%s', `created` = '%s',
`priority` = %d, `pid` = 0 WHERE `pid` = %d",
dbesc(NULL_DATE),
dbesc(datetime_convert()),
intval(PRIORITY_NEGLIGIBLE),
intval($pid["pid"]));
dba::update('workerqueue',
array('executed' => NULL_DATE, 'created' => datetime_convert(), 'priority' => PRIORITY_NEGLIGIBLE, 'pid' => 0),
array('pid' => $pid["pid"]));
} else {
logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG);
}

4
include/threads.php
View file

@ -245,9 +245,7 @@ function delete_thread($itemid, $itemuri = "") {
intval($item["uid"])
);
if (!dbm::is_result($r)) {
$r = q("DELETE FROM `item` WHERE `uri` = '%s' AND `uid` = 0",
dbesc($itemuri)
);
dba::delete('item', array('uri' => $itemuri, 'uid' => 0));
logger("delete_thread: Deleted shadow for item ".$itemuri." - ".print_r($result, true), LOGGER_DEBUG);
}
}

12
include/uimport.php
View file

@ -63,16 +63,6 @@ function db_import_assoc($table, $arr) {
return q($query);
}
function import_cleanup($newuid) {
q("DELETE FROM `user` WHERE uid = %d", $newuid);
q("DELETE FROM `contact` WHERE uid = %d", $newuid);
q("DELETE FROM `profile` WHERE uid = %d", $newuid);
q("DELETE FROM `photo` WHERE uid = %d", $newuid);
q("DELETE FROM `group` WHERE uid = %d", $newuid);
q("DELETE FROM `group_member` WHERE uid = %d", $newuid);
q("DELETE FROM `pconfig` WHERE uid = %d", $newuid);
}
/**
* @brief Import account file exported from mod/uexport
*
@ -174,7 +164,7 @@ function import_account(App $a, $file) {
if ($r === false) {
logger("uimport:insert profile " . $profile['profile-name'] . " : ERROR : " . last_error(), LOGGER_NORMAL);
info(t("User profile creation error"));
import_cleanup($newuid);
dba::delete('user', array('uid' => $newuid));
return;
}
}

7
include/user.php
View file

@ -226,9 +226,7 @@ function create_user($arr) {
);
if ((dbm::is_result($r)) && (count($r) > 1) && $newuid) {
$result['message'] .= t('Nickname is already registered. Please choose another.') . EOL;
q("DELETE FROM `user` WHERE `uid` = %d",
intval($newuid)
);
dba::delete('user', array('uid' => $newuid));
return $result;
}
@ -248,8 +246,7 @@ function create_user($arr) {
if ($r === false) {
$result['message'] .= t('An error occurred creating your default profile. Please try again.') . EOL;
// Start fresh next time.
$r = q("DELETE FROM `user` WHERE `uid` = %d",
intval($newuid));
dba::delete('user', array('uid' => $newuid));
return $result;
}

19
mod/regmod.php
View file

@ -74,27 +74,18 @@ function user_deny($hash) {
dbesc($hash)
);
if(! dbm::is_result($register))
if (!dbm::is_result($register)) {
return false;
}
$user = q("SELECT * FROM `user` WHERE `uid` = %d LIMIT 1",
intval($register[0]['uid'])
);
$r = q("DELETE FROM `user` WHERE `uid` = %d",
intval($register[0]['uid'])
);
$r = q("DELETE FROM `contact` WHERE `uid` = %d",
intval($register[0]['uid'])
);
$r = q("DELETE FROM `profile` WHERE `uid` = %d",
intval($register[0]['uid'])
);
dba::delete('user', array('uid' => $register[0]['uid']));
dba::delete('register', array('hash' => $register[0]['hash']));
$r = q("DELETE FROM `register` WHERE `hash` = '%s'",
dbesc($register[0]['hash'])
);
notice( sprintf(t('Registration revoked for %s'), $user[0]['username']) . EOL);
notice(sprintf(t('Registration revoked for %s'), $user[0]['username']) . EOL);
return true;
}