mirror of https://github.com/friendica/friendica
Merge pull request #3461 from annando/1705-some-more-dba
DBA: poller / SQL errors / dbclean
This commit is contained in:
Hypolite Petovan
GitHub
commit
db0c3620c0
8
boot.php
8
boot.php
|
|
@ -1077,14 +1077,10 @@ function proc_run($cmd) {
|
|||
array_shift($argv);
|
||||
|
||||
$parameters = json_encode($argv);
|
||||
$found = q("SELECT `id` FROM `workerqueue` WHERE `parameter` = '%s'", dbesc($parameters));
|
||||
$found = dba::select('workerqueue', array('id'), array('parameter' => $parameters), array('limit' => 1));
|
||||
|
||||
if (!dbm::is_result($found)) {
|
||||
q("INSERT INTO `workerqueue` (`parameter`, `created`, `priority`)
|
||||
VALUES ('%s', '%s', %d)",
|
||||
dbesc($parameters),
|
||||
dbesc(datetime_convert()),
|
||||
intval($priority));
|
||||
dba::insert('workerqueue', array('parameter' => $parameters, 'created' => datetime_convert(), 'priority' => $priority));
|
||||
}
|
||||
|
||||
// Should we quit and wait for the poller to be called as a cronjob?
|
||||
|
|
|
|||
|
|
@ -23,29 +23,7 @@ function user_remove($uid) {
|
|||
$r[0]['nickname']
|
||||
);
|
||||
|
||||
/// @todo Should be done in a background job since this likely will run into a time out
|
||||
// don't delete yet, will be done later when contacts have deleted my stuff
|
||||
// q("DELETE FROM `contact` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `gcign` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `group` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `group_member` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `intro` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `event` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `item` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `item_id` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `mail` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `mailacct` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `manage` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `notify` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `photo` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `attach` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `profile` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `profile_check` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `pconfig` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `search` WHERE `uid` = %d", intval($uid));
|
||||
q("DELETE FROM `spam` WHERE `uid` = %d", intval($uid));
|
||||
// don't delete yet, will be done later when contacts have deleted my stuff
|
||||
// q("DELETE FROM `user` WHERE `uid` = %d", intval($uid));
|
||||
// The user and related data will be deleted in "cron_expire_and_remove_users" (cronjobs.php)
|
||||
q("UPDATE `user` SET `account_removed` = 1, `account_expires_on` = UTC_TIMESTAMP() WHERE `uid` = %d", intval($uid));
|
||||
proc_run(PRIORITY_HIGH, "include/notifier.php", "removeme", $uid);
|
||||
|
||||
|
|
|
|||
|
|
@ -110,12 +110,11 @@ function cron_expire_and_remove_users() {
|
|||
AND `account_expires_on` > '%s'
|
||||
AND `account_expires_on` < UTC_TIMESTAMP()", dbesc(NULL_DATE));
|
||||
|
||||
// delete user and contact records for recently removed accounts
|
||||
// delete user records for recently removed accounts
|
||||
$r = q("SELECT * FROM `user` WHERE `account_removed` AND `account_expires_on` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
|
||||
if (dbm::is_result($r)) {
|
||||
foreach ($r as $user) {
|
||||
q("DELETE FROM `contact` WHERE `uid` = %d", intval($user['uid']));
|
||||
q("DELETE FROM `user` WHERE `uid` = %d", intval($user['uid']));
|
||||
dba::delete('user', array('uid' => $user['uid']));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,13 +18,11 @@ function dbclean_run(&$argv, &$argc) {
|
|||
}
|
||||
|
||||
if ($stage == 0) {
|
||||
proc_run(PRIORITY_LOW, 'include/dbclean.php', 1);
|
||||
proc_run(PRIORITY_LOW, 'include/dbclean.php', 2);
|
||||
proc_run(PRIORITY_LOW, 'include/dbclean.php', 3);
|
||||
proc_run(PRIORITY_LOW, 'include/dbclean.php', 4);
|
||||
proc_run(PRIORITY_LOW, 'include/dbclean.php', 5);
|
||||
proc_run(PRIORITY_LOW, 'include/dbclean.php', 6);
|
||||
proc_run(PRIORITY_LOW, 'include/dbclean.php', 7);
|
||||
for ($i = 1; $i <= 7; $i++) {
|
||||
if (!Config::get('system', 'finished-dbclean-'.$i)) {
|
||||
proc_run(PRIORITY_LOW, 'include/dbclean.php', $i);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
remove_orphans($stage);
|
||||
}
|
||||
|
|
@ -54,9 +52,15 @@ function remove_orphans($stage = 0) {
|
|||
}
|
||||
} else {
|
||||
logger("No global item orphans found");
|
||||
|
||||
}
|
||||
dba::close($r);
|
||||
logger("Done deleting ".$count." old global item entries from item table without user copy");
|
||||
|
||||
// We will eventually set this value when we found a good way to delete these items in another way.
|
||||
// if ($count < $limit) {
|
||||
// Config::set('system', 'finished-dbclean-1', true);
|
||||
// }
|
||||
} elseif ($stage == 2) {
|
||||
logger("Deleting items without parents");
|
||||
$r = dba::p("SELECT `id` FROM `item` WHERE NOT EXISTS (SELECT `id` FROM `item` AS `i` WHERE `item`.`parent` = `i`.`id`) LIMIT ".intval($limit));
|
||||
|
|
@ -71,6 +75,10 @@ function remove_orphans($stage = 0) {
|
|||
}
|
||||
dba::close($r);
|
||||
logger("Done deleting ".$count." items without parents");
|
||||
|
||||
if ($count < $limit) {
|
||||
Config::set('system', 'finished-dbclean-2', true);
|
||||
}
|
||||
} elseif ($stage == 3) {
|
||||
logger("Deleting orphaned data from thread table");
|
||||
$r = dba::p("SELECT `iid` FROM `thread` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`) LIMIT ".intval($limit));
|
||||
|
|
@ -83,9 +91,12 @@ function remove_orphans($stage = 0) {
|
|||
} else {
|
||||
logger("No thread orphans found");
|
||||
}
|
||||
|
||||
dba::close($r);
|
||||
logger("Done deleting ".$count." orphaned data from thread table");
|
||||
|
||||
if ($count < $limit) {
|
||||
Config::set('system', 'finished-dbclean-3', true);
|
||||
}
|
||||
} elseif ($stage == 4) {
|
||||
logger("Deleting orphaned data from notify table");
|
||||
$r = dba::p("SELECT `iid` FROM `notify` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`) LIMIT ".intval($limit));
|
||||
|
|
@ -100,6 +111,10 @@ function remove_orphans($stage = 0) {
|
|||
}
|
||||
dba::close($r);
|
||||
logger("Done deleting ".$count." orphaned data from notify table");
|
||||
|
||||
if ($count < $limit) {
|
||||
Config::set('system', 'finished-dbclean-4', true);
|
||||
}
|
||||
} elseif ($stage == 5) {
|
||||
logger("Deleting orphaned data from notify-threads table");
|
||||
$r = dba::p("SELECT `id` FROM `notify-threads` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `notify-threads`.`master-parent-item`) LIMIT ".intval($limit));
|
||||
|
|
@ -114,6 +129,10 @@ function remove_orphans($stage = 0) {
|
|||
}
|
||||
dba::close($r);
|
||||
logger("Done deleting ".$count." orphaned data from notify-threads table");
|
||||
|
||||
if ($count < $limit) {
|
||||
Config::set('system', 'finished-dbclean-5', true);
|
||||
}
|
||||
} elseif ($stage == 6) {
|
||||
logger("Deleting orphaned data from sign table");
|
||||
$r = dba::p("SELECT `iid` FROM `sign` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`) LIMIT ".intval($limit));
|
||||
|
|
@ -128,6 +147,10 @@ function remove_orphans($stage = 0) {
|
|||
}
|
||||
dba::close($r);
|
||||
logger("Done deleting ".$count." orphaned data from sign table");
|
||||
|
||||
if ($count < $limit) {
|
||||
Config::set('system', 'finished-dbclean-6', true);
|
||||
}
|
||||
} elseif ($stage == 7) {
|
||||
logger("Deleting orphaned data from term table");
|
||||
$r = dba::p("SELECT `oid` FROM `term` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`) LIMIT ".intval($limit));
|
||||
|
|
@ -142,6 +165,10 @@ function remove_orphans($stage = 0) {
|
|||
}
|
||||
dba::close($r);
|
||||
logger("Done deleting ".$count." orphaned data from term table");
|
||||
|
||||
if ($count < $limit) {
|
||||
Config::set('system', 'finished-dbclean-7', true);
|
||||
}
|
||||
}
|
||||
|
||||
// Call it again if not all entries were purged
|
||||
|
|
|
|||
|
|
@ -501,6 +501,20 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
|
|||
}
|
||||
}
|
||||
|
||||
if ($notify) {
|
||||
$guid_prefix = "";
|
||||
} elseif ((trim($arr['guid']) == "") AND (trim($arr['plink']) != "")) {
|
||||
$arr['guid'] = uri_to_guid($arr['plink']);
|
||||
} elseif ((trim($arr['guid']) == "") AND (trim($arr['uri']) != "")) {
|
||||
$arr['guid'] = uri_to_guid($arr['uri']);
|
||||
} else {
|
||||
$parsed = parse_url($arr["author-link"]);
|
||||
$guid_prefix = hash("crc32", $parsed["host"]);
|
||||
}
|
||||
|
||||
$arr['guid'] = ((x($arr, 'guid')) ? notags(trim($arr['guid'])) : get_guid(32, $guid_prefix));
|
||||
$arr['uri'] = ((x($arr, 'uri')) ? notags(trim($arr['uri'])) : item_new_uri($a->get_hostname(), $uid, $arr['guid']));
|
||||
|
||||
// Store conversation data
|
||||
$arr = store_conversation($arr);
|
||||
|
||||
|
|
@ -585,20 +599,7 @@ function item_store($arr, $force_parent = false, $notify = false, $dontcache = f
|
|||
|
||||
item_add_language_opt($arr);
|
||||
|
||||
if ($notify) {
|
||||
$guid_prefix = "";
|
||||
} elseif ((trim($arr['guid']) == "") AND (trim($arr['plink']) != "")) {
|
||||
$arr['guid'] = uri_to_guid($arr['plink']);
|
||||
} elseif ((trim($arr['guid']) == "") AND (trim($arr['uri']) != "")) {
|
||||
$arr['guid'] = uri_to_guid($arr['uri']);
|
||||
} else {
|
||||
$parsed = parse_url($arr["author-link"]);
|
||||
$guid_prefix = hash("crc32", $parsed["host"]);
|
||||
}
|
||||
|
||||
$arr['wall'] = ((x($arr, 'wall')) ? intval($arr['wall']) : 0);
|
||||
$arr['guid'] = ((x($arr, 'guid')) ? notags(trim($arr['guid'])) : get_guid(32, $guid_prefix));
|
||||
$arr['uri'] = ((x($arr, 'uri')) ? notags(trim($arr['uri'])) : item_new_uri($a->get_hostname(), $uid, $arr['guid']));
|
||||
$arr['extid'] = ((x($arr, 'extid')) ? notags(trim($arr['extid'])) : '');
|
||||
$arr['author-name'] = ((x($arr, 'author-name')) ? trim($arr['author-name']) : '');
|
||||
$arr['author-link'] = ((x($arr, 'author-link')) ? notags(trim($arr['author-link'])) : '');
|
||||
|
|
|
|||
|
|
@ -121,12 +121,8 @@ function poller_execute($queue) {
|
|||
return false;
|
||||
}
|
||||
|
||||
$upd = q("UPDATE `workerqueue` SET `executed` = '%s', `pid` = %d WHERE `id` = %d AND `pid` = 0",
|
||||
dbesc(datetime_convert()),
|
||||
intval($mypid),
|
||||
intval($queue["id"]));
|
||||
|
||||
if (!$upd) {
|
||||
if (!dba::update('workerqueue', array('executed' => datetime_convert(), 'pid' => $mypid),
|
||||
array('id' => $queue["id"], 'pid' => 0))) {
|
||||
logger("Couldn't update queue entry ".$queue["id"]." - skip this execution", LOGGER_DEBUG);
|
||||
dba::commit();
|
||||
return true;
|
||||
|
|
@ -156,7 +152,7 @@ function poller_execute($queue) {
|
|||
|
||||
if (!validate_include($include)) {
|
||||
logger("Include file ".$argv[0]." is not valid!");
|
||||
q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($queue["id"]));
|
||||
dba::delete('workerqueue', array('id' => $queue["id"]));
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -168,7 +164,7 @@ function poller_execute($queue) {
|
|||
|
||||
poller_exec_function($queue, $funcname, $argv);
|
||||
|
||||
q("DELETE FROM `workerqueue` WHERE `id` = %d", intval($queue["id"]));
|
||||
dba::delete('workerqueue', array('id' => $queue["id"]));
|
||||
} else {
|
||||
logger("Function ".$funcname." does not exist");
|
||||
}
|
||||
|
|
@ -376,8 +372,8 @@ function poller_kill_stale_workers() {
|
|||
|
||||
foreach ($r AS $pid) {
|
||||
if (!posix_kill($pid["pid"], 0)) {
|
||||
q("UPDATE `workerqueue` SET `executed` = '%s', `pid` = 0 WHERE `pid` = %d",
|
||||
dbesc(NULL_DATE), intval($pid["pid"]));
|
||||
dba::update('workerqueue', array('executed' => NULL_DATE, 'pid' => 0),
|
||||
array('pid' => $pid["pid"]));
|
||||
} else {
|
||||
// Kill long running processes
|
||||
|
||||
|
|
@ -401,12 +397,9 @@ function poller_kill_stale_workers() {
|
|||
// We killed the stale process.
|
||||
// To avoid a blocking situation we reschedule the process at the beginning of the queue.
|
||||
// Additionally we are lowering the priority.
|
||||
q("UPDATE `workerqueue` SET `executed` = '%s', `created` = '%s',
|
||||
`priority` = %d, `pid` = 0 WHERE `pid` = %d",
|
||||
dbesc(NULL_DATE),
|
||||
dbesc(datetime_convert()),
|
||||
intval(PRIORITY_NEGLIGIBLE),
|
||||
intval($pid["pid"]));
|
||||
dba::update('workerqueue',
|
||||
array('executed' => NULL_DATE, 'created' => datetime_convert(), 'priority' => PRIORITY_NEGLIGIBLE, 'pid' => 0),
|
||||
array('pid' => $pid["pid"]));
|
||||
} else {
|
||||
logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -245,9 +245,7 @@ function delete_thread($itemid, $itemuri = "") {
|
|||
intval($item["uid"])
|
||||
);
|
||||
if (!dbm::is_result($r)) {
|
||||
$r = q("DELETE FROM `item` WHERE `uri` = '%s' AND `uid` = 0",
|
||||
dbesc($itemuri)
|
||||
);
|
||||
dba::delete('item', array('uri' => $itemuri, 'uid' => 0));
|
||||
logger("delete_thread: Deleted shadow for item ".$itemuri." - ".print_r($result, true), LOGGER_DEBUG);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -63,16 +63,6 @@ function db_import_assoc($table, $arr) {
|
|||
return q($query);
|
||||
}
|
||||
|
||||
function import_cleanup($newuid) {
|
||||
q("DELETE FROM `user` WHERE uid = %d", $newuid);
|
||||
q("DELETE FROM `contact` WHERE uid = %d", $newuid);
|
||||
q("DELETE FROM `profile` WHERE uid = %d", $newuid);
|
||||
q("DELETE FROM `photo` WHERE uid = %d", $newuid);
|
||||
q("DELETE FROM `group` WHERE uid = %d", $newuid);
|
||||
q("DELETE FROM `group_member` WHERE uid = %d", $newuid);
|
||||
q("DELETE FROM `pconfig` WHERE uid = %d", $newuid);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Import account file exported from mod/uexport
|
||||
*
|
||||
|
|
@ -174,7 +164,7 @@ function import_account(App $a, $file) {
|
|||
if ($r === false) {
|
||||
logger("uimport:insert profile " . $profile['profile-name'] . " : ERROR : " . last_error(), LOGGER_NORMAL);
|
||||
info(t("User profile creation error"));
|
||||
import_cleanup($newuid);
|
||||
dba::delete('user', array('uid' => $newuid));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -226,9 +226,7 @@ function create_user($arr) {
|
|||
);
|
||||
if ((dbm::is_result($r)) && (count($r) > 1) && $newuid) {
|
||||
$result['message'] .= t('Nickname is already registered. Please choose another.') . EOL;
|
||||
q("DELETE FROM `user` WHERE `uid` = %d",
|
||||
intval($newuid)
|
||||
);
|
||||
dba::delete('user', array('uid' => $newuid));
|
||||
return $result;
|
||||
}
|
||||
|
||||
|
|
@ -248,8 +246,7 @@ function create_user($arr) {
|
|||
if ($r === false) {
|
||||
$result['message'] .= t('An error occurred creating your default profile. Please try again.') . EOL;
|
||||
// Start fresh next time.
|
||||
$r = q("DELETE FROM `user` WHERE `uid` = %d",
|
||||
intval($newuid));
|
||||
dba::delete('user', array('uid' => $newuid));
|
||||
return $result;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -74,27 +74,18 @@ function user_deny($hash) {
|
|||
dbesc($hash)
|
||||
);
|
||||
|
||||
if(! dbm::is_result($register))
|
||||
if (!dbm::is_result($register)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
$user = q("SELECT * FROM `user` WHERE `uid` = %d LIMIT 1",
|
||||
intval($register[0]['uid'])
|
||||
);
|
||||
|
||||
$r = q("DELETE FROM `user` WHERE `uid` = %d",
|
||||
intval($register[0]['uid'])
|
||||
);
|
||||
$r = q("DELETE FROM `contact` WHERE `uid` = %d",
|
||||
intval($register[0]['uid'])
|
||||
);
|
||||
$r = q("DELETE FROM `profile` WHERE `uid` = %d",
|
||||
intval($register[0]['uid'])
|
||||
);
|
||||
dba::delete('user', array('uid' => $register[0]['uid']));
|
||||
dba::delete('register', array('hash' => $register[0]['hash']));
|
||||
|
||||
$r = q("DELETE FROM `register` WHERE `hash` = '%s'",
|
||||
dbesc($register[0]['hash'])
|
||||
);
|
||||
notice( sprintf(t('Registration revoked for %s'), $user[0]['username']) . EOL);
|
||||
notice(sprintf(t('Registration revoked for %s'), $user[0]['username']) . EOL);
|
||||
return true;
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue