From 04cc9bb4dee379b957059960e26c65734b4f631b Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 18 Nov 2017 06:22:41 +0000 Subject: [PATCH] CronJobs moved as well --- include/cronjobs.php | 281 --------------------------------------- src/Worker/Cron.php | 16 +-- src/Worker/CronJobs.php | 284 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 292 insertions(+), 289 deletions(-) delete mode 100644 include/cronjobs.php create mode 100644 src/Worker/CronJobs.php diff --git a/include/cronjobs.php b/include/cronjobs.php deleted file mode 100644 index 2f6ade9c9..000000000 --- a/include/cronjobs.php +++ /dev/null @@ -1,281 +0,0 @@ - '%s' - AND `account_expires_on` < UTC_TIMESTAMP()", dbesc(NULL_DATE)); - - // delete user records for recently removed accounts - $r = q("SELECT * FROM `user` WHERE `account_removed` AND `account_expires_on` < UTC_TIMESTAMP() - INTERVAL 3 DAY"); - if (DBM::is_result($r)) { - foreach ($r as $user) { - dba::delete('user', array('uid' => $user['uid'])); - } - } -} - -/** - * @brief Clear cache entries - * - * @param App $a - */ -function cron_clear_cache(App $a) { - - $last = Config::get('system','cache_last_cleared'); - - if ($last) { - $next = $last + (3600); // Once per hour - $clear_cache = ($next <= time()); - } else { - $clear_cache = true; - } - - if (!$clear_cache) { - return; - } - - // clear old cache - Cache::clear(); - - // clear old item cache files - clear_cache(); - - // clear cache for photos - clear_cache($a->get_basepath(), $a->get_basepath()."/photo"); - - // clear smarty cache - clear_cache($a->get_basepath()."/view/smarty3/compiled", $a->get_basepath()."/view/smarty3/compiled"); - - // clear cache for image proxy - if (!Config::get("system", "proxy_disabled")) { - clear_cache($a->get_basepath(), $a->get_basepath()."/proxy"); - - $cachetime = Config::get('system','proxy_cache_time'); - if (!$cachetime) { - $cachetime = PROXY_DEFAULT_TIME; - } - q('DELETE FROM `photo` WHERE `uid` = 0 AND `resource-id` LIKE "pic:%%" AND `created` < NOW() - INTERVAL %d SECOND', $cachetime); - } - - // Delete the cached OEmbed entries that are older than one year - q("DELETE FROM `oembed` WHERE `created` < NOW() - INTERVAL 3 MONTH"); - - // Delete the cached "parse_url" entries that are older than one year - q("DELETE FROM `parsed_url` WHERE `created` < NOW() - INTERVAL 3 MONTH"); - - // Maximum table size in megabyte - $max_tablesize = intval(Config::get('system','optimize_max_tablesize')) * 1000000; - if ($max_tablesize == 0) { - $max_tablesize = 100 * 1000000; // Default are 100 MB - } - if ($max_tablesize > 0) { - // Minimum fragmentation level in percent - $fragmentation_level = intval(Config::get('system','optimize_fragmentation')) / 100; - if ($fragmentation_level == 0) { - $fragmentation_level = 0.3; // Default value is 30% - } - - // Optimize some tables that need to be optimized - $r = q("SHOW TABLE STATUS"); - foreach ($r as $table) { - - // Don't optimize tables that are too large - if ($table["Data_length"] > $max_tablesize) { - continue; - } - - // Don't optimize empty tables - if ($table["Data_length"] == 0) { - continue; - } - - // Calculate fragmentation - $fragmentation = $table["Data_free"] / ($table["Data_length"] + $table["Index_length"]); - - logger("Table ".$table["Name"]." - Fragmentation level: ".round($fragmentation * 100, 2), LOGGER_DEBUG); - - // Don't optimize tables that needn't to be optimized - if ($fragmentation < $fragmentation_level) { - continue; - } - - // So optimize it - logger("Optimize Table ".$table["Name"], LOGGER_DEBUG); - q("OPTIMIZE TABLE `%s`", dbesc($table["Name"])); - } - } - - Config::set('system','cache_last_cleared', time()); -} - -/** - * @brief Repair missing values in Diaspora contacts - * - * @param App $a - */ -function cron_repair_diaspora(App $a) { - - $starttime = time(); - - $r = q("SELECT `id`, `url` FROM `contact` - WHERE `network` = '%s' AND (`batch` = '' OR `notify` = '' OR `poll` = '' OR pubkey = '') - ORDER BY RAND() LIMIT 50", dbesc(NETWORK_DIASPORA)); - if (!DBM::is_result($r)) { - return; - } - - foreach ($r AS $contact) { - // Quit the loop after 3 minutes - if (time() > ($starttime + 180)) { - return; - } - - if (!PortableContact::reachable($contact["url"])) { - continue; - } - - $data = Probe::uri($contact["url"]); - if ($data["network"] != NETWORK_DIASPORA) { - continue; - } - - logger("Repair contact ".$contact["id"]." ".$contact["url"], LOGGER_DEBUG); - q("UPDATE `contact` SET `batch` = '%s', `notify` = '%s', `poll` = '%s', pubkey = '%s' WHERE `id` = %d", - dbesc($data["batch"]), dbesc($data["notify"]), dbesc($data["poll"]), dbesc($data["pubkey"]), - intval($contact["id"])); - } -} - -/** - * @brief Do some repairs in database entries - * - */ -function cron_repair_database() { - - // Sometimes there seem to be issues where the "self" contact vanishes. - // We haven't found the origin of the problem by now. - $r = q("SELECT `uid` FROM `user` WHERE NOT EXISTS (SELECT `uid` FROM `contact` WHERE `contact`.`uid` = `user`.`uid` AND `contact`.`self`)"); - if (DBM::is_result($r)) { - foreach ($r AS $user) { - logger('Create missing self contact for user '.$user['uid']); - user_create_self_contact($user['uid']); - } - } - - // Set the parent if it wasn't set. (Shouldn't happen - but does sometimes) - // This call is very "cheap" so we can do it at any time without a problem - q("UPDATE `item` INNER JOIN `item` AS `parent` ON `parent`.`uri` = `item`.`parent-uri` AND `parent`.`uid` = `item`.`uid` SET `item`.`parent` = `parent`.`id` WHERE `item`.`parent` = 0"); - - // There was an issue where the nick vanishes from the contact table - q("UPDATE `contact` INNER JOIN `user` ON `contact`.`uid` = `user`.`uid` SET `nick` = `nickname` WHERE `self` AND `nick`=''"); - - // Update the global contacts for local users - $r = q("SELECT `uid` FROM `user` WHERE `verified` AND NOT `blocked` AND NOT `account_removed` AND NOT `account_expired`"); - if (DBM::is_result($r)) { - foreach ($r AS $user) { - GlobalContact::updateForUser($user["uid"]); - } - } - - /// @todo - /// - remove thread entries without item - /// - remove sign entries without item - /// - remove children when parent got lost - /// - set contact-id in item when not present -} diff --git a/src/Worker/Cron.php b/src/Worker/Cron.php index 7b6e2ede1..1893a862e 100644 --- a/src/Worker/Cron.php +++ b/src/Worker/Cron.php @@ -45,22 +45,22 @@ Class Cron { Worker::add(PRIORITY_LOW, "discover_poco", "checkcontact"); // Expire and remove user entries - Worker::add(PRIORITY_MEDIUM, "cronjobs", "expire_and_remove_users"); + Worker::add(PRIORITY_MEDIUM, "CronJobs", "expire_and_remove_users"); // Call possible post update functions - Worker::add(PRIORITY_LOW, "cronjobs", "post_update"); + Worker::add(PRIORITY_LOW, "CronJobs", "post_update"); // update nodeinfo data - Worker::add(PRIORITY_LOW, "cronjobs", "nodeinfo"); + Worker::add(PRIORITY_LOW, "CronJobs", "nodeinfo"); // Clear cache entries - Worker::add(PRIORITY_LOW, "cronjobs", "clear_cache"); + Worker::add(PRIORITY_LOW, "CronJobs", "clear_cache"); // Repair missing Diaspora values in contacts - Worker::add(PRIORITY_LOW, "cronjobs", "repair_diaspora"); + Worker::add(PRIORITY_LOW, "CronJobs", "repair_diaspora"); // Repair entries in the database - Worker::add(PRIORITY_LOW, "cronjobs", "repair_database"); + Worker::add(PRIORITY_LOW, "CronJobs", "repair_database"); // once daily run birthday_updates and then expire in background $d1 = Config::get('system', 'last_expire_day'); @@ -68,7 +68,7 @@ Class Cron { if ($d2 != intval($d1)) { - Worker::add(PRIORITY_LOW, "cronjobs", "update_contact_birthdays"); + Worker::add(PRIORITY_LOW, "CronJobs", "update_contact_birthdays"); Worker::add(PRIORITY_LOW, "discover_poco", "update_server"); @@ -80,7 +80,7 @@ Class Cron { Worker::add(PRIORITY_MEDIUM, 'DBClean'); - Worker::add(PRIORITY_LOW, "cronjobs", "update_photo_albums"); + Worker::add(PRIORITY_LOW, "CronJobs", "update_photo_albums"); // Delete all done workerqueue entries dba::delete('workerqueue', array('`done` AND `executed` < UTC_TIMESTAMP() - INTERVAL 12 HOUR')); diff --git a/src/Worker/CronJobs.php b/src/Worker/CronJobs.php new file mode 100644 index 000000000..5e1d65d3b --- /dev/null +++ b/src/Worker/CronJobs.php @@ -0,0 +1,284 @@ + '%s' + AND `account_expires_on` < UTC_TIMESTAMP()", dbesc(NULL_DATE)); + + // delete user records for recently removed accounts + $r = q("SELECT * FROM `user` WHERE `account_removed` AND `account_expires_on` < UTC_TIMESTAMP() - INTERVAL 3 DAY"); + if (DBM::is_result($r)) { + foreach ($r as $user) { + dba::delete('user', array('uid' => $user['uid'])); + } + } + } + + /** + * @brief Clear cache entries + * + * @param App $a + */ + private static function clearCache(App $a) { + + $last = Config::get('system','cache_last_cleared'); + + if ($last) { + $next = $last + (3600); // Once per hour + $clear_cache = ($next <= time()); + } else { + $clear_cache = true; + } + + if (!$clear_cache) { + return; + } + + // clear old cache + Cache::clear(); + + // clear old item cache files + clear_cache(); + + // clear cache for photos + clear_cache($a->get_basepath(), $a->get_basepath()."/photo"); + + // clear smarty cache + clear_cache($a->get_basepath()."/view/smarty3/compiled", $a->get_basepath()."/view/smarty3/compiled"); + + // clear cache for image proxy + if (!Config::get("system", "proxy_disabled")) { + clear_cache($a->get_basepath(), $a->get_basepath()."/proxy"); + + $cachetime = Config::get('system','proxy_cache_time'); + if (!$cachetime) { + $cachetime = PROXY_DEFAULT_TIME; + } + q('DELETE FROM `photo` WHERE `uid` = 0 AND `resource-id` LIKE "pic:%%" AND `created` < NOW() - INTERVAL %d SECOND', $cachetime); + } + + // Delete the cached OEmbed entries that are older than one year + q("DELETE FROM `oembed` WHERE `created` < NOW() - INTERVAL 3 MONTH"); + + // Delete the cached "parse_url" entries that are older than one year + q("DELETE FROM `parsed_url` WHERE `created` < NOW() - INTERVAL 3 MONTH"); + + // Maximum table size in megabyte + $max_tablesize = intval(Config::get('system','optimize_max_tablesize')) * 1000000; + if ($max_tablesize == 0) { + $max_tablesize = 100 * 1000000; // Default are 100 MB + } + if ($max_tablesize > 0) { + // Minimum fragmentation level in percent + $fragmentation_level = intval(Config::get('system','optimize_fragmentation')) / 100; + if ($fragmentation_level == 0) { + $fragmentation_level = 0.3; // Default value is 30% + } + + // Optimize some tables that need to be optimized + $r = q("SHOW TABLE STATUS"); + foreach ($r as $table) { + + // Don't optimize tables that are too large + if ($table["Data_length"] > $max_tablesize) { + continue; + } + + // Don't optimize empty tables + if ($table["Data_length"] == 0) { + continue; + } + + // Calculate fragmentation + $fragmentation = $table["Data_free"] / ($table["Data_length"] + $table["Index_length"]); + + logger("Table ".$table["Name"]." - Fragmentation level: ".round($fragmentation * 100, 2), LOGGER_DEBUG); + + // Don't optimize tables that needn't to be optimized + if ($fragmentation < $fragmentation_level) { + continue; + } + + // So optimize it + logger("Optimize Table ".$table["Name"], LOGGER_DEBUG); + q("OPTIMIZE TABLE `%s`", dbesc($table["Name"])); + } + } + + Config::set('system','cache_last_cleared', time()); + } + + /** + * @brief Repair missing values in Diaspora contacts + * + * @param App $a + */ + private static function repairDiaspora(App $a) { + + $starttime = time(); + + $r = q("SELECT `id`, `url` FROM `contact` + WHERE `network` = '%s' AND (`batch` = '' OR `notify` = '' OR `poll` = '' OR pubkey = '') + ORDER BY RAND() LIMIT 50", dbesc(NETWORK_DIASPORA)); + if (!DBM::is_result($r)) { + return; + } + + foreach ($r AS $contact) { + // Quit the loop after 3 minutes + if (time() > ($starttime + 180)) { + return; + } + + if (!PortableContact::reachable($contact["url"])) { + continue; + } + + $data = Probe::uri($contact["url"]); + if ($data["network"] != NETWORK_DIASPORA) { + continue; + } + + logger("Repair contact ".$contact["id"]." ".$contact["url"], LOGGER_DEBUG); + q("UPDATE `contact` SET `batch` = '%s', `notify` = '%s', `poll` = '%s', pubkey = '%s' WHERE `id` = %d", + dbesc($data["batch"]), dbesc($data["notify"]), dbesc($data["poll"]), dbesc($data["pubkey"]), + intval($contact["id"])); + } + } + + /** + * @brief Do some repairs in database entries + * + */ + private static function repairDatabase() { + + // Sometimes there seem to be issues where the "self" contact vanishes. + // We haven't found the origin of the problem by now. + $r = q("SELECT `uid` FROM `user` WHERE NOT EXISTS (SELECT `uid` FROM `contact` WHERE `contact`.`uid` = `user`.`uid` AND `contact`.`self`)"); + if (DBM::is_result($r)) { + foreach ($r AS $user) { + logger('Create missing self contact for user '.$user['uid']); + user_create_self_contact($user['uid']); + } + } + + // Set the parent if it wasn't set. (Shouldn't happen - but does sometimes) + // This call is very "cheap" so we can do it at any time without a problem + q("UPDATE `item` INNER JOIN `item` AS `parent` ON `parent`.`uri` = `item`.`parent-uri` AND `parent`.`uid` = `item`.`uid` SET `item`.`parent` = `parent`.`id` WHERE `item`.`parent` = 0"); + + // There was an issue where the nick vanishes from the contact table + q("UPDATE `contact` INNER JOIN `user` ON `contact`.`uid` = `user`.`uid` SET `nick` = `nickname` WHERE `self` AND `nick`=''"); + + // Update the global contacts for local users + $r = q("SELECT `uid` FROM `user` WHERE `verified` AND NOT `blocked` AND NOT `account_removed` AND NOT `account_expired`"); + if (DBM::is_result($r)) { + foreach ($r AS $user) { + GlobalContact::updateForUser($user["uid"]); + } + } + + /// @todo + /// - remove thread entries without item + /// - remove sign entries without item + /// - remove children when parent got lost + /// - set contact-id in item when not present + } +}