Merge pull request #5157 from annando/daemon

Workers can now be started exclusively from the daemon and other workers
This commit is contained in:
Hypolite Petovan 2018-06-03 12:18:47 -04:00 committed by GitHub
commit 973ece09c5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 236 additions and 127 deletions

View file

@ -6,8 +6,38 @@
*
* This script was taken from http://php.net/manual/en/function.pcntl-fork.php
*/
function shutdown() {
posix_kill(posix_getpid(), SIGHUP);
use Friendica\App;
use Friendica\BaseObject;
use Friendica\Core\Config;
use Friendica\Core\Worker;
// Ensure that daemon.php is executed from the base path of the installation
if (!file_exists("boot.php") && (sizeof($_SERVER["argv"]) != 0)) {
$directory = dirname($_SERVER["argv"][0]);
if (substr($directory, 0, 1) != "/") {
$directory = $_SERVER["PWD"]."/".$directory;
}
$directory = realpath($directory."/..");
chdir($directory);
}
require_once "boot.php";
require_once "include/dba.php";
$a = new App(dirname(__DIR__));
BaseObject::setApp($a);
require_once ".htconfig.php";
dba::connect($db_host, $db_user, $db_pass, $db_data);
Config::load();
if (!isset($pidfile)) {
die('Please specify a pid file in the variable $pidfile in the .htconfig.php. For example:'."\n".
'$pidfile = "/path/to/daemon.pid";'."\n");
}
if (in_array("start", $_SERVER["argv"])) {
@ -30,27 +60,11 @@ if (empty($_SERVER["argv"][0])) {
die("Unexpected script behaviour. This message should never occur.\n");
}
// Fetch the base directory
$directory = dirname($_SERVER["argv"][0]);
$pid = @file_get_contents($pidfile);
if (substr($directory, 0, 1) != "/") {
$directory = $_SERVER["PWD"]."/".$directory;
}
$directory = realpath($directory."/..");
include $directory."/.htconfig.php";
if (!isset($pidfile)) {
die('Please specify a pid file in the variable $pidfile in the .htconfig.php. For example:'."\n".
'$pidfile = "/path/to/daemon.pid";'."\n");
}
if (in_array($mode, array("stop", "status"))) {
$pid = @file_get_contents($pidfile);
if (!$pid) {
die("Pidfile wasn't found. Is the daemon running?\n");
}
if (empty($pid) && in_array($mode, ["stop", "status"])) {
Config::set('system', 'worker_daemon_mode', false);
die("Pidfile wasn't found. Is the daemon running?\n");
}
if ($mode == "status") {
@ -60,6 +74,7 @@ if ($mode == "status") {
unlink($pidfile);
Config::set('system', 'worker_daemon_mode', false);
die("Daemon process $pid isn't running.\n");
}
@ -68,17 +83,19 @@ if ($mode == "stop") {
unlink($pidfile);
logger("Worker daemon process $pid was killed.", LOGGER_DEBUG);
Config::set('system', 'worker_daemon_mode', false);
die("Worker daemon process $pid was killed.\n");
}
echo "Starting worker daemon.\n";
if (isset($a->config['php_path'])) {
$php = $a->config['php_path'];
} else {
$php = "php";
if (!empty($pid) && posix_kill($pid, 0)) {
die("Daemon process $pid is already running.\n");
}
logger('Starting worker daemon.', LOGGER_DEBUG);
echo "Starting worker daemon.\n";
// Switch over to daemon mode.
if ($pid = pcntl_fork())
return; // Parent
@ -95,32 +112,32 @@ if (posix_setsid() < 0)
if ($pid = pcntl_fork())
return; // Parent
// We lose the database connection upon forking
dba::connect($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
Config::set('system', 'worker_daemon_mode', true);
// Just to be sure that this script really runs endlessly
set_time_limit(0);
$pid = getmypid();
file_put_contents($pidfile, $pid);
$wait_interval = intval(Config::get('system', 'cron_interval', 5)) * 60;
// Now running as a daemon.
while (true) {
// Just to be sure that this script really runs endlessly
set_time_limit(0);
logger('Call the worker', LOGGER_DEBUG);
Worker::spawnWorker();
// Call the worker
$cmdline = $php.' bin/worker.php';
$executed = false;
if (function_exists('proc_open')) {
$resource = proc_open($cmdline . ' &', array(), $foo, $directory);
if (is_resource($resource)) {
$executed = true;
proc_close($resource);
}
}
if (!$executed) {
exec($cmdline.' spawn');
}
// Now sleep for 5 minutes
sleep(300);
logger("Sleep for $wait_interval seconds - or when a worker needs to be called", LOGGER_DEBUG);
$i = 0;
do {
sleep(1);
} while (($i++ < $wait_interval) && !Worker::IPCJobsExists());
}
function shutdown() {
posix_kill(posix_getpid(), SIGHUP);
}

View file

@ -41,7 +41,7 @@ define('FRIENDICA_PLATFORM', 'Friendica');
define('FRIENDICA_CODENAME', 'The Tazmans Flax-lily');
define('FRIENDICA_VERSION', '2018.08-dev');
define('DFRN_PROTOCOL_VERSION', '2.23');
define('DB_UPDATE_VERSION', 1267);
define('DB_UPDATE_VERSION', 1268);
define('NEW_UPDATE_ROUTINE_VERSION', 1170);
/**

View file

@ -1,6 +1,6 @@
-- ------------------------------------------
-- Friendica 2018.08-dev (The Tazmans Flax-lily)
-- DB_UPDATE_VERSION 1267
-- DB_UPDATE_VERSION 1268
-- ------------------------------------------
@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS `addon` (
`plugin_admin` boolean NOT NULL DEFAULT '0' COMMENT '1 = has admin config, 0 = has no admin config',
PRIMARY KEY(`id`),
UNIQUE INDEX `name` (`name`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='registered addons';
--
-- TABLE attach
@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS `attach` (
`deny_cid` mediumtext COMMENT 'Access Control - list of denied contact.id',
`deny_gid` mediumtext COMMENT 'Access Control - list of denied groups',
PRIMARY KEY(`id`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='file attachments';
--
-- TABLE auth_codes
@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS `auth_codes` (
`expires` int NOT NULL DEFAULT 0 COMMENT '',
`scope` varchar(250) NOT NULL DEFAULT '' COMMENT '',
PRIMARY KEY(`id`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='OAuth usage';
--
-- TABLE cache
@ -61,7 +61,7 @@ CREATE TABLE IF NOT EXISTS `cache` (
`updated` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'datetime of cache insertion',
PRIMARY KEY(`k`),
INDEX `k_expires` (`k`,`expires`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Stores temporary data';
--
-- TABLE challenge
@ -74,7 +74,7 @@ CREATE TABLE IF NOT EXISTS `challenge` (
`type` varchar(255) NOT NULL DEFAULT '' COMMENT '',
`last_update` varchar(255) NOT NULL DEFAULT '' COMMENT '',
PRIMARY KEY(`id`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='';
--
-- TABLE clients
@ -87,7 +87,7 @@ CREATE TABLE IF NOT EXISTS `clients` (
`icon` text COMMENT '',
`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
PRIMARY KEY(`client_id`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='OAuth usage';
--
-- TABLE config
@ -99,7 +99,7 @@ CREATE TABLE IF NOT EXISTS `config` (
`v` mediumtext COMMENT '',
PRIMARY KEY(`id`),
UNIQUE INDEX `cat_k` (`cat`,`k`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='main configuration storage';
--
-- TABLE contact
@ -186,7 +186,7 @@ CREATE TABLE IF NOT EXISTS `contact` (
INDEX `nick_uid` (`nick`(32),`uid`),
INDEX `dfrn-id` (`dfrn-id`(64)),
INDEX `issued-id` (`issued-id`(64))
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='contact table';
--
-- TABLE conv
@ -202,7 +202,7 @@ CREATE TABLE IF NOT EXISTS `conv` (
`subject` text COMMENT 'subject of initial message',
PRIMARY KEY(`id`),
INDEX `uid` (`uid`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='private messages';
--
-- TABLE conversation
@ -218,7 +218,7 @@ CREATE TABLE IF NOT EXISTS `conversation` (
PRIMARY KEY(`item-uri`),
INDEX `conversation-uri` (`conversation-uri`),
INDEX `received` (`received`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Raw data and structure information for messages';
--
-- TABLE event
@ -246,7 +246,7 @@ CREATE TABLE IF NOT EXISTS `event` (
`deny_gid` mediumtext COMMENT 'Access Control - list of denied groups',
PRIMARY KEY(`id`),
INDEX `uid_start` (`uid`,`start`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Events';
--
-- TABLE fcontact
@ -272,7 +272,7 @@ CREATE TABLE IF NOT EXISTS `fcontact` (
PRIMARY KEY(`id`),
INDEX `addr` (`addr`(32)),
UNIQUE INDEX `url` (`url`(190))
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Diaspora compatible contacts - used in the Diaspora implementation';
--
-- TABLE fsuggest
@ -288,7 +288,7 @@ CREATE TABLE IF NOT EXISTS `fsuggest` (
`note` text COMMENT '',
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
PRIMARY KEY(`id`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='friend suggestion stuff';
--
-- TABLE gcign
@ -300,7 +300,7 @@ CREATE TABLE IF NOT EXISTS `gcign` (
PRIMARY KEY(`id`),
INDEX `uid` (`uid`),
INDEX `gcid` (`gcid`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='contacts ignored by friend suggestions';
--
-- TABLE gcontact
@ -339,7 +339,7 @@ CREATE TABLE IF NOT EXISTS `gcontact` (
INDEX `addr` (`addr`(64)),
INDEX `hide_network_updated` (`hide`,`network`,`updated`),
INDEX `updated` (`updated`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='global contacts';
--
-- TABLE glink
@ -354,7 +354,7 @@ CREATE TABLE IF NOT EXISTS `glink` (
PRIMARY KEY(`id`),
UNIQUE INDEX `cid_uid_gcid_zcid` (`cid`,`uid`,`gcid`,`zcid`),
INDEX `gcid` (`gcid`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='\'friends of friends\' linkages derived from poco';
--
-- TABLE group
@ -367,7 +367,7 @@ CREATE TABLE IF NOT EXISTS `group` (
`name` varchar(255) NOT NULL DEFAULT '' COMMENT 'human readable name of group',
PRIMARY KEY(`id`),
INDEX `uid` (`uid`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='privacy groups, group info';
--
-- TABLE group_member
@ -379,7 +379,7 @@ CREATE TABLE IF NOT EXISTS `group_member` (
PRIMARY KEY(`id`),
INDEX `contactid` (`contact-id`),
UNIQUE INDEX `gid_contactid` (`gid`,`contact-id`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='privacy groups, member info';
--
-- TABLE gserver
@ -405,7 +405,7 @@ CREATE TABLE IF NOT EXISTS `gserver` (
`last_failure` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
PRIMARY KEY(`id`),
UNIQUE INDEX `nurl` (`nurl`(190))
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Global servers';
--
-- TABLE gserver-tag
@ -415,7 +415,7 @@ CREATE TABLE IF NOT EXISTS `gserver-tag` (
`tag` varchar(100) NOT NULL DEFAULT '' COMMENT 'Tag that the server has subscribed',
PRIMARY KEY(`gserver-id`,`tag`),
INDEX `tag` (`tag`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Tags that the server has subscribed';
--
-- TABLE hook
@ -428,7 +428,7 @@ CREATE TABLE IF NOT EXISTS `hook` (
`priority` smallint unsigned NOT NULL DEFAULT 0 COMMENT 'not yet implemented - can be used to sort conflicts in hook handling by calling handlers in priority order',
PRIMARY KEY(`id`),
UNIQUE INDEX `hook_file_function` (`hook`,`file`,`function`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='addon hook registry';
--
-- TABLE intro
@ -446,7 +446,7 @@ CREATE TABLE IF NOT EXISTS `intro` (
`blocked` boolean NOT NULL DEFAULT '1' COMMENT '',
`ignore` boolean NOT NULL DEFAULT '0' COMMENT '',
PRIMARY KEY(`id`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='';
--
-- TABLE item
@ -465,7 +465,7 @@ CREATE TABLE IF NOT EXISTS `item` (
`extid` varchar(255) NOT NULL DEFAULT '' COMMENT '',
`thr-parent` varchar(255) NOT NULL DEFAULT '' COMMENT 'If the parent of this item is not the top-level item in the conversation, the uri of the immediate parent; otherwise set to parent-uri',
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation timestamp.',
`edited` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last edit (default is created)',
`edited` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last edit (default is created)',
`commented` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last comment/reply to this item',
`received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'datetime',
`changed` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date that something in the conversation changed, indicating clients should fetch the conversation again',
@ -487,7 +487,7 @@ CREATE TABLE IF NOT EXISTS `item` (
`target-type` varchar(100) NOT NULL DEFAULT '' COMMENT 'ActivityStreams target type if applicable (URI)',
`target` text COMMENT 'JSON encoded target structure if used',
`postopts` text COMMENT 'External post connectors add their network name to this comma-separated string to identify that they should be delivered to these networks during delivery',
`plink` varchar(255) NOT NULL DEFAULT '' COMMENT 'permalink or URL toa displayable copy of the message at its source',
`plink` varchar(255) NOT NULL DEFAULT '' COMMENT 'permalink or URL to a displayable copy of the message at its source',
`resource-id` varchar(32) NOT NULL DEFAULT '' COMMENT 'Used to link other tables to items, it identifies the linked resource (e.g. photo) and if set must also set resource_type',
`event-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Used to link to the event.id',
`tag` mediumtext COMMENT '',
@ -544,7 +544,7 @@ CREATE TABLE IF NOT EXISTS `item` (
INDEX `uid_eventid` (`uid`,`event-id`),
INDEX `uid_authorlink` (`uid`,`author-link`(190)),
INDEX `uid_ownerlink` (`uid`,`owner-link`(190))
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='All posts';
--
-- TABLE locks
@ -555,7 +555,7 @@ CREATE TABLE IF NOT EXISTS `locks` (
`locked` boolean NOT NULL DEFAULT '0' COMMENT '',
`pid` int unsigned NOT NULL DEFAULT 0 COMMENT 'Process ID',
PRIMARY KEY(`id`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='';
--
-- TABLE mail
@ -584,7 +584,7 @@ CREATE TABLE IF NOT EXISTS `mail` (
INDEX `uri` (`uri`(64)),
INDEX `parent-uri` (`parent-uri`(64)),
INDEX `contactid` (`contact-id`(32))
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='private messages';
--
-- TABLE mailacct
@ -604,7 +604,7 @@ CREATE TABLE IF NOT EXISTS `mailacct` (
`pubmail` boolean NOT NULL DEFAULT '0' COMMENT '',
`last_check` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
PRIMARY KEY(`id`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Mail account data for fetching mails';
--
-- TABLE manage
@ -615,7 +615,7 @@ CREATE TABLE IF NOT EXISTS `manage` (
`mid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
PRIMARY KEY(`id`),
UNIQUE INDEX `uid_mid` (`uid`,`mid`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='table of accounts that can manage each other';
--
-- TABLE notify
@ -643,7 +643,7 @@ CREATE TABLE IF NOT EXISTS `notify` (
INDEX `seen_uid_date` (`seen`,`uid`,`date`),
INDEX `uid_date` (`uid`,`date`),
INDEX `uid_type_link` (`uid`,`type`,`link`(190))
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='notifications';
--
-- TABLE notify-threads
@ -655,7 +655,7 @@ CREATE TABLE IF NOT EXISTS `notify-threads` (
`parent-item` int unsigned NOT NULL DEFAULT 0 COMMENT '',
`receiver-uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
PRIMARY KEY(`id`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='';
--
-- TABLE oembed
@ -667,7 +667,7 @@ CREATE TABLE IF NOT EXISTS `oembed` (
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'datetime of creation',
PRIMARY KEY(`url`,`maxwidth`),
INDEX `created` (`created`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='cache for OEmbed queries';
--
-- TABLE parsed_url
@ -680,7 +680,7 @@ CREATE TABLE IF NOT EXISTS `parsed_url` (
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'datetime of creation',
PRIMARY KEY(`url`,`guessing`,`oembed`),
INDEX `created` (`created`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='cache for \'parse_url\' queries';
--
-- TABLE participation
@ -691,7 +691,7 @@ CREATE TABLE IF NOT EXISTS `participation` (
`cid` int unsigned NOT NULL COMMENT '',
`fid` int unsigned NOT NULL COMMENT '',
PRIMARY KEY(`iid`,`server`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Storage for participation messages from Diaspora';
--
-- TABLE pconfig
@ -704,7 +704,7 @@ CREATE TABLE IF NOT EXISTS `pconfig` (
`v` mediumtext COMMENT '',
PRIMARY KEY(`id`),
UNIQUE INDEX `uid_cat_k` (`uid`,`cat`,`k`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='personal (per user) configuration storage';
--
-- TABLE photo
@ -739,7 +739,7 @@ CREATE TABLE IF NOT EXISTS `photo` (
INDEX `uid_album_scale_created` (`uid`,`album`(32),`scale`,`created`),
INDEX `uid_album_resource-id_created` (`uid`,`album`(32),`resource-id`,`created`),
INDEX `resource-id` (`resource-id`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='photo storage';
--
-- TABLE poll
@ -759,7 +759,7 @@ CREATE TABLE IF NOT EXISTS `poll` (
`q9` text COMMENT '',
PRIMARY KEY(`id`),
INDEX `uid` (`uid`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Currently unused table for storing poll results';
--
-- TABLE poll_result
@ -770,7 +770,7 @@ CREATE TABLE IF NOT EXISTS `poll_result` (
`choice` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
PRIMARY KEY(`id`),
INDEX `poll_id` (`poll_id`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='data for polls - currently unused';
--
-- TABLE process
@ -781,7 +781,7 @@ CREATE TABLE IF NOT EXISTS `process` (
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
PRIMARY KEY(`pid`),
INDEX `command` (`command`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Currently running system processes';
--
-- TABLE profile
@ -831,7 +831,7 @@ CREATE TABLE IF NOT EXISTS `profile` (
`net-publish` boolean NOT NULL DEFAULT '0' COMMENT 'publish profile in global directory',
PRIMARY KEY(`id`),
INDEX `uid_is-default` (`uid`,`is-default`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='user profiles data';
--
-- TABLE profile_check
@ -844,7 +844,7 @@ CREATE TABLE IF NOT EXISTS `profile_check` (
`sec` varchar(255) NOT NULL DEFAULT '' COMMENT '',
`expire` int unsigned NOT NULL DEFAULT 0 COMMENT '',
PRIMARY KEY(`id`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='DFRN remote auth use';
--
-- TABLE push_subscriber
@ -862,7 +862,7 @@ CREATE TABLE IF NOT EXISTS `push_subscriber` (
`secret` varchar(255) NOT NULL DEFAULT '' COMMENT '',
PRIMARY KEY(`id`),
INDEX `next_try` (`next_try`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Used for OStatus: Contains feed subscribers';
--
-- TABLE queue
@ -881,7 +881,7 @@ CREATE TABLE IF NOT EXISTS `queue` (
PRIMARY KEY(`id`),
INDEX `last` (`last`),
INDEX `next` (`next`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Queue for messages that couldn\'t be delivered';
--
-- TABLE register
@ -895,7 +895,7 @@ CREATE TABLE IF NOT EXISTS `register` (
`language` varchar(16) NOT NULL DEFAULT '' COMMENT '',
`note` text COMMENT '',
PRIMARY KEY(`id`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='registrations requiring admin approval';
--
-- TABLE search
@ -906,7 +906,7 @@ CREATE TABLE IF NOT EXISTS `search` (
`term` varchar(255) NOT NULL DEFAULT '' COMMENT '',
PRIMARY KEY(`id`),
INDEX `uid` (`uid`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='';
--
-- TABLE session
@ -919,7 +919,7 @@ CREATE TABLE IF NOT EXISTS `session` (
PRIMARY KEY(`id`),
INDEX `sid` (`sid`(64)),
INDEX `expire` (`expire`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='web session storage';
--
-- TABLE sign
@ -932,7 +932,7 @@ CREATE TABLE IF NOT EXISTS `sign` (
`signer` varchar(255) NOT NULL DEFAULT '' COMMENT '',
PRIMARY KEY(`id`),
UNIQUE INDEX `iid` (`iid`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Diaspora signatures';
--
-- TABLE term
@ -955,7 +955,7 @@ CREATE TABLE IF NOT EXISTS `term` (
INDEX `uid_otype_type_term_global_created` (`uid`,`otype`,`type`,`term`(32),`global`,`created`),
INDEX `uid_otype_type_url` (`uid`,`otype`,`type`,`url`(64)),
INDEX `guid` (`guid`(64))
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='item taxonomy (categories, tags, etc.) table';
--
-- TABLE thread
@ -998,7 +998,7 @@ CREATE TABLE IF NOT EXISTS `thread` (
INDEX `uid_commented` (`uid`,`commented`),
INDEX `uid_wall_created` (`uid`,`wall`,`created`),
INDEX `private_wall_origin_commented` (`private`,`wall`,`origin`,`commented`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Thread related data';
--
-- TABLE tokens
@ -1011,7 +1011,7 @@ CREATE TABLE IF NOT EXISTS `tokens` (
`scope` varchar(200) NOT NULL DEFAULT '' COMMENT '',
`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
PRIMARY KEY(`id`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='OAuth usage';
--
-- TABLE user
@ -1064,7 +1064,7 @@ CREATE TABLE IF NOT EXISTS `user` (
`openidserver` text COMMENT '',
PRIMARY KEY(`uid`),
INDEX `nickname` (`nickname`(32))
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='The local users';
--
-- TABLE userd
@ -1074,7 +1074,7 @@ CREATE TABLE IF NOT EXISTS `userd` (
`username` varchar(255) NOT NULL COMMENT '',
PRIMARY KEY(`id`),
INDEX `username` (`username`(32))
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Deleted usernames';
--
-- TABLE user-item
@ -1082,9 +1082,18 @@ CREATE TABLE IF NOT EXISTS `userd` (
CREATE TABLE IF NOT EXISTS `user-item` (
`iid` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item id',
`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
`hidden` boolean NOT NULL DEFAULT '0' COMMENT 'Hidden marker',
`hidden` boolean NOT NULL DEFAULT '0' COMMENT 'Marker to hide an item from the user',
PRIMARY KEY(`uid`,`iid`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='User specific item data';
--
-- TABLE worker-ipc
--
CREATE TABLE IF NOT EXISTS `worker-ipc` (
`key` int NOT NULL COMMENT '',
`jobs` boolean COMMENT 'Flag for outstanding jobs',
PRIMARY KEY(`key`)
) ENGINE=MEMORY DEFAULT COLLATE utf8mb4_general_ci COMMENT='Inter process communication between the frontend and the worker';
--
-- TABLE workerqueue
@ -1102,6 +1111,6 @@ CREATE TABLE IF NOT EXISTS `workerqueue` (
INDEX `parameter` (`parameter`(64)),
INDEX `priority_created` (`priority`,`created`),
INDEX `done_executed` (`done`,`executed`)
) DEFAULT COLLATE utf8mb4_general_ci;
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Background tasks queue entries';

View file

@ -26,7 +26,7 @@ class dba {
private static $relation = [];
public static function connect($serveraddr, $user, $pass, $db) {
if (!is_null(self::$db)) {
if (!is_null(self::$db) && self::connected()) {
return true;
}

View file

@ -53,10 +53,15 @@ class Worker
// We now start the process. This is done after the load check since this could increase the load.
self::startProcess();
// The daemon doesn't need to fork new workers anymore, since we started a process
if (Config::get('system', 'worker_daemon_mode', false)) {
self::IPCSetJobState(false);
}
// Kill stale processes every 5 minutes
$last_cleanup = Config::get('system', 'poller_last_cleaned', 0);
$last_cleanup = Config::get('system', 'worker_last_cleaned', 0);
if (time() > ($last_cleanup + 300)) {
Config::set('system', 'poller_last_cleaned', time());
Config::set('system', 'worker_last_cleaned', time());
self::killStaleWorkers();
}
@ -108,16 +113,16 @@ class Worker
}
// If possible we will fetch new jobs for this worker
if (!$refetched && Lock::set('poller_worker_process', 0)) {
if (!$refetched && Lock::set('worker_process', 0)) {
$stamp = (float)microtime(true);
$refetched = self::findWorkerProcesses($passing_slow);
self::$db_duration += (microtime(true) - $stamp);
Lock::remove('poller_worker_process');
Lock::remove('worker_process');
}
}
// To avoid the quitting of multiple workers only one worker at a time will execute the check
if (Lock::set('poller_worker', 0)) {
if (Lock::set('worker', 0)) {
$stamp = (float)microtime(true);
// Count active workers and compare them with a maximum value that depends on the load
if (self::tooMuchWorkers()) {
@ -130,7 +135,7 @@ class Worker
logger('Memory limit reached, quitting.', LOGGER_DEBUG);
return;
}
Lock::remove('poller_worker');
Lock::remove('worker');
self::$db_duration += (microtime(true) - $stamp);
}
@ -140,6 +145,9 @@ class Worker
return;
}
}
if (Config::get('system', 'worker_daemon_mode', false)) {
self::IPCSetJobState(false);
}
logger("Couldn't select a workerqueue entry, quitting.", LOGGER_DEBUG);
}
@ -244,7 +252,7 @@ class Worker
$stamp = (float)microtime(true);
if (dba::update('workerqueue', ['done' => true], ['id' => $queue["id"]])) {
Config::set('system', 'last_poller_execution', DateTimeFormat::utcNow());
Config::set('system', 'last_worker_execution', DateTimeFormat::utcNow());
}
self::$db_duration = (microtime(true) - $stamp);
@ -285,7 +293,7 @@ class Worker
$stamp = (float)microtime(true);
if (dba::update('workerqueue', ['done' => true], ['id' => $queue["id"]])) {
Config::set('system', 'last_poller_execution', DateTimeFormat::utcNow());
Config::set('system', 'last_worker_execution', DateTimeFormat::utcNow());
}
self::$db_duration = (microtime(true) - $stamp);
} else {
@ -851,6 +859,11 @@ class Worker
dba::update('workerqueue', ['executed' => DateTimeFormat::utcNow(), 'pid' => $mypid], $ids);
}
// The daemon doesn't need to fork new workers anymore, since we are inside the worker
if (Config::get('system', 'worker_daemon_mode', false)) {
self::IPCSetJobState(false);
}
return $found;
}
@ -873,7 +886,7 @@ class Worker
dba::close($r);
$stamp = (float)microtime(true);
if (!Lock::set('poller_worker_process')) {
if (!Lock::set('worker_process')) {
return false;
}
self::$lock_duration = (microtime(true) - $stamp);
@ -882,7 +895,7 @@ class Worker
$found = self::findWorkerProcesses($passing_slow);
self::$db_duration += (microtime(true) - $stamp);
Lock::remove('poller_worker_process');
Lock::remove('worker_process');
if ($found) {
$r = dba::select('workerqueue', [], ['pid' => getmypid(), 'done' => false]);
@ -1076,14 +1089,20 @@ class Worker
return true;
}
// We tell the daemon that a new job entry exists
if (Config::get('system', 'worker_daemon_mode', false)) {
self::IPCSetJobState(true);
return true;
}
// If there is a lock then we don't have to check for too much worker
if (!Lock::set('poller_worker', 0)) {
if (!Lock::set('worker', 0)) {
return true;
}
// If there are already enough workers running, don't fork another one
$quit = self::tooMuchWorkers();
Lock::remove('poller_worker');
Lock::remove('worker');
if ($quit) {
return true;
@ -1121,4 +1140,33 @@ class Worker
{
return Process::deleteByPid();
}
/**
* Set the flag if some job is waiting
*
* @brief Set the flag if some job is waiting
* @param boolean $jobs Is there a waiting job?
*/
public static function IPCSetJobState($jobs)
{
dba::update('worker-ipc', ['jobs' => $jobs], ['key' => 1], true);
}
/**
* Checks if some worker job waits to be executed
*
* @brief Checks if some worker job waits to be executed
* @return bool
*/
public static function IPCJobsExists()
{
$row = dba::selectFirst('worker-ipc', ['jobs'], ['key' => 1]);
// When we don't have a row, no job is running
if (!DBM::is_result($row)) {
return false;
}
return (bool)$row['jobs'];
}
}

View file

@ -174,7 +174,7 @@ class DBStructure
echo "--\n";
echo "-- TABLE $name\n";
echo "--\n";
self::createTable($name, $structure['fields'], true, false, $structure["indexes"]);
self::createTable($name, $structure, true, false);
echo "\n";
}
@ -251,7 +251,7 @@ class DBStructure
$is_unique = false;
$temp_name = $name;
if (!isset($database[$name])) {
$r = self::createTable($name, $structure["fields"], $verbose, $action, $structure['indexes']);
$r = self::createTable($name, $structure, $verbose, $action);
if (!DBM::is_result($r)) {
$errors .= self::printUpdateError($name);
}
@ -378,6 +378,18 @@ class DBStructure
}
}
if (isset($database[$name]["table_status"]["Engine"]) && isset($structure['engine'])) {
if ($database[$name]["table_status"]["Engine"] != $structure['engine']) {
$sql2 = "ENGINE = '".dbesc($structure['engine'])."'";
if ($sql3 == "") {
$sql3 = "ALTER" . $ignore . " TABLE `".$temp_name."` ".$sql2;
} else {
$sql3 .= ", ".$sql2;
}
}
}
if (isset($database[$name]["table_status"]["Collation"])) {
if ($database[$name]["table_status"]["Collation"] != 'utf8mb4_general_ci') {
$sql2 = "DEFAULT COLLATE utf8mb4_general_ci";
@ -554,20 +566,22 @@ class DBStructure
return($fieldstruct);
}
private static function createTable($name, $fields, $verbose, $action, $indexes=null) {
private static function createTable($name, $structure, $verbose, $action) {
$r = true;
$engine = "";
$comment = "";
$sql_rows = [];
$primary_keys = [];
foreach ($fields AS $fieldname => $field) {
foreach ($structure["fields"] AS $fieldname => $field) {
$sql_rows[] = "`".dbesc($fieldname)."` ".self::FieldCommand($field);
if (x($field,'primary') && $field['primary']!='') {
$primary_keys[] = $fieldname;
}
}
if (!is_null($indexes)) {
foreach ($indexes AS $indexname => $fieldnames) {
if (!is_null($structure["indexes"])) {
foreach ($structure["indexes"] AS $indexname => $fieldnames) {
$sql_index = self::createIndex($indexname, $fieldnames, "");
if (!is_null($sql_index)) {
$sql_rows[] = $sql_index;
@ -575,9 +589,18 @@ class DBStructure
}
}
if (!is_null($structure["engine"])) {
$engine = " ENGINE=" . $structure["engine"];
}
if (!is_null($structure["comment"])) {
$comment = " COMMENT='" . dbesc($structure["comment"]) . "'";
}
$sql = implode(",\n\t", $sql_rows);
$sql = sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n\t", dbesc($name)).$sql."\n) DEFAULT COLLATE utf8mb4_general_ci";
$sql = sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n\t", dbesc($name)).$sql.
"\n)" . $engine . " DEFAULT COLLATE utf8mb4_general_ci" . $comment;
if ($verbose) {
echo $sql.";\n";
}
@ -1797,6 +1820,18 @@ class DBStructure
"PRIMARY" => ["uid", "iid"],
]
];
$database["worker-ipc"] = [
"comment" => "Inter process communication between the frontend and the worker",
"fields" => [
"key" => ["type" => "int", "not null" => "1", "primary" => "1", "comment" => ""],
"jobs" => ["type" => "boolean", "comment" => "Flag for outstanding jobs"],
],
"indexes" => [
"PRIMARY" => ["key"],
],
"engine" => "MEMORY",
];
$database["workerqueue"] = [
"comment" => "Background tasks queue entries",
"fields" => [