From 28f8beebfc147fdcf14afcf8d56e8888dfbc0f0d Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Fri, 1 Jun 2018 22:09:27 +0000
Subject: [PATCH 1/3] Workers can now be started exclusively from the daemon
 and other workers

---
 bin/daemon.php      | 117 +++++++++++++++++++++++++-------------------
 include/dba.php     |   2 +-
 src/Core/Worker.php |  69 +++++++++++++++++++++-----
 3 files changed, 125 insertions(+), 63 deletions(-)

diff --git a/bin/daemon.php b/bin/daemon.php
index b51dd392e..8ba86b3de 100755
--- a/bin/daemon.php
+++ b/bin/daemon.php
@@ -6,8 +6,38 @@
  *
  * This script was taken from http://php.net/manual/en/function.pcntl-fork.php
  */
-function shutdown() {
-	posix_kill(posix_getpid(), SIGHUP);
+
+use Friendica\App;
+use Friendica\BaseObject;
+use Friendica\Core\Config;
+use Friendica\Core\Worker;
+
+// Ensure that daemon.php is executed from the base path of the installation
+if (!file_exists("boot.php") && (sizeof($_SERVER["argv"]) != 0)) {
+	$directory = dirname($_SERVER["argv"][0]);
+
+	if (substr($directory, 0, 1) != "/") {
+		$directory = $_SERVER["PWD"]."/".$directory;
+	}
+	$directory = realpath($directory."/..");
+
+	chdir($directory);
+}
+
+require_once "boot.php";
+require_once "include/dba.php";
+
+$a = new App(dirname(__DIR__));
+BaseObject::setApp($a);
+
+require_once ".htconfig.php";
+dba::connect($db_host, $db_user, $db_pass, $db_data);
+
+Config::load();
+
+if (!isset($pidfile)) {
+	die('Please specify a pid file in the variable $pidfile in the .htconfig.php. For example:'."\n".
+		'$pidfile = "/path/to/daemon.pid";'."\n");
 }
 
 if (in_array("start", $_SERVER["argv"])) {
@@ -30,27 +60,11 @@ if (empty($_SERVER["argv"][0])) {
 	die("Unexpected script behaviour. This message should never occur.\n");
 }
 
-// Fetch the base directory
-$directory = dirname($_SERVER["argv"][0]);
+$pid = @file_get_contents($pidfile);
 
-if (substr($directory, 0, 1) != "/") {
-	$directory = $_SERVER["PWD"]."/".$directory;
-}
-$directory = realpath($directory."/..");
-
-include $directory."/.htconfig.php";
-
-if (!isset($pidfile)) {
-	die('Please specify a pid file in the variable $pidfile in the .htconfig.php. For example:'."\n".
-		'$pidfile = "/path/to/daemon.pid";'."\n");
-}
-
-if (in_array($mode, array("stop", "status"))) {
-	$pid = @file_get_contents($pidfile);
-
-	if (!$pid) {
-		die("Pidfile wasn't found. Is the daemon running?\n");
-	}
+if (empty($pid) && in_array($mode, ["stop", "status"])) {
+	Config::set('system', 'worker_daemon_mode', false);
+	die("Pidfile wasn't found. Is the daemon running?\n");
 }
 
 if ($mode == "status") {
@@ -60,6 +74,7 @@ if ($mode == "status") {
 
 	unlink($pidfile);
 
+	Config::set('system', 'worker_daemon_mode', false);
 	die("Daemon process $pid isn't running.\n");
 }
 
@@ -68,17 +83,19 @@ if ($mode == "stop") {
 
 	unlink($pidfile);
 
+	logger("Worker daemon process $pid was killed.", LOGGER_DEBUG);
+
+	Config::set('system', 'worker_daemon_mode', false);
 	die("Worker daemon process $pid was killed.\n");
 }
 
-echo "Starting worker daemon.\n";
-
-if (isset($a->config['php_path'])) {
-	$php = $a->config['php_path'];
-} else {
-	$php = "php";
+if (!empty($pid) && posix_kill($pid, 0)) {
+	die("Daemon process $pid is already running.\n");
 }
 
+logger('Starting worker daemon.', LOGGER_DEBUG);
+echo "Starting worker daemon.\n";
+
 // Switch over to daemon mode.
 if ($pid = pcntl_fork())
 	return;     // Parent
@@ -95,32 +112,32 @@ if (posix_setsid() < 0)
 if ($pid = pcntl_fork())
 	return;     // Parent
 
+// We lose the database connection upon forking
+dba::connect($db_host, $db_user, $db_pass, $db_data);
+unset($db_host, $db_user, $db_pass, $db_data);
+
+Config::set('system', 'worker_daemon_mode', true);
+
+// Just to be sure that this script really runs endlessly
+set_time_limit(0);
+
 $pid = getmypid();
 file_put_contents($pidfile, $pid);
 
+$wait_interval = intval(Config::get('system', 'cron_interval', 5)) * 60;
+
 // Now running as a daemon.
 while (true) {
-	// Just to be sure that this script really runs endlessly
-	set_time_limit(0);
+	logger('Call the worker', LOGGER_DEBUG);
+	Worker::spawnWorker();
 
-	// Call the worker
-	$cmdline = $php.' bin/worker.php';
-
-	$executed = false;
-
-	if (function_exists('proc_open')) {
-		$resource = proc_open($cmdline . ' &', array(), $foo, $directory);
-
-		if (is_resource($resource)) {
-			$executed = true;
-			proc_close($resource);
-		}
-	}
-
-	if (!$executed) {
-		exec($cmdline.' spawn');
-	}
-
-	// Now sleep for 5 minutes
-	sleep(300);
+	logger("Sleep for $wait_interval seconds - or when a worker needs to be called", LOGGER_DEBUG);
+	$i = 0;
+	do {
+		sleep(1);
+	} while (($i++ < $wait_interval) && !Worker::IPCJobsExists());
+}
+
+function shutdown() {
+	posix_kill(posix_getpid(), SIGHUP);
 }
diff --git a/include/dba.php b/include/dba.php
index aa63a5b36..5245538ce 100644
--- a/include/dba.php
+++ b/include/dba.php
@@ -26,7 +26,7 @@ class dba {
 	private static $relation = [];
 
 	public static function connect($serveraddr, $user, $pass, $db) {
-		if (!is_null(self::$db)) {
+		if (!is_null(self::$db) && self::connected()) {
 			return true;
 		}
 
diff --git a/src/Core/Worker.php b/src/Core/Worker.php
index 81cdf1bbd..e2135f0da 100644
--- a/src/Core/Worker.php
+++ b/src/Core/Worker.php
@@ -53,10 +53,15 @@ class Worker
 		// We now start the process. This is done after the load check since this could increase the load.
 		self::startProcess();
 
+		// The daemon doesn't need to fork new workers anymore, since we started a process
+		if (Config::get('system', 'worker_daemon_mode', false)) {
+			self::IPCSetJobState(false);
+		}
+
 		// Kill stale processes every 5 minutes
-		$last_cleanup = Config::get('system', 'poller_last_cleaned', 0);
+		$last_cleanup = Config::get('system', 'worker_last_cleaned', 0);
 		if (time() > ($last_cleanup + 300)) {
-			Config::set('system', 'poller_last_cleaned', time());
+			Config::set('system', 'worker_last_cleaned', time());
 			self::killStaleWorkers();
 		}
 
@@ -108,16 +113,16 @@ class Worker
 				}
 
 				// If possible we will fetch new jobs for this worker
-				if (!$refetched && Lock::set('poller_worker_process', 0)) {
+				if (!$refetched && Lock::set('worker_process', 0)) {
 					$stamp = (float)microtime(true);
 					$refetched = self::findWorkerProcesses($passing_slow);
 					self::$db_duration += (microtime(true) - $stamp);
-					Lock::remove('poller_worker_process');
+					Lock::remove('worker_process');
 				}
 			}
 
 			// To avoid the quitting of multiple workers only one worker at a time will execute the check
-			if (Lock::set('poller_worker', 0)) {
+			if (Lock::set('worker', 0)) {
 				$stamp = (float)microtime(true);
 				// Count active workers and compare them with a maximum value that depends on the load
 				if (self::tooMuchWorkers()) {
@@ -130,7 +135,7 @@ class Worker
 					logger('Memory limit reached, quitting.', LOGGER_DEBUG);
 					return;
 				}
-				Lock::remove('poller_worker');
+				Lock::remove('worker');
 				self::$db_duration += (microtime(true) - $stamp);
 			}
 
@@ -140,6 +145,9 @@ class Worker
 				return;
 			}
 		}
+		if (Config::get('system', 'worker_daemon_mode', false)) {
+			self::IPCSetJobState(false);
+		}
 		logger("Couldn't select a workerqueue entry, quitting.", LOGGER_DEBUG);
 	}
 
@@ -244,7 +252,7 @@ class Worker
 
 			$stamp = (float)microtime(true);
 			if (dba::update('workerqueue', ['done' => true], ['id' => $queue["id"]])) {
-				Config::set('system', 'last_poller_execution', DateTimeFormat::utcNow());
+				Config::set('system', 'last_worker_execution', DateTimeFormat::utcNow());
 			}
 			self::$db_duration = (microtime(true) - $stamp);
 
@@ -285,7 +293,7 @@ class Worker
 
 			$stamp = (float)microtime(true);
 			if (dba::update('workerqueue', ['done' => true], ['id' => $queue["id"]])) {
-				Config::set('system', 'last_poller_execution', DateTimeFormat::utcNow());
+				Config::set('system', 'last_worker_execution', DateTimeFormat::utcNow());
 			}
 			self::$db_duration = (microtime(true) - $stamp);
 		} else {
@@ -851,6 +859,11 @@ class Worker
 			dba::update('workerqueue', ['executed' => DateTimeFormat::utcNow(), 'pid' => $mypid], $ids);
 		}
 
+		// The daemon doesn't need to fork new workers anymore, since we are inside the worker
+		if (Config::get('system', 'worker_daemon_mode', false)) {
+			self::IPCSetJobState(false);
+		}
+
 		return $found;
 	}
 
@@ -873,7 +886,7 @@ class Worker
 		dba::close($r);
 
 		$stamp = (float)microtime(true);
-		if (!Lock::set('poller_worker_process')) {
+		if (!Lock::set('worker_process')) {
 			return false;
 		}
 		self::$lock_duration = (microtime(true) - $stamp);
@@ -882,7 +895,7 @@ class Worker
 		$found = self::findWorkerProcesses($passing_slow);
 		self::$db_duration += (microtime(true) - $stamp);
 
-		Lock::remove('poller_worker_process');
+		Lock::remove('worker_process');
 
 		if ($found) {
 			$r = dba::select('workerqueue', [], ['pid' => getmypid(), 'done' => false]);
@@ -1071,19 +1084,25 @@ class Worker
 			dba::insert('workerqueue', ['parameter' => $parameters, 'created' => $created, 'priority' => $priority]);
 		}
 
+		// We tell the daemon that a new job entry exists
+		if (Config::get('system', 'worker_daemon_mode', false)) {
+			self::IPCSetJobState(true);
+			return true;
+		}
+
 		// Should we quit and wait for the worker to be called as a cronjob?
 		if ($dont_fork) {
 			return true;
 		}
 
 		// If there is a lock then we don't have to check for too much worker
-		if (!Lock::set('poller_worker', 0)) {
+		if (!Lock::set('worker', 0)) {
 			return true;
 		}
 
 		// If there are already enough workers running, don't fork another one
 		$quit = self::tooMuchWorkers();
-		Lock::remove('poller_worker');
+		Lock::remove('worker');
 
 		if ($quit) {
 			return true;
@@ -1121,4 +1140,30 @@ class Worker
 	{
 		return Process::deleteByPid();
 	}
+
+	private static function checkIPC()
+	{
+		dba::e("CREATE TABLE IF NOT EXISTS `worker-ipc` (`key` integer, `jobs` boolean) ENGINE = MEMORY;");
+	}
+
+	public static function IPCSetJobState($jobs)
+	{
+		self::checkIPC();
+
+		dba::update('worker-ipc', ['jobs' => $jobs], ['key' => 1], true);
+	}
+
+	public static function IPCJobsExists()
+	{
+		self::checkIPC();
+
+		$row = dba::selectFirst('worker-ipc', ['jobs'], ['key' => 1]);
+
+		// When we don't have a row, no job is running
+		if (!DBM::is_result($row)) {
+			return false;
+		}
+
+		return (bool)$row['jobs'];
+	}
 }

From 30341700884d52707bf81e195858a48f1f4494d1 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Sat, 2 Jun 2018 05:03:23 +0000
Subject: [PATCH 2/3] DBStruxture can now create "memory" tables as well

---
 boot.php                     |   2 +-
 database.sql                 | 121 +++++++++++++++++++----------------
 src/Core/Worker.php          |  21 +++---
 src/Database/DBStructure.php |  49 ++++++++++++--
 4 files changed, 120 insertions(+), 73 deletions(-)

diff --git a/boot.php b/boot.php
index 57176296e..79ec53abf 100644
--- a/boot.php
+++ b/boot.php
@@ -41,7 +41,7 @@ define('FRIENDICA_PLATFORM',     'Friendica');
 define('FRIENDICA_CODENAME',     'The Tazmans Flax-lily');
 define('FRIENDICA_VERSION',      '2018.08-dev');
 define('DFRN_PROTOCOL_VERSION',  '2.23');
-define('DB_UPDATE_VERSION',      1267);
+define('DB_UPDATE_VERSION',      1268);
 define('NEW_UPDATE_ROUTINE_VERSION', 1170);
 
 /**
diff --git a/database.sql b/database.sql
index cc6ab16e1..bd36bdb70 100644
--- a/database.sql
+++ b/database.sql
@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2018.08-dev (The Tazmans Flax-lily)
--- DB_UPDATE_VERSION 1267
+-- DB_UPDATE_VERSION 1268
 -- ------------------------------------------
 
 
@@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS `addon` (
 	`plugin_admin` boolean NOT NULL DEFAULT '0' COMMENT '1 = has admin config, 0 = has no admin config',
 	 PRIMARY KEY(`id`),
 	 UNIQUE INDEX `name` (`name`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='registered addons';
 
 --
 -- TABLE attach
@@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS `attach` (
 	`deny_cid` mediumtext COMMENT 'Access Control - list of denied contact.id',
 	`deny_gid` mediumtext COMMENT 'Access Control - list of denied groups',
 	 PRIMARY KEY(`id`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='file attachments';
 
 --
 -- TABLE auth_codes
@@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS `auth_codes` (
 	`expires` int NOT NULL DEFAULT 0 COMMENT '',
 	`scope` varchar(250) NOT NULL DEFAULT '' COMMENT '',
 	 PRIMARY KEY(`id`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='OAuth usage';
 
 --
 -- TABLE cache
@@ -61,7 +61,7 @@ CREATE TABLE IF NOT EXISTS `cache` (
 	`updated` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'datetime of cache insertion',
 	 PRIMARY KEY(`k`),
 	 INDEX `k_expires` (`k`,`expires`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Stores temporary data';
 
 --
 -- TABLE challenge
@@ -74,7 +74,7 @@ CREATE TABLE IF NOT EXISTS `challenge` (
 	`type` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	`last_update` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	 PRIMARY KEY(`id`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='';
 
 --
 -- TABLE clients
@@ -87,7 +87,7 @@ CREATE TABLE IF NOT EXISTS `clients` (
 	`icon` text COMMENT '',
 	`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
 	 PRIMARY KEY(`client_id`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='OAuth usage';
 
 --
 -- TABLE config
@@ -99,7 +99,7 @@ CREATE TABLE IF NOT EXISTS `config` (
 	`v` mediumtext COMMENT '',
 	 PRIMARY KEY(`id`),
 	 UNIQUE INDEX `cat_k` (`cat`,`k`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='main configuration storage';
 
 --
 -- TABLE contact
@@ -186,7 +186,7 @@ CREATE TABLE IF NOT EXISTS `contact` (
 	 INDEX `nick_uid` (`nick`(32),`uid`),
 	 INDEX `dfrn-id` (`dfrn-id`(64)),
 	 INDEX `issued-id` (`issued-id`(64))
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='contact table';
 
 --
 -- TABLE conv
@@ -202,7 +202,7 @@ CREATE TABLE IF NOT EXISTS `conv` (
 	`subject` text COMMENT 'subject of initial message',
 	 PRIMARY KEY(`id`),
 	 INDEX `uid` (`uid`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='private messages';
 
 --
 -- TABLE conversation
@@ -218,7 +218,7 @@ CREATE TABLE IF NOT EXISTS `conversation` (
 	 PRIMARY KEY(`item-uri`),
 	 INDEX `conversation-uri` (`conversation-uri`),
 	 INDEX `received` (`received`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Raw data and structure information for messages';
 
 --
 -- TABLE event
@@ -246,7 +246,7 @@ CREATE TABLE IF NOT EXISTS `event` (
 	`deny_gid` mediumtext COMMENT 'Access Control - list of denied groups',
 	 PRIMARY KEY(`id`),
 	 INDEX `uid_start` (`uid`,`start`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Events';
 
 --
 -- TABLE fcontact
@@ -272,7 +272,7 @@ CREATE TABLE IF NOT EXISTS `fcontact` (
 	 PRIMARY KEY(`id`),
 	 INDEX `addr` (`addr`(32)),
 	 UNIQUE INDEX `url` (`url`(190))
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Diaspora compatible contacts - used in the Diaspora implementation';
 
 --
 -- TABLE fsuggest
@@ -288,7 +288,7 @@ CREATE TABLE IF NOT EXISTS `fsuggest` (
 	`note` text COMMENT '',
 	`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
 	 PRIMARY KEY(`id`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='friend suggestion stuff';
 
 --
 -- TABLE gcign
@@ -300,7 +300,7 @@ CREATE TABLE IF NOT EXISTS `gcign` (
 	 PRIMARY KEY(`id`),
 	 INDEX `uid` (`uid`),
 	 INDEX `gcid` (`gcid`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='contacts ignored by friend suggestions';
 
 --
 -- TABLE gcontact
@@ -339,7 +339,7 @@ CREATE TABLE IF NOT EXISTS `gcontact` (
 	 INDEX `addr` (`addr`(64)),
 	 INDEX `hide_network_updated` (`hide`,`network`,`updated`),
 	 INDEX `updated` (`updated`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='global contacts';
 
 --
 -- TABLE glink
@@ -354,7 +354,7 @@ CREATE TABLE IF NOT EXISTS `glink` (
 	 PRIMARY KEY(`id`),
 	 UNIQUE INDEX `cid_uid_gcid_zcid` (`cid`,`uid`,`gcid`,`zcid`),
 	 INDEX `gcid` (`gcid`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='\'friends of friends\' linkages derived from poco';
 
 --
 -- TABLE group
@@ -367,7 +367,7 @@ CREATE TABLE IF NOT EXISTS `group` (
 	`name` varchar(255) NOT NULL DEFAULT '' COMMENT 'human readable name of group',
 	 PRIMARY KEY(`id`),
 	 INDEX `uid` (`uid`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='privacy groups, group info';
 
 --
 -- TABLE group_member
@@ -379,7 +379,7 @@ CREATE TABLE IF NOT EXISTS `group_member` (
 	 PRIMARY KEY(`id`),
 	 INDEX `contactid` (`contact-id`),
 	 UNIQUE INDEX `gid_contactid` (`gid`,`contact-id`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='privacy groups, member info';
 
 --
 -- TABLE gserver
@@ -405,7 +405,7 @@ CREATE TABLE IF NOT EXISTS `gserver` (
 	`last_failure` datetime DEFAULT '0001-01-01 00:00:00' COMMENT '',
 	 PRIMARY KEY(`id`),
 	 UNIQUE INDEX `nurl` (`nurl`(190))
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Global servers';
 
 --
 -- TABLE gserver-tag
@@ -415,7 +415,7 @@ CREATE TABLE IF NOT EXISTS `gserver-tag` (
 	`tag` varchar(100) NOT NULL DEFAULT '' COMMENT 'Tag that the server has subscribed',
 	 PRIMARY KEY(`gserver-id`,`tag`),
 	 INDEX `tag` (`tag`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Tags that the server has subscribed';
 
 --
 -- TABLE hook
@@ -428,7 +428,7 @@ CREATE TABLE IF NOT EXISTS `hook` (
 	`priority` smallint unsigned NOT NULL DEFAULT 0 COMMENT 'not yet implemented - can be used to sort conflicts in hook handling by calling handlers in priority order',
 	 PRIMARY KEY(`id`),
 	 UNIQUE INDEX `hook_file_function` (`hook`,`file`,`function`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='addon hook registry';
 
 --
 -- TABLE intro
@@ -446,7 +446,7 @@ CREATE TABLE IF NOT EXISTS `intro` (
 	`blocked` boolean NOT NULL DEFAULT '1' COMMENT '',
 	`ignore` boolean NOT NULL DEFAULT '0' COMMENT '',
 	 PRIMARY KEY(`id`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='';
 
 --
 -- TABLE item
@@ -465,7 +465,7 @@ CREATE TABLE IF NOT EXISTS `item` (
 	`extid` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	`thr-parent` varchar(255) NOT NULL DEFAULT '' COMMENT 'If the parent of this item is not the top-level item in the conversation, the uri of the immediate parent; otherwise set to parent-uri',
 	`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation timestamp.',
-	`edited` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of  last edit (default is created)',
+	`edited` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last edit (default is created)',
 	`commented` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date of last comment/reply to this item',
 	`received` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'datetime',
 	`changed` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Date that something in the conversation changed, indicating clients should fetch the conversation again',
@@ -487,7 +487,7 @@ CREATE TABLE IF NOT EXISTS `item` (
 	`target-type` varchar(100) NOT NULL DEFAULT '' COMMENT 'ActivityStreams target type if applicable (URI)',
 	`target` text COMMENT 'JSON encoded target structure if used',
 	`postopts` text COMMENT 'External post connectors add their network name to this comma-separated string to identify that they should be delivered to these networks during delivery',
-	`plink` varchar(255) NOT NULL DEFAULT '' COMMENT 'permalink or URL toa displayable copy  of the message at its source',
+	`plink` varchar(255) NOT NULL DEFAULT '' COMMENT 'permalink or URL to a displayable copy of the message at its source',
 	`resource-id` varchar(32) NOT NULL DEFAULT '' COMMENT 'Used to link other tables to items, it identifies the linked resource (e.g. photo) and if set must also set resource_type',
 	`event-id` int unsigned NOT NULL DEFAULT 0 COMMENT 'Used to link to the event.id',
 	`tag` mediumtext COMMENT '',
@@ -544,7 +544,7 @@ CREATE TABLE IF NOT EXISTS `item` (
 	 INDEX `uid_eventid` (`uid`,`event-id`),
 	 INDEX `uid_authorlink` (`uid`,`author-link`(190)),
 	 INDEX `uid_ownerlink` (`uid`,`owner-link`(190))
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='All posts';
 
 --
 -- TABLE locks
@@ -555,7 +555,7 @@ CREATE TABLE IF NOT EXISTS `locks` (
 	`locked` boolean NOT NULL DEFAULT '0' COMMENT '',
 	`pid` int unsigned NOT NULL DEFAULT 0 COMMENT 'Process ID',
 	 PRIMARY KEY(`id`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='';
 
 --
 -- TABLE mail
@@ -584,7 +584,7 @@ CREATE TABLE IF NOT EXISTS `mail` (
 	 INDEX `uri` (`uri`(64)),
 	 INDEX `parent-uri` (`parent-uri`(64)),
 	 INDEX `contactid` (`contact-id`(32))
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='private messages';
 
 --
 -- TABLE mailacct
@@ -604,7 +604,7 @@ CREATE TABLE IF NOT EXISTS `mailacct` (
 	`pubmail` boolean NOT NULL DEFAULT '0' COMMENT '',
 	`last_check` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
 	 PRIMARY KEY(`id`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Mail account data for fetching mails';
 
 --
 -- TABLE manage
@@ -615,7 +615,7 @@ CREATE TABLE IF NOT EXISTS `manage` (
 	`mid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
 	 PRIMARY KEY(`id`),
 	 UNIQUE INDEX `uid_mid` (`uid`,`mid`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='table of accounts that can manage each other';
 
 --
 -- TABLE notify
@@ -643,7 +643,7 @@ CREATE TABLE IF NOT EXISTS `notify` (
 	 INDEX `seen_uid_date` (`seen`,`uid`,`date`),
 	 INDEX `uid_date` (`uid`,`date`),
 	 INDEX `uid_type_link` (`uid`,`type`,`link`(190))
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='notifications';
 
 --
 -- TABLE notify-threads
@@ -655,7 +655,7 @@ CREATE TABLE IF NOT EXISTS `notify-threads` (
 	`parent-item` int unsigned NOT NULL DEFAULT 0 COMMENT '',
 	`receiver-uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
 	 PRIMARY KEY(`id`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='';
 
 --
 -- TABLE oembed
@@ -667,7 +667,7 @@ CREATE TABLE IF NOT EXISTS `oembed` (
 	`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'datetime of creation',
 	 PRIMARY KEY(`url`,`maxwidth`),
 	 INDEX `created` (`created`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='cache for OEmbed queries';
 
 --
 -- TABLE parsed_url
@@ -680,7 +680,7 @@ CREATE TABLE IF NOT EXISTS `parsed_url` (
 	`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'datetime of creation',
 	 PRIMARY KEY(`url`,`guessing`,`oembed`),
 	 INDEX `created` (`created`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='cache for \'parse_url\' queries';
 
 --
 -- TABLE participation
@@ -691,7 +691,7 @@ CREATE TABLE IF NOT EXISTS `participation` (
 	`cid` int unsigned NOT NULL COMMENT '',
 	`fid` int unsigned NOT NULL COMMENT '',
 	 PRIMARY KEY(`iid`,`server`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Storage for participation messages from Diaspora';
 
 --
 -- TABLE pconfig
@@ -704,7 +704,7 @@ CREATE TABLE IF NOT EXISTS `pconfig` (
 	`v` mediumtext COMMENT '',
 	 PRIMARY KEY(`id`),
 	 UNIQUE INDEX `uid_cat_k` (`uid`,`cat`,`k`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='personal (per user) configuration storage';
 
 --
 -- TABLE photo
@@ -739,7 +739,7 @@ CREATE TABLE IF NOT EXISTS `photo` (
 	 INDEX `uid_album_scale_created` (`uid`,`album`(32),`scale`,`created`),
 	 INDEX `uid_album_resource-id_created` (`uid`,`album`(32),`resource-id`,`created`),
 	 INDEX `resource-id` (`resource-id`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='photo storage';
 
 --
 -- TABLE poll
@@ -759,7 +759,7 @@ CREATE TABLE IF NOT EXISTS `poll` (
 	`q9` text COMMENT '',
 	 PRIMARY KEY(`id`),
 	 INDEX `uid` (`uid`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Currently unused table for storing poll results';
 
 --
 -- TABLE poll_result
@@ -770,7 +770,7 @@ CREATE TABLE IF NOT EXISTS `poll_result` (
 	`choice` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '',
 	 PRIMARY KEY(`id`),
 	 INDEX `poll_id` (`poll_id`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='data for polls - currently unused';
 
 --
 -- TABLE process
@@ -781,7 +781,7 @@ CREATE TABLE IF NOT EXISTS `process` (
 	`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT '',
 	 PRIMARY KEY(`pid`),
 	 INDEX `command` (`command`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Currently running system processes';
 
 --
 -- TABLE profile
@@ -831,7 +831,7 @@ CREATE TABLE IF NOT EXISTS `profile` (
 	`net-publish` boolean NOT NULL DEFAULT '0' COMMENT 'publish profile in global directory',
 	 PRIMARY KEY(`id`),
 	 INDEX `uid_is-default` (`uid`,`is-default`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='user profiles data';
 
 --
 -- TABLE profile_check
@@ -844,7 +844,7 @@ CREATE TABLE IF NOT EXISTS `profile_check` (
 	`sec` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	`expire` int unsigned NOT NULL DEFAULT 0 COMMENT '',
 	 PRIMARY KEY(`id`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='DFRN remote auth use';
 
 --
 -- TABLE push_subscriber
@@ -862,7 +862,7 @@ CREATE TABLE IF NOT EXISTS `push_subscriber` (
 	`secret` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	 PRIMARY KEY(`id`),
 	 INDEX `next_try` (`next_try`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Used for OStatus: Contains feed subscribers';
 
 --
 -- TABLE queue
@@ -881,7 +881,7 @@ CREATE TABLE IF NOT EXISTS `queue` (
 	 PRIMARY KEY(`id`),
 	 INDEX `last` (`last`),
 	 INDEX `next` (`next`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Queue for messages that couldn\'t be delivered';
 
 --
 -- TABLE register
@@ -895,7 +895,7 @@ CREATE TABLE IF NOT EXISTS `register` (
 	`language` varchar(16) NOT NULL DEFAULT '' COMMENT '',
 	`note` text COMMENT '',
 	 PRIMARY KEY(`id`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='registrations requiring admin approval';
 
 --
 -- TABLE search
@@ -906,7 +906,7 @@ CREATE TABLE IF NOT EXISTS `search` (
 	`term` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	 PRIMARY KEY(`id`),
 	 INDEX `uid` (`uid`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='';
 
 --
 -- TABLE session
@@ -919,7 +919,7 @@ CREATE TABLE IF NOT EXISTS `session` (
 	 PRIMARY KEY(`id`),
 	 INDEX `sid` (`sid`(64)),
 	 INDEX `expire` (`expire`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='web session storage';
 
 --
 -- TABLE sign
@@ -932,7 +932,7 @@ CREATE TABLE IF NOT EXISTS `sign` (
 	`signer` varchar(255) NOT NULL DEFAULT '' COMMENT '',
 	 PRIMARY KEY(`id`),
 	 UNIQUE INDEX `iid` (`iid`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Diaspora signatures';
 
 --
 -- TABLE term
@@ -955,7 +955,7 @@ CREATE TABLE IF NOT EXISTS `term` (
 	 INDEX `uid_otype_type_term_global_created` (`uid`,`otype`,`type`,`term`(32),`global`,`created`),
 	 INDEX `uid_otype_type_url` (`uid`,`otype`,`type`,`url`(64)),
 	 INDEX `guid` (`guid`(64))
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='item taxonomy (categories, tags, etc.) table';
 
 --
 -- TABLE thread
@@ -998,7 +998,7 @@ CREATE TABLE IF NOT EXISTS `thread` (
 	 INDEX `uid_commented` (`uid`,`commented`),
 	 INDEX `uid_wall_created` (`uid`,`wall`,`created`),
 	 INDEX `private_wall_origin_commented` (`private`,`wall`,`origin`,`commented`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Thread related data';
 
 --
 -- TABLE tokens
@@ -1011,7 +1011,7 @@ CREATE TABLE IF NOT EXISTS `tokens` (
 	`scope` varchar(200) NOT NULL DEFAULT '' COMMENT '',
 	`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
 	 PRIMARY KEY(`id`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='OAuth usage';
 
 --
 -- TABLE user
@@ -1064,7 +1064,7 @@ CREATE TABLE IF NOT EXISTS `user` (
 	`openidserver` text COMMENT '',
 	 PRIMARY KEY(`uid`),
 	 INDEX `nickname` (`nickname`(32))
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='The local users';
 
 --
 -- TABLE userd
@@ -1074,7 +1074,7 @@ CREATE TABLE IF NOT EXISTS `userd` (
 	`username` varchar(255) NOT NULL COMMENT '',
 	 PRIMARY KEY(`id`),
 	 INDEX `username` (`username`(32))
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Deleted usernames';
 
 --
 -- TABLE user-item
@@ -1082,9 +1082,18 @@ CREATE TABLE IF NOT EXISTS `userd` (
 CREATE TABLE IF NOT EXISTS `user-item` (
 	`iid` int unsigned NOT NULL DEFAULT 0 COMMENT 'Item id',
 	`uid` mediumint unsigned NOT NULL DEFAULT 0 COMMENT 'User id',
-	`hidden` boolean NOT NULL DEFAULT '0' COMMENT 'Hidden marker',
+	`hidden` boolean NOT NULL DEFAULT '0' COMMENT 'Marker to hide an item from the user',
 	 PRIMARY KEY(`uid`,`iid`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='User specific item data';
+
+--
+-- TABLE worker-ipc
+--
+CREATE TABLE IF NOT EXISTS `worker-ipc` (
+	`key` int NOT NULL COMMENT '',
+	`jobs` boolean COMMENT 'Flag for outstanding jobs',
+	 PRIMARY KEY(`key`)
+) ENGINE=MEMORY DEFAULT COLLATE utf8mb4_general_ci COMMENT='Inter process communication between the frontend and the worker';
 
 --
 -- TABLE workerqueue
@@ -1102,6 +1111,6 @@ CREATE TABLE IF NOT EXISTS `workerqueue` (
 	 INDEX `parameter` (`parameter`(64)),
 	 INDEX `priority_created` (`priority`,`created`),
 	 INDEX `done_executed` (`done`,`executed`)
-) DEFAULT COLLATE utf8mb4_general_ci;
+) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Background tasks queue entries';
 
 
diff --git a/src/Core/Worker.php b/src/Core/Worker.php
index e2135f0da..b75bbdf28 100644
--- a/src/Core/Worker.php
+++ b/src/Core/Worker.php
@@ -1141,22 +1141,25 @@ class Worker
 		return Process::deleteByPid();
 	}
 
-	private static function checkIPC()
-	{
-		dba::e("CREATE TABLE IF NOT EXISTS `worker-ipc` (`key` integer, `jobs` boolean) ENGINE = MEMORY;");
-	}
-
+	/**
+	 * Set the flag if some job is waiting
+	 *
+	 * @brief Set the flag if some job is waiting
+	 * @param boolean $jobs Is there a waiting job?
+	 */
 	public static function IPCSetJobState($jobs)
 	{
-		self::checkIPC();
-
 		dba::update('worker-ipc', ['jobs' => $jobs], ['key' => 1], true);
 	}
 
+	/**
+	 * Checks if some worker job waits to be executed
+	 *
+	 * @brief Checks if some worker job waits to be executed
+	 * @return bool
+	 */
 	public static function IPCJobsExists()
 	{
-		self::checkIPC();
-
 		$row = dba::selectFirst('worker-ipc', ['jobs'], ['key' => 1]);
 
 		// When we don't have a row, no job is running
diff --git a/src/Database/DBStructure.php b/src/Database/DBStructure.php
index 45c98f0ed..b8b5a63b9 100644
--- a/src/Database/DBStructure.php
+++ b/src/Database/DBStructure.php
@@ -174,7 +174,7 @@ class DBStructure
 			echo "--\n";
 			echo "-- TABLE $name\n";
 			echo "--\n";
-			self::createTable($name, $structure['fields'], true, false, $structure["indexes"]);
+			self::createTable($name, $structure, true, false);
 
 			echo "\n";
 		}
@@ -251,7 +251,7 @@ class DBStructure
 			$is_unique = false;
 			$temp_name = $name;
 			if (!isset($database[$name])) {
-				$r = self::createTable($name, $structure["fields"], $verbose, $action, $structure['indexes']);
+				$r = self::createTable($name, $structure, $verbose, $action);
 				if (!DBM::is_result($r)) {
 					$errors .= self::printUpdateError($name);
 				}
@@ -378,6 +378,18 @@ class DBStructure
 					}
 				}
 
+				if (isset($database[$name]["table_status"]["Engine"]) && isset($structure['engine'])) {
+					if ($database[$name]["table_status"]["Engine"] != $structure['engine']) {
+						$sql2 = "ENGINE = '".dbesc($structure['engine'])."'";
+
+						if ($sql3 == "") {
+							$sql3 = "ALTER" . $ignore . " TABLE `".$temp_name."` ".$sql2;
+						} else {
+							$sql3 .= ", ".$sql2;
+						}
+					}
+				}
+
 				if (isset($database[$name]["table_status"]["Collation"])) {
 					if ($database[$name]["table_status"]["Collation"] != 'utf8mb4_general_ci') {
 						$sql2 = "DEFAULT COLLATE utf8mb4_general_ci";
@@ -554,20 +566,22 @@ class DBStructure
 		return($fieldstruct);
 	}
 
-	private static function createTable($name, $fields, $verbose, $action, $indexes=null) {
+	private static function createTable($name, $structure, $verbose, $action) {
 		$r = true;
 
+		$engine = "";
+		$comment = "";
 		$sql_rows = [];
 		$primary_keys = [];
-		foreach ($fields AS $fieldname => $field) {
+		foreach ($structure["fields"] AS $fieldname => $field) {
 			$sql_rows[] = "`".dbesc($fieldname)."` ".self::FieldCommand($field);
 			if (x($field,'primary') && $field['primary']!='') {
 				$primary_keys[] = $fieldname;
 			}
 		}
 
-		if (!is_null($indexes)) {
-			foreach ($indexes AS $indexname => $fieldnames) {
+		if (!is_null($structure["indexes"])) {
+			foreach ($structure["indexes"] AS $indexname => $fieldnames) {
 				$sql_index = self::createIndex($indexname, $fieldnames, "");
 				if (!is_null($sql_index)) {
 					$sql_rows[] = $sql_index;
@@ -575,9 +589,18 @@ class DBStructure
 			}
 		}
 
+		if (!is_null($structure["engine"])) {
+			$engine = " ENGINE=" . $structure["engine"];
+		}
+
+		if (!is_null($structure["comment"])) {
+			$comment = " COMMENT='" . dbesc($structure["comment"]) . "'";
+		}
+
 		$sql = implode(",\n\t", $sql_rows);
 
-		$sql = sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n\t", dbesc($name)).$sql."\n) DEFAULT COLLATE utf8mb4_general_ci";
+		$sql = sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n\t", dbesc($name)).$sql.
+				"\n)" . $engine . " DEFAULT COLLATE utf8mb4_general_ci" . $comment;
 		if ($verbose) {
 			echo $sql.";\n";
 		}
@@ -1797,6 +1820,18 @@ class DBStructure
 						"PRIMARY" => ["uid", "iid"],
 						]
 				];
+		$database["worker-ipc"] = [
+				"comment" => "Inter process communication between the frontend and the worker",
+				"fields" => [
+						"key" => ["type" => "int", "not null" => "1", "primary" => "1", "comment" => ""],
+						"jobs" => ["type" => "boolean", "comment" => "Flag for outstanding jobs"],
+						],
+				"indexes" => [
+						"PRIMARY" => ["key"],
+						],
+				"engine" => "MEMORY",
+				];
+
 		$database["workerqueue"] = [
 				"comment" => "Background tasks queue entries",
 				"fields" => [

From 371f511954d8b33e639c21dd16c9bc365a5054e5 Mon Sep 17 00:00:00 2001
From: Michael <heluecht@pirati.ca>
Date: Sat, 2 Jun 2018 05:17:32 +0000
Subject: [PATCH 3/3] Respect "don't fork"

---
 src/Core/Worker.php | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/Core/Worker.php b/src/Core/Worker.php
index b75bbdf28..5ca815a7a 100644
--- a/src/Core/Worker.php
+++ b/src/Core/Worker.php
@@ -1084,14 +1084,14 @@ class Worker
 			dba::insert('workerqueue', ['parameter' => $parameters, 'created' => $created, 'priority' => $priority]);
 		}
 
-		// We tell the daemon that a new job entry exists
-		if (Config::get('system', 'worker_daemon_mode', false)) {
-			self::IPCSetJobState(true);
+		// Should we quit and wait for the worker to be called as a cronjob?
+		if ($dont_fork) {
 			return true;
 		}
 
-		// Should we quit and wait for the worker to be called as a cronjob?
-		if ($dont_fork) {
+		// We tell the daemon that a new job entry exists
+		if (Config::get('system', 'worker_daemon_mode', false)) {
+			self::IPCSetJobState(true);
 			return true;
 		}