Preparation for delayed posts

This commit is contained in:
Michael 2021-07-28 22:22:00 +00:00
parent f89cc6bd45
commit 2951243b07
5 changed files with 92 additions and 64 deletions

View file

@ -1,6 +1,6 @@
-- ------------------------------------------ -- ------------------------------------------
-- Friendica 2021.09-dev (Siberian Iris) -- Friendica 2021.09-dev (Siberian Iris)
-- DB_UPDATE_VERSION 1430 -- DB_UPDATE_VERSION 1431
-- ------------------------------------------ -- ------------------------------------------
@ -492,16 +492,47 @@ CREATE TABLE IF NOT EXISTS `conversation` (
INDEX `received` (`received`) INDEX `received` (`received`)
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Raw data and structure information for messages'; ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Raw data and structure information for messages';
--
-- TABLE workerqueue
--
CREATE TABLE IF NOT EXISTS `workerqueue` (
`id` int unsigned NOT NULL auto_increment COMMENT 'Auto incremented worker task id',
`command` varchar(100) COMMENT 'Task command',
`parameter` mediumtext COMMENT 'Task parameter',
`priority` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'Task priority',
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation date',
`pid` int unsigned NOT NULL DEFAULT 0 COMMENT 'Process id of the worker',
`executed` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Execution date',
`next_try` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Next retrial date',
`retrial` tinyint NOT NULL DEFAULT 0 COMMENT 'Retrial counter',
`done` boolean NOT NULL DEFAULT '0' COMMENT 'Marked 1 when the task was done - will be deleted later',
PRIMARY KEY(`id`),
INDEX `command` (`command`),
INDEX `done_command_parameter` (`done`,`command`,`parameter`(64)),
INDEX `done_executed` (`done`,`executed`),
INDEX `done_priority_retrial_created` (`done`,`priority`,`retrial`,`created`),
INDEX `done_priority_next_try` (`done`,`priority`,`next_try`),
INDEX `done_pid_next_try` (`done`,`pid`,`next_try`),
INDEX `done_pid_retrial` (`done`,`pid`,`retrial`),
INDEX `done_pid_priority_created` (`done`,`pid`,`priority`,`created`)
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Background tasks queue entries';
-- --
-- TABLE delayed-post -- TABLE delayed-post
-- --
CREATE TABLE IF NOT EXISTS `delayed-post` ( CREATE TABLE IF NOT EXISTS `delayed-post` (
`id` int unsigned NOT NULL auto_increment, `id` int unsigned NOT NULL auto_increment,
`uri` varchar(255) COMMENT 'URI of the post that will be distributed later', `uri` varchar(255) COMMENT 'URI of the post that will be distributed later',
`title` varchar(255) COMMENT 'post title',
`body` mediumtext COMMENT 'post body content',
`private` tinyint unsigned COMMENT '0=public, 1=private, 2=unlisted',
`wid` int unsigned COMMENT 'Workerqueue id',
`uid` mediumint unsigned COMMENT 'Owner User id', `uid` mediumint unsigned COMMENT 'Owner User id',
`delayed` datetime COMMENT 'delay time', `delayed` datetime COMMENT 'delay time',
PRIMARY KEY(`id`), PRIMARY KEY(`id`),
UNIQUE INDEX `uid_uri` (`uid`,`uri`(190)), UNIQUE INDEX `uid_uri` (`uid`,`uri`(190)),
INDEX `wid` (`wid`),
FOREIGN KEY (`wid`) REFERENCES `workerqueue` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE FOREIGN KEY (`uid`) REFERENCES `user` (`uid`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Posts that are about to be distributed at a later time'; ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Posts that are about to be distributed at a later time';
@ -1474,31 +1505,6 @@ CREATE TABLE IF NOT EXISTS `worker-ipc` (
PRIMARY KEY(`key`) PRIMARY KEY(`key`)
) ENGINE=MEMORY DEFAULT COLLATE utf8mb4_general_ci COMMENT='Inter process communication between the frontend and the worker'; ) ENGINE=MEMORY DEFAULT COLLATE utf8mb4_general_ci COMMENT='Inter process communication between the frontend and the worker';
--
-- TABLE workerqueue
--
CREATE TABLE IF NOT EXISTS `workerqueue` (
`id` int unsigned NOT NULL auto_increment COMMENT 'Auto incremented worker task id',
`command` varchar(100) COMMENT 'Task command',
`parameter` mediumtext COMMENT 'Task parameter',
`priority` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'Task priority',
`created` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Creation date',
`pid` int unsigned NOT NULL DEFAULT 0 COMMENT 'Process id of the worker',
`executed` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Execution date',
`next_try` datetime NOT NULL DEFAULT '0001-01-01 00:00:00' COMMENT 'Next retrial date',
`retrial` tinyint NOT NULL DEFAULT 0 COMMENT 'Retrial counter',
`done` boolean NOT NULL DEFAULT '0' COMMENT 'Marked 1 when the task was done - will be deleted later',
PRIMARY KEY(`id`),
INDEX `command` (`command`),
INDEX `done_command_parameter` (`done`,`command`,`parameter`(64)),
INDEX `done_executed` (`done`,`executed`),
INDEX `done_priority_retrial_created` (`done`,`priority`,`retrial`,`created`),
INDEX `done_priority_next_try` (`done`,`priority`,`next_try`),
INDEX `done_pid_next_try` (`done`,`pid`,`next_try`),
INDEX `done_pid_retrial` (`done`,`pid`,`retrial`),
INDEX `done_pid_priority_created` (`done`,`pid`,`priority`,`created`)
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Background tasks queue entries';
-- --
-- VIEW application-view -- VIEW application-view
-- --

View file

@ -10,6 +10,10 @@ Fields
| ------- | ---------------------------------------------- | ------------------ | ---- | --- | ------- | -------------- | | ------- | ---------------------------------------------- | ------------------ | ---- | --- | ------- | -------------- |
| id | | int unsigned | NO | PRI | NULL | auto_increment | | id | | int unsigned | NO | PRI | NULL | auto_increment |
| uri | URI of the post that will be distributed later | varchar(255) | YES | | NULL | | | uri | URI of the post that will be distributed later | varchar(255) | YES | | NULL | |
| title | post title | varchar(255) | YES | | NULL | |
| body | post body content | mediumtext | YES | | NULL | |
| private | 0=public, 1=private, 2=unlisted | tinyint unsigned | YES | | NULL | |
| wid | Workerqueue id | int unsigned | YES | | NULL | |
| uid | Owner User id | mediumint unsigned | YES | | NULL | | | uid | Owner User id | mediumint unsigned | YES | | NULL | |
| delayed | delay time | datetime | YES | | NULL | | | delayed | delay time | datetime | YES | | NULL | |
@ -20,12 +24,14 @@ Indexes
| ------- | --------------------- | | ------- | --------------------- |
| PRIMARY | id | | PRIMARY | id |
| uid_uri | UNIQUE, uid, uri(190) | | uid_uri | UNIQUE, uid, uri(190) |
| wid | wid |
Foreign Keys Foreign Keys
------------ ------------
| Field | Target Table | Target Field | | Field | Target Table | Target Field |
|-------|--------------|--------------| |-------|--------------|--------------|
| wid | [workerqueue](help/database/db_workerqueue) | id |
| uid | [user](help/database/db_user) | uid | | uid | [user](help/database/db_user) | uid |
Return to [database documentation](help/database) Return to [database documentation](help/database)

View file

@ -1200,7 +1200,7 @@ class Worker
* or: Worker::add(PRIORITY_HIGH, "Notifier", Delivery::DELETION, $drop_id); * or: Worker::add(PRIORITY_HIGH, "Notifier", Delivery::DELETION, $drop_id);
* or: Worker::add(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "Delivery", $post_id); * or: Worker::add(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "Delivery", $post_id);
* *
* @return boolean "false" if worker queue entry already existed or there had been an error * @return int "0" if worker queue entry already existed or there had been an error, otherwise the ID of the worker task
* @throws \Friendica\Network\HTTPException\InternalServerErrorException * @throws \Friendica\Network\HTTPException\InternalServerErrorException
* @note $cmd and string args are surrounded with "" * @note $cmd and string args are surrounded with ""
* *
@ -1213,14 +1213,14 @@ class Worker
$args = func_get_args(); $args = func_get_args();
if (!count($args)) { if (!count($args)) {
return false; return 0;
} }
$arr = ['args' => $args, 'run_cmd' => true]; $arr = ['args' => $args, 'run_cmd' => true];
Hook::callAll("proc_run", $arr); Hook::callAll("proc_run", $arr);
if (!$arr['run_cmd'] || !count($args)) { if (!$arr['run_cmd'] || !count($args)) {
return true; return 1;
} }
$priority = PRIORITY_MEDIUM; $priority = PRIORITY_MEDIUM;
@ -1255,7 +1255,7 @@ class Worker
$command = array_shift($args); $command = array_shift($args);
$parameters = json_encode($args); $parameters = json_encode($args);
$found = DBA::exists('workerqueue', ['command' => $command, 'parameter' => $parameters, 'done' => false]); $found = DBA::exists('workerqueue', ['command' => $command, 'parameter' => $parameters, 'done' => false]);
$added = false; $added = 0;
if (!in_array($priority, PRIORITIES)) { if (!in_array($priority, PRIORITIES)) {
Logger::warning('Invalid priority', ['priority' => $priority, 'command' => $command, 'callstack' => System::callstack(20)]); Logger::warning('Invalid priority', ['priority' => $priority, 'command' => $command, 'callstack' => System::callstack(20)]);
@ -1264,15 +1264,15 @@ class Worker
// Quit if there was a database error - a precaution for the update process to 3.5.3 // Quit if there was a database error - a precaution for the update process to 3.5.3
if (DBA::errorNo() != 0) { if (DBA::errorNo() != 0) {
return false; return 0;
} }
if (!$found) { if (!$found) {
$added = DBA::insert('workerqueue', ['command' => $command, 'parameter' => $parameters, 'created' => $created, if (!DBA::insert('workerqueue', ['command' => $command, 'parameter' => $parameters, 'created' => $created,
'priority' => $priority, 'next_try' => $delayed]); 'priority' => $priority, 'next_try' => $delayed])) {
if (!$added) { return 0;
return false;
} }
$added = DBA::lastInsertId();
} elseif ($force_priority) { } elseif ($force_priority) {
DBA::update('workerqueue', ['priority' => $priority], ['command' => $command, 'parameter' => $parameters, 'done' => false, 'pid' => 0]); DBA::update('workerqueue', ['priority' => $priority], ['command' => $command, 'parameter' => $parameters, 'done' => false, 'pid' => 0]);
} }

View file

@ -64,13 +64,24 @@ class Delayed
Logger::notice('Adding post for delayed publishing', ['uid' => $item['uid'], 'delayed' => $delayed, 'uri' => $uri]); Logger::notice('Adding post for delayed publishing', ['uid' => $item['uid'], 'delayed' => $delayed, 'uri' => $uri]);
if (!Worker::add(['priority' => PRIORITY_HIGH, 'delayed' => $delayed], 'DelayedPublish', $item, $notify, $taglist, $attachments, $unprepared, $uri)) { $wid = Worker::add(['priority' => PRIORITY_HIGH, 'delayed' => $delayed], 'DelayedPublish', $item, $notify, $taglist, $attachments, $unprepared, $uri);
if (!$wid) {
return false; return false;
} }
DI::pConfig()->set($item['uid'], 'system', 'last_publish', $next_publish); DI::pConfig()->set($item['uid'], 'system', 'last_publish', $next_publish);
return DBA::insert('delayed-post', ['uri' => $uri, 'uid' => $item['uid'], 'delayed' => $delayed], Database::INSERT_IGNORE); $delayed_post = [
'uri' => $uri,
'title' => $item['title'],
'body' => $item['body'],
'private' => $item['private'],
'wid' => $item['wid'],
'uid' => $item['uid'],
'delayed' => $delayed,
];
return DBA::insert('delayed-post', $delayed_post, Database::INSERT_IGNORE);
} }
/** /**

View file

@ -55,7 +55,7 @@
use Friendica\Database\DBA; use Friendica\Database\DBA;
if (!defined('DB_UPDATE_VERSION')) { if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1430); define('DB_UPDATE_VERSION', 1431);
} }
return [ return [
@ -554,17 +554,48 @@ return [
"received" => ["received"], "received" => ["received"],
] ]
], ],
"workerqueue" => [
"comment" => "Background tasks queue entries",
"fields" => [
"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => "Auto incremented worker task id"],
"command" => ["type" => "varchar(100)", "comment" => "Task command"],
"parameter" => ["type" => "mediumtext", "comment" => "Task parameter"],
"priority" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "Task priority"],
"created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Creation date"],
"pid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "comment" => "Process id of the worker"],
"executed" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Execution date"],
"next_try" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Next retrial date"],
"retrial" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => "Retrial counter"],
"done" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Marked 1 when the task was done - will be deleted later"],
],
"indexes" => [
"PRIMARY" => ["id"],
"command" => ["command"],
"done_command_parameter" => ["done", "command", "parameter(64)"],
"done_executed" => ["done", "executed"],
"done_priority_retrial_created" => ["done", "priority", "retrial", "created"],
"done_priority_next_try" => ["done", "priority", "next_try"],
"done_pid_next_try" => ["done", "pid", "next_try"],
"done_pid_retrial" => ["done", "pid", "retrial"],
"done_pid_priority_created" => ["done", "pid", "priority", "created"]
]
],
"delayed-post" => [ "delayed-post" => [
"comment" => "Posts that are about to be distributed at a later time", "comment" => "Posts that are about to be distributed at a later time",
"fields" => [ "fields" => [
"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"], "id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"],
"uri" => ["type" => "varchar(255)", "comment" => "URI of the post that will be distributed later"], "uri" => ["type" => "varchar(255)", "comment" => "URI of the post that will be distributed later"],
"title" => ["type" => "varchar(255)", "comment" => "post title"],
"body" => ["type" => "mediumtext", "comment" => "post body content"],
"private" => ["type" => "tinyint unsigned", "comment" => "0=public, 1=private, 2=unlisted"],
"wid" => ["type" => "int unsigned", "foreign" => ["workerqueue" => "id"], "comment" => "Workerqueue id"],
"uid" => ["type" => "mediumint unsigned", "foreign" => ["user" => "uid"], "comment" => "Owner User id"], "uid" => ["type" => "mediumint unsigned", "foreign" => ["user" => "uid"], "comment" => "Owner User id"],
"delayed" => ["type" => "datetime", "comment" => "delay time"], "delayed" => ["type" => "datetime", "comment" => "delay time"],
], ],
"indexes" => [ "indexes" => [
"PRIMARY" => ["id"], "PRIMARY" => ["id"],
"uid_uri" => ["UNIQUE", "uid", "uri(190)"], "uid_uri" => ["UNIQUE", "uid", "uri(190)"],
"wid" => ["wid"],
] ]
], ],
"diaspora-interaction" => [ "diaspora-interaction" => [
@ -1496,30 +1527,4 @@ return [
], ],
"engine" => "MEMORY", "engine" => "MEMORY",
], ],
"workerqueue" => [
"comment" => "Background tasks queue entries",
"fields" => [
"id" => ["type" => "int unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1", "comment" => "Auto incremented worker task id"],
"command" => ["type" => "varchar(100)", "comment" => "Task command"],
"parameter" => ["type" => "mediumtext", "comment" => "Task parameter"],
"priority" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "Task priority"],
"created" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Creation date"],
"pid" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "comment" => "Process id of the worker"],
"executed" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Execution date"],
"next_try" => ["type" => "datetime", "not null" => "1", "default" => DBA::NULL_DATETIME, "comment" => "Next retrial date"],
"retrial" => ["type" => "tinyint", "not null" => "1", "default" => "0", "comment" => "Retrial counter"],
"done" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => "Marked 1 when the task was done - will be deleted later"],
],
"indexes" => [
"PRIMARY" => ["id"],
"command" => ["command"],
"done_command_parameter" => ["done", "command", "parameter(64)"],
"done_executed" => ["done", "executed"],
"done_priority_retrial_created" => ["done", "priority", "retrial", "created"],
"done_priority_next_try" => ["done", "priority", "next_try"],
"done_pid_next_try" => ["done", "pid", "next_try"],
"done_pid_retrial" => ["done", "pid", "retrial"],
"done_pid_priority_created" => ["done", "pid", "priority", "created"]
]
],
]; ];