Browse Source

Merge pull request #3226 from annando/1702-only-worker

The worker is now the only background processing system
pull/3244/head
rabuzarus 5 years ago
committed by GitHub
parent
commit
3cef98caf6
  1. 100
      boot.php
  2. 14
      database.sql
  3. 22
      include/create_shadowentry.php
  4. 93
      include/cron.php
  5. 42
      include/cronhooks.php
  6. 37
      include/cronjobs.php
  7. 35
      include/dbclean.php
  8. 12
      include/dbstructure.php
  9. 24
      include/dbupdate.php
  10. 58
      include/delivery.php
  11. 39
      include/directory.php
  12. 43
      include/discover_poco.php
  13. 26
      include/expire.php
  14. 33
      include/gprobe.php
  15. 111
      include/notifier.php
  16. 35
      include/onepoll.php
  17. 12
      include/poller.php
  18. 96
      include/pubsubpublish.php
  19. 263
      include/queue.php
  20. 22
      include/remove_contact.php
  21. 32
      include/shadowupdate.php
  22. 21
      include/spool_post.php
  23. 22
      include/tagupdate.php
  24. 24
      include/threadupdate.php
  25. 38
      include/update_gcontact.php
  26. 22
      mod/admin.php
  27. 2
      mod/worker.php
  28. 16
      view/templates/admin_site.tpl
  29. 2
      view/templates/admin_summary.tpl

100
boot.php

@ -32,7 +32,6 @@ require_once('include/cache.php');
require_once('library/Mobile_Detect/Mobile_Detect.php');
require_once('include/features.php');
require_once('include/identity.php');
require_once('include/pidfile.php');
require_once('update.php');
require_once('include/dbstructure.php');
@ -1346,60 +1345,25 @@ class App {
return false;
}
/**
* @brief Checks if the process is already running
*
* @param string $taskname The name of the task that will be used for the name of the lockfile
* @param string $task The path and name of the php script
* @param int $timeout The timeout after which a task should be killed
*
* @return bool Is the process running?
*/
function is_already_running($taskname, $task = "", $timeout = 540) {
$lockpath = get_lockpath();
if ($lockpath != '') {
$pidfile = new pidfile($lockpath, $taskname);
if ($pidfile->is_already_running()) {
logger("Already running");
if ($pidfile->running_time() > $timeout) {
$pidfile->kill();
logger("killed stale process");
// Calling a new instance
if ($task != "")
proc_run(PRIORITY_MEDIUM, $task);
}
return true;
}
}
return false;
}
function proc_run($args) {
if (!function_exists("proc_open")) {
return;
}
// Add the php path if it is a php call
if (count($args) && ($args[0] === 'php' OR !is_string($args[0]))) {
// If the last worker fork was less than 10 seconds before then don't fork another one.
// This should prevent the forking of masses of workers.
if (get_config("system", "worker")) {
$cachekey = "app:proc_run:started";
$result = Cache::get($cachekey);
if (!is_null($result)) {
if ((time() - $result) < 10) {
return;
}
}
// Set the timestamp of the last proc_run
Cache::set($cachekey, time(), CACHE_MINUTE);
// If the last worker fork was less than 10 seconds before then don't fork another one.
// This should prevent the forking of masses of workers.
$cachekey = "app:proc_run:started";
$result = Cache::get($cachekey);
if (!is_null($result)) {
if ((time() - $result) < 10) {
return;
}
$args[0] = ((x($this->config,'php_path')) && (strlen($this->config['php_path'])) ? $this->config['php_path'] : 'php');
}
// Set the timestamp of the last proc_run
Cache::set($cachekey, time(), CACHE_MINUTE);
array_unshift($args, ((x($this->config,'php_path')) && (strlen($this->config['php_path'])) ? $this->config['php_path'] : 'php'));
// add baseurl to args. cli scripts can't construct it
$args[] = $this->get_baseurl();
@ -1976,10 +1940,9 @@ function get_max_import_size() {
* @brief Wrap calls to proc_close(proc_open()) and call hook
* so plugins can take part in process :)
*
* @param (string|integer|array) $cmd program to run, priority or parameter array
* @param (integer|array) priority or parameter array, $cmd atrings are deprecated and are ignored
*
* next args are passed as $cmd command line
* e.g.: proc_run("ls","-la","/tmp");
* or: proc_run(PRIORITY_HIGH, "include/notifier.php", "drop", $drop_id);
* or: proc_run(array('priority' => PRIORITY_HIGH, 'dont_fork' => true), "include/create_shadowentry.php", $post_id);
*
@ -2024,11 +1987,6 @@ function proc_run($cmd){
if (!$arr['run_cmd'] OR !count($args))
return;
if (!get_config("system", "worker") OR (is_string($run_parameter) AND ($run_parameter != 'php'))) {
$a->proc_run($args);
return;
}
$priority = PRIORITY_MEDIUM;
$dont_fork = get_config("system", "worker_dont_fork");
@ -2076,7 +2034,7 @@ function proc_run($cmd){
return;
// Now call the poller to execute the jobs that we just added to the queue
$args = array("php", "include/poller.php", "no_cron");
$args = array("include/poller.php", "no_cron");
$a->proc_run($args);
}
@ -2410,38 +2368,6 @@ function get_itemcachepath() {
return "";
}
function get_lockpath() {
$lockpath = get_config('system','lockpath');
if (($lockpath != "") AND App::directory_usable($lockpath)) {
// We have a lock path and it is usable
return $lockpath;
}
// We don't have a working preconfigured lock path, so we take the temp path.
$temppath = get_temppath();
if ($temppath != "") {
// To avoid any interferences with other systems we create our own directory
$lockpath = $temppath."/lock";
if (!is_dir($lockpath)) {
mkdir($lockpath);
}
if (App::directory_usable($lockpath)) {
// The new path is usable, we are happy
set_config("system", "lockpath", $lockpath);
return $lockpath;
} else {
// We can't create a subdirectory, strange.
// But the directory seems to work, so we use it but don't store it.
return $temppath;
}
}
// Reaching this point means that the operating system is configured badly.
return "";
}
/**
* @brief Returns the path where spool files are stored
*

14
database.sql

@ -1,5 +1,5 @@
-- ------------------------------------------
-- Friendica 3.5.1-rc (Asparagus)
-- Friendica 3.5.2-dev (Asparagus)
-- DB_UPDATE_VERSION 1215
-- ------------------------------------------
@ -204,18 +204,6 @@ CREATE TABLE IF NOT EXISTS `conv` (
INDEX `uid` (`uid`)
) DEFAULT CHARSET=utf8mb4;
--
-- TABLE deliverq
--
CREATE TABLE IF NOT EXISTS `deliverq` (
`id` int(10) unsigned NOT NULL auto_increment,
`cmd` varbinary(32) NOT NULL DEFAULT '',
`item` int(11) NOT NULL DEFAULT 0,
`contact` int(11) NOT NULL DEFAULT 0,
PRIMARY KEY(`id`),
UNIQUE INDEX `cmd_item_contact` (`cmd`,`item`,`contact`)
) DEFAULT CHARSET=utf8mb4;
--
-- TABLE event
--

22
include/create_shadowentry.php

@ -6,26 +6,9 @@
* This script is started from mod/item.php to save some time when doing a post.
*/
use \Friendica\Core\Config;
require_once("boot.php");
require_once("include/threads.php");
function create_shadowentry_run($argv, $argc) {
global $a, $db;
if (is_null($a))
$a = new App;
if (is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
}
Config::load();
if ($argc != 2) {
return;
}
@ -34,9 +17,4 @@ function create_shadowentry_run($argv, $argc) {
add_shadow_entry($message_id);
}
if (array_search(__file__,get_included_files())===0){
create_shadowentry_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
}
?>

93
include/cron.php

@ -1,35 +1,11 @@
<?php
if (!file_exists("boot.php") AND (sizeof($_SERVER["argv"]) != 0)) {
$directory = dirname($_SERVER["argv"][0]);
if (substr($directory, 0, 1) != "/")
$directory = $_SERVER["PWD"]."/".$directory;
$directory = realpath($directory."/..");
chdir($directory);
}
use \Friendica\Core\Config;
require_once("boot.php");
require_once("include/photos.php");
require_once("include/user.php");
require_once('include/photos.php');
require_once('include/user.php');
function cron_run(&$argv, &$argc){
global $a, $db;
if(is_null($a)) {
$a = new App;
}
if(is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
};
global $a;
require_once('include/session.php');
require_once('include/datetime.php');
@ -40,16 +16,6 @@ function cron_run(&$argv, &$argc){
require_once('mod/nodeinfo.php');
require_once('include/post_update.php');
Config::load();
// Don't check this stuff if the function is called by the poller
if (App::callstack() != "poller_run") {
if ($a->maxload_reached())
return;
if (App::is_already_running('cron', 'include/cron.php', 540))
return;
}
$last = get_config('system','last_cron');
$poll_interval = intval(get_config('system','cron_interval'));
@ -64,10 +30,6 @@ function cron_run(&$argv, &$argc){
}
}
$a->set_baseurl(get_config('system','url'));
load_hooks();
logger('cron: start');
// run queue delivery process in the background
@ -85,34 +47,17 @@ function cron_run(&$argv, &$argc){
// Expire and remove user entries
cron_expire_and_remove_users();
// If the worker is active, split the jobs in several sub processes
if (get_config("system", "worker")) {
// Check OStatus conversations
proc_run(PRIORITY_MEDIUM, "include/cronjobs.php", "ostatus_mentions");
// Check every conversation
proc_run(PRIORITY_MEDIUM, "include/cronjobs.php", "ostatus_conversations");
// Call possible post update functions
proc_run(PRIORITY_LOW, "include/cronjobs.php", "post_update");
// update nodeinfo data
proc_run(PRIORITY_LOW, "include/cronjobs.php", "nodeinfo");
} else {
// Check OStatus conversations
// Check only conversations with mentions (for a longer time)
ostatus::check_conversations(true);
// Check OStatus conversations
proc_run(PRIORITY_MEDIUM, "include/cronjobs.php", "ostatus_mentions");
// Check every conversation
ostatus::check_conversations(false);
// Check every conversation
proc_run(PRIORITY_MEDIUM, "include/cronjobs.php", "ostatus_conversations");
// Call possible post update functions
// see include/post_update.php for more details
post_update();
// Call possible post update functions
proc_run(PRIORITY_LOW, "include/cronjobs.php", "post_update");
// update nodeinfo data
nodeinfo_cron();
}
// update nodeinfo data
proc_run(PRIORITY_LOW, "include/cronjobs.php", "nodeinfo");
// once daily run birthday_updates and then expire in background
@ -213,14 +158,6 @@ function cron_poll_contacts($argc, $argv) {
$force = true;
}
$interval = intval(get_config('system','poll_interval'));
if (!$interval)
$interval = ((get_config('system','delivery_interval') === false) ? 3 : intval(get_config('system','delivery_interval')));
// If we are using the worker we don't need a delivery interval
if (get_config("system", "worker"))
$interval = false;
$sql_extra = (($manual_id) ? " AND `id` = $manual_id " : "");
reload_plugins();
@ -335,9 +272,6 @@ function cron_poll_contacts($argc, $argv) {
} else {
proc_run(PRIORITY_LOW, 'include/onepoll.php', $contact['id']);
}
if($interval)
@time_sleep_until(microtime(true) + (float) $interval);
}
}
}
@ -488,8 +422,3 @@ function cron_repair_database() {
/// - remove children when parent got lost
/// - set contact-id in item when not present
}
if (array_search(__file__,get_included_files())===0){
cron_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
}

42
include/cronhooks.php

@ -2,37 +2,11 @@
use \Friendica\Core\Config;
require_once("boot.php");
function cronhooks_run(&$argv, &$argc){
global $a, $db;
if(is_null($a)) {
$a = new App;
}
if(is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
};
global $a;
require_once('include/session.php');
require_once('include/datetime.php');
Config::load();
// Don't check this stuff if the function is called by the poller
if (App::callstack() != "poller_run") {
if ($a->maxload_reached())
return;
if (App::is_already_running('cronhooks', 'include/cronhooks.php', 1140))
return;
}
load_hooks();
if (($argc == 2) AND is_array($a->hooks) AND array_key_exists("cron", $a->hooks)) {
foreach ($a->hooks["cron"] as $hook)
if ($hook[1] == $argv[1]) {
@ -42,7 +16,7 @@ function cronhooks_run(&$argv, &$argc){
return;
}
$last = get_config('system','last_cronhook');
$last = get_config('system', 'last_cronhook');
$poll_interval = intval(get_config('system','cronhook_interval'));
if(! $poll_interval)
@ -62,22 +36,16 @@ function cronhooks_run(&$argv, &$argc){
$d = datetime_convert();
if (get_config("system", "worker") AND is_array($a->hooks) AND array_key_exists("cron", $a->hooks)) {
if (is_array($a->hooks) AND array_key_exists("cron", $a->hooks)) {
foreach ($a->hooks["cron"] as $hook) {
logger("Calling cronhooks for '".$hook[1]."'", LOGGER_DEBUG);
proc_run(PRIORITY_MEDIUM, "include/cronhooks.php", $hook[1]);
}
} else
call_hooks('cron', $d);
}
logger('cronhooks: end');
set_config('system','last_cronhook', time());
set_config('system', 'last_cronhook', time());
return;
}
if (array_search(__file__,get_included_files())===0){
cronhooks_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
}

37
include/cronjobs.php

@ -1,44 +1,14 @@
<?php
use \Friendica\Core\Config;
if (!file_exists("boot.php") AND (sizeof($_SERVER["argv"]) != 0)) {
$directory = dirname($_SERVER["argv"][0]);
if (substr($directory, 0, 1) != "/")
$directory = $_SERVER["PWD"]."/".$directory;
$directory = realpath($directory."/..");
chdir($directory);
}
require_once("boot.php");
function cronjobs_run(&$argv, &$argc){
global $a, $db;
global $a;
if(is_null($a)) {
$a = new App;
}
if(is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
};
require_once('include/session.php');
require_once('include/datetime.php');
require_once('include/ostatus.php');
require_once('include/post_update.php');
require_once('mod/nodeinfo.php');
Config::load();
$a->set_baseurl(get_config('system','url'));
// No parameter set? So return
if ($argc <= 1)
return;
@ -71,8 +41,3 @@ function cronjobs_run(&$argv, &$argc){
return;
}
if (array_search(__file__,get_included_files())===0){
cronjobs_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
}

35
include/dbclean.php

@ -5,26 +5,8 @@
*/
use \Friendica\Core\Config;
use \Friendica\Core\PConfig;
require_once("boot.php");
function dbclean_run(&$argv, &$argc) {
global $a, $db;
if (is_null($a)) {
$a = new App;
}
if (is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
}
Config::load();
if (!Config::get('system', 'dbclean', false)) {
return;
}
@ -35,7 +17,7 @@ function dbclean_run(&$argv, &$argc) {
$stage = 0;
}
if (Config::get("system", "worker") AND ($stage == 0)) {
if ($stage == 0) {
proc_run(PRIORITY_LOW, 'include/dbclean.php', 1);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 2);
proc_run(PRIORITY_LOW, 'include/dbclean.php', 3);
@ -56,12 +38,8 @@ function remove_orphans($stage = 0) {
$count = 0;
// With activated worker we split the deletion in many small tasks
if (Config::get("system", "worker")) {
$limit = 1000;
} else {
$limit = 10000;
}
// We split the deletion in many small tasks
$limit = 1000;
if (($stage == 1) OR ($stage == 0)) {
logger("Deleting old global item entries from item table without user copy");
@ -159,14 +137,9 @@ function remove_orphans($stage = 0) {
}
// Call it again if not all entries were purged
if (($stage != 0) AND ($count > 0) AND Config::get("system", "worker")) {
if (($stage != 0) AND ($count > 0)) {
proc_run(PRIORITY_MEDIUM, 'include/dbclean.php');
}
}
if (array_search(__file__,get_included_files())===0){
dbclean_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
}
?>

12
include/dbstructure.php

@ -712,18 +712,6 @@ function db_definition($charset) {
"uid" => array("uid"),
)
);
$database["deliverq"] = array(
"fields" => array(
"id" => array("type" => "int(10) unsigned", "not null" => "1", "extra" => "auto_increment", "primary" => "1"),
"cmd" => array("type" => "varbinary(32)", "not null" => "1", "default" => ""),
"item" => array("type" => "int(11)", "not null" => "1", "default" => "0"),
"contact" => array("type" => "int(11)", "not null" => "1", "default" => "0"),
),
"indexes" => array(
"PRIMARY" => array("id"),
"cmd_item_contact" => array("UNIQUE", "cmd", "item", "contact"),
)
);
$database["event"] = array(
"fields" => array(
"id" => array("type" => "int(11)", "not null" => "1", "extra" => "auto_increment", "primary" => "1"),

24
include/dbupdate.php

@ -2,32 +2,12 @@
use \Friendica\Core\Config;
require_once("boot.php");
function dbupdate_run(&$argv, &$argc) {
global $a, $db;
if(is_null($a)){
$a = new App;
}
if(is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
}
Config::load();
global $a;
// We are deleting the latest dbupdate entry.
// This is done to avoid endless loops because the update was interupted.
Config::delete('database','dbupdate_'.DB_UPDATE_VERSION);
Config::delete('database', 'dbupdate_'.DB_UPDATE_VERSION);
update_db($a);
}
if (array_search(__file__,get_included_files())===0){
dbupdate_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
}

58
include/delivery.php

@ -2,44 +2,25 @@
use \Friendica\Core\Config;
require_once("boot.php");
require_once('include/queue_fn.php');
require_once('include/html2plain.php');
require_once("include/Scrape.php");
require_once('include/Scrape.php');
require_once('include/diaspora.php');
require_once("include/ostatus.php");
require_once("include/dfrn.php");
require_once('include/ostatus.php');
require_once('include/dfrn.php');
function delivery_run(&$argv, &$argc){
global $a, $db;
global $a;
if (is_null($a)) {
$a = new App;
}
if (is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
}
require_once("include/session.php");
require_once("include/datetime.php");
require_once('include/datetime.php');
require_once('include/items.php');
require_once('include/bbcode.php');
require_once('include/email.php');
Config::load();
load_hooks();
if ($argc < 3) {
return;
}
$a->set_baseurl(get_config('system','url'));
logger('delivery: invoked: '. print_r($argv,true), LOGGER_DEBUG);
$cmd = $argv[1];
@ -49,30 +30,6 @@ function delivery_run(&$argv, &$argc){
$contact_id = intval($argv[$x]);
/// @todo When switching completely to the worker we won't need this anymore
// Some other process may have delivered this item already.
$r = q("SELECT * FROM `deliverq` WHERE `cmd` = '%s' AND `item` = %d AND `contact` = %d LIMIT 1",
dbesc($cmd),
dbesc($item_id),
dbesc($contact_id)
);
if (!dbm::is_result($r)) {
continue;
}
if ($a->maxload_reached()) {
return;
}
// It's ours to deliver. Remove it from the queue.
q("DELETE FROM `deliverq` WHERE `cmd` = '%s' AND `item` = %d AND `contact` = %d",
dbesc($cmd),
dbesc($item_id),
dbesc($contact_id)
);
if (!$item_id || !$contact_id) {
continue;
}
@ -577,8 +534,3 @@ function delivery_run(&$argv, &$argc){
return;
}
if (array_search(__file__,get_included_files())===0){
delivery_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
}

39
include/directory.php

@ -1,36 +1,16 @@
<?php
require_once("boot.php");
use \Friendica\Core\Config;
function directory_run(&$argv, &$argc){
global $a, $db;
if(is_null($a)) {
$a = new App;
}
if(is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
};
Config::load();
if($argc != 2)
if ($argc != 2) {
return;
}
load_hooks();
$a->set_baseurl(get_config('system','url'));
$dir = get_config('system','directory');
$dir = get_config('system', 'directory');
if(! strlen($dir))
if (!strlen($dir)) {
return;
}
$dir .= "/submit";
@ -39,13 +19,8 @@ function directory_run(&$argv, &$argc){
call_hooks('globaldir_update', $arr);
logger('Updating directory: ' . $arr['url'], LOGGER_DEBUG);
if(strlen($arr['url']))
if (strlen($arr['url'])) {
fetch_url($dir . '?url=' . bin2hex($arr['url']));
}
return;
}
if (array_search(__file__,get_included_files())===0){
directory_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
}

43
include/discover_poco.php

@ -2,34 +2,12 @@
use \Friendica\Core\Config;
require_once("boot.php");
require_once("include/socgraph.php");
require_once('include/socgraph.php');
require_once('include/datetime.php');
function discover_poco_run(&$argv, &$argc){
global $a, $db;
if(is_null($a)) {
$a = new App;
}
if(is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
};
require_once('include/session.php');
require_once('include/datetime.php');
Config::load();
// Don't check this stuff if the function is called by the poller
if (App::callstack() != "poller_run")
if ($a->maxload_reached())
return;
if(($argc > 2) && ($argv[1] == "dirsearch")) {
if (($argc > 2) && ($argv[1] == "dirsearch")) {
$search = urldecode($argv[2]);
$mode = 1;
} elseif(($argc == 2) && ($argv[1] == "checkcontact")) {
@ -42,15 +20,6 @@ function discover_poco_run(&$argv, &$argc){
} else
die("Unknown or missing parameter ".$argv[1]."\n");
// Don't check this stuff if the function is called by the poller
if (App::callstack() != "poller_run")
if (App::is_already_running('discover_poco'.$mode.urlencode($search), 'include/discover_poco.php', 1140))
return;
$a->set_baseurl(get_config('system','url'));
load_hooks();
logger('start '.$search);
if ($mode==3)
@ -211,9 +180,3 @@ function gs_search_user($search) {
}
}
}
if (array_search(__file__,get_included_files())===0){
discover_poco_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
}

26
include/expire.php

@ -2,32 +2,13 @@
use \Friendica\Core\Config;
require_once("boot.php");
function expire_run(&$argv, &$argc){
global $a, $db;
if(is_null($a)) {
$a = new App;
}
global $a;
if(is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
};
require_once('include/session.php');
require_once('include/datetime.php');
require_once('include/items.php');
require_once('include/Contact.php');
Config::load();
$a->set_baseurl(get_config('system','url'));
// physically remove anything that has been deleted for more than two months
$r = q("delete from item where deleted = 1 and changed < UTC_TIMESTAMP() - INTERVAL 60 DAY");
@ -53,8 +34,3 @@ function expire_run(&$argv, &$argc){
return;
}
if (array_search(__file__,get_included_files())===0){
expire_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
}

33
include/gprobe.php

@ -2,36 +2,14 @@
use \Friendica\Core\Config;
require_once("boot.php");
require_once('include/Scrape.php');
require_once('include/socgraph.php');
require_once('include/datetime.php');
function gprobe_run(&$argv, &$argc){
global $a, $db;
if(is_null($a)) {
$a = new App;
}
if(is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
};
require_once('include/session.php');
require_once('include/datetime.php');
Config::load();
$a->set_baseurl(get_config('system','url'));
load_hooks();
if($argc != 2)
if ($argc != 2) {
return;
}
$url = hex2bin($argv[1]);
$r = q("SELECT `id`, `url`, `network` FROM `gcontact` WHERE `nurl` = '%s' ORDER BY `id` LIMIT 1",
@ -74,8 +52,3 @@ function gprobe_run(&$argv, &$argc){
logger("gprobe end for ".normalise_link($url), LOGGER_DEBUG);
return;
}
if (array_search(__file__,get_included_files())===0){
gprobe_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
}

111
include/notifier.php

@ -2,12 +2,11 @@
use \Friendica\Core\Config;
require_once("boot.php");
require_once('include/queue_fn.php');
require_once('include/html2plain.php');
require_once("include/Scrape.php");
require_once('include/Scrape.php');
require_once('include/diaspora.php');
require_once("include/ostatus.php");
require_once('include/ostatus.php');
require_once('include/salmon.php');
/*
@ -44,35 +43,17 @@ require_once('include/salmon.php');
function notifier_run(&$argv, &$argc){
global $a, $db;
global $a;
if (is_null($a)) {
$a = new App;
}
if (is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
}
require_once("include/session.php");
require_once("include/datetime.php");
require_once('include/datetime.php');
require_once('include/items.php');
require_once('include/bbcode.php');
require_once('include/email.php');
Config::load();
load_hooks();
if ($argc < 3) {
return;
}
$a->set_baseurl(get_config('system','url'));
logger('notifier: invoked: ' . print_r($argv,true), LOGGER_DEBUG);
$cmd = $argv[1];
@ -490,79 +471,22 @@ function notifier_run(&$argv, &$argc){
if ($relocate) {
$r = $recipients_relocate;
} else {
$r = q("SELECT * FROM `contact` WHERE `id` IN (%s) AND NOT `blocked` AND NOT `pending` AND NOT `archive`".$sql_extra,
$r = q("SELECT `id`, `url`, `network`, `self` FROM `contact`
WHERE `id` IN (%s) AND NOT `blocked` AND NOT `pending` AND NOT `archive`".$sql_extra,
dbesc($recip_str)
);
}
$interval = ((get_config('system','delivery_interval') === false) ? 2 : intval(get_config('system','delivery_interval')));
// If we are using the worker we don't need a delivery interval
if (get_config("system", "worker")) {
$interval = false;
}
// delivery loop
if (dbm::is_result($r)) {
foreach ($r as $contact) {
if (!$contact['self']) {
if (($contact['network'] === NETWORK_DIASPORA) && ($public_message)) {
continue;
}
q("INSERT INTO `deliverq` (`cmd`,`item`,`contact`) VALUES ('%s', %d, %d)",
dbesc($cmd),
intval($item_id),
intval($contact['id'])
);
}
}
// This controls the number of deliveries to execute with each separate delivery process.
// By default we'll perform one delivery per process. Assuming a hostile shared hosting
// provider, this provides the greatest chance of deliveries if processes start getting
// killed. We can also space them out with the delivery_interval to also help avoid them
// getting whacked.
// If $deliveries_per_process > 1, we will chain this number of multiple deliveries
// together into a single process. This will reduce the overall number of processes
// spawned for each delivery, but they will run longer.
// When using the workerqueue, we don't need this functionality.
$deliveries_per_process = intval(get_config('system','delivery_batch_count'));
if (($deliveries_per_process <= 0) OR get_config("system", "worker")) {
$deliveries_per_process = 1;
}
$this_batch = array();
for ($x = 0; $x < count($r); $x ++) {
$contact = $r[$x];
if ($contact['self']) {
continue;
}
logger("Deliver ".$target_item["guid"]." to ".$contact['url']." via network ".$contact['network'], LOGGER_DEBUG);
// potentially more than one recipient. Start a new process and space them out a bit.
// we will deliver single recipient types of message and email recipients here.
$this_batch[] = $contact['id'];
if (count($this_batch) >= $deliveries_per_process) {
proc_run(PRIORITY_HIGH,'include/delivery.php',$cmd,$item_id,$this_batch);
$this_batch = array();
if ($interval) {
@time_sleep_until(microtime(true) + (float) $interval);
}
}
continue;
}
// be sure to pick up any stragglers
if (count($this_batch)) {
proc_run(PRIORITY_HIGH,'include/delivery.php',$cmd,$item_id,$this_batch);
proc_run(PRIORITY_HIGH,'include/delivery.php', $cmd, $item_id, $contact['id']);
}
}
@ -614,18 +538,6 @@ function notifier_run(&$argv, &$argc){
if (dbm::is_result($r)) {
logger('pubdeliver '.$target_item["guid"].': '.print_r($r,true), LOGGER_DEBUG);
// throw everything into the queue in case we get killed
foreach ($r as $rr) {
if ((! $mail) && (! $fsuggest) && (! $followup)) {
q("INSERT INTO `deliverq` (`cmd`,`item`,`contact`) VALUES ('%s', %d, %d)
ON DUPLICATE KEY UPDATE `cmd` = '%s', `item` = %d, `contact` = %d",
dbesc($cmd), intval($item_id), intval($rr['id']),
dbesc($cmd), intval($item_id), intval($rr['id'])
);
}
}
foreach ($r as $rr) {
// except for Diaspora batch jobs
@ -639,9 +551,6 @@ function notifier_run(&$argv, &$argc){
if ((! $mail) && (! $fsuggest) && (! $followup)) {
logger('notifier: delivery agent: '.$rr['name'].' '.$rr['id'].' '.$rr['network'].' '.$target_item["guid"]);
proc_run(PRIORITY_HIGH,'include/delivery.php',$cmd,$item_id,$rr['id']);
if ($interval) {
@time_sleep_until(microtime(true) + (float) $interval);
}
}
}
}
@ -694,9 +603,3 @@ function notifier_run(&$argv, &$argc){
return;
}
if (array_search(__file__,get_included_files())===0){
notifier_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
}

35
include/onepoll.php

@ -2,8 +2,7 @@
use \Friendica\Core\Config;
require_once("boot.php");
require_once("include/follow.php");
require_once('include/follow.php');
function RemoveReply($subject) {
while (in_array(strtolower(substr($subject, 0, 3)), array("re:", "aw:")))
@ -13,20 +12,8 @@ function RemoveReply($subject) {
}
function onepoll_run(&$argv, &$argc){
global $a, $db;
global $a;
if (is_null($a)) {
$a = new App;
}
if (is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
};
require_once('include/session.php');
require_once('include/datetime.php');
require_once('include/items.php');
require_once('include/Contact.php');
@ -34,12 +21,6 @@ function onepoll_run(&$argv, &$argc){
require_once('include/socgraph.php');
require_once('include/queue_fn.php');
Config::load();
$a->set_baseurl(get_config('system','url'));
load_hooks();
logger('onepoll: start');
$manual_id = 0;
@ -61,13 +42,6 @@ function onepoll_run(&$argv, &$argc){
return;
}
// Don't check this stuff if the function is called by the poller
if (App::callstack() != "poller_run") {
if (App::is_already_running('onepoll'.$contact_id, '', 540)) {
return;
}
}
$d = datetime_convert();
// Only poll from those with suitable relationships,
@ -697,8 +671,3 @@ function onepoll_run(&$argv, &$argc){
return;
}
if (array_search(__file__,get_included_files())===0) {
onepoll_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
}

12
include/poller.php

@ -35,6 +35,10 @@ function poller_run($argv, $argc){
return;
}
$a->set_baseurl(Config::get('system', 'url'));
load_hooks();
$a->start_process();
if (poller_max_connections_reached()) {
@ -464,7 +468,7 @@ function poller_too_much_workers() {
// Are there fewer workers running as possible? Then fork a new one.
if (!Config::get("system", "worker_dont_fork") AND ($queues > ($active + 1)) AND ($entries > 1)) {
logger("Active workers: ".$active."/".$queues." Fork a new worker.", LOGGER_DEBUG);
$args = array("php", "include/poller.php", "no_cron");
$args = array("include/poller.php", "no_cron");
$a = get_app();
$a->proc_run($args);
}
@ -571,7 +575,7 @@ function poller_worker_process() {
* @brief Call the front end worker
*/
function call_worker() {
if (!Config::get("system", "frontend_worker") OR !Config::get("system", "worker")) {
if (!Config::get("system", "frontend_worker")) {
return;
}
@ -583,7 +587,7 @@ function call_worker() {
* @brief Call the front end worker if there aren't any active
*/
function call_worker_if_idle() {
if (!Config::get("system", "frontend_worker") OR !Config::get("system", "worker")) {
if (!Config::get("system", "frontend_worker")) {
return;
}
@ -610,7 +614,7 @@ function call_worker_if_idle() {
logger('Call poller', LOGGER_DEBUG);
$args = array("php", "include/poller.php", "no_cron");
$args = array("include/poller.php", "no_cron");
$a = get_app();
$a->proc_run($args);
return;

96
include/pubsubpublish.php

@ -1,12 +1,31 @@
<?php
require_once("boot.php");
require_once("include/ostatus.php");
use \Friendica\Core\Config;
use \Friendica\Core\PConfig;
require_once('include/items.php');
require_once('include/ostatus.php');
function pubsubpublish_run(&$argv, &$argc){
if ($argc > 1) {
$pubsubpublish_id = intval($argv[1]);
} else {
// We'll push to each subscriber that has push > 0,
// i.e. there has been an update (set in notifier.php).
$r = q("SELECT `id`, `callback_url` FROM `push_subscriber` WHERE `push` > 0");
foreach ($r as $rr) {
logger("Publish feed to ".$rr["callback_url"], LOGGER_DEBUG);
proc_run(PRIORITY_HIGH, 'include/pubsubpublish.php', $rr["id"]);
}
}
handle_pubsubhubbub($pubsubpublish_id);
return;
}
function handle_pubsubhubbub($id) {
global $a, $db;
global $a;
$r = q("SELECT * FROM `push_subscriber` WHERE `id` = %d", intval($id));
if (!$r)
@ -54,70 +73,3 @@ function handle_pubsubhubbub($id) {
intval($rr['id']));
}
}
function pubsubpublish_run(&$argv, &$argc){
global $a, $db;
if(is_null($a)){
$a = new App;
}
if(is_null($db)){
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
};
require_once('include/items.php');
Config::load();
// Don't check this stuff if the function is called by the poller
if (App::callstack() != "poller_run") {
if (App::is_already_running("pubsubpublish", "include/pubsubpublish.php", 540)) {
return;
}
}
$a->set_baseurl(get_config('system','url'));
load_hooks();
if ($argc > 1) {
$pubsubpublish_id = intval($argv[1]);
}
else {
// We'll push to each subscriber that has push > 0,
// i.e. there has been an update (set in notifier.php).
$r = q("SELECT `id`, `callback_url` FROM `push_subscriber` WHERE `push` > 0");
// Use the delivery interval that is also used for the notifier
$interval = Config::get("system", "delivery_interval", 2);
// If we are using the worker we don't need a delivery interval
if (get_config("system", "worker")) {
$interval = false;
}
foreach ($r as $rr) {
logger("Publish feed to ".$rr["callback_url"], LOGGER_DEBUG);
proc_run(PRIORITY_HIGH, 'include/pubsubpublish.php', $rr["id"]);
if($interval)
@time_sleep_until(microtime(true) + (float) $interval);
}
}
handle_pubsubhubbub($pubsubpublish_id);
return;
}
if (array_search(__file__,get_included_files())===0){
pubsubpublish_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
}

263
include/queue.php

@ -2,41 +2,16 @@
use \Friendica\Core\Config;
require_once("boot.php");
require_once('include/queue_fn.php');
require_once('include/dfrn.php');
require_once("include/datetime.php");
require_once('include/items.php');
require_once('include/bbcode.php');
require_once('include/socgraph.php');
require_once('include/cache.php');
function queue_run(&$argv, &$argc){
global $a, $db;
if (is_null($a)){
$a = new App;
}
if (is_null($db)){
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);
};
require_once("include/session.php");
require_once("include/datetime.php");
require_once('include/items.php');
require_once('include/bbcode.php');
require_once('include/socgraph.php');
Config::load();
// Don't check this stuff if the function is called by the poller
if (App::callstack() != "poller_run")
if (App::is_already_running('queue', 'include/queue.php', 540))
return;
$a->set_baseurl(get_config('system','url'));
load_hooks();
global $a;
if ($argc > 1) {
$queue_id = intval($argv[1]);
@ -54,30 +29,13 @@ function queue_run(&$argv, &$argc){
// Handling the pubsubhubbub requests
proc_run(PRIORITY_HIGH,'include/pubsubpublish.php');
$interval = ((get_config('system','delivery_interval') === false) ? 2 : intval(get_config('system','delivery_interval')));
// If we are using the worker we don't need a delivery interval
if (get_config("system", "worker"))
$interval = false;
$r = q("select * from deliverq where 1");
if ($r) {
foreach ($r as $rr) {
logger('queue: deliverq');
proc_run(PRIORITY_HIGH,'include/delivery.php',$rr['cmd'],$rr['item'],$rr['contact']);
if ($interval) {
time_sleep_until(microtime(true) + (float) $interval);
}
}
}
$r = q("SELECT `queue`.*, `contact`.`name`, `contact`.`uid` FROM `queue`
INNER JOIN `contact` ON `queue`.`cid` = `contact`.`id`
WHERE `queue`.`created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
if ($r) {
foreach ($r as $rr) {
logger('Removing expired queue item for ' . $rr['name'] . ', uid=' . $rr['uid']);
logger('Expired queue data :' . $rr['content'], LOGGER_DATA);
logger('Expired queue data: ' . $rr['content'], LOGGER_DATA);
}
q("DELETE FROM `queue` WHERE `created` < UTC_TIMESTAMP() - INTERVAL 3 DAY");
}
@ -86,158 +44,137 @@ function queue_run(&$argv, &$argc){
// After that, we'll only attempt delivery once per hour.
$r = q("SELECT `id` FROM `queue` WHERE ((`created` > UTC_TIMESTAMP() - INTERVAL 12 HOUR && `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE) OR (`last` < UTC_TIMESTAMP() - INTERVAL 1 HOUR)) ORDER BY `cid`, `created`");
} else {
logger('queue: start for id '.$queue_id);
$r = q("SELECT `id` FROM `queue` WHERE `id` = %d LIMIT 1",
intval($queue_id)
);
}
call_hooks('queue_predeliver', $a, $r);
if (!$r){
if (dbm::is_result($r)) {
foreach ($r as $q_item) {
logger('Call queue for id '.$q_item['id']);
proc_run(PRIORITY_LOW, "include/queue.php", $q_item['id']);
}
}
return;
}
if (!$queue_id)
call_hooks('queue_predeliver', $a, $r);
// delivery loop
// delivering
require_once('include/salmon.php');
require_once('include/diaspora.php');
foreach($r as $q_item) {
// queue_predeliver hooks may have changed the queue db details,
// so check again if this entry still needs processing
$r = q("SELECT * FROM `queue` WHERE `id` = %d LIMIT 1",
intval($queue_id));
if ($queue_id) {
$qi = q("SELECT * FROM `queue` WHERE `id` = %d LIMIT 1",
intval($queue_id));
} elseif (get_config("system", "worker")) {
logger('Call queue for id '.$q_item['id']);
proc_run(PRIORITY_LOW, "include/queue.php", $q_item['id']);
continue;
} else
$qi = q("SELECT * FROM `queue` WHERE `id` = %d AND `last` < UTC_TIMESTAMP() - INTERVAL 15 MINUTE ",
intval($q_item['id']));
if (!dbm::is_result($qi)) {
continue;
}
if (!dbm::is_result($r)) {
return;
}
$q_item = $r[0];
$c = q("SELECT * FROM `contact` WHERE `id` = %d LIMIT 1",
intval($qi[0]['cid'])
);
if (! dbm::is_result($c)) {
remove_queue_item($q_item['id']);
continue;
}
$c = q("SELECT * FROM `contact` WHERE `id` = %d LIMIT 1",
intval($q_item['cid'])
);
$dead = Cache::get($cachekey_deadguy.$c[0]['notify']);
if (!dbm::is_result($c)) {
remove_queue_item($q_item['id']);
return;
}
if (!is_null($dead) AND $dead) {
logger('queue: skipping known dead url: '.$c[0]['notify']);
update_queue_time($q_item['id']);
continue;
}
$dead = Cache::get($cachekey_deadguy.$c[0]['notify']);
$server = poco_detect_server($c[0]['url']);
if (!is_null($dead) AND $dead) {
logger('queue: skipping known dead url: '.$c[0]['notify']);
update_queue_time($q_item['id']);
return;
}
if ($server != "") {
$vital = Cache::get($cachekey_server.$server);
$server = poco_detect_server($c[0]['url']);
if (is_null($vital)) {
logger("Check server ".$server." (".$c[0]["network"].")");
if ($server != "") {
$vital = Cache::get($cachekey_server.$server);
$vital = poco_check_server($server, $c[0]["network"], true);
Cache::set($cachekey_server.$server, $vital, CACHE_QUARTER_HOUR);
}
if (is_null($vital)) {
logger("Check server ".$server." (".$c[0]["network"].")");
if (!is_null($vital) AND !$vital) {
logger('queue: skipping dead server: '.$server);
update_queue_time($q_item['id']);
continue;
}
$vital = poco_check_server($server, $c[0]["network"], true);
Cache::set($cachekey_server.$server, $vital, CACHE_QUARTER_HOUR);
}
$u = q("SELECT `user`.*, `user`.`pubkey` AS `upubkey`, `user`.`prvkey` AS `uprvkey`
FROM `user` WHERE `uid` = %d LIMIT 1",
intval($c[0]['uid'])
);
if (! dbm::is_result($u)) {
remove_queue_item($q_item['id']);
continue;
if (!is_null($vital) AND !$vital) {
logger('queue: skipping dead server: '.$server);
update_queue_time($q_item['id']);
return;
}
}
$data = $qi[0]['content'];
$public = $qi[0]['batch'];
$contact = $c[0];
$owner = $u[0];
$u = q("SELECT `user`.*, `user`.`pubkey` AS `upubkey`, `user`.`prvkey` AS `uprvkey`
FROM `user` WHERE `uid` = %d LIMIT 1",
intval($c[0]['uid'])
);
if (!dbm::is_result($u)) {
remove_queue_item($q_item['id']);
return;
}
$data = $q_item['content'];
$public = $q_item['batch'];
$contact = $c[0];
$owner = $u[0];
$deliver_status = 0;
$deliver_status = 0;
switch ($contact['network']) {
case NETWORK_DFRN:
logger('queue: dfrndelivery: item '.$q_item['id'].' for '.$contact['name'].' <'.$contact['url'].'>');
$deliver_status = dfrn::deliver($owner, $contact, $data);
switch($contact['network']) {
case NETWORK_DFRN:
logger('queue: dfrndelivery: item '.$q_item['id'].' for '.$contact['name'].' <'.$contact['url'].'>');
$deliver_status = dfrn::deliver($owner,$contact,$data);
if ($deliver_status == (-1)) {
update_queue_time($q_item['id']);
Cache::set($cachekey_deadguy.$contact['notify'], true, CACHE_QUARTER_HOUR);
} else {
remove_queue_item($q_item['id']);
}
break;
case NETWORK_OSTATUS:
if ($contact['notify']) {
logger('queue: slapdelivery: item '.$q_item['id'].' for '.$contact['name'].' <'.$contact['url'].'>');
$deliver_status = slapper($owner, $contact['notify'], $data);
if ($deliver_status < 0) {
if ($deliver_status == (-1)) {
update_queue_time($q_item['id']);
Cache::set($cachekey_deadguy.$contact['notify'], true, CACHE_QUARTER_HOUR);
} else {
remove_queue_item($q_item['id']);
}
break;
case NETWORK_OSTATUS:
if ($contact['notify']) {
logger('queue: slapdelivery: item '.$q_item['id'].' for '.$contact['name'].' <'.$contact['url'].'>');
$deliver_status = slapper($owner,$contact['notify'],$data);
if ($deliver_status == (-1)) {
update_queue_time($q_item['id']);
Cache::set($cachekey_deadguy.$contact['notify'], true, CACHE_QUARTER_HOUR);
} else {
remove_queue_item($q_item['id']);
}
}
break;
case NETWORK_DIASPORA:
if ($contact['notify']) {
logger('queue: diaspora_delivery: item '.$q_item['id'].' for '.$contact['name'].' <'.$contact['url'].'>');
$deliver_status = Diaspora::transmit($owner,$contact,$data,$public,true);
if ($deliver_status == (-1)) {
update_queue_time($q_item['id']);
Cache::set($cachekey_deadguy.$contact['notify'], true, CACHE_QUARTER_HOUR);
} else {
remove_queue_item($q_item['id']);
}
}
break;
default:
$params = array('owner' => $owner, 'contact' => $contact, 'queue' => $q_item, 'result' => false);
call_hooks('queue_deliver', $a, $params);
}
break;
case NETWORK_DIASPORA:
if ($contact['notify']) {
logger('queue: diaspora_delivery: item '.$q_item['id'].' for '.$contact['name'].' <'.$contact['url'].'>');
$deliver_status = Diaspora::transmit($owner, $contact, $data, $public, true);
if ($params['result']) {
remove_queue_item($q_item['id']);
} else {
if ($deliver_status == (-1)) {
update_queue_time($q_item['id']);
Cache::set($cachekey_deadguy.$contact['notify'], true, CACHE_QUARTER_HOUR);
} else {
remove_queue_item($q_item['id']);
}
break;
}
break;
}
logger('Deliver status '.(int)$deliver_status.' for item '.$q_item['id'].' to '.$contact['name'].' <'.$contact['url'].'>');
}
default:
$params = array('owner' => $owner, 'contact' => $contact, 'queue' => $q_item, 'result' => false);
call_hooks('queue_deliver', $a, $params);
return;
if ($params['result']) {
remove_queue_item($q_item['id']);
} else {
update_queue_time($q_item['id']);
}
break;
}
}
logger('Deliver status '.(int)$deliver_status.' for item '.$q_item['id'].' to '.$contact['name'].' <'.$contact['url'].'>');
if (array_search(__file__,get_included_files())===0){
queue_run($_SERVER["argv"],$_SERVER["argc"]);
killme();
return;
}

22
include/remove_contact.php

@ -6,24 +6,7 @@
use \Friendica\Core\Config;
require_once("boot.php");
function remove_contact_run($argv, $argc) {
global $a, $db;
if (is_null($a)) {
$a = new App;
}
if (is_null($db)) {
@include(".htconfig.php");
require_once("include/dba.php");
$db = new dba($db_host, $db_user, $db_pass, $db_data);
unset($db_host, $db_user, $db_pass, $db_data);