Merge pull request #2709 from annando/1608-push-splitted

pubsubpublish is now split into separate calls per entry

include/notifier.php

@@ -642,8 +642,8 @@ function notifier_run(&$argv, &$argc){
 				if ($h === '[internal]') {
 					// Set push flag for PuSH subscribers to this topic,
 					// they will be notified in queue.php
-					q("UPDATE `push_subscriber` SET `push` = 1 " .
+					q("UPDATE `push_subscriber` SET `push` = 1 ".
-					  "WHERE `nickname` = '%s'", dbesc($owner['nickname']));
+					  "WHERE `nickname` = '%s' AND `push` = 0", dbesc($owner['nickname']));
 
 					logger('Activating internal PuSH for item '.$item_id, LOGGER_DEBUG);
 

include/poller.php

@@ -235,9 +235,13 @@ function poller_kill_stale_workers() {
 				logger("Worker process ".$pid["pid"]." took more than 3 hours. It will be killed now.");
 				posix_kill($pid["pid"], SIGTERM);
 
-				// Question: If a process is stale: Should we remove it or should we reschedule it?
+				// We killed the stale process.
-				// By now we rescheduling it. It's maybe not the wisest decision?
+				// To avoid a blocking situation we reschedule the process at the beginning of the queue.
-				q("UPDATE `workerqueue` SET `executed` = '0000-00-00 00:00:00', `pid` = 0 WHERE `pid` = %d",
+				// Additionally we are lowering the priority.
+				q("UPDATE `workerqueue` SET `executed` = '0000-00-00 00:00:00', `created` = '%s',
+							`priority` = %d, `pid` = 0 WHERE `pid` = %d",
+					dbesc(datetime_convert()),
+					intval(PRIORITY_LOW),
 					intval($pid["pid"]));
 			} else
 				logger("Worker process ".$pid["pid"]." now runs for ".round($duration)." minutes. That's okay.", LOGGER_DEBUG);

include/pubsubpublish.php

@@ -2,60 +2,57 @@
 require_once("boot.php");
 require_once("include/ostatus.php");
 
-function handle_pubsubhubbub() {
+use \Friendica\Core\Config;
+use \Friendica\Core\PConfig;
+
+function handle_pubsubhubbub($id) {
 	global $a, $db;
 
-	logger('start');
+	$r = q("SELECT * FROM `push_subscriber` WHERE `id` = %d", intval($id));
+	if (!$r)
+		return;
+	else
+		$rr = $r[0];
 
-	// We'll push to each subscriber that has push > 0,
+	logger("Generate feed of user ".$rr['nickname']." to ".$rr['callback_url']." - last updated ".$rr['last_update'], LOGGER_DEBUG);
-	// i.e. there has been an update (set in notifier.php).
 
-	$r = q("SELECT * FROM `push_subscriber` WHERE `push` > 0");
+	$params = ostatus::feed($a, $rr['nickname'], $rr['last_update']);
+	$hmac_sig = hash_hmac("sha1", $params, $rr['secret']);
 
-	foreach($r as $rr) {
+	$headers = array("Content-type: application/atom+xml",
+			sprintf("Link: <%s>;rel=hub,<%s>;rel=self",
+				$a->get_baseurl().'/pubsubhubbub',
+				$rr['topic']),
+			"X-Hub-Signature: sha1=".$hmac_sig);
 
-		logger("Generate feed for user ".$rr['nickname']." - last updated ".$rr['last_update'], LOGGER_DEBUG);
+	logger('POST '.print_r($headers, true)."\n".$params, LOGGER_DEBUG);
 
-		$params = ostatus::feed($a, $rr['nickname'], $rr['last_update']);
+	post_url($rr['callback_url'], $params, $headers);
-		$hmac_sig = hash_hmac("sha1", $params, $rr['secret']);
+	$ret = $a->get_curl_code();
 
-		$headers = array("Content-type: application/atom+xml",
+	if ($ret >= 200 && $ret <= 299) {
-				sprintf("Link: <%s>;rel=hub,<%s>;rel=self",
+		logger('successfully pushed to '.$rr['callback_url']);
-					$a->get_baseurl().'/pubsubhubbub',
-					$rr['topic']),
-				"X-Hub-Signature: sha1=".$hmac_sig);
 
-		logger('POST '.print_r($headers, true)."\n".$params, LOGGER_DEBUG);
+		// set last_update to "now", and reset push=0
+		$date_now = datetime_convert('UTC','UTC','now','Y-m-d H:i:s');
+		q("UPDATE `push_subscriber` SET `push` = 0, last_update = '%s' WHERE id = %d",
+			dbesc($date_now),
+			intval($rr['id']));
 
-		post_url($rr['callback_url'], $params, $headers);
+	} else {
-		$ret = $a->get_curl_code();
+		logger('error when pushing to '.$rr['callback_url'].' HTTP: '.$ret);
 
-		if ($ret >= 200 && $ret <= 299) {
+		// we use the push variable also as a counter, if we failed we
-			logger('successfully pushed to '.$rr['callback_url']);
+		// increment this until some upper limit where we give up
+		$new_push = intval($rr['push']) + 1;
 
-			// set last_update to "now", and reset push=0
+		if ($new_push > 30) // OK, let's give up
-			$date_now = datetime_convert('UTC','UTC','now','Y-m-d H:i:s');
+			$new_push = 0;
-			q("UPDATE `push_subscriber` SET `push` = 0, last_update = '%s' WHERE id = %d",
-				dbesc($date_now),
-				intval($rr['id']));
 
-		} else {
+		q("UPDATE `push_subscriber` SET `push` = %d WHERE id = %d",
-			logger('error when pushing to '.$rr['callback_url'].' HTTP: '.$ret);
+			$new_push,
-
+			intval($rr['id']));
-			// we use the push variable also as a counter, if we failed we
-			// increment this until some upper limit where we give up
-			$new_push = intval($rr['push']) + 1;
-
-			if ($new_push > 30) // OK, let's give up
-				$new_push = 0;
-
-			q("UPDATE `push_subscriber` SET `push` = %d WHERE id = %d",
-				$new_push,
-				intval($rr['id']));
-		}
 	}
-
-	logger('done');
 }
 
 
@@ -89,10 +86,28 @@ function pubsubpublish_run(&$argv, &$argc){
 
 	if($argc > 1)
 		$pubsubpublish_id = intval($argv[1]);
-	else
+	else {
-		$pubsubpublish_id = 0;
+		// We'll push to each subscriber that has push > 0,
+		// i.e. there has been an update (set in notifier.php).
+		$r = q("SELECT `id`, `callback_url` FROM `push_subscriber` WHERE `push` > 0");
 
-	handle_pubsubhubbub();
+		// Use the delivery interval that is also used for the notifier
+		$interval = Config::get("system", "delivery_interval", 2);
+
+		// If we are using the worker we don't need a delivery interval
+		if (get_config("system", "worker"))
+			$interval = false;
+
+		foreach($r as $rr) {
+			logger("Publish feed to ".$rr["callback_url"], LOGGER_DEBUG);
+			proc_run(PRIORITY_HIGH, 'include/pubsubpublish.php', $rr["id"]);
+
+			if($interval)
+				@time_sleep_until(microtime(true) + (float) $interval);
+		}
+	}
+
+	handle_pubsubhubbub($pubsubpublish_id);
 
 	return;