Merge branch 'develop' into semaphore_warning

INSTALL.txt

@@ -140,7 +140,17 @@ $a->config['system']['addon'] = 'js_upload,poormancron';
 
 and save your changes.
 
-9. (Optional) Reverse-proxying and HTTPS
+9. (Recommended) Set up a backup plan
+
+Bad things will happen. Let there be a hardware failure, a corrupted
+database or whatever you can think of. So once the installation of your
+Friendica node is done, you should make yoursef a backup plan.
+
+The most important file is the `.htconfig.php` file in the base directory.
+As it stores all your data, you should also have a recent dump of your
+Friendica database at hand, should you have to recover your node.
+
+10. (Optional) Reverse-proxying and HTTPS
 
 Friendica looks for some well-known HTTP headers indicating a reverse-proxy
 terminating an HTTPS connection. While the standard from RFC 7239 specifies

boot.php

@@ -42,7 +42,7 @@ define ( 'FRIENDICA_PLATFORM',     'Friendica');
 define ( 'FRIENDICA_CODENAME',     'Asparagus');
 define ( 'FRIENDICA_VERSION',      '3.5.3-dev' );
 define ( 'DFRN_PROTOCOL_VERSION',  '2.23'    );
-define ( 'DB_UPDATE_VERSION',      1231      );
+define ( 'DB_UPDATE_VERSION',      1232      );
 
 /**
  * @brief Constant with a HTML line break.

doc/Account-Basics.md

@@ -89,7 +89,10 @@ A ['Tips for New Members'](newmember) link will show up on your network and home
 Retrieving Personal Data
 ---
 
-You can export a copy of your personal data in XML format from the "Export personal data" link at the top of your settings page.
+You can export a copy of your personal data in JSON format from the "Export personal data" link at the top of your settings page.
+
+You need this file to relocate your Friendica account to another node.
+This might be necessary, e.g. if your node suffers a severe hardware problem and is not recoverable.
 
 See Also
 ---

doc/Install.md

@@ -144,3 +144,11 @@ The addon tree has to be updated separately like so:
 
     cd mywebsite/addon
     git pull
+
+###Set up a backup plan
+Bad things will happen.
+Let there be a hardware failure, a corrupted database or whatever you can think of.
+So once the installation of your Friendica node is done, you should make yoursef a backup plan.
+
+The most important file is the `.htconfig.php` file in the base directory.
+As it stores all your data, you should also have a recent dump of your Friendica database at hand, should you have to recover your node.

doc/de/Account-Basics.md

@@ -95,9 +95,11 @@ Ein ['Tipp für neue Mitglieder'](newmember)-Link zeigt sich in den ersten beide
 
 **Persönliche Daten exportieren**
 
-Du kannst eine Kopie Deiner persönlichen Daten in einer XML-Datei exportieren.
+Du kannst eine Kopie Deiner persönlichen Daten in einer JSON-Datei exportieren.
 Gehe hierzu in Deinen Einstellungen auf "Persönliche Daten exportieren".
 
+Dies ist z.B. dann nützlich wenn du mit deinem Account auf einen anderen Friendica Knoten umziehen möchstest.
+Ein Grund hierfür könnte sein, dass der Server auf dem dieser Friendica Knoten läuft dauerhaft wegen eines Hardware Problems ausfällt. 
 
 **Schau Dir ebenfalls folgende Seiten an**
 

doc/de/Install.md

@@ -108,3 +108,11 @@ Du kannst auch weitere Addons/Plugins ergänzen. Ändere den Eintrag folgenderma
 `$a->config['system']['addon'] = 'js_upload,poormancron';`
 
 und speichere deine Änderungen.
+
+###Erstelle einen Backup Plan
+Es werden schlimme Dinge geschehen.
+Sei es nun ein Hardwareversage oder eine korrumpierte Datenbank.
+Deshalb solltest du dir nachdem die Installation deines Friendica Knotens abgeschlossen ist einen Backup Plan erstellen.
+
+Die wichtigste Datei ist die `.htconfig.php` im Stammverzeichnis deiner Friendica Installation.
+Und da alle Daten in der Datenbank gespeichert werden, solltest du einen nicht all zu alten Dump der Friendica Datenbank zur Hand haben, solltest du deinen Knoten wieder herstellen müssen.

include/dba.php

@@ -627,6 +627,12 @@ class dba {
 					self::$dbo->errorno = mysql_errno(self::$dbo->db);
 				} else {
 					self::$dbo->affected_rows = mysql_affected_rows($retval);
+
+					// Due to missing mysql_* support this here wasn't tested at all
+					// See here: http://php.net/manual/en/function.mysql-num-rows.php
+					if (self::$dbo->affected_rows <= 0) {
+						self::$dbo->affected_rows = mysql_num_rows($retval);
+					}
 				}
 				break;
 		}
@@ -1038,7 +1044,7 @@ class dba {
 					$sql = "DELETE FROM `".$command['table']."` WHERE `".
 						implode("` = ? AND `", array_keys($command['param']))."` = ?";
 
-					logger(dba::replace_parameters($sql, $command['param']), LOGGER_DATA);
+					logger(self::replace_parameters($sql, $command['param']), LOGGER_DATA);
 
 					if (!self::e($sql, $command['param'])) {
 						if ($do_transaction) {
@@ -1068,7 +1074,7 @@ class dba {
 						$sql = "DELETE FROM `".$table."` WHERE `".$field."` IN (".
 							substr(str_repeat("?, ", count($field_values)), 0, -2).");";
 
-						logger(dba::replace_parameters($sql, $field_values), LOGGER_DATA);
+						logger(self::replace_parameters($sql, $field_values), LOGGER_DATA);
 
 						if (!self::e($sql, $field_values)) {
 							if ($do_transaction) {

include/items.php

@@ -2051,7 +2051,7 @@ function item_expire($uid, $days, $network = "", $force = false) {
 	$expire_photos = get_pconfig($uid, 'expire', 'photos');
 	$expire_photos = (($expire_photos === false) ? 0 : intval($expire_photos)); // default if not set: 0
 
-	logger('expire: # items=' . count($r). "; expire items: $expire_items, expire notes: $expire_notes, expire starred: $expire_starred, expire photos: $expire_photos");
+	logger('User '.$uid.': expire: # items=' . count($r). "; expire items: $expire_items, expire notes: $expire_notes, expire starred: $expire_starred, expire photos: $expire_photos");
 
 	foreach ($r as $item) {
 

include/poller.php

@@ -18,7 +18,7 @@ if (!file_exists("boot.php") && (sizeof($_SERVER["argv"]) != 0)) {
 require_once("boot.php");
 
 function poller_run($argv, $argc){
-	global $a, $db, $poller_up_start;
+	global $a, $db, $poller_up_start, $poller_db_duration;
 
 	$poller_up_start = microtime(true);
 
@@ -85,12 +85,13 @@ function poller_run($argv, $argc){
 		poller_run_cron();
 	}
 
-	$refetched = false;
-
 	$starttime = time();
 
 	// We fetch the next queue entry that is about to be executed
 	while ($r = poller_worker_process()) {
+
+		$refetched = false;
+
 		foreach ($r AS $entry) {
 			// Assure that the priority is an integer value
 			$entry['priority'] = (int)$entry['priority'];
@@ -100,10 +101,19 @@ function poller_run($argv, $argc){
 				logger('Process execution failed, quitting.', LOGGER_DEBUG);
 				return;
 			}
+
+			// If possible we will fetch new jobs for this worker
+			if (!$refetched && Lock::set('poller_worker_process', 0)) {
+				$stamp = (float)microtime(true);
+				$refetched = find_worker_processes();
+				$poller_db_duration += (microtime(true) - $stamp);
+				Lock::remove('poller_worker_process');
+			}
 		}
 
 		// To avoid the quitting of multiple pollers only one poller at a time will execute the check
 		if (Lock::set('poller_worker', 0)) {
+			$stamp = (float)microtime(true);
 			// Count active workers and compare them with a maximum value that depends on the load
 			if (poller_too_much_workers()) {
 				logger('Active worker limit reached, quitting.', LOGGER_DEBUG);
@@ -116,6 +126,7 @@ function poller_run($argv, $argc){
 				return;
 			}
 			Lock::remove('poller_worker');
+			$poller_db_duration += (microtime(true) - $stamp);
 		}
 
 		// Quit the poller once every 5 minutes
@@ -123,13 +134,6 @@ function poller_run($argv, $argc){
 			logger('Process lifetime reached, quitting.', LOGGER_DEBUG);
 			return;
 		}
-
-		// If possible we will fetch new jobs for this worker
-		if (!$refetched && Lock::set('poller_worker_process', 0)) {
-			$refetched = find_worker_processes();
-			Lock::remove('poller_worker_process');
-		}
-
 	}
 	logger("Couldn't select a workerqueue entry, quitting.", LOGGER_DEBUG);
 }
@@ -183,7 +187,7 @@ function poller_process_with_priority_active($priority) {
  * @return boolean "true" if further processing should be stopped
  */
 function poller_execute($queue) {
-	global $poller_db_duration;
+	global $poller_db_duration, $poller_last_update;
 
 	$a = get_app();
 
@@ -223,6 +227,21 @@ function poller_execute($queue) {
 	$funcname = str_replace(".php", "", basename($argv[0]))."_run";
 
 	if (function_exists($funcname)) {
+
+		// We constantly update the "executed" date every minute to avoid being killed too soon
+		if (!isset($poller_last_update)) {
+			$poller_last_update = strtotime($queue["executed"]);
+		}
+
+		$age = (time() - $poller_last_update) / 60;
+		$poller_last_update = time();
+
+		if ($age > 1) {
+			$stamp = (float)microtime(true);
+			dba::update('workerqueue', array('executed' => datetime_convert()), array('pid' => $mypid, 'done' => false));
+			$poller_db_duration += (microtime(true) - $stamp);
+		}
+
 		poller_exec_function($queue, $funcname, $argv);
 
 		$stamp = (float)microtime(true);
@@ -451,46 +470,47 @@ function poller_max_connections_reached() {
  *
  */
 function poller_kill_stale_workers() {
-	$r = q("SELECT `pid`, `executed`, `priority`, `parameter` FROM `workerqueue` WHERE `executed` > '%s' AND NOT `done`", dbesc(NULL_DATE));
+	$entries = dba::p("SELECT `id`, `pid`, `executed`, `priority`, `parameter` FROM `workerqueue` WHERE `executed` > ? AND NOT `done` AND `pid` != 0 ORDER BY `priority`, `created`", NULL_DATE);
 
-	if (!dbm::is_result($r)) {
-		// No processing here needed
-		return;
-	}
-
-	foreach ($r AS $pid) {
-		if (!posix_kill($pid["pid"], 0)) {
+	while ($entry = dba::fetch($entries)) {
+		if (!posix_kill($entry["pid"], 0)) {
 			dba::update('workerqueue', array('executed' => NULL_DATE, 'pid' => 0),
-					array('pid' => $pid["pid"], 'done' => false));
+					array('id' => $entry["id"]));
 		} else {
 			// Kill long running processes
-
 			// Check if the priority is in a valid range
-			if (!in_array($pid["priority"], array(PRIORITY_CRITICAL, PRIORITY_HIGH, PRIORITY_MEDIUM, PRIORITY_LOW, PRIORITY_NEGLIGIBLE))) {
-				$pid["priority"] = PRIORITY_MEDIUM;
+			if (!in_array($entry["priority"], array(PRIORITY_CRITICAL, PRIORITY_HIGH, PRIORITY_MEDIUM, PRIORITY_LOW, PRIORITY_NEGLIGIBLE))) {
+				$entry["priority"] = PRIORITY_MEDIUM;
 			}
 
 			// Define the maximum durations
-			$max_duration_defaults = array(PRIORITY_CRITICAL => 360, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180, PRIORITY_NEGLIGIBLE => 360);
-			$max_duration = $max_duration_defaults[$pid["priority"]];
+			$max_duration_defaults = array(PRIORITY_CRITICAL => 720, PRIORITY_HIGH => 10, PRIORITY_MEDIUM => 60, PRIORITY_LOW => 180, PRIORITY_NEGLIGIBLE => 720);
+			$max_duration = $max_duration_defaults[$entry["priority"]];
 
-			$argv = json_decode($pid["parameter"]);
+			$argv = json_decode($entry["parameter"]);
 			$argv[0] = basename($argv[0]);
 
 			// How long is the process already running?
-			$duration = (time() - strtotime($pid["executed"])) / 60;
+			$duration = (time() - strtotime($entry["executed"])) / 60;
 			if ($duration > $max_duration) {
-				logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") took more than ".$max_duration." minutes. It will be killed now.");
-				posix_kill($pid["pid"], SIGTERM);
+				logger("Worker process ".$entry["pid"]." (".implode(" ", $argv).") took more than ".$max_duration." minutes. It will be killed now.");
+				posix_kill($entry["pid"], SIGTERM);
 
 				// We killed the stale process.
 				// To avoid a blocking situation we reschedule the process at the beginning of the queue.
-				// Additionally we are lowering the priority.
+				// Additionally we are lowering the priority. (But not PRIORITY_CRITICAL)
+				if ($entry["priority"] == PRIORITY_HIGH) {
+					$new_priority = PRIORITY_MEDIUM;
+				} elseif ($entry["priority"] == PRIORITY_MEDIUM) {
+					$new_priority = PRIORITY_LOW;
+				} elseif ($entry["priority"] != PRIORITY_CRITICAL) {
+					$new_priority = PRIORITY_NEGLIGIBLE;
+				}
 				dba::update('workerqueue',
-						array('executed' => NULL_DATE, 'created' => datetime_convert(), 'priority' => PRIORITY_NEGLIGIBLE, 'pid' => 0),
-						array('pid' => $pid["pid"], 'done' => false));
+						array('executed' => NULL_DATE, 'created' => datetime_convert(), 'priority' => $new_priority, 'pid' => 0),
+						array('id' => $entry["id"]));
 			} else {
-				logger("Worker process ".$pid["pid"]." (".implode(" ", $argv).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG);
+				logger("Worker process ".$entry["pid"]." (".implode(" ", $argv).") now runs for ".round($duration)." of ".$max_duration." allowed minutes. That's okay.", LOGGER_DEBUG);
 			}
 		}
 	}
@@ -544,15 +564,16 @@ function poller_too_much_workers() {
 			}
 			dba::close($entries);
 
-			$jobs_per_minute = 0;
-
-			$jobs = dba::p("SELECT COUNT(*) AS `jobs` FROM `workerqueue` WHERE `done` AND `executed` > UTC_TIMESTAMP() - INTERVAL 10 MINUTE");
-			if ($job = dba::fetch($jobs)) {
-				$jobs_per_minute = number_format($job['jobs'] / 10, 0);
+			$intervals = array(1, 10, 60);
+			$jobs_per_minute = array();
+			foreach ($intervals AS $interval) {
+				$jobs = dba::p("SELECT COUNT(*) AS `jobs` FROM `workerqueue` WHERE `done` AND `executed` > UTC_TIMESTAMP() - INTERVAL ".intval($interval)." MINUTE");
+				if ($job = dba::fetch($jobs)) {
+					$jobs_per_minute[$interval] = number_format($job['jobs'] / $interval, 0);
+				}
+				dba::close($jobs);
 			}
-			dba::close($jobs);
-
-			$processlist = ' - jpm: '.$jobs_per_minute.' ('.implode(', ', $listitem).')';
+			$processlist = ' - jpm: '.implode('/', $jobs_per_minute).' ('.implode(', ', $listitem).')';
 		}
 
 		$entries = poller_total_entries();
@@ -573,8 +594,7 @@ function poller_too_much_workers() {
 		if (!Config::get("system", "worker_dont_fork") && ($queues > ($active + 1)) && ($entries > 1)) {
 			logger("Active workers: ".$active."/".$queues." Fork a new worker.", LOGGER_DEBUG);
 			$args = array("include/poller.php", "no_cron");
-			$a = get_app();
-			$a->proc_run($args);
+			get_app()->proc_run($args);
 		}
 	}
 
@@ -649,44 +669,65 @@ function poller_passing_slow(&$highest_priority) {
  * @return boolean Have we found something?
  */
 function find_worker_processes() {
+
+	$mypid = getmypid();
+
 	// Check if we should pass some low priority process
 	$highest_priority = 0;
 	$found = false;
 
 	// The higher the number of parallel workers, the more we prefetch to prevent concurring access
-	$limit = Config::get("system", "worker_queues", 4) * 2;
+	$limit = Config::get("system", "worker_queues", 4);
 	$limit = Config::get('system', 'worker_fetch_limit', $limit);
 
 	if (poller_passing_slow($highest_priority)) {
 		// Are there waiting processes with a higher priority than the currently highest?
-		$result = dba::e("UPDATE `workerqueue` SET `executed` = ?, `pid` = ?
+		$result = dba::p("SELECT `id` FROM `workerqueue`
 					WHERE `executed` <= ? AND `priority` < ? AND NOT `done`
 					ORDER BY `priority`, `created` LIMIT ".intval($limit),
-				datetime_convert(), getmypid(), NULL_DATE, $highest_priority);
-		if ($result) {
-			$found = (dba::affected_rows() > 0);
+				NULL_DATE, $highest_priority);
+
+		while ($id = dba::fetch($result)) {
+			$ids[] = $id["id"];
 		}
+		dba::close($result);
+
+		$found = (count($ids) > 0);
 
 		if (!$found) {
 			// Give slower processes some processing time
-			$result = dba::e("UPDATE `workerqueue` SET `executed` = ?, `pid` = ?
+			$result = dba::p("SELECT `id` FROM `workerqueue`
 						WHERE `executed` <= ? AND `priority` > ? AND NOT `done`
 						ORDER BY `priority`, `created` LIMIT ".intval($limit),
-					datetime_convert(), getmypid(), NULL_DATE, $highest_priority);
-			if ($result) {
-				$found = (dba::affected_rows() > 0);
+					NULL_DATE, $highest_priority);
+
+			while ($id = dba::fetch($result)) {
+				$ids[] = $id["id"];
 			}
+			dba::close($result);
+
+			$found = (count($ids) > 0);
 		}
 	}
 
 	// If there is no result (or we shouldn't pass lower processes) we check without priority limit
 	if (!$found) {
-		$result = dba::e("UPDATE `workerqueue` SET `executed` = ?, `pid` = ? WHERE `executed` <= ? AND NOT `done` ORDER BY `priority`, `created` LIMIT ".intval($limit),
-				datetime_convert(), getmypid(), NULL_DATE);
-		if ($result) {
-			$found = (dba::affected_rows() > 0);
+		$result = dba::p("SELECT `id` FROM `workerqueue` WHERE `executed` <= ? AND NOT `done` ORDER BY `priority`, `created` LIMIT ".intval($limit), NULL_DATE);
+
+		while ($id = dba::fetch($result)) {
+			$ids[] = $id["id"];
 		}
+		dba::close($result);
+
+		$found = (count($ids) > 0);
 	}
+
+	if ($found) {
+		$sql = "UPDATE `workerqueue` SET `executed` = ?, `pid` = ? WHERE `id` IN (".substr(str_repeat("?, ", count($ids)), 0, -2).") AND `pid` = 0 AND NOT `done`;";
+		array_unshift($ids, datetime_convert(), $mypid);
+		dba::e($sql, $ids);
+	}
+
 	return $found;
 }
 
@@ -778,8 +819,7 @@ function call_worker_if_idle() {
 		logger('Call poller', LOGGER_DEBUG);
 
 		$args = array("include/poller.php", "no_cron");
-		$a = get_app();
-		$a->proc_run($args);
+		get_app()->proc_run($args);
 		return;
 	}
 
@@ -834,8 +874,5 @@ if (array_search(__file__,get_included_files())===0){
 
 	get_app()->end_process();
 
-	Lock::remove('poller_worker');
-	Lock::remove('poller_worker_process');
-
 	killme();
 }

mod/register.php

@@ -8,6 +8,7 @@ require_once('include/user.php');
 
 if(! function_exists('register_post')) {
 function register_post(App $a) {
+	check_form_security_token_redirectOnErr('/register', 'register');
 
 	global $lang;
 
@@ -296,7 +297,7 @@ function register_content(App $a) {
 		'$sitename'  => $a->get_hostname(),
 		'$importh'   => t('Import'),
 		'$importt'   => t('Import your profile to this friendica instance'),
-
+		'$form_security_token'	=> get_form_security_token("register")
 	));
 	return $o;
 

mod/search.php

@@ -92,12 +92,12 @@ function search_post(App $a) {
 
 function search_content(App $a) {
 
-	if((get_config('system','block_public')) && (! local_user()) && (! remote_user())) {
-		notice( t('Public access denied.') . EOL);
+	if (get_config('system','block_public') && !local_user() && !remote_user()) {
+		notice(t('Public access denied.') . EOL);
 		return;
 	}
 
-	if(get_config('system','local_search') && !local_user()) {
+	if (get_config('system','local_search') && !local_user() && !remote_user()) {
 		http_status_exit(403,
 				array("title" => t("Public access denied."),
 					"description" => t("Only logged in users are permitted to perform a search.")));
@@ -106,7 +106,7 @@ function search_content(App $a) {
 		//return;
 	}
 
-	if (get_config('system','permit_crawling') && !local_user()) {
+	if (get_config('system','permit_crawling') && !local_user() && !remote_user()) {
 		// Default values:
 		// 10 requests are "free", after the 11th only a call per minute is allowed
 

update.php

@@ -1,6 +1,6 @@
 <?php
 
-define('UPDATE_VERSION' , 1231);
+define('UPDATE_VERSION' , 1232);
 
 /**
  *
@@ -1730,7 +1730,7 @@ function update_1202() {
 		dbesc(ACCOUNT_TYPE_COMMUNITY), dbesc(PAGE_COMMUNITY), dbesc(PAGE_PRVGROUP));
 }
 
-function update_1230() {
+function update_1231() {
 	// For this special case we have to use the old update routine
-	$r = q("ALTER TABLE `workerqueue` ADD `done2` tinyint(1) NOT NULL DEFAULT 0");
+	$r = q("ALTER TABLE `workerqueue` ADD `done` tinyint(1) NOT NULL DEFAULT 0");
 }

view/templates/register.tpl

@@ -3,6 +3,7 @@
 <form action="register" method="post" id="register-form">
 
 	<input type="hidden" name="photo" value="{{$photo}}" />
+	<input type="hidden" name="form_security_token" value="{{$form_security_token}}">
 
 	{{if $registertext != ""}}<div class="error-message">{{$registertext}} </div>{{/if}}