Merge branch 'develop' into improvement/ping-performance

boot.php

@@ -127,6 +127,10 @@ define ( 'CACHE_MONTH',            0 );
 define ( 'CACHE_WEEK',             1 );
 define ( 'CACHE_DAY',              2 );
 define ( 'CACHE_HOUR',             3 );
+define ( 'CACHE_HALF_HOUR',        4 );
+define ( 'CACHE_QUARTER_HOUR',     5 );
+define ( 'CACHE_FIVE_MINUTES',     6 );
+define ( 'CACHE_MINUTE',           7 );
 /* @}*/
 
 /**
@@ -1159,23 +1163,33 @@ class App {
 
 		$this->remove_inactive_processes();
 
+		q("START TRANSACTION");
+
 		$r = q("SELECT `pid` FROM `process` WHERE `pid` = %d", intval(getmypid()));
-		if(!dbm::is_result($r))
+		if(!dbm::is_result($r)) {
 			q("INSERT INTO `process` (`pid`,`command`,`created`) VALUES (%d, '%s', '%s')",
 				intval(getmypid()),
 				dbesc($command),
 				dbesc(datetime_convert()));
+		}
+		q("COMMIT");
 	}
 
 	/**
 	 * @brief Remove inactive processes
 	 */
 	function remove_inactive_processes() {
+		q("START TRANSACTION");
+
 		$r = q("SELECT `pid` FROM `process`");
-		if(dbm::is_result($r))
+		if(dbm::is_result($r)) {
-			foreach ($r AS $process)
+			foreach ($r AS $process) {
-				if (!posix_kill($process["pid"], 0))
+				if (!posix_kill($process["pid"], 0)) {
 					q("DELETE FROM `process` WHERE `pid` = %d", intval($process["pid"]));
+				}
+			}
+		}
+		q("COMMIT");
 	}
 
 	/**

doc/htconfig.md

@@ -39,6 +39,9 @@ Example: To set the directory value please add this line to your .htconfig.php:
 * max_batch_queue - Default value is 1000.
 * max_processes_backend - Maximum number of concurrent database processes for background tasks. Default value is 5.
 * max_processes_frontend - Maximum number of concurrent database processes for foreground tasks. Default value is 20.
+* memcache (Boolean) - Use memcache. To use memcache the PECL extension "memcache" has to be installed and activated.
+* memcache_host - Hostname of the memcache daemon. Default is '127.0.0.1'.
+* memcache_port- Portnumberof the memcache daemon. Default is 11211.
 * no_oembed (Boolean) - Don't use OEmbed to fetch more information about a link.
 * no_oembed_rich_content (Boolean) - Don't show the rich content (e.g. embedded PDF).
 * no_smilies (Boolean) - Don't show smilies.

include/Contact.php

@@ -487,9 +487,9 @@ function get_contact($url, $uid = 0) {
 	if ($contactid == 0) {
 		q("INSERT INTO `contact` (`uid`, `created`, `url`, `nurl`, `addr`, `alias`, `notify`, `poll`,
 					`name`, `nick`, `photo`, `network`, `pubkey`, `rel`, `priority`,
-					`batch`, `request`, `confirm`, `poco`,
+					`batch`, `request`, `confirm`, `poco`, `name-date`, `uri-date`,
 					`writable`, `blocked`, `readonly`, `pending`)
-					VALUES (%d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, %d, '%s', '%s', '%s', '%s', 1, 0, 0, 0)",
+					VALUES (%d, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, %d, '%s', '%s', '%s', '%s', '%s', '%s', 1, 0, 0, 0)",
 			intval($uid),
 			dbesc(datetime_convert()),
 			dbesc($data["url"]),
@@ -508,7 +508,9 @@ function get_contact($url, $uid = 0) {
 			dbesc($data["batch"]),
 			dbesc($data["request"]),
 			dbesc($data["confirm"]),
-			dbesc($data["poco"])
+			dbesc($data["poco"]),
+			dbesc(datetime_convert()),
+			dbesc(datetime_convert())
 		);
 
 		$contact = q("SELECT `id` FROM `contact` WHERE `nurl` = '%s' AND `uid` = %d ORDER BY `id` LIMIT 2",
@@ -539,16 +541,27 @@ function get_contact($url, $uid = 0) {
 
 	update_contact_avatar($data["photo"],$uid,$contactid);
 
-	q("UPDATE `contact` SET `addr` = '%s', `alias` = '%s', `name` = '%s', `nick` = '%s',
+	$r = q("SELECT `addr`, `alias`, `name`, `nick` FROM `contact`  WHERE `id` = %d", intval($contactid));
-		`name-date` = '%s', `uri-date` = '%s' WHERE `id` = %d",
+
-		dbesc($data["addr"]),
+	// This condition should always be true
-		dbesc($data["alias"]),
+	if (!dbm::is_result($r))
-		dbesc($data["name"]),
+		return $contactid;
-		dbesc($data["nick"]),
+
-		dbesc(datetime_convert()),
+	// Only update if there had something been changed
-		dbesc(datetime_convert()),
+	if (($data["addr"] != $r[0]["addr"]) OR
-		intval($contactid)
+		($data["alias"] != $r[0]["alias"]) OR
-	);
+		($data["name"] != $r[0]["name"]) OR
+		($data["nick"] != $r[0]["nick"]))
+		q("UPDATE `contact` SET `addr` = '%s', `alias` = '%s', `name` = '%s', `nick` = '%s',
+			`name-date` = '%s', `uri-date` = '%s' WHERE `id` = %d",
+			dbesc($data["addr"]),
+			dbesc($data["alias"]),
+			dbesc($data["name"]),
+			dbesc($data["nick"]),
+			dbesc(datetime_convert()),
+			dbesc(datetime_convert()),
+			intval($contactid)
+		);
 
 	return $contactid;
 }

include/Core/Config.php

@@ -33,8 +33,8 @@ class Config {
 		global $a;
 
 		$r = q("SELECT `v`, `k` FROM `config` WHERE `cat` = '%s' ORDER BY `cat`, `k`, `id`", dbesc($family));
-		if(count($r)) {
+		if (count($r)) {
-			foreach($r as $rr) {
+			foreach ($r as $rr) {
 				$k = $rr['k'];
 				if ($family === 'config') {
 					$a->config[$k] = $rr['v'];
@@ -74,16 +74,16 @@ class Config {
 
 		global $a;
 
-		if(! $refresh) {
+		if (!$refresh) {
 			// Looking if the whole family isn't set
-			if(isset($a->config[$family])) {
+			if (isset($a->config[$family])) {
-				if($a->config[$family] === '!<unset>!') {
+				if ($a->config[$family] === '!<unset>!') {
 					return $default_value;
 				}
 			}
 
-			if(isset($a->config[$family][$key])) {
+			if (isset($a->config[$family][$key])) {
-				if($a->config[$family][$key] === '!<unset>!') {
+				if ($a->config[$family][$key] === '!<unset>!') {
 					return $default_value;
 				}
 				return $a->config[$family][$key];
@@ -94,14 +94,13 @@ class Config {
 			dbesc($family),
 			dbesc($key)
 		);
-		if(count($ret)) {
+		if (count($ret)) {
 			// manage array value
 			$val = (preg_match("|^a:[0-9]+:{.*}$|s", $ret[0]['v'])?unserialize( $ret[0]['v']):$ret[0]['v']);
 			$a->config[$family][$key] = $val;
 
 			return $val;
-		}
+		} else {
-		else {
 			$a->config[$family][$key] = '!<unset>!';
 		}
 		return $default_value;
@@ -126,19 +125,32 @@ class Config {
 	public static function set($family, $key, $value) {
 		global $a;
 
+		$stored = self::get($family, $key);
+
+		if ($stored == $value) {
+			return true;
+		}
+
 		$a->config[$family][$key] = $value;
 
 		// manage array value
-		$dbvalue = is_array($value) ? serialize($value) : $value;
+		$dbvalue = (is_array($value) ? serialize($value) : $value);
-		$dbvalue = is_bool($dbvalue) ? intval($dbvalue) : $dbvalue;
+		$dbvalue = (is_bool($dbvalue) ? intval($dbvalue) : $dbvalue);
 
-		$ret = q("INSERT INTO `config` ( `cat`, `k`, `v` ) VALUES ( '%s', '%s', '%s' )
+		if (is_null($stored)) {
-ON DUPLICATE KEY UPDATE `v` = '%s'",
+			$ret = q("INSERT INTO `config` (`cat`, `k`, `v`) VALUES ('%s', '%s', '%s') ON DUPLICATE KEY UPDATE `v` = '%s'",
-			dbesc($family),
+				dbesc($family),
-			dbesc($key),
+				dbesc($key),
-			dbesc($dbvalue),
+				dbesc($dbvalue),
-			dbesc($dbvalue)
+				dbesc($dbvalue)
-		);
+			);
+		} else {
+			$ret = q("UPDATE `config` SET `v` = '%s' WHERE `cat` = '%s' AND `k` = '%s'",
+				dbesc($dbvalue),
+				dbesc($family),
+				dbesc($key)
+			);
+		}
 		if ($ret) {
 			return $value;
 		}
@@ -157,11 +169,12 @@ ON DUPLICATE KEY UPDATE `v` = '%s'",
 	 *  The configuration key to delete
 	 * @return mixed
 	 */
-	public static function delete($family,$key) {
+	public static function delete($family, $key) {
 
 		global $a;
-		if(x($a->config[$family],$key))
+		if (x($a->config[$family],$key)) {
 			unset($a->config[$family][$key]);
+		}
 		$ret = q("DELETE FROM `config` WHERE `cat` = '%s' AND `k` = '%s'",
 			dbesc($family),
 			dbesc($key)

include/Core/PConfig.php

@@ -27,14 +27,14 @@ class PConfig {
 	 *  The category of the configuration value
 	 * @return void
 	 */
-	public static function load($uid,$family) {
+	public static function load($uid, $family) {
 		global $a;
 		$r = q("SELECT `v`,`k` FROM `pconfig` WHERE `cat` = '%s' AND `uid` = %d ORDER BY `cat`, `k`, `id`",
 			dbesc($family),
 			intval($uid)
 		);
-		if(count($r)) {
+		if (count($r)) {
-			foreach($r as $rr) {
+			foreach ($r as $rr) {
 				$k = $rr['k'];
 				$a->config[$uid][$family][$k] = $rr['v'];
 			}
@@ -67,16 +67,16 @@ class PConfig {
 
 		global $a;
 
-		if(! $refresh) {
+		if (!$refresh) {
 			// Looking if the whole family isn't set
-			if(isset($a->config[$uid][$family])) {
+			if (isset($a->config[$uid][$family])) {
-				if($a->config[$uid][$family] === '!<unset>!') {
+				if ($a->config[$uid][$family] === '!<unset>!') {
 					return $default_value;
 				}
 			}
 
-			if(isset($a->config[$uid][$family][$key])) {
+			if (isset($a->config[$uid][$family][$key])) {
-				if($a->config[$uid][$family][$key] === '!<unset>!') {
+				if ($a->config[$uid][$family][$key] === '!<unset>!') {
 					return $default_value;
 				}
 				return $a->config[$uid][$family][$key];
@@ -89,13 +89,12 @@ class PConfig {
 			dbesc($key)
 		);
 
-		if(count($ret)) {
+		if (count($ret)) {
 			$val = (preg_match("|^a:[0-9]+:{.*}$|s", $ret[0]['v'])?unserialize( $ret[0]['v']):$ret[0]['v']);
 			$a->config[$uid][$family][$key] = $val;
 
 			return $val;
-		}
+		} else {
-		else {
 			$a->config[$uid][$family][$key] = '!<unset>!';
 		}
 		return $default_value;
@@ -123,19 +122,34 @@ class PConfig {
 
 		global $a;
 
+		$stored = self::get($uid, $family, $key);
+
+		if ($stored == $value) {
+			return true;
+		}
+
 		// manage array value
-		$dbvalue = (is_array($value)?serialize($value):$value);
+		$dbvalue = (is_array($value) ? serialize($value):$value);
 
 		$a->config[$uid][$family][$key] = $value;
 
-		$ret = q("INSERT INTO `pconfig` ( `uid`, `cat`, `k`, `v` ) VALUES ( %d, '%s', '%s', '%s' )
+                if (is_null($stored)) {
-ON DUPLICATE KEY UPDATE `v` = '%s'",
+			$ret = q("INSERT INTO `pconfig` (`uid`, `cat`, `k`, `v`) VALUES (%d, '%s', '%s', '%s') ON DUPLICATE KEY UPDATE `v` = '%s'",
-			intval($uid),
+				intval($uid),
-			dbesc($family),
+				dbesc($family),
-			dbesc($key),
+				dbesc($key),
-			dbesc($dbvalue),
+				dbesc($dbvalue),
-			dbesc($dbvalue)
+				dbesc($dbvalue)
-		);
+			);
+		} else {
+			$ret = q("UPDATE `pconfig` SET `v` = '%s' WHERE `uid` = %d AND `cat` = '%s' AND `k` = '%s'",
+				dbesc($dbvalue),
+				intval($uid),
+				dbesc($family),
+				dbesc($key)
+			);
+		}
+
 		if ($ret) {
 			return $value;
 		}
@@ -158,13 +172,17 @@ ON DUPLICATE KEY UPDATE `v` = '%s'",
 	public static function delete($uid,$family,$key) {
 
 		global $a;
-		if(x($a->config[$uid][$family],$key))
+
+		if (x($a->config[$uid][$family], $key)) {
 			unset($a->config[$uid][$family][$key]);
+		}
+
 		$ret = q("DELETE FROM `pconfig` WHERE `uid` = %d AND `cat` = '%s' AND `k` = '%s'",
 			intval($uid),
 			dbesc($family),
 			dbesc($key)
 		);
+
 		return $ret;
 	}
 }

include/Photo.php

@@ -670,6 +670,12 @@ class Photo {
 		dbesc($deny_gid)
 	    );
 	}
+
+	// Update the cached values
+	if ($album != 'Contact Photos') {
+		photo_albums($uid, true);
+	}
+
 	return $r;
     }
 }}
@@ -1059,3 +1065,33 @@ function store_photo($a, $uid, $imagedata = "", $url = "") {
 	return($image);
 }
 
+/**
+ * @brief Fetch the photo albums that are available for a viewer
+ *
+ * The query in this function is cost intensive, so it is cached.
+ *
+ * @param int $uid User id of the photos
+ * @param bool $update Update the cache
+ *
+ * @return array Returns array of the photo albums
+ */
+function photo_albums($uid, $update = false) {
+	$sql_extra = permissions_sql($uid);
+
+	$key = "photo_albums:".$uid.":".local_user().":".remote_user();
+	$albums = Cache::get($key);
+	if (is_null($albums) OR $update) {
+		/// @todo This query needs to be renewed. It is really slow
+		// At this time we just store the data in the cache
+		$albums = qu("SELECT COUNT(DISTINCT `resource-id`) AS `total`, `album`
+			FROM `photo` USE INDEX (`uid_album_created`)
+			WHERE `uid` = %d  AND `album` != '%s' AND `album` != '%s' $sql_extra
+			GROUP BY `album` ORDER BY `created` DESC",
+			intval($uid),
+			dbesc('Contact Photos'),
+			dbesc(t('Contact Photos'))
+		);
+		Cache::set($key, $albums, CACHE_DAY);
+	}
+	return $albums;
+}

include/api.php

@@ -565,9 +565,7 @@
 
 			//AND `allow_cid`='' AND `allow_gid`='' AND `deny_cid`='' AND `deny_gid`=''",
 			// count public wall messages
-			$r = q("SELECT count(*) as `count` FROM `item`
+			$r = q("SELECT COUNT(*) as `count` FROM `item` WHERE `uid` = %d AND `wall`",
-					WHERE  `uid` = %d
-					AND `type`='wall'",
 					intval($uinfo[0]['uid'])
 			);
 			$countitms = $r[0]['count'];

include/cache.php

@@ -1,72 +1,195 @@
 <?php
+/**
+ * @file include/cache.php
+ *
+ * @brief Class for storing data for a short time
+ */
+
+use \Friendica\Core\Config;
+use \Friendica\Core\PConfig;
+
+class Cache {
 	/**
-	 *  cache api
+	 * @brief Check for memcache and open a connection if configured
+	 *
+	 * @return object|boolean The memcache object - or "false" if not successful
 	 */
+	public static function memcache() {
+		if (!function_exists('memcache_connect')) {
+			return false;
+		}
 
-	class Cache {
+		if (!Config::get('system', 'memcache')) {
-		public static function get($key) {
+			return false;
+		}
 
-			$r = q("SELECT `v` FROM `cache` WHERE `k`='%s' limit 1",
+		$memcache_host = Config::get('system', 'memcache_host', '127.0.0.1');
-				dbesc($key)
+		$memcache_port = Config::get('system', 'memcache_port', 11211);
-			);
 
-			if (count($r))
+		$memcache = new Memcache;
-				return $r[0]['v'];
+
+		if (!$memcache->connect($memcache_host, $memcache_port)) {
+			return false;
+		}
+
+		return $memcache;
+	}
+
+	/**
+	 * @brief Return the duration for a given cache level
+	 *
+	 * @param integer $level Cache level
+	 *
+	 * @return integer The cache duration in seconds
+	 */
+	private function duration($level) {
+		switch($level) {
+			case CACHE_MONTH;
+				$seconds = 2592000;
+				break;
+			case CACHE_WEEK;
+				$seconds = 604800;
+				break;
+			case CACHE_DAY;
+				$seconds = 86400;
+				break;
+			case CACHE_HOUR;
+				$seconds = 3600;
+				break;
+			case CACHE_HALF_HOUR;
+				$seconds = 1800;
+				break;
+			case CACHE_QUARTER_HOUR;
+				$seconds = 900;
+				break;
+			case CACHE_FIVE_MINUTES;
+				$seconds = 300;
+				break;
+			case CACHE_MINUTE;
+				$seconds = 60;
+				break;
+		}
+		return $seconds;
+	}
+
+	/**
+	 * @brief Fetch cached data according to the key
+	 *
+	 * @param string $key The key to the cached data
+	 *
+	 * @return mixed Cached $value or "null" if not found
+	 */
+	public static function get($key) {
+
+		$memcache = self::memcache();
+		if (is_object($memcache)) {
+			// We fetch with the hostname as key to avoid problems with other applications
+			$value = $memcache->get(get_app()->get_hostname().":".$key);
+			if (!is_bool($value)) {
+				return unserialize($value);
+			}
 
 			return null;
 		}
 
-		public static function set($key,$value, $duration = CACHE_MONTH) {
+		// Frequently clear cache
+		self::clear($duration);
 
-			q("REPLACE INTO `cache` (`k`,`v`,`expire_mode`,`updated`) VALUES ('%s','%s',%d,'%s')",
+		$r = q("SELECT `v` FROM `cache` WHERE `k`='%s' LIMIT 1",
-					dbesc($key),
+			dbesc($key)
-					dbesc($value),
+		);
-					intval($duration),
-					dbesc(datetime_convert()));
 
+		if (count($r)) {
+			return $r[0]['v'];
 		}
 
-
+		return null;
-/*
- *
- * Leaving this legacy code temporaily to see how REPLACE fares
- * as opposed to non-atomic checks when faced with fast moving key duplication.
- * As a MySQL extension it isn't portable, but we're not yet very portable.
- */
-
-/*
- *			$r = q("SELECT * FROM `cache` WHERE `k`='%s' limit 1",
- *				dbesc($key)
- *			);
- *			if(count($r)) {
- *				q("UPDATE `cache` SET `v` = '%s', `updated = '%s' WHERE `k` = '%s'",
- *					dbesc($value),
- *					dbesc(datetime_convert()),
- *					dbesc($key));
- *			}
- *			else {
- *				q("INSERT INTO `cache` (`k`,`v`,`updated`) VALUES ('%s','%s','%s')",
- *					dbesc($key),
- *					dbesc($value),
- *					dbesc(datetime_convert()));
- *			}
- *		}
- */
-
-
-		public static function clear(){
-			q("DELETE FROM `cache` WHERE `updated` < '%s' AND `expire_mode` = %d",
-				dbesc(datetime_convert('UTC','UTC',"now - 30 days")), intval(CACHE_MONTH));
-
-			q("DELETE FROM `cache` WHERE `updated` < '%s' AND `expire_mode` = %d",
-				dbesc(datetime_convert('UTC','UTC',"now - 7 days")), intval(CACHE_WEEK));
-
-			q("DELETE FROM `cache` WHERE `updated` < '%s' AND `expire_mode` = %d",
-				dbesc(datetime_convert('UTC','UTC',"now - 1 days")), intval(CACHE_DAY));
-
-			q("DELETE FROM `cache` WHERE `updated` < '%s' AND `expire_mode` = %d",
-				dbesc(datetime_convert('UTC','UTC',"now - 1 hours")), intval(CACHE_HOUR));
-		}
-
 	}
 
+	/**
+	 * @brief Put data in the cache according to the key
+	 *
+	 * @param string $key The key to the cached data
+	 * @param mixed $valie The value that is about to be stored
+	 * @param integer $duration The cache lifespan
+	 */
+	public static function set($key, $value, $duration = CACHE_MONTH) {
+
+		// Do we have an installed memcache? Use it instead.
+		$memcache = self::memcache();
+		if (is_object($memcache)) {
+			// We store with the hostname as key to avoid problems with other applications
+			$memcache->set(get_app()->get_hostname().":".$key, serialize($value), MEMCACHE_COMPRESSED, self::duration($duration));
+			return;
+		}
+
+		/// @todo store the cache data in the same way like the config data
+		q("REPLACE INTO `cache` (`k`,`v`,`expire_mode`,`updated`) VALUES ('%s','%s',%d,'%s')",
+				dbesc($key),
+				dbesc($value),
+				intval($duration),
+				dbesc(datetime_convert()));
+	}
+
+	/**
+	 * @brief Remove outdated data from the cache
+	 *
+	 * @param integer $maxlevel The maximum cache level that is to be cleared
+	 */
+	public static function clear($max_level = CACHE_MONTH) {
+
+		// Clear long lasting cache entries only once a day
+		if (get_config("system", "cache_cleared_day") < time() - self::duration(CACHE_DAY)) {
+			if ($max_level == CACHE_MONTH) {
+				q("DELETE FROM `cache` WHERE `updated` < '%s' AND `expire_mode` = %d",
+					dbesc(datetime_convert('UTC','UTC',"now - 30 days")), intval(CACHE_MONTH));
+			}
+
+			if ($max_level <= CACHE_WEEK) {
+				q("DELETE FROM `cache` WHERE `updated` < '%s' AND `expire_mode` = %d",
+					dbesc(datetime_convert('UTC','UTC',"now - 7 days")), intval(CACHE_WEEK));
+			}
+
+			if ($max_level <= CACHE_DAY) {
+				q("DELETE FROM `cache` WHERE `updated` < '%s' AND `expire_mode` = %d",
+				dbesc(datetime_convert('UTC','UTC',"now - 1 days")), intval(CACHE_DAY));
+			}
+			set_config("system", "cache_cleared_day", time());
+		}
+
+		if (($max_level <= CACHE_HOUR) AND (get_config("system", "cache_cleared_hour")) < time() - self::duration(CACHE_HOUR)) {
+			q("DELETE FROM `cache` WHERE `updated` < '%s' AND `expire_mode` = %d",
+				dbesc(datetime_convert('UTC','UTC',"now - 1 hours")), intval(CACHE_HOUR));
+
+			set_config("system", "cache_cleared_hour", time());
+		}
+
+		if (($max_level <= CACHE_HALF_HOUR) AND (get_config("system", "cache_cleared_half_hour")) < time() - self::duration(CACHE_HALF_HOUR)) {
+			q("DELETE FROM `cache` WHERE `updated` < '%s' AND `expire_mode` = %d",
+				dbesc(datetime_convert('UTC','UTC',"now - 30 minutes")), intval(CACHE_HALF_HOUR));
+
+			set_config("system", "cache_cleared_half_hour", time());
+		}
+
+		if (($max_level <= CACHE_QUARTER_HOUR) AND (get_config("system", "cache_cleared_hour")) < time() - self::duration(CACHE_QUARTER_HOUR)) {
+			q("DELETE FROM `cache` WHERE `updated` < '%s' AND `expire_mode` = %d",
+				dbesc(datetime_convert('UTC','UTC',"now - 15 minutes")), intval(CACHE_QUARTER_HOUR));
+
+			set_config("system", "cache_cleared_quarter_hour", time());
+		}
+
+		if (($max_level <= CACHE_FIVE_MINUTES) AND (get_config("system", "cache_cleared_five_minute")) < time() - self::duration(CACHE_FIVE_MINUTES)) {
+			q("DELETE FROM `cache` WHERE `updated` < '%s' AND `expire_mode` = %d",
+				dbesc(datetime_convert('UTC','UTC',"now - 5 minutes")), intval(CACHE_FIVE_MINUTES));
+
+			set_config("system", "cache_cleared_five_minute", time());
+		}
+
+		if (($max_level <= CACHE_MINUTE) AND (get_config("system", "cache_cleared_minute")) < time() - self::duration(CACHE_MINUTE)) {
+			q("DELETE FROM `cache` WHERE `updated` < '%s' AND `expire_mode` = %d",
+				dbesc(datetime_convert('UTC','UTC',"now - 1 minutes")), intval(CACHE_MINUTE));
+
+			set_config("system", "cache_cleared_minute", time());
+		}
+	}
+}

include/cron.php

@@ -69,15 +69,15 @@ function cron_run(&$argv, &$argc){
 
 	// run queue delivery process in the background
 
-	proc_run(PRIORITY_NEGLIGIBLE,"include/queue.php");
+	proc_run(PRIORITY_NEGLIGIBLE, "include/queue.php");
 
 	// run the process to discover global contacts in the background
 
-	proc_run(PRIORITY_LOW,"include/discover_poco.php");
+	proc_run(PRIORITY_LOW, "include/discover_poco.php");
 
 	// run the process to update locally stored global contacts in the background
 
-	proc_run(PRIORITY_LOW,"include/discover_poco.php", "checkcontact");
+	proc_run(PRIORITY_LOW, "include/discover_poco.php", "checkcontact");
 
 	// Expire and remove user entries
 	cron_expire_and_remove_users();
@@ -120,11 +120,22 @@ function cron_run(&$argv, &$argc){
 
 		update_contact_birthdays();
 
-		proc_run(PRIORITY_LOW,"include/discover_poco.php", "suggestions");
+		proc_run(PRIORITY_LOW, "include/discover_poco.php", "suggestions");
 
 		set_config('system','last_expire_day',$d2);
 
-		proc_run(PRIORITY_LOW,'include/expire.php');
+		proc_run(PRIORITY_LOW, 'include/expire.php');
+
+		if (get_config("system", "worker")) {
+			proc_run(PRIORITY_LOW, 'include/dbclean.php', 1);
+			proc_run(PRIORITY_LOW, 'include/dbclean.php', 2);
+			proc_run(PRIORITY_LOW, 'include/dbclean.php', 3);
+			proc_run(PRIORITY_LOW, 'include/dbclean.php', 4);
+		} else {
+			proc_run(PRIORITY_LOW, 'include/dbclean.php');
+		}
+
+		cron_update_photo_albums();
 	}
 
 	// Clear cache entries
@@ -146,6 +157,19 @@ function cron_run(&$argv, &$argc){
 	return;
 }
 
+/**
+ * @brief Update the cached values for the number of photo albums per user
+ */
+function cron_update_photo_albums() {
+	$r = q("SELECT `uid` FROM `user` WHERE NOT `account_expired` AND NOT `account_removed`");
+	if (!dbm::is_result($r))
+		return;
+
+	foreach ($r AS $user) {
+		photo_albums($user['uid'], true);
+	}
+}
+
 /**
  * @brief Expire and remove user entries
  */
@@ -306,7 +330,7 @@ function cron_poll_contacts($argc, $argv) {
 
 			logger("Polling ".$contact["network"]." ".$contact["id"]." ".$contact["nick"]." ".$contact["name"]);
 
-			proc_run(PRIORITY_MEDIUM,'include/onepoll.php',$contact['id']);
+			proc_run(PRIORITY_MEDIUM, 'include/onepoll.php', $contact['id']);
 
 			if($interval)
 				@time_sleep_until(microtime(true) + (float) $interval);
@@ -355,10 +379,10 @@ function cron_clear_cache(&$a) {
 	}
 
 	// Delete the cached OEmbed entries that are older than one year
-	q("DELETE FROM `oembed` WHERE `created` < NOW() - INTERVAL 1 YEAR");
+	q("DELETE FROM `oembed` WHERE `created` < NOW() - INTERVAL 3 MONTH");
 
 	// Delete the cached "parse_url" entries that are older than one year
-	q("DELETE FROM `parsed_url` WHERE `created` < NOW() - INTERVAL 1 YEAR");
+	q("DELETE FROM `parsed_url` WHERE `created` < NOW() - INTERVAL 3 MONTH");
 
 	// Maximum table size in megabyte
 	$max_tablesize = intval(get_config('system','optimize_max_tablesize')) * 1000000;

include/dba.php

@@ -5,7 +5,7 @@ require_once("dbm.php");
 # TODO: PDO is disabled for release 3.3. We need to investigate why
 # the update from 3.2 fails with pdo
 /*
-if(class_exists('\PDO') && in_array('mysql', PDO::getAvailableDrivers())) {
+if (class_exists('\PDO') && in_array('mysql', PDO::getAvailableDrivers())) {
   require_once("library/dddbl2/dddbl.php");
   require_once("include/dba_pdo.php");
 }
@@ -24,7 +24,7 @@ require_once('include/datetime.php');
  *
  */
 
-if(! class_exists('dba')) {
+if (! class_exists('dba')) {
 class dba {
 
 	private $debug = 0;
@@ -34,7 +34,7 @@ class dba {
 	public  $connected = false;
 	public  $error = false;
 
-	function __construct($server,$user,$pass,$db,$install = false) {
+	function __construct($server, $user, $pass, $db, $install = false) {
 		global $a;
 
 		$stamp1 = microtime(true);
@@ -44,15 +44,15 @@ class dba {
 		$pass = trim($pass);
 		$db = trim($db);
 
-		if (!(strlen($server) && strlen($user))){
+		if (!(strlen($server) && strlen($user))) {
 			$this->connected = false;
 			$this->db = null;
 			return;
 		}
 
-		if($install) {
+		if ($install) {
-			if(strlen($server) && ($server !== 'localhost') && ($server !== '127.0.0.1')) {
+			if (strlen($server) && ($server !== 'localhost') && ($server !== '127.0.0.1')) {
-				if(! dns_get_record($server, DNS_A + DNS_CNAME + DNS_PTR)) {
+				if (! dns_get_record($server, DNS_A + DNS_CNAME + DNS_PTR)) {
 					$this->error = sprintf( t('Cannot locate DNS info for database server \'%s\''), $server);
 					$this->connected = false;
 					$this->db = null;
@@ -61,27 +61,28 @@ class dba {
 			}
 		}
 
-		if(class_exists('mysqli')) {
+		if (class_exists('mysqli')) {
 			$this->db = @new mysqli($server,$user,$pass,$db);
-			if(! mysqli_connect_errno()) {
+			if (! mysqli_connect_errno()) {
 				$this->connected = true;
 			}
-			if (isset($a->config["system"]["db_charset"]))
+			if (isset($a->config["system"]["db_charset"])) {
 				$this->db->set_charset($a->config["system"]["db_charset"]);
-		}
+			}
-		else {
+		} else {
 			$this->mysqli = false;
 			$this->db = mysql_connect($server,$user,$pass);
-			if($this->db && mysql_select_db($db,$this->db)) {
+			if ($this->db && mysql_select_db($db,$this->db)) {
 				$this->connected = true;
 			}
 			if (isset($a->config["system"]["db_charset"]))
 				mysql_set_charset($a->config["system"]["db_charset"], $this->db);
 		}
-		if(! $this->connected) {
+		if (!$this->connected) {
 			$this->db = null;
-			if(! $install)
+			if (!$install) {
 				system_unavailable();
+			}
 		}
 
 		$a->save_timestamp($stamp1, "network");
@@ -108,38 +109,63 @@ class dba {
 		return $return;
 	}
 
+	/**
+	 * @brief Returns the number of rows
+	 *
+	 * @return integer
+	 */
+	public function num_rows() {
+		if (!$this->result) {
+			return 0;
+		}
+
+		if ($this->mysqli) {
+			$return = $this->result->num_rows;
+		} else {
+			$return = mysql_num_rows($this->result);
+		}
+		return $return;
+	}
+
 	public function q($sql, $onlyquery = false) {
 		global $a;
 
-		if((! $this->db) || (! $this->connected))
+		if (!$this->db || !$this->connected) {
 			return false;
+		}
 
 		$this->error = '';
 
 		// Check the connection (This can reconnect the connection - if configured)
-		if ($this->mysqli)
+		if ($this->mysqli) {
 			$connected = $this->db->ping();
-		else
+		} else {
 			$connected = mysql_ping($this->db);
-
+		}
-		$connstr = ($connected ? "Connected": "Disonnected");
+		$connstr = ($connected ? "Connected" : "Disonnected");
 
 		$stamp1 = microtime(true);
 
-		if($this->mysqli)
+		$orig_sql = $sql;
-			$result = @$this->db->query($sql);
-		else
-			$result = @mysql_query($sql,$this->db);
 
+		if (x($a->config,'system') && x($a->config['system'], 'db_callstack')) {
+			$sql = "/*".$a->callstack()." */ ".$sql;
+		}
+
+		if ($this->mysqli) {
+			$result = @$this->db->query($sql);
+		} else {
+			$result = @mysql_query($sql,$this->db);
+		}
 		$stamp2 = microtime(true);
 		$duration = (float)($stamp2-$stamp1);
 
 		$a->save_timestamp($stamp1, "database");
 
-		if (strtolower(substr($sql, 0, 6)) != "select")
+		if (strtolower(substr($orig_sql, 0, 6)) != "select") {
 			$a->save_timestamp($stamp1, "database_write");
-
+		}
-		if(x($a->config,'system') && x($a->config['system'],'db_log')) {
+		if (x($a->config,'system') && x($a->config['system'],'db_log')) {
 			if (($duration > $a->config["system"]["db_loglimit"])) {
 				$duration = round($duration, 3);
 				$backtrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS);
@@ -150,33 +176,34 @@ class dba {
 			}
 		}
 
-		if($this->mysqli) {
+		if ($this->mysqli) {
-			if($this->db->errno) {
+			if ($this->db->errno) {
 				$this->error = $this->db->error;
 				$this->errorno = $this->db->errno;
 			}
-		} elseif(mysql_errno($this->db)) {
+		} elseif (mysql_errno($this->db)) {
 			$this->error = mysql_error($this->db);
 			$this->errorno = mysql_errno($this->db);
 		}
 
-		if(strlen($this->error)) {
+		if (strlen($this->error)) {
 			logger('DB Error ('.$connstr.') '.$this->errorno.': '.$this->error);
 		}
 
-		if($this->debug) {
+		if ($this->debug) {
 
 			$mesg = '';
 
-			if($result === false)
+			if ($result === false) {
 				$mesg = 'false';
-			elseif($result === true)
+			} elseif ($result === true) {
 				$mesg = 'true';
-			else {
+			} else {
-				if($this->mysqli)
+				if ($this->mysqli) {
 					$mesg = $result->num_rows . ' results' . EOL;
-    			else
+				} else {
 					$mesg = mysql_num_rows($result) . ' results' . EOL;
+				}
 			}
 
 			$str =  'SQL = ' . printable($sql) . EOL . 'SQL returned ' . $mesg
@@ -192,30 +219,30 @@ class dba {
 		 * These usually indicate SQL syntax errors that need to be resolved.
 		 */
 
-		if($result === false) {
+		if ($result === false) {
 			logger('dba: ' . printable($sql) . ' returned false.' . "\n" . $this->error);
-			if(file_exists('dbfail.out'))
+			if (file_exists('dbfail.out')) {
 				file_put_contents('dbfail.out', datetime_convert() . "\n" . printable($sql) . ' returned false' . "\n" . $this->error . "\n", FILE_APPEND);
+			}
 		}
 
-		if(($result === true) || ($result === false))
+		if (($result === true) || ($result === false)) {
 			return $result;
-
+		}
 		if ($onlyquery) {
 			$this->result = $result;
 			return true;
 		}
 
 		$r = array();
-		if($this->mysqli) {
+		if ($this->mysqli) {
-			if($result->num_rows) {
+			if ($result->num_rows) {
 				while($x = $result->fetch_array(MYSQLI_ASSOC))
 					$r[] = $x;
 				$result->free_result();
 			}
-		}
+		} else {
-		else {
+			if (mysql_num_rows($result)) {
-			if(mysql_num_rows($result)) {
 				while($x = mysql_fetch_array($result, MYSQL_ASSOC))
 					$r[] = $x;
 				mysql_free_result($result);
@@ -224,33 +251,35 @@ class dba {
 
 		//$a->save_timestamp($stamp1, "database");
 
-		if($this->debug)
+		if ($this->debug) {
 			logger('dba: ' . printable(print_r($r, true)));
+		}
 		return($r);
 	}
 
 	public function qfetch() {
 		$x = false;
 
-		if ($this->result)
+		if ($this->result) {
-			if($this->mysqli) {
+			if ($this->mysqli) {
-				if($this->result->num_rows)
+				if ($this->result->num_rows)
 					$x = $this->result->fetch_array(MYSQLI_ASSOC);
 			} else {
-				if(mysql_num_rows($this->result))
+				if (mysql_num_rows($this->result))
 					$x = mysql_fetch_array($this->result, MYSQL_ASSOC);
 			}
-
+		}
 		return($x);
 	}
 
 	public function qclose() {
-		if ($this->result)
+		if ($this->result) {
-			if($this->mysqli) {
+			if ($this->mysqli) {
 				$this->result->free_result();
 			} else {
 				mysql_free_result($this->result);
 			}
+		}
 	}
 
 	public function dbg($dbg) {
@@ -258,56 +287,62 @@ class dba {
 	}
 
 	public function escape($str) {
-		if($this->db && $this->connected) {
+		if ($this->db && $this->connected) {
-			if($this->mysqli)
+			if ($this->mysqli) {
 				return @$this->db->real_escape_string($str);
-			else
+			} else {
 				return @mysql_real_escape_string($str,$this->db);
+			}
 		}
 	}
 
 	function connected() {
-		if ($this->mysqli)
+		if ($this->mysqli) {
 			$connected = $this->db->ping();
-		else
+		} else {
 			$connected = mysql_ping($this->db);
-
+		}
 		return $connected;
 	}
 
 	function __destruct() {
-		if ($this->db)
+		if ($this->db) {
-			if($this->mysqli)
+			if ($this->mysqli) {
 				$this->db->close();
-			else
+			} else {
 				mysql_close($this->db);
+			}
+		}
 	}
 }}
 
-if(! function_exists('printable')) {
+if (! function_exists('printable')) {
 function printable($s) {
 	$s = preg_replace("~([\x01-\x08\x0E-\x0F\x10-\x1F\x7F-\xFF])~",".", $s);
 	$s = str_replace("\x00",'.',$s);
-	if(x($_SERVER,'SERVER_NAME'))
+	if (x($_SERVER,'SERVER_NAME')) {
 		$s = escape_tags($s);
+	}
 	return $s;
 }}
 
 // Procedural functions
-if(! function_exists('dbg')) {
+if (! function_exists('dbg')) {
 function dbg($state) {
 	global $db;
-	if($db)
+	if ($db) {
-	$db->dbg($state);
+		$db->dbg($state);
+	}
 }}
 
-if(! function_exists('dbesc')) {
+if (! function_exists('dbesc')) {
 function dbesc($str) {
 	global $db;
-	if($db && $db->connected)
+	if ($db && $db->connected) {
 		return($db->escape($str));
-	else
+	} else {
 		return(str_replace("'","\\'",$str));
+	}
 }}
 
 
@@ -317,17 +352,17 @@ function dbesc($str) {
 // Example: $r = q("SELECT * FROM `%s` WHERE `uid` = %d",
 //                   'user', 1);
 
-if(! function_exists('q')) {
+if (! function_exists('q')) {
 function q($sql) {
 
 	global $db;
 	$args = func_get_args();
 	unset($args[0]);
 
-	if($db && $db->connected) {
+	if ($db && $db->connected) {
 		$stmt = @vsprintf($sql,$args); // Disabled warnings
 		//logger("dba: q: $stmt", LOGGER_ALL);
-		if($stmt === false)
+		if ($stmt === false)
 			logger('dba: vsprintf error: ' . print_r(debug_backtrace(),true), LOGGER_DEBUG);
 		return $db->q($stmt);
 	}
@@ -343,20 +378,57 @@ function q($sql) {
 
 }}
 
+/**
+ * @brief Performs a query with "dirty reads"
+ *
+ * By doing dirty reads (reading uncommitted data) no locks are performed
+ * This function can be used to fetch data that doesn't need to be reliable.
+ *
+ * @param $args Query parameters (1 to N parameters of different types)
+ * @return array Query array
+ */
+function qu($sql) {
+
+	global $db;
+	$args = func_get_args();
+	unset($args[0]);
+
+	if ($db && $db->connected) {
+		$stmt = @vsprintf($sql,$args); // Disabled warnings
+		if ($stmt === false)
+			logger('dba: vsprintf error: ' . print_r(debug_backtrace(),true), LOGGER_DEBUG);
+		$db->q("SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;");
+		$retval = $db->q($stmt);
+		$db->q("SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ;");
+		return $retval;
+	}
+
+	/**
+	 *
+	 * This will happen occasionally trying to store the
+	 * session data after abnormal program termination
+	 *
+	 */
+	logger('dba: no database: ' . print_r($args,true));
+	return false;
+
+}
+
 /**
  *
  * Raw db query, no arguments
  *
  */
 
-if(! function_exists('dbq')) {
+if (! function_exists('dbq')) {
 function dbq($sql) {
 
 	global $db;
-	if($db && $db->connected)
+	if ($db && $db->connected) {
 		$ret = $db->q($sql);
-	else
+	} else {
 		$ret = false;
+	}
 	return $ret;
 }}
 
@@ -367,16 +439,16 @@ function dbq($sql) {
 // cast to int to avoid trouble.
 
 
-if(! function_exists('dbesc_array_cb')) {
+if (! function_exists('dbesc_array_cb')) {
 function dbesc_array_cb(&$item, $key) {
-	if(is_string($item))
+	if (is_string($item))
 		$item = dbesc($item);
 }}
 
 
-if(! function_exists('dbesc_array')) {
+if (! function_exists('dbesc_array')) {
 function dbesc_array(&$arr) {
-	if(is_array($arr) && count($arr)) {
+	if (is_array($arr) && count($arr)) {
 		array_walk($arr,'dbesc_array_cb');
 	}
 }}

include/dbclean.php

@@ -0,0 +1,94 @@
+<?php
+/**
+ * @file include/dbclean.php
+ * @brief The script is called from time to time to clean the database entries and remove orphaned data.
+ */
+require_once("boot.php");
+
+function dbclean_run(&$argv, &$argc) {
+	global $a, $db;
+
+	if (is_null($a))
+		$a = new App;
+
+	if (is_null($db)) {
+		@include(".htconfig.php");
+		require_once("include/dba.php");
+		$db = new dba($db_host, $db_user, $db_pass, $db_data);
+		unset($db_host, $db_user, $db_pass, $db_data);
+	}
+
+	load_config('config');
+	load_config('system');
+
+	if ($argc == 2) {
+		$stage = intval($argv[1]);
+	} else {
+		$stage = 0;
+	}
+	remove_orphans($stage);
+}
+
+/**
+ * @brief Remove orphaned database entries
+ */
+function remove_orphans($stage = 0) {
+	global $db;
+
+	if (($stage == 1) OR ($stage == 0)) {
+		logger("Deleting orphaned data from thread table");
+		if ($db->q("SELECT `iid` FROM `thread` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`parent` = `thread`.`iid`)", true)) {
+			logger("found thread orphans: ".$db->num_rows());
+			while ($orphan = $db->qfetch()) {
+				q("DELETE FROM `thread` WHERE `iid` = %d", intval($orphan["iid"]));
+			}
+		}
+		$db->qclose();
+	}
+
+	if (($stage == 2) OR ($stage == 0)) {
+		logger("Deleting orphaned data from notify table");
+		if ($db->q("SELECT `iid` FROM `notify` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `notify`.`iid`)", true)) {
+			logger("found notify orphans: ".$db->num_rows());
+			while ($orphan = $db->qfetch()) {
+				q("DELETE FROM `notify` WHERE `iid` = %d", intval($orphan["iid"]));
+			}
+		}
+		$db->qclose();
+	}
+
+
+	if (($stage == 3) OR ($stage == 0)) {
+		logger("Deleting orphaned data from sign table");
+		if ($db->q("SELECT `iid` FROM `sign` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `sign`.`iid`)", true)) {
+			logger("found sign orphans: ".$db->num_rows());
+			while ($orphan = $db->qfetch()) {
+				q("DELETE FROM `sign` WHERE `iid` = %d", intval($orphan["iid"]));
+			}
+		}
+		$db->qclose();
+	}
+
+
+	if (($stage == 4) OR ($stage == 0)) {
+		logger("Deleting orphaned data from term table");
+		if ($db->q("SELECT `oid` FROM `term` WHERE NOT EXISTS (SELECT `id` FROM `item` WHERE `item`.`id` = `term`.`oid`)", true)) {
+			logger("found term orphans: ".$db->num_rows());
+			while ($orphan = $db->qfetch()) {
+				q("DELETE FROM `term` WHERE `oid` = %d", intval($orphan["oid"]));
+			}
+		}
+		$db->qclose();
+	}
+
+	/// @todo Based on the following query we should remove some more data
+	// SELECT `id`, `received`, `created`, `guid` FROM `item` WHERE `uid` = 0 AND NOT EXISTS (SELECT `guid` FROM `item` AS `i` WHERE `item`.`guid` = `i`.`guid` AND `i`.`uid` != 0) LIMIT 1;
+
+	logger("Done deleting orphaned data from tables");
+}
+
+if (array_search(__file__,get_included_files())===0){
+  dbclean_run($_SERVER["argv"],$_SERVER["argc"]);
+  killme();
+}
+?>

include/dbm.php

@@ -43,6 +43,10 @@ class dbm {
 	 * @return      Whether $array is a filled array
 	 */
 	public static function is_result($array) {
+		// It could be a return value from an update statement
+		if (is_bool($array)) {
+			return $array;
+		}
 		return (is_array($array) && count($array) > 0);
 	}
 }

include/dbstructure.php

@@ -443,6 +443,7 @@ function db_definition($charset) {
 			"indexes" => array(
 					"PRIMARY" => array("k".db_index_suffix($charset)),
 					"updated" => array("updated"),
+					"expire_mode_updated" => array("expire_mode", "updated"),
 					)
 			);
 	$database["challenge"] = array(
@@ -557,6 +558,7 @@ function db_definition($charset) {
 			"indexes" => array(
 					"PRIMARY" => array("id"),
 					"uid" => array("uid"),
+					"addr_uid" => array("addr", "uid"),
 					"nurl" => array("nurl"),
 					)
 			);
@@ -585,6 +587,7 @@ function db_definition($charset) {
 					),
 			"indexes" => array(
 					"PRIMARY" => array("id"),
+					"cmd_item_contact" => array("UNIQUE", "cmd", "item", "contact"),
 					)
 			);
 	$database["event"] = array(
@@ -1122,7 +1125,9 @@ function db_definition($charset) {
 					),
 			"indexes" => array(
 					"PRIMARY" => array("id"),
-					"uid" => array("uid"),
+					"uid_contactid" => array("uid", "contact-id"),
+					"uid_profile" => array("uid", "profile"),
+					"uid_album_created" => array("uid", "album", "created"),
 					"resource-id" => array("resource-id"),
 					"guid" => array("guid"),
 					)
@@ -1357,6 +1362,7 @@ function db_definition($charset) {
 					"type_term" => array("type","term"),
 					"uid_otype_type_term_global_created" => array("uid","otype","type","term","global","created"),
 					"otype_type_term_tid" => array("otype","type","term","tid"),
+					"uid_otype_type_url" => array("uid","otype","type","url"),
 					"guid" => array("guid"),
 					)
 			);

include/dfrn.php

@@ -112,7 +112,6 @@ class dfrn {
 		$owner_nick = $owner['nickname'];
 
 		$sql_post_table = "";
-		$visibility = "";
 
 		if(! $public_feed) {
 
@@ -171,9 +170,6 @@ class dfrn {
 		else
 			$sort = 'ASC';
 
-		$date_field = "`changed`";
-		$sql_order = "`item`.`parent` ".$sort.", `item`.`created` ASC";
-
 		if(! strlen($last_update))
 			$last_update = 'now -30 days';
 
@@ -190,22 +186,19 @@ class dfrn {
 
 		$check_date = datetime_convert('UTC','UTC',$last_update,'Y-m-d H:i:s');
 
-		//	AND ( `item`.`edited` > '%s' OR `item`.`changed` > '%s' )
+		$r = q("SELECT `item`.*, `item`.`id` AS `item_id`,
-		//	dbesc($check_date),
-
-		$r = q("SELECT STRAIGHT_JOIN `item`.*, `item`.`id` AS `item_id`,
 			`contact`.`name`, `contact`.`network`, `contact`.`photo`, `contact`.`url`,
 			`contact`.`name-date`, `contact`.`uri-date`, `contact`.`avatar-date`,
 			`contact`.`thumb`, `contact`.`dfrn-id`, `contact`.`self`,
 			`sign`.`signed_text`, `sign`.`signature`, `sign`.`signer`
-			FROM `item` $sql_post_table
+			FROM `item` USE INDEX (`uid_wall_changed`, `uid_type_changed`) $sql_post_table
-			INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
+			STRAIGHT_JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
-			AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
+			AND NOT `contact`.`blocked`
 			LEFT JOIN `sign` ON `sign`.`iid` = `item`.`id`
-			WHERE `item`.`uid` = %d AND `item`.`visible` = 1 and `item`.`moderated` = 0 AND `item`.`parent` != 0
+			WHERE `item`.`uid` = %d AND `item`.`visible` AND NOT `item`.`moderated` AND `item`.`parent` != 0
-			AND ((`item`.`wall` = 1) $visibility) AND `item`.$date_field > '%s'
+			AND `item`.`wall` AND `item`.`changed` > '%s'
 			$sql_extra
-			ORDER BY $sql_order LIMIT 0, 300",
+			ORDER BY `item`.`parent` ".$sort.", `item`.`created` ASC LIMIT 0, 300",
 			intval($owner_id),
 			dbesc($check_date),
 			dbesc($sort)

include/event.php

@@ -509,7 +509,7 @@ function event_by_id($owner_uid = 0, $event_params, $sql_extra = '') {
 	// query for the event by event id
 	$r = q("SELECT `event`.*, `item`.`id` AS `itemid`,`item`.`plink`,
 			`item`.`author-name`, `item`.`author-avatar`, `item`.`author-link` FROM `event`
-		LEFT JOIN `item` ON `item`.`event-id` = `event`.`id` AND `item`.`uid` = `event`.`uid`
+		STRAIGHT_JOIN `item` ON `item`.`event-id` = `event`.`id` AND `item`.`uid` = `event`.`uid`
 		WHERE `event`.`uid` = %d AND `event`.`id` = %d $sql_extra",
 		intval($owner_uid),
 		intval($event_params["event_id"])
@@ -542,7 +542,7 @@ function events_by_date($owner_uid = 0, $event_params, $sql_extra = '') {
 	// query for the event by date
 	$r = q("SELECT `event`.*, `item`.`id` AS `itemid`,`item`.`plink`,
 				`item`.`author-name`, `item`.`author-avatar`, `item`.`author-link` FROM `event`
-			LEFT JOIN `item` ON `item`.`event-id` = `event`.`id` AND `item`.`uid` = `event`.`uid`
+			STRAIGHT_JOIN `item` ON `item`.`event-id` = `event`.`id` AND `item`.`uid` = `event`.`uid`
 			WHERE `event`.`uid` = %d AND event.ignore = %d
 			AND ((`adjust` = 0 AND (`finish` >= '%s' OR (nofinish AND start >= '%s')) AND `start` <= '%s')
 			OR  (`adjust` = 1 AND (`finish` >= '%s' OR (nofinish AND start >= '%s')) AND `start` <= '%s'))

include/identity.php

@@ -149,17 +149,23 @@ function get_profiledata_by_nick($nickname, $uid = 0, $profile = 0) {
 
 	if($profile) {
 		$profile_int = intval($profile);
-		$r = q("SELECT `profile`.`uid` AS `profile_uid`, `profile`.* , `contact`.`avatar-date` AS picdate, `contact`.`addr`, `user`.* FROM `profile`
+		$r = q("SELECT `contact`.`id` AS `contact_id`, `profile`.`uid` AS `profile_uid`, `profile`.*,
-				INNER JOIN `contact` on `contact`.`uid` = `profile`.`uid` INNER JOIN `user` ON `profile`.`uid` = `user`.`uid`
+				`contact`.`avatar-date` AS picdate, `contact`.`addr`, `user`.*
-				WHERE `user`.`nickname` = '%s' AND `profile`.`id` = %d AND `contact`.`self` = 1 LIMIT 1",
+			FROM `profile`
+			INNER JOIN `contact` on `contact`.`uid` = `profile`.`uid` AND `contact`.`self`
+			INNER JOIN `user` ON `profile`.`uid` = `user`.`uid`
+			WHERE `user`.`nickname` = '%s' AND `profile`.`id` = %d LIMIT 1",
 				dbesc($nickname),
 				intval($profile_int)
 		);
 	}
 	if((!$r) && (!count($r))) {
-		$r = q("SELECT `profile`.`uid` AS `profile_uid`, `profile`.* , `contact`.`avatar-date` AS picdate, `contact`.`addr`, `user`.* FROM `profile`
+		$r = q("SELECT `contact`.`id` AS `contact_id`, `profile`.`uid` AS `profile_uid`, `profile`.*,
-				INNER JOIN `contact` ON `contact`.`uid` = `profile`.`uid` INNER JOIN `user` ON `profile`.`uid` = `user`.`uid`
+				`contact`.`avatar-date` AS picdate, `contact`.`addr`, `user`.*
-				WHERE `user`.`nickname` = '%s' AND `profile`.`is-default` = 1 AND `contact`.`self` = 1 LIMIT 1",
+			FROM `profile`
+			INNER JOIN `contact` ON `contact`.`uid` = `profile`.`uid` AND `contact`.`self`
+			INNER JOIN `user` ON `profile`.`uid` = `user`.`uid`
+			WHERE `user`.`nickname` = '%s' AND `profile`.`is-default` LIMIT 1",
 				dbesc($nickname)
 		);
 	}

include/onepoll.php

@@ -443,7 +443,7 @@ function onepoll_run(&$argv, &$argc){
 									$refs_arr[$x] = "'" . msgid2iri(str_replace(array('<','>',' '),array('','',''),dbesc($refs_arr[$x]))) . "'";
 							}
 							$qstr = implode(',',$refs_arr);
-							$r = q("SELECT `uri` , `parent-uri` FROM `item` WHERE `uri` IN ( $qstr ) AND `uid` = %d LIMIT 1",
+							$r = q("SELECT `uri` , `parent-uri` FROM `item` USE INDEX (`uid_uri`) WHERE `uri` IN ($qstr) AND `uid` = %d LIMIT 1",
 								intval($importer_uid)
 							);
 							if(count($r))

include/ostatus.php

@@ -806,11 +806,20 @@ class ostatus {
 		}
 
 		// Get the parent
+		$parents = q("SELECT `item`.`id`, `item`.`parent`, `item`.`uri`, `item`.`contact-id`, `item`.`type`,
+				`item`.`verb`, `item`.`visible` FROM `term`
+				STRAIGHT_JOIN `item` AS `thritem` ON `thritem`.`parent` = `term`.`oid`
+				STRAIGHT_JOIN `item` ON `item`.`parent` = `thritem`.`parent`
+				WHERE `term`.`uid` = %d AND `term`.`otype` = %d AND `term`.`type` = %d AND `term`.`url` = '%s'",
+				intval($uid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION), dbesc($conversation_url));
+
+/*		2016-10-23: The old query will be kept until we are sure that the query above is a good and fast replacement
+
 		$parents = q("SELECT `id`, `parent`, `uri`, `contact-id`, `type`, `verb`, `visible` FROM `item` WHERE `id` IN
 				(SELECT `parent` FROM `item` WHERE `id` IN
 					(SELECT `oid` FROM `term` WHERE `uid` = %d AND `otype` = %d AND `type` = %d AND `url` = '%s'))",
 				intval($uid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION), dbesc($conversation_url));
-
+*/
 		if ($parents)
 			$parent = $parents[0];
 		elseif (count($item) > 0) {
@@ -1961,9 +1970,23 @@ class ostatus {
 			$last_update = 'now -30 days';
 
 		$check_date = datetime_convert('UTC','UTC',$last_update,'Y-m-d H:i:s');
+		$authorid = get_contact($owner["url"], 0);
 
-		$items = q("SELECT STRAIGHT_JOIN `item`.*, `item`.`id` AS `item_id` FROM `item`
+		$items = q("SELECT `item`.*, `item`.`id` AS `item_id` FROM `item` USE INDEX (`uid_contactid_created`)
-				INNER JOIN `thread` ON `thread`.`iid` = `item`.`parent`
+				STRAIGHT_JOIN `thread` ON `thread`.`iid` = `item`.`parent`
+				WHERE `item`.`uid` = %d AND `item`.`contact-id` = %d AND
+					`item`.`author-id` = %d AND `item`.`created` > '%s' AND
+					NOT `item`.`deleted` AND NOT `item`.`private` AND
+					`thread`.`network` IN ('%s', '%s')
+				ORDER BY `item`.`created` DESC LIMIT 300",
+				intval($owner["uid"]), intval($owner["id"]),
+				intval($authorid), dbesc($check_date),
+				dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DFRN));
+
+/*		2016-10-23: The old query will be kept until we are sure that the query above is a good and fast replacement
+
+		$items = q("SELECT `item`.*, `item`.`id` AS `item_id` FROM `item`
+				STRAIGHT_JOIN `thread` ON `thread`.`iid` = `item`.`parent`
 				LEFT JOIN `item` AS `thritem` ON `thritem`.`uri`=`item`.`thr-parent` AND `thritem`.`uid`=`item`.`uid`
 				WHERE `item`.`uid` = %d AND `item`.`received` > '%s' AND NOT `item`.`private` AND NOT `item`.`deleted`
 					AND `item`.`allow_cid` = '' AND `item`.`allow_gid` = '' AND `item`.`deny_cid`  = '' AND `item`.`deny_gid`  = ''
@@ -1981,7 +2004,7 @@ class ostatus {
 				dbesc($owner["nurl"]), dbesc(str_replace("http://", "https://", $owner["nurl"])),
 				dbesc($owner["nurl"]), dbesc(str_replace("http://", "https://", $owner["nurl"]))
 			);
-
+*/
 		$doc = new DOMDocument('1.0', 'utf-8');
 		$doc->formatOutput = true;
 

include/poller.php

@@ -91,23 +91,33 @@ function poller_run(&$argv, &$argc){
 		if (poller_too_much_workers())
 			return;
 
-		q("UPDATE `workerqueue` SET `executed` = '%s', `pid` = %d WHERE `id` = %d AND `executed` = '0000-00-00 00:00:00'",
+		$upd = q("UPDATE `workerqueue` SET `executed` = '%s', `pid` = %d WHERE `id` = %d AND `pid` = 0",
 			dbesc(datetime_convert()),
 			intval($mypid),
 			intval($r[0]["id"]));
 
+		if (!$upd) {
+			logger("Couldn't update queue entry ".$r[0]["id"]." - skip this execution", LOGGER_DEBUG);
+			q("COMMIT");
+			continue;
+		}
+
 		// Assure that there are no tasks executed twice
 		$id = q("SELECT `pid`, `executed` FROM `workerqueue` WHERE `id` = %d", intval($r[0]["id"]));
 		if (!$id) {
 			logger("Queue item ".$r[0]["id"]." vanished - skip this execution", LOGGER_DEBUG);
+			q("COMMIT");
 			continue;
 		} elseif ((strtotime($id[0]["executed"]) <= 0) OR ($id[0]["pid"] == 0)) {
-			logger("Entry for queue item ".$r[0]["id"]." wasn't stored - we better stop here", LOGGER_DEBUG);
+			logger("Entry for queue item ".$r[0]["id"]." wasn't stored - skip this execution", LOGGER_DEBUG);
-			return;
+			q("COMMIT");
+			continue;
 		} elseif ($id[0]["pid"] != $mypid) {
 			logger("Queue item ".$r[0]["id"]." is to be executed by process ".$id[0]["pid"]." and not by me (".$mypid.") - skip this execution", LOGGER_DEBUG);
+			q("COMMIT");
 			continue;
 		}
+		q("COMMIT");
 
 		$argv = json_decode($r[0]["parameter"]);
 
@@ -428,6 +438,8 @@ function poller_passing_slow(&$highest_priority) {
 
 function poller_worker_process() {
 
+	q("START TRANSACTION;");
+
 	// Check if we should pass some low priority process
 	$highest_priority = 0;
 

include/post_update.php

@@ -8,14 +8,18 @@
  */
 function post_update() {
 
-	if (!post_update_1192())
+	if (!post_update_1192()) {
 		return;
-
+	}
-	if (!post_update_1194())
+	if (!post_update_1194()) {
 		return;
-
+	}
-	if (!post_update_1198())
+	if (!post_update_1198()) {
 		return;
+	}
+	if (!post_update_1206()) {
+		return;
+	}
 }
 
 /**
@@ -174,13 +178,18 @@ function post_update_1198() {
 		}
 
 		// Update the thread table from the item table
-		q("UPDATE `thread` INNER JOIN `item` ON `item`.`id`=`thread`.`iid`
+		$r = q("UPDATE `thread` INNER JOIN `item` ON `item`.`id`=`thread`.`iid`
 				SET `thread`.`author-id` = `item`.`author-id`,
 				`thread`.`owner-id` = `item`.`owner-id`
 			WHERE `thread`.`author-id` = 0 AND `thread`.`owner-id` = 0 AND
 				(`thread`.`uid` IN (SELECT `uid` from `user`) OR `thread`.`uid` = 0)");
 
 		logger("Updated threads", LOGGER_DEBUG);
+		if (dbm::is_result($r)) {
+			set_config("system", "post_update_version", 1198);
+			logger("Done", LOGGER_DEBUG);
+			return true;
+		}
 		return false;
 	}
 
@@ -215,4 +224,39 @@ function post_update_1198() {
 	logger("Updated items", LOGGER_DEBUG);
 	return false;
 }
+
+/**
+ * @brief update the "last-item" field in the "self" contact
+ *
+ * This field avoids cost intensive calls in the admin panel and in "nodeinfo"
+ *
+ * @return bool "true" when the job is done
+ */
+function post_update_1206() {
+	// Was the script completed?
+	if (get_config("system", "post_update_version") >= 1206)
+		return true;
+
+	logger("Start", LOGGER_DEBUG);
+	$r = q("SELECT `contact`.`id`, `contact`.`last-item`,
+		(SELECT MAX(`changed`) FROM `item` FORCE INDEX (`uid_wall_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date`
+		FROM `user`
+		INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`self`");
+
+	if (!dbm::is_result($r)) {
+		return false;
+	}
+	foreach ($r AS $user) {
+		if (!empty($user["lastitem_date"]) AND ($user["lastitem_date"] > $user["last-item"])) {
+			q("UPDATE `contact` SET `last-item` = '%s' WHERE `id` = %d",
+				dbesc($user["lastitem_date"]),
+				intval($user["id"]));
+		}
+	}
+
+	set_config("system", "post_update_version", 1206);
+	logger("Done", LOGGER_DEBUG);
+	return true;
+}
+
 ?>

include/session.php

@@ -1,30 +1,44 @@
 <?php
-
 // Session management functions. These provide database storage of PHP
 // session info.
 
+require_once('include/cache.php');
+
 $session_exists = 0;
 $session_expire = 180000;
 
-if(! function_exists('ref_session_open')) {
+function ref_session_open($s, $n) {
-function ref_session_open ($s,$n) {
 	return true;
-}}
+}
 
-if(! function_exists('ref_session_read')) {
+function ref_session_read($id) {
-function ref_session_read ($id) {
 	global $session_exists;
-	if(x($id))
-		$r = q("SELECT `data` FROM `session` WHERE `sid`= '%s'", dbesc($id));
 
-	if(count($r)) {
+	if (!x($id)) {
+		return '';
+	}
+
+	$memcache = cache::memcache();
+	if (is_object($memcache)) {
+		$data = $memcache->get(get_app()->get_hostname().":session:".$id);
+		if (!is_bool($data)) {
+			return $data;
+		}
+		logger("no data for session $id", LOGGER_TRACE);
+		return '';
+	}
+
+	$r = q("SELECT `data` FROM `session` WHERE `sid`= '%s'", dbesc($id));
+
+	if (dbm::is_result($r)) {
 		$session_exists = true;
 		return $r[0]['data'];
 	} else {
 		logger("no data for session $id", LOGGER_TRACE);
 	}
+
 	return '';
-}}
+}
 
 /**
  * @brief Standard PHP session write callback
@@ -49,6 +63,12 @@ function ref_session_write($id, $data) {
 	$expire = time() + $session_expire;
 	$default_expire = time() + 300;
 
+	$memcache = cache::memcache();
+	if (is_object($memcache)) {
+		$memcache->set(get_app()->get_hostname().":session:".$id, $data, MEMCACHE_COMPRESSED, $expire);
+		return true;
+	}
+
 	if ($session_exists) {
 		$r = q("UPDATE `session`
 				SET `data` = '%s'
@@ -68,22 +88,28 @@ function ref_session_write($id, $data) {
 	return true;
 }
 
-if(! function_exists('ref_session_close')) {
 function ref_session_close() {
 	return true;
-}}
+}
+
+function ref_session_destroy($id) {
+	$memcache = cache::memcache();
+
+	if (is_object($memcache)) {
+		$memcache->delete(get_app()->get_hostname().":session:".$id);
+		return true;
+	}
 
-if(! function_exists('ref_session_destroy')) {
-function ref_session_destroy ($id) {
 	q("DELETE FROM `session` WHERE `sid` = '%s'", dbesc($id));
-	return true;
-}}
 
-if(! function_exists('ref_session_gc')) {
+	return true;
+}
+
 function ref_session_gc($expire) {
 	q("DELETE FROM `session` WHERE `expire` < %d", dbesc(time()));
+
 	return true;
-}}
+}
 
 $gc_probability = 50;
 
@@ -91,7 +117,8 @@ ini_set('session.gc_probability', $gc_probability);
 ini_set('session.use_only_cookies', 1);
 ini_set('session.cookie_httponly', 1);
 
-if (!get_config('system', 'disable_database_session'))
+if (!get_config('system', 'disable_database_session')) {
 	session_set_save_handler('ref_session_open', 'ref_session_close',
 				'ref_session_read', 'ref_session_write',
 				'ref_session_destroy', 'ref_session_gc');
+}

include/socgraph.php

@@ -1075,8 +1075,14 @@ function all_friends($uid,$cid,$start = 0, $limit = 80) {
 
 function suggestion_query($uid, $start = 0, $limit = 80) {
 
-	if(! $uid)
+	if (!$uid) {
 		return array();
+	}
+
+	$list = Cache::get("suggestion_query:".$uid.":".$start.":".$limit);
+	if (!is_null($list)) {
+		return $list;
+	}
 
 	$network = array(NETWORK_DFRN);
 
@@ -1087,9 +1093,10 @@ function suggestion_query($uid, $start = 0, $limit = 80) {
 		$network[] = NETWORK_OSTATUS;
 
 	$sql_network = implode("', '", $network);
-	//$sql_network = "'".$sql_network."', ''";
 	$sql_network = "'".$sql_network."'";
 
+	/// @todo This query is really slow
+	// By now we cache the data for five minutes
 	$r = q("SELECT count(glink.gcid) as `total`, gcontact.* from gcontact
 		INNER JOIN `glink` ON `glink`.`gcid` = `gcontact`.`id`
 		where uid = %d and not gcontact.nurl in ( select nurl from contact where uid = %d )
@@ -1108,8 +1115,10 @@ function suggestion_query($uid, $start = 0, $limit = 80) {
 		intval($limit)
 	);
 
-	if(count($r) && count($r) >= ($limit -1))
+	if (count($r) && count($r) >= ($limit -1)) {
+		Cache::set("suggestion_query:".$uid.":".$start.":".$limit, $r, CACHE_FIVE_MINUTES);
 		return $r;
+	}
 
 	$r2 = q("SELECT gcontact.* FROM gcontact
 		INNER JOIN `glink` ON `glink`.`gcid` = `gcontact`.`id`
@@ -1138,6 +1147,7 @@ function suggestion_query($uid, $start = 0, $limit = 80) {
 	while (sizeof($list) > ($limit))
 		array_pop($list);
 
+	Cache::set("suggestion_query:".$uid.":".$start.":".$limit, $list, CACHE_FIVE_MINUTES);
 	return $list;
 }
 

include/threads.php

@@ -18,42 +18,70 @@ function add_thread($itemid, $onlyshadow = false) {
 				.implode("', '", array_values($item))
 				."')");
 
-		logger("add_thread: Add thread for item ".$itemid." - ".print_r($result, true), LOGGER_DEBUG);
+		logger("Add thread for item ".$itemid." - ".print_r($result, true), LOGGER_DEBUG);
+	}
+}
+
+/**
+ * @brief Add a shadow entry for a given item id
+ *
+ * We store every public item entry additionally with the user id "0".
+ * This is used for the community page and for the search.
+ * It is planned that in the future we will store public item entries only once.
+ *
+ * @param integer $itemid Item ID that should be added
+ */
+function add_shadow_thread($itemid) {
+	$items = q("SELECT `uid`, `wall`, `private`, `moderated`, `visible`, `contact-id`, `deleted`, `network`
+		FROM `item` WHERE `id` = %d AND (`parent` = %d OR `parent` = 0) LIMIT 1", intval($itemid), intval($itemid));
+
+	if (!dbm::is_result($items)) {
+		return;
 	}
 
+	$item = $items[0];
+
 	// is it already a copy?
-	if (($itemid == 0) OR ($item['uid'] == 0))
+	if (($itemid == 0) OR ($item['uid'] == 0)) {
 		return;
+	}
 
 	// Is it a visible public post?
-	if (!$item["visible"] OR $item["deleted"] OR $item["moderated"] OR $item["private"])
+	if (!$item["visible"] OR $item["deleted"] OR $item["moderated"] OR $item["private"]) {
 		return;
+	}
 
 	// is it an entry from a connector? Only add an entry for natively connected networks
-	if (!in_array($item["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, "")))
+	if (!in_array($item["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, ""))) {
 		return;
+	}
 
 	// Only do these checks if the post isn't a wall post
 	if (!$item["wall"]) {
 		// Check, if hide-friends is activated - then don't do a shadow entry
 		$r = q("SELECT `hide-friends` FROM `profile` WHERE `is-default` AND `uid` = %d AND NOT `hide-friends`",
 			$item['uid']);
-		if (!count($r))
+
+		if (!dbm::is_result($r)) {
 			return;
+		}
+
 		// Check if the contact is hidden or blocked
 		$r = q("SELECT `id` FROM `contact` WHERE NOT `hidden` AND NOT `blocked` AND `id` = %d",
 			$item['contact-id']);
-		if (!count($r))
+
+		if (!dbm::is_result($r)) {
 			return;
+		}
 	}
 
 	// Only add a shadow, if the profile isn't hidden
 	$r = q("SELECT `uid` FROM `user` where `uid` = %d AND NOT `hidewall`", $item['uid']);
-	if (!count($r))
+	if (!dbm::is_result($r)) {
 		return;
+	}
 
-	$item = q("SELECT * FROM `item` WHERE `id` = %d",
+	$item = q("SELECT * FROM `item` WHERE `id` = %d", intval($itemid));
-		intval($itemid));
 
 	if (count($item) AND ($item[0]["allow_cid"] == '')  AND ($item[0]["allow_gid"] == '') AND
 		($item[0]["deny_cid"] == '') AND ($item[0]["deny_gid"] == '')) {
@@ -61,7 +89,7 @@ function add_thread($itemid, $onlyshadow = false) {
 		$r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND `uid` = 0 LIMIT 1",
 			dbesc($item['uri']));
 
-		if (!$r) {
+		if (!dbm::is_result($r)) {
 			// Preparing public shadow (removing user specific data)
 			require_once("include/items.php");
 			require_once("include/Contact.php");
@@ -72,7 +100,7 @@ function add_thread($itemid, $onlyshadow = false) {
 			$item[0]['contact-id'] = get_contact($item[0]['author-link'], 0);
 			$public_shadow = item_store($item[0], false, false, true);
 
-			logger("add_thread: Stored public shadow for post ".$itemid." under id ".$public_shadow, LOGGER_DEBUG);
+			logger("Stored public shadow for thread ".$itemid." under id ".$public_shadow, LOGGER_DEBUG);
 		}
 	}
 }
@@ -193,8 +221,10 @@ function update_threads() {
 
 	logger("update_threads: fetched messages: ".count($messages));
 
-	while ($message = $db->qfetch())
+	while ($message = $db->qfetch()) {
 		add_thread($message["id"]);
+		add_shadow_thread($message["id"]);
+	}
 	$db->qclose();
 }
 
@@ -227,7 +257,7 @@ function update_shadow_copy() {
 
 	logger("fetched messages: ".count($messages));
 	while ($message = $db->qfetch())
-		add_thread($message["iid"], true);
+		add_shadow_thread($message["iid"]);
 
 	$db->qclose();
 }

mod/admin.php

@@ -282,14 +282,14 @@ function admin_page_federation(&$a) {
 	foreach ($platforms as $p) {
 		// get a total count for the platform, the name and version of the
 		// highest version and the protocol tpe
-		$c = q('SELECT COUNT(*) AS `total`, `platform`, `network`, `version` FROM `gserver`
+		$c = qu('SELECT COUNT(*) AS `total`, `platform`, `network`, `version` FROM `gserver`
 				WHERE `platform` LIKE "%s" AND `last_contact` > `last_failure` AND `version` != ""
 				ORDER BY `version` ASC;', $p);
 		$total = $total + $c[0]['total'];
 
 		// what versions for that platform do we know at all?
 		// again only the active nodes
-		$v = q('SELECT COUNT(*) AS `total`, `version` FROM `gserver`
+		$v = qu('SELECT COUNT(*) AS `total`, `version` FROM `gserver`
 				WHERE `last_contact` > `last_failure` AND `platform` LIKE "%s"  AND `version` != ""
 				GROUP BY `version`
 				ORDER BY `version`;', $p);
@@ -434,17 +434,17 @@ function admin_page_summary(&$a) {
 
 	logger('accounts: '.print_r($accounts,true),LOGGER_DATA);
 
-	$r = q("SELECT COUNT(`id`) AS `count` FROM `register`");
+	$r = qu("SELECT COUNT(`id`) AS `count` FROM `register`");
 	$pending = $r[0]['count'];
 
-	$r = q("SELECT COUNT(*) AS `total` FROM `deliverq` WHERE 1");
+	$r = qu("SELECT COUNT(*) AS `total` FROM `deliverq` WHERE 1");
 	$deliverq = (($r) ? $r[0]['total'] : 0);
 
-	$r = q("SELECT COUNT(*) AS `total` FROM `queue` WHERE 1");
+	$r = qu("SELECT COUNT(*) AS `total` FROM `queue` WHERE 1");
 	$queue = (($r) ? $r[0]['total'] : 0);
 
 	if (get_config('system','worker')) {
-		$r = q("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE 1");
+		$r = qu("SELECT COUNT(*) AS `total` FROM `workerqueue` WHERE 1");
 		$workerqueue = (($r) ? $r[0]['total'] : 0);
 	} else {
 		$workerqueue = 0;
@@ -1271,7 +1271,7 @@ function admin_page_users(&$a){
 
 
 	/* get users */
-	$total = q("SELECT COUNT(*) AS `total` FROM `user` WHERE 1");
+	$total = qu("SELECT COUNT(*) AS `total` FROM `user` WHERE 1");
 	if(count($total)) {
 		$a->set_pager_total($total[0]['total']);
 		$a->set_pager_itemspage(100);
@@ -1306,8 +1306,7 @@ function admin_page_users(&$a){
 	$sql_order = "`".str_replace('.','`.`',$order)."`";
 	$sql_order_direction = ($order_direction==="+")?"ASC":"DESC";
 
-	$users = q("SELECT `user`.*, `contact`.`name`, `contact`.`url`, `contact`.`micro`, `user`.`account_expired`,
+	$users = qu("SELECT `user`.*, `contact`.`name`, `contact`.`url`, `contact`.`micro`, `user`.`account_expired`, `contact`.`last-item` AS `lastitem_date`
-				(SELECT MAX(`changed`) FROM `item` FORCE INDEX (`uid_wall_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date`
 				FROM `user`
 				INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`self`
 				WHERE `user`.`verified`

mod/community.php

@@ -49,7 +49,7 @@ function community_content(&$a, $update = 0) {
 	// OR your own posts if you are a logged in member
 
 	if(get_config('system', 'old_pager')) {
-		$r = q("SELECT COUNT(distinct(`item`.`uri`)) AS `total`
+		$r = qu("SELECT COUNT(distinct(`item`.`uri`)) AS `total`
 			FROM `item` INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
 			AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
 			INNER JOIN `user` ON `user`.`uid` = `item`.`uid` AND `user`.`hidewall` = 0
@@ -120,7 +120,7 @@ function community_getitems($start, $itemspage) {
 	if (get_config('system','community_page_style') == CP_GLOBAL_COMMUNITY)
 		return(community_getpublicitems($start, $itemspage));
 
-	$r = q("SELECT %s
+	$r = qu("SELECT %s
 		FROM `thread` FORCE INDEX (`wall_private_received`)
 		INNER JOIN `user` ON `user`.`uid` = `thread`.`uid` AND NOT `user`.`hidewall`
 		INNER JOIN `item` ON `item`.`id` = `thread`.`iid`
@@ -140,7 +140,7 @@ function community_getitems($start, $itemspage) {
 
 function community_getpublicitems($start, $itemspage) {
 
-	$r = q("SELECT %s
+	$r = qu("SELECT %s
 		FROM `thread`
 		INNER JOIN `item` ON `item`.`id` = `thread`.`iid` %s
 		WHERE `thread`.`uid` = 0

mod/display.php

@@ -2,7 +2,7 @@
 
 function display_init(&$a) {
 
-	if((get_config('system','block_public')) && (! local_user()) && (! remote_user())) {
+	if ((get_config('system','block_public')) && (! local_user()) && (! remote_user())) {
 		return;
 	}
 
@@ -16,10 +16,10 @@ function display_init(&$a) {
 
 		// Does the local user have this item?
 		if (local_user()) {
-			$r = q("SELECT `id`, `parent`, `author-name`, `author-link`, `author-avatar`, `network`, `body`, `uid`, `owner-link` FROM `item`
+			$r = qu("SELECT `id`, `parent`, `author-name`, `author-link`, `author-avatar`, `network`, `body`, `uid`, `owner-link` FROM `item`
 				WHERE `item`.`visible` AND NOT `item`.`deleted` AND NOT `item`.`moderated`
 					AND `guid` = '%s' AND `uid` = %d", dbesc($a->argv[1]), local_user());
-			if (count($r)) {
+			if (dbm::is_result($r)) {
 				$nick = $a->user["nickname"];
 				$itemuid = local_user();
 			}
@@ -27,16 +27,15 @@ function display_init(&$a) {
 
 		// Or is it anywhere on the server?
 		if ($nick == "") {
-			$r = q("SELECT `user`.`nickname`, `item`.`id`, `item`.`parent`, `item`.`author-name`,
+			$r = qu("SELECT `user`.`nickname`, `item`.`id`, `item`.`parent`, `item`.`author-name`,
 				`item`.`author-link`, `item`.`author-avatar`, `item`.`network`, `item`.`uid`, `item`.`owner-link`, `item`.`body`
-				FROM `item` INNER JOIN `user` ON `user`.`uid` = `item`.`uid`
+				FROM `item` STRAIGHT_JOIN `user` ON `user`.`uid` = `item`.`uid`
 				WHERE `item`.`visible` AND NOT `item`.`deleted` AND NOT `item`.`moderated`
 					AND `item`.`allow_cid` = ''  AND `item`.`allow_gid` = ''
 					AND `item`.`deny_cid`  = '' AND `item`.`deny_gid`  = ''
 					AND NOT `item`.`private` AND NOT `user`.`hidewall`
 					AND `item`.`guid` = '%s'", dbesc($a->argv[1]));
-				//	AND NOT `item`.`private` AND `item`.`wall`
+			if (dbm::is_result($r)) {
-			if (count($r)) {
 				$nick = $r[0]["nickname"];
 				$itemuid = $r[0]["uid"];
 			}
@@ -44,33 +43,32 @@ function display_init(&$a) {
 
 		// Is it an item with uid=0?
 		if ($nick == "") {
-			$r = q("SELECT `item`.`id`, `item`.`parent`, `item`.`author-name`, `item`.`author-link`,
+			$r = qu("SELECT `item`.`id`, `item`.`parent`, `item`.`author-name`, `item`.`author-link`,
 				`item`.`author-avatar`, `item`.`network`, `item`.`uid`, `item`.`owner-link`, `item`.`body`
 				FROM `item` WHERE `item`.`visible` AND NOT `item`.`deleted` AND NOT `item`.`moderated`
 					AND `item`.`allow_cid` = ''  AND `item`.`allow_gid` = ''
 					AND `item`.`deny_cid`  = '' AND `item`.`deny_gid`  = ''
 					AND NOT `item`.`private` AND `item`.`uid` = 0
 					AND `item`.`guid` = '%s'", dbesc($a->argv[1]));
-				//	AND NOT `item`.`private` AND `item`.`wall`
 		}
-		if (count($r)) {
+		if (dbm::is_result($r)) {
-			if ($r[0]["id"] != $r[0]["parent"])
+			if ($r[0]["id"] != $r[0]["parent"]) {
-				$r = q("SELECT `id`, `author-name`, `author-link`, `author-avatar`, `network`, `body`, `uid`, `owner-link` FROM `item`
+				$r = qu("SELECT `id`, `author-name`, `author-link`, `author-avatar`, `network`, `body`, `uid`, `owner-link` FROM `item`
 					WHERE `item`.`visible` AND NOT `item`.`deleted` AND NOT `item`.`moderated`
 						AND `id` = %d", $r[0]["parent"]);
-
+			}
 			if (($itemuid != local_user()) AND local_user()) {
 				// Do we know this contact but we haven't got this item?
 				// Copy the wohle thread to our local storage so that we can interact.
 				// We really should change this need for the future since it scales very bad.
 				$contactid = get_contact($r[0]['owner-link'], local_user());
 				if ($contactid) {
-					$items = q("SELECT * FROM `item` WHERE `parent` = %d ORDER BY `id`", intval($r[0]["id"]));
+					$items = qu("SELECT * FROM `item` WHERE `parent` = %d ORDER BY `id`", intval($r[0]["id"]));
 					foreach ($items AS $item) {
 						$itemcontactid = get_contact($item['owner-link'], local_user());
-						if (!$itemcontactid)
+						if (!$itemcontactid) {
 							$itemcontactid = $contactid;
-
+						}
 						unset($item['id']);
 						$item['uid'] = local_user();
 						$item['origin'] = 0;
@@ -87,21 +85,22 @@ function display_init(&$a) {
 				$nickname = str_replace(normalise_link($a->get_baseurl())."/profile/", "", normalise_link($profiledata["url"]));
 
 				if (($nickname != $a->user["nickname"])) {
-					$r = q("SELECT `profile`.`uid` AS `profile_uid`, `profile`.* , `contact`.`avatar-date` AS picdate, `user`.* FROM `profile`
+					$r = qu("SELECT `profile`.`uid` AS `profile_uid`, `profile`.* , `contact`.`avatar-date` AS picdate, `user`.* FROM `profile`
 						INNER JOIN `contact` on `contact`.`uid` = `profile`.`uid` INNER JOIN `user` ON `profile`.`uid` = `user`.`uid`
 						WHERE `user`.`nickname` = '%s' AND `profile`.`is-default` AND `contact`.`self` LIMIT 1",
 						dbesc($nickname)
 					);
-					if (count($r))
+					if (dbm::is_result($r)) {
 						$profiledata = $r[0];
-
+					}
 					$profiledata["network"] = NETWORK_DFRN;
-				} else
+				} else {
 					$profiledata = array();
+				}
 			}
 		} else {
 			$a->error = 404;
-			notice( t('Item not found.') . EOL);
+			notice(t('Item not found.') . EOL);
 			return;
 		}
 	}
@@ -129,48 +128,49 @@ function display_fetchauthor($a, $item) {
 
 	// Skip if it isn't a pure repeated messages
 	// Does it start with a share?
-	if (!$skip AND strpos($body, "[share") > 0)
+	if (!$skip AND strpos($body, "[share") > 0) {
 		$skip = true;
-
+	}
 	// Does it end with a share?
-	if (!$skip AND (strlen($body) > (strrpos($body, "[/share]") + 8)))
+	if (!$skip AND (strlen($body) > (strrpos($body, "[/share]") + 8))) {
 		$skip = true;
-
+	}
 	if (!$skip) {
 		$attributes = preg_replace("/\[share(.*?)\]\s?(.*?)\s?\[\/share\]\s?/ism","$1",$body);
 		// Skip if there is no shared message in there
-		if ($body == $attributes)
+		if ($body == $attributes) {
 			$skip = true;
+		}
 	}
 
 	if (!$skip) {
 		$author = "";
 		preg_match("/author='(.*?)'/ism", $attributes, $matches);
-		if ($matches[1] != "")
+		if ($matches[1] != "") {
 			$profiledata["name"] = html_entity_decode($matches[1],ENT_QUOTES,'UTF-8');
-
+		}
 		preg_match('/author="(.*?)"/ism', $attributes, $matches);
-		if ($matches[1] != "")
+		if ($matches[1] != "") {
 			$profiledata["name"] = html_entity_decode($matches[1],ENT_QUOTES,'UTF-8');
-
+		}
 		$profile = "";
 		preg_match("/profile='(.*?)'/ism", $attributes, $matches);
-		if ($matches[1] != "")
+		if ($matches[1] != "") {
 			$profiledata["url"] = $matches[1];
-
+		}
 		preg_match('/profile="(.*?)"/ism', $attributes, $matches);
-		if ($matches[1] != "")
+		if ($matches[1] != "") {
 			$profiledata["url"] = $matches[1];
-
+		}
 		$avatar = "";
 		preg_match("/avatar='(.*?)'/ism", $attributes, $matches);
-		if ($matches[1] != "")
+		if ($matches[1] != "") {
 			$profiledata["photo"] = $matches[1];
-
+		}
 		preg_match('/avatar="(.*?)"/ism', $attributes, $matches);
-		if ($matches[1] != "")
+		if ($matches[1] != "") {
 			$profiledata["photo"] = $matches[1];
-
+		}
 		$profiledata["nickname"] = $profiledata["name"];
 		$profiledata["network"] = GetProfileUsername($profiledata["url"], "", false, true);
 
@@ -183,8 +183,9 @@ function display_fetchauthor($a, $item) {
 	$profiledata["photo"] = App::remove_baseurl($profiledata["photo"]);
 
 	if (local_user()) {
-		if (in_array($profiledata["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS)))
+		if (in_array($profiledata["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS))) {
 			$profiledata["remoteconnect"] = $a->get_baseurl()."/follow?url=".urlencode($profiledata["url"]);
+		}
 	} elseif ($profiledata["network"] == NETWORK_DFRN) {
 		$connect = str_replace("/profile/", "/dfrn_request/", $profiledata["url"]);
 		$profiledata["remoteconnect"] = $connect;
@@ -195,8 +196,8 @@ function display_fetchauthor($a, $item) {
 
 function display_content(&$a, $update = 0) {
 
-	if((get_config('system','block_public')) && (! local_user()) && (! remote_user())) {
+	if ((get_config('system','block_public')) && (! local_user()) && (! remote_user())) {
-		notice( t('Public access denied.') . EOL);
+		notice(t('Public access denied.') . EOL);
 		return;
 	}
 
@@ -210,55 +211,53 @@ function display_content(&$a, $update = 0) {
 	$a->page['htmlhead'] .= replace_macros(get_markup_template('display-head.tpl'), array());
 
 
-	if($update) {
+	if ($update) {
 		$nick = $_REQUEST['nick'];
-	}
+	} else {
-	else {
 		$nick = (($a->argc > 1) ? $a->argv[1] : '');
 	}
 
-	if($update) {
+	if ($update) {
 		$item_id = $_REQUEST['item_id'];
 		$a->profile = array('uid' => intval($update), 'profile_uid' => intval($update));
-	}
+	} else {
-	else {
 		$item_id = (($a->argc > 2) ? $a->argv[2] : 0);
 
 		if ($a->argc == 2) {
 			$nick = "";
 
 			if (local_user()) {
-				$r = q("SELECT `id` FROM `item`
+				$r = qu("SELECT `id` FROM `item`
 					WHERE `item`.`visible` AND NOT `item`.`deleted` AND NOT `item`.`moderated`
 						AND `guid` = '%s' AND `uid` = %d", dbesc($a->argv[1]), local_user());
-				if (count($r)) {
+				if (dbm::is_result($r)) {
 					$item_id = $r[0]["id"];
 					$nick = $a->user["nickname"];
 				}
 			}
 
 			if ($nick == "") {
-				$r = q("SELECT `user`.`nickname`, `item`.`id` FROM `item` INNER JOIN `user` ON `user`.`uid` = `item`.`uid`
+				$r = qu("SELECT `user`.`nickname`, `item`.`id` FROM `item` STRAIGHT_JOIN `user` ON `user`.`uid` = `item`.`uid`
 					WHERE `item`.`visible` AND NOT `item`.`deleted` AND NOT `item`.`moderated`
 						AND `item`.`allow_cid` = ''  AND `item`.`allow_gid` = ''
 						AND `item`.`deny_cid`  = '' AND `item`.`deny_gid`  = ''
 						AND NOT `item`.`private` AND NOT `user`.`hidewall`
 						AND `item`.`guid` = '%s'", dbesc($a->argv[1]));
 					//	AND NOT `item`.`private` AND `item`.`wall`
-				if (count($r)) {
+				if (dbm::is_result($r)) {
 					$item_id = $r[0]["id"];
 					$nick = $r[0]["nickname"];
 				}
 			}
 			if ($nick == "") {
-				$r = q("SELECT `item`.`id` FROM `item`
+				$r = qu("SELECT `item`.`id` FROM `item`
 					WHERE `item`.`visible` AND NOT `item`.`deleted` AND NOT `item`.`moderated`
 						AND `item`.`allow_cid` = ''  AND `item`.`allow_gid` = ''
 						AND `item`.`deny_cid`  = '' AND `item`.`deny_gid`  = ''
 						AND NOT `item`.`private` AND `item`.`uid` = 0
 						AND `item`.`guid` = '%s'", dbesc($a->argv[1]));
 					//	AND NOT `item`.`private` AND `item`.`wall`
-				if (count($r)) {
+				if (dbm::is_result($r)) {
 					$item_id = $r[0]["id"];
 				}
 			}
@@ -266,12 +265,13 @@ function display_content(&$a, $update = 0) {
 	}
 
 	if ($item_id AND !is_numeric($item_id)) {
-		$r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
+		$r = qu("SELECT `id` FROM `item` WHERE `uri` = '%s' AND `uid` = %d LIMIT 1",
 			dbesc($item_id), intval($a->profile['uid']));
-		if ($r)
+		if (dbm::is_result($r)) {
 			$item_id = $r[0]["id"];
-		else
+		} else {
 			$item_id = false;
+		}
 	}
 
 	if (!$item_id) {
@@ -288,44 +288,44 @@ function display_content(&$a, $update = 0) {
 
 	$contact_id = 0;
 
-	if(is_array($_SESSION['remote'])) {
+	if (is_array($_SESSION['remote'])) {
-		foreach($_SESSION['remote'] as $v) {
+		foreach ($_SESSION['remote'] as $v) {
-			if($v['uid'] == $a->profile['uid']) {
+			if ($v['uid'] == $a->profile['uid']) {
 				$contact_id = $v['cid'];
 				break;
 			}
 		}
 	}
 
-	if($contact_id) {
+	if ($contact_id) {
 		$groups = init_groups_visitor($contact_id);
-		$r = q("SELECT * FROM `contact` WHERE `id` = %d AND `uid` = %d LIMIT 1",
+		$r = qu("SELECT * FROM `contact` WHERE `id` = %d AND `uid` = %d LIMIT 1",
 			intval($contact_id),
 			intval($a->profile['uid'])
 		);
-		if(count($r)) {
+		if (dbm::is_result($r)) {
 			$contact = $r[0];
 			$remote_contact = true;
 		}
 	}
 
-	if(! $remote_contact) {
+	if (!$remote_contact) {
-		if(local_user()) {
+		if (local_user()) {
 			$contact_id = $_SESSION['cid'];
 			$contact = $a->contact;
 		}
 	}
 
-	$r = q("SELECT * FROM `contact` WHERE `uid` = %d AND `self` LIMIT 1",
+	$r = qu("SELECT * FROM `contact` WHERE `uid` = %d AND `self` LIMIT 1",
 		intval($a->profile['uid'])
 	);
-	if(count($r))
+	if (dbm::is_result($r)) {
 		$a->page_contact = $r[0];
-
+	}
 	$is_owner = ((local_user()) && (local_user() == $a->profile['profile_uid']) ? true : false);
 
-	if($a->profile['hidewall'] && (! $is_owner) && (! $remote_contact)) {
+	if ($a->profile['hidewall'] && (! $is_owner) && (! $remote_contact)) {
-		notice( t('Access to this profile has been restricted.') . EOL);
+		notice(t('Access to this profile has been restricted.') . EOL);
 		return;
 	}
 
@@ -349,20 +349,21 @@ function display_content(&$a, $update = 0) {
 
 	$sql_extra = item_permissions_sql($a->profile['uid'],$remote_contact,$groups);
 
-	if($update) {
+	if ($update) {
 
-		$r = q("SELECT `id` FROM `item` WHERE `item`.`uid` = %d
+		$r = qu("SELECT `id` FROM `item` WHERE `item`.`uid` = %d
 			AND `item`.`parent` = (SELECT `parent` FROM `item` WHERE `id` = %d)
 			$sql_extra AND `unseen`",
 			intval($a->profile['uid']),
 			intval($item_id)
 		);
 
-		if(!$r)
+		if (!$r) {
 			return '';
+		}
 	}
 
-	$r = q(item_query()." AND `item`.`uid` = %d
+	$r = qu(item_query()." AND `item`.`uid` = %d
 		AND `item`.`parent` = (SELECT `parent` FROM `item` WHERE `id` = %d)
 		$sql_extra
 		ORDER BY `parent` DESC, `gravity` ASC, `id` ASC",
@@ -371,18 +372,18 @@ function display_content(&$a, $update = 0) {
 	);
 
 
-	if(!$r && local_user()) {
+	if (!$r && local_user()) {
 		// Check if this is another person's link to a post that we have
-		$r = q("SELECT `item`.uri FROM `item`
+		$r = qu("SELECT `item`.uri FROM `item`
 			WHERE (`item`.`id` = %d OR `item`.`uri` = '%s')
 			LIMIT 1",
 			intval($item_id),
 			dbesc($item_id)
 		);
-		if($r) {
+		if (dbm::is_result($r)) {
 			$item_uri = $r[0]['uri'];
 
-			$r = q(item_query()." AND `item`.`uid` = %d
+			$r = qu(item_query()." AND `item`.`uid` = %d
 				AND `item`.`parent` = (SELECT `parent` FROM `item` WHERE `uri` = '%s' AND uid = %d)
 				ORDER BY `parent` DESC, `gravity` ASC, `id` ASC ",
 				intval(local_user()),
@@ -392,23 +393,24 @@ function display_content(&$a, $update = 0) {
 		}
 	}
 
-	if($r) {
+	if ($r) {
 
-		if((local_user()) && (local_user() == $a->profile['uid'])) {
+		if ((local_user()) && (local_user() == $a->profile['uid'])) {
 			$unseen = q("SELECT `id` FROM `item` WHERE `unseen` AND `parent` = %d",
 					intval($r[0]['parent']));
 
-			if ($unseen)
+			if ($unseen) {
-				q("UPDATE `item` SET `unseen` = 0
+				q("UPDATE `item` SET `unseen` = 0 WHERE `parent` = %d AND `unseen`",
-					WHERE `parent` = %d AND `unseen`",
 					intval($r[0]['parent'])
 				);
+			}
 		}
 
 		$items = conv_sort($r,"`commented`");
 
-		if(!$update)
+		if (!$update) {
 			$o .= "<script> var netargs = '?f=&nick=" . $nick . "&item_id=" . $item_id . "'; </script>";
+		}
 		$o .= conversation($a,$items,'display', $update);
 
 		// Preparing the meta header
@@ -420,9 +422,9 @@ function display_content(&$a, $update = 0) {
 
 		$image = $a->remove_baseurl($r[0]["thumb"]);
 
-		if ($title == "")
+		if ($title == "") {
 			$title = $author_name;
-
+		}
 		$description = htmlspecialchars($description, ENT_COMPAT, 'UTF-8', true); // allow double encoding here
 		$title = htmlspecialchars($title, ENT_COMPAT, 'UTF-8', true); // allow double encoding here
 		$author_name = htmlspecialchars($author_name, ENT_COMPAT, 'UTF-8', true); // allow double encoding here
@@ -462,20 +464,18 @@ function display_content(&$a, $update = 0) {
 		return $o;
 	}
 
-	$r = q("SELECT `id`,`deleted` FROM `item` WHERE `id` = '%s' OR `uri` = '%s' LIMIT 1",
+	$r = qu("SELECT `id`,`deleted` FROM `item` WHERE `id` = '%s' OR `uri` = '%s' LIMIT 1",
 		dbesc($item_id),
 		dbesc($item_id)
 	);
-	if($r) {
+	if ($r) {
-		if($r[0]['deleted']) {
+		if ($r[0]['deleted']) {
-			notice( t('Item has been removed.') . EOL );
+			notice(t('Item has been removed.') . EOL );
+		} else {
+			notice(t('Permission denied.') . EOL );
 		}
-		else {
+	} else {
-			notice( t('Permission denied.') . EOL );
+		notice(t('Item not found.') . EOL );
-		}
-	}
-	else {
-		notice( t('Item not found.') . EOL );
 	}
 
 	return $o;

mod/item.php

@@ -801,6 +801,9 @@ function item_post(&$a) {
 	} else
 		$post_id = 0;
 
+	q("COMMIT");
+	q("START TRANSACTION;");
+
 	$r = q("INSERT INTO `item` (`guid`, `extid`, `uid`,`type`,`wall`,`gravity`, `network`, `contact-id`,
 					`owner-name`,`owner-link`,`owner-avatar`, `owner-id`,
 					`author-name`, `author-link`, `author-avatar`, `author-id`,
@@ -877,6 +880,7 @@ function item_post(&$a) {
 	$r = q("SELECT `id` FROM `item` WHERE `uri` = '%s' LIMIT 1",
 		dbesc($datarray['uri']));
 	if(!count($r)) {
+		q("COMMIT");
 		logger('mod_item: unable to retrieve post that was just stored.');
 		notice( t('System error. Post not saved.') . EOL);
 		goaway($a->get_baseurl() . "/" . $return_path );
@@ -888,6 +892,8 @@ function item_post(&$a) {
 
 	$datarray["id"] = $post_id;
 
+	item_set_last_item($datarray);
+
 	// update filetags in pconfig
 	file_tag_update_pconfig($uid,$categories_old,$categories_new,'category');
 
@@ -997,10 +1003,14 @@ function item_post(&$a) {
 	create_tags_from_item($post_id);
 	create_files_from_item($post_id);
 
-	if ($post_id == $parent)
+	if ($post_id == $parent) {
 		add_thread($post_id);
-	else {
+		q("COMMIT");
+
+		add_shadow_thread($post_id);
+	} else {
 		update_thread($parent, true);
+		q("COMMIT");
 
 		// Insert an item entry for UID=0 for global entries
 		// We have to remove or change some data before that,

mod/network.php

@@ -122,7 +122,7 @@ function network_init(&$a) {
 	$search = ((x($_GET,'search')) ? escape_tags($_GET['search']) : '');
 
 	if(x($_GET,'save')) {
-		$r = q("SELECT * FROM `search` WHERE `uid` = %d AND `term` = '%s' LIMIT 1",
+		$r = qu("SELECT * FROM `search` WHERE `uid` = %d AND `term` = '%s' LIMIT 1",
 			intval(local_user()),
 			dbesc($search)
 		);
@@ -176,7 +176,7 @@ function saved_searches($search) {
 
 	$o = '';
 
-	$r = q("SELECT `id`,`term` FROM `search` WHERE `uid` = %d",
+	$r = qu("SELECT `id`,`term` FROM `search` WHERE `uid` = %d",
 		intval(local_user())
 	);
 
@@ -375,7 +375,7 @@ function network_content(&$a, $update = 0) {
 		$def_acl = array('allow_cid' => '<' . intval($cid) . '>');
 
 	if($nets) {
-		$r = q("SELECT `id` FROM `contact` WHERE `uid` = %d AND network = '%s' AND `self` = 0",
+		$r = qu("SELECT `id` FROM `contact` WHERE `uid` = %d AND network = '%s' AND `self` = 0",
 			intval(local_user()),
 			dbesc($nets)
 		);
@@ -408,7 +408,7 @@ function network_content(&$a, $update = 0) {
 
 		if ($cid) {
 			// If $cid belongs to a communitity forum or a privat goup,.add a mention to the status editor
-			$contact = q("SELECT `nick` FROM `contact` WHERE `id` = %d AND `uid` = %d AND (`forum` OR `prv`) ",
+			$contact = qu("SELECT `nick` FROM `contact` WHERE `id` = %d AND `uid` = %d AND (`forum` OR `prv`) ",
 				intval($cid),
 				intval(local_user())
 			);
@@ -458,7 +458,7 @@ function network_content(&$a, $update = 0) {
 	$sql_nets = (($nets) ? sprintf(" and $sql_table.`network` = '%s' ", dbesc($nets)) : '');
 
 	if($group) {
-		$r = q("SELECT `name`, `id` FROM `group` WHERE `id` = %d AND `uid` = %d LIMIT 1",
+		$r = qu("SELECT `name`, `id` FROM `group` WHERE `id` = %d AND `uid` = %d LIMIT 1",
 			intval($group),
 			intval($_SESSION['uid'])
 		);
@@ -479,7 +479,7 @@ function network_content(&$a, $update = 0) {
 
 			$contact_str = implode(',',$contacts);
 			$gcontact_str = implode(',',$gcontacts);
-			$self = q("SELECT `contact`.`id`, `gcontact`.`id` AS `gid` FROM `contact`
+			$self = qu("SELECT `contact`.`id`, `gcontact`.`id` AS `gid` FROM `contact`
 					INNER JOIN `gcontact` ON `gcontact`.`nurl` = `contact`.`nurl`
 					WHERE `uid` = %d AND `self`", intval($_SESSION['uid']));
 			if (count($self)) {
@@ -502,7 +502,7 @@ function network_content(&$a, $update = 0) {
 	}
 	elseif($cid) {
 
-		$r = q("SELECT `id`,`name`,`network`,`writable`,`nurl`, `forum`, `prv`, `contact-type`, `addr`, `thumb`, `location` FROM `contact` WHERE `id` = %d
+		$r = qu("SELECT `id`,`name`,`network`,`writable`,`nurl`, `forum`, `prv`, `contact-type`, `addr`, `thumb`, `location` FROM `contact` WHERE `id` = %d
 				AND `blocked` = 0 AND `pending` = 0 LIMIT 1",
 			intval($cid)
 		);
@@ -600,7 +600,7 @@ function network_content(&$a, $update = 0) {
 	}
 	else {
 		if(get_config('system', 'old_pager')) {
-			$r = q("SELECT COUNT(*) AS `total`
+			$r = qu("SELECT COUNT(*) AS `total`
 				FROM $sql_table $sql_post_table INNER JOIN `contact` ON `contact`.`id` = $sql_table.`contact-id`
 				AND NOT `contact`.`blocked` AND NOT `contact`.`pending`
 				WHERE $sql_table.`uid` = %d AND $sql_table.`visible` AND NOT $sql_table.`deleted`
@@ -640,7 +640,7 @@ function network_content(&$a, $update = 0) {
 			$sql_order = "`item`.`received`";
 
 		// "New Item View" - show all items unthreaded in reverse created date order
-		$items = q("SELECT %s FROM $sql_table $sql_post_table %s
+		$items = qu("SELECT %s FROM $sql_table $sql_post_table %s
 			WHERE %s AND `item`.`uid` = %d
 			$simple_update
 			$sql_extra $sql_nets
@@ -678,7 +678,7 @@ function network_content(&$a, $update = 0) {
 			else
 				$sql_extra4 = "";
 
-			$r = q("SELECT `item`.`parent` AS `item_id`, `item`.`network` AS `item_network`, `contact`.`uid` AS `contact_uid`
+			$r = qu("SELECT `item`.`parent` AS `item_id`, `item`.`network` AS `item_network`, `contact`.`uid` AS `contact_uid`
 				FROM $sql_table $sql_post_table INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
 				AND NOT `contact`.`blocked` AND NOT `contact`.`pending`
 				WHERE `item`.`uid` = %d AND `item`.`visible` AND NOT `item`.`deleted` $sql_extra4
@@ -688,7 +688,7 @@ function network_content(&$a, $update = 0) {
 				intval(local_user())
 			);
 		} else {
-			$r = q("SELECT `thread`.`iid` AS `item_id`, `thread`.`network` AS `item_network`, `contact`.`uid` AS `contact_uid`
+			$r = qu("SELECT `thread`.`iid` AS `item_id`, `thread`.`network` AS `item_network`, `contact`.`uid` AS `contact_uid`
 				FROM $sql_table $sql_post_table STRAIGHT_JOIN `contact` ON `contact`.`id` = `thread`.`contact-id`
 				AND NOT `contact`.`blocked` AND NOT `contact`.`pending`
 				WHERE `thread`.`uid` = %d AND `thread`.`visible` AND NOT `thread`.`deleted`
@@ -722,7 +722,7 @@ function network_content(&$a, $update = 0) {
 			$items = array();
 
 			foreach ($parents_arr AS $parents) {
-				$thread_items = q(item_query()." AND `item`.`uid` = %d
+				$thread_items = qu(item_query()." AND `item`.`uid` = %d
 					AND `item`.`parent` = %d
 					ORDER BY `item`.`commented` DESC LIMIT %d",
 					intval(local_user()),

mod/nodeinfo.php

@@ -185,10 +185,10 @@ function nodeinfo_cron() {
 	}
         logger("cron_start");
 
-	$users = q("SELECT `user`.`uid`, `user`.`login_date`,
+	$users = qu("SELECT `user`.`uid`, `user`.`login_date`, `contact`.`last-item`
-			(SELECT MAX(`changed`) FROM `item` FORCE INDEX (`uid_wall_changed`) WHERE `wall` AND `uid` = `user`.`uid`) AS `lastitem_date`
 			FROM `user`
 			INNER JOIN `profile` ON `profile`.`uid` = `user`.`uid` AND `profile`.`is-default`
+			INNER JOIN `contact` ON `contact`.`uid` = `user`.`uid` AND `contact`.`self`
 			WHERE (`profile`.`publish` OR `profile`.`net-publish`) AND `user`.`verified`
 				AND NOT `user`.`blocked` AND NOT `user`.`account_removed`
 				AND NOT `user`.`account_expired`");
@@ -202,11 +202,11 @@ function nodeinfo_cron() {
 
 			foreach ($users AS $user) {
 				if ((strtotime($user['login_date']) > $halfyear) OR
-					(strtotime($user['lastitem_date']) > $halfyear))
+					(strtotime($user['last-item']) > $halfyear))
 					++$active_users_halfyear;
 
 				if ((strtotime($user['login_date']) > $month) OR
-					(strtotime($user['lastitem_date']) > $month))
+					(strtotime($user['last-item']) > $month))
 					++$active_users_monthly;
 
 			}
@@ -217,8 +217,7 @@ function nodeinfo_cron() {
 			set_config('nodeinfo','active_users_monthly', $active_users_monthly);
 	}
 
-	//$posts = q("SELECT COUNT(*) AS local_posts FROM `item` WHERE `wall` AND `uid` != 0 AND `id` = `parent` AND left(body, 6) != '[share'");
+	$posts = qu("SELECT COUNT(*) AS `local_posts` FROM `item`
-	$posts = q("SELECT COUNT(*) AS `local_posts` FROM `item`
 			INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
 			WHERE `contact`.`self` and `item`.`id` = `item`.`parent` AND left(body, 6) != '[share' AND `item`.`network` IN ('%s', '%s', '%s')",
 			dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_DFRN));
@@ -232,7 +231,7 @@ function nodeinfo_cron() {
 
         logger("local_posts: ".$local_posts, LOGGER_DEBUG);
 
-	$posts = q("SELECT COUNT(*) AS `local_comments` FROM `item`
+	$posts = qu("SELECT COUNT(*) AS `local_comments` FROM `item`
 			INNER JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
 			WHERE `contact`.`self` and `item`.`id` != `item`.`parent` AND `item`.`network` IN ('%s', '%s', '%s')",
 			dbesc(NETWORK_OSTATUS), dbesc(NETWORK_DIASPORA), dbesc(NETWORK_DFRN));

mod/photo.php

@@ -72,7 +72,7 @@ function photo_init(&$a) {
 
 		$uid = str_replace(array('.jpg','.png'),array('',''), $person);
 
-		$r = q("SELECT * FROM `photo` WHERE `scale` = %d AND `uid` = %d AND `profile` = 1 LIMIT 1",
+		$r = qu("SELECT * FROM `photo` WHERE `scale` = %d AND `uid` = %d AND `profile` = 1 LIMIT 1",
 			intval($resolution),
 			intval($uid)
 		);
@@ -102,7 +102,7 @@ function photo_init(&$a) {
 		}
 
         // check if the photo exists and get the owner of the photo
-		$r = q("SELECT `uid` FROM `photo` WHERE `resource-id` = '%s' LIMIT 1",
+		$r = qu("SELECT `uid` FROM `photo` WHERE `resource-id` = '%s' LIMIT 1",
 			dbesc($photo),
 			intval($resolution)
 		);
@@ -112,7 +112,7 @@ function photo_init(&$a) {
 
 			// Now we'll see if we can access the photo
 
-			$r = q("SELECT * FROM `photo` WHERE `resource-id` = '%s' AND `scale` <= %d $sql_extra ORDER BY scale DESC LIMIT 1",
+			$r = qu("SELECT * FROM `photo` WHERE `resource-id` = '%s' AND `scale` <= %d $sql_extra ORDER BY scale DESC LIMIT 1",
 				dbesc($photo),
 				intval($resolution)
 			);

mod/profile.php

@@ -282,16 +282,20 @@ function profile_content(&$a, $update = 0) {
 		$pager_sql = sprintf(" LIMIT %d, %d ",intval($a->pager['start']), intval($a->pager['itemspage']));
 
 		$r = q("SELECT `thread`.`iid` AS `item_id`, `thread`.`network` AS `item_network`
-			FROM `thread` FORCE INDEX (`uid_created`) INNER JOIN `item` ON `item`.`id` = `thread`.`iid`
+			FROM `thread`
-			$sql_post_table INNER JOIN `contact` ON `contact`.`id` = `thread`.`contact-id`
+			STRAIGHT_JOIN `item` ON `item`.`id` = `thread`.`iid`
-			AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
+			$sql_post_table
-			WHERE `thread`.`uid` = %d AND `thread`.`visible` = 1 AND `thread`.`deleted` = 0
+			STRAIGHT_JOIN `contact` ON `contact`.`id` = `thread`.`contact-id`
-			and `thread`.`moderated` = 0
+				AND NOT `contact`.`blocked` AND NOT `contact`.`pending`
-			AND `thread`.`wall` = 1
+			WHERE `thread`.`uid` = %d AND `thread`.`visible`
-			$sql_extra $sql_extra2
+				AND `thread`.`contact-id` = %d
-			ORDER BY `thread`.`created` DESC $pager_sql ",
+				AND NOT `thread`.`deleted`
-			intval($a->profile['profile_uid'])
+				AND NOT `thread`.`moderated`
-
+				AND `thread`.`wall`
+				$sql_extra $sql_extra2
+			ORDER BY `thread`.`created` DESC $pager_sql",
+			intval($a->profile['profile_uid']),
+			intval($a->profile['contact_id'])
 		);
 	}
 

mod/proxy.php

@@ -135,7 +135,7 @@ function proxy_init() {
 	$valid = true;
 
 	if (!$direct_cache AND ($cachefile == "")) {
-		$r = q("SELECT * FROM `photo` WHERE `resource-id` = '%s' LIMIT 1", $urlhash);
+		$r = qu("SELECT * FROM `photo` WHERE `resource-id` = '%s' LIMIT 1", $urlhash);
 		if (count($r)) {
         		$img_str = $r[0]['data'];
 			$mime = $r[0]["desc"];