Merge pull request #621 from annando/master

Using "term" table to improve performance, OStatus autocomplete, some more performance stuff

.gitignore

@@ -9,6 +9,7 @@ favicon.*
 home.html
 addon
 *~
+robots.txt
 
 #ignore documentation, it should be newly built 
 doc/api

boot.php

@@ -14,8 +14,7 @@ require_once('include/features.php');
 define ( 'FRIENDICA_PLATFORM',     'Friendica');
 define ( 'FRIENDICA_VERSION',      '3.1.1614' );
 define ( 'DFRN_PROTOCOL_VERSION',  '2.23'    );
-define ( 'DB_UPDATE_VERSION',      1159      );
+define ( 'DB_UPDATE_VERSION',      1162      );
-
 define ( 'EOL',                    "<br />\r\n"     );
 define ( 'ATOM_TIME',              'Y-m-d\TH:i:s\Z' );
 
@@ -208,6 +207,8 @@ define ( 'TERM_MENTION',   2 );
 define ( 'TERM_CATEGORY',  3 );
 define ( 'TERM_PCATEGORY', 4 );
 define ( 'TERM_FILE',      5 );
+define ( 'TERM_SAVEDSEARCH', 6 );
+define ( 'TERM_CONVERSATION', 7 );
 
 define ( 'TERM_OBJ_POST',  1 );
 define ( 'TERM_OBJ_PHOTO', 2 );
@@ -355,6 +356,7 @@ if(! class_exists('App')) {
 		public  $identities;
 		public	$is_mobile;
 		public	$is_tablet;
+		public	$performance = array();
 
 		public $nav_sel;
 
@@ -412,6 +414,14 @@ if(! class_exists('App')) {
 
 			date_default_timezone_set($this->timezone);
 
+			$this->performance["start"] = microtime(true);
+			$this->performance["database"] = 0;
+			$this->performance["network"] = 0;
+			$this->performance["rendering"] = 0;
+			$this->performance["parser"] = 0;
+			$this->performance["marktime"] = 0;
+			$this->performance["markstart"] = microtime(true);
+
 			$this->config = array();
 			$this->page = array();
 			$this->pager= array();
@@ -719,6 +729,17 @@ if(! class_exists('App')) {
 			return $this->rdelim[$engine];
 		}
 
+		function save_timestamp($stamp, $value) {
+			$duration = (float)(microtime(true)-$stamp);
+
+			$this->performance[$value] += (float)$duration;
+			$this->performance["marktime"] += (float)$duration;
+		}
+
+		function mark_timestamp($mark) {
+			//$this->performance["markstart"] -= microtime(true) - $this->performance["marktime"];
+			$this->performance["markstart"] = microtime(true) - $this->performance["markstart"] - $this->performance["marktime"];
+		}
 	}
 }
 

database.sql

@@ -560,6 +560,7 @@ CREATE TABLE IF NOT EXISTS `item` (
   `deleted` tinyint(1) NOT NULL DEFAULT '0',
   `origin` tinyint(1) NOT NULL DEFAULT '0',
   `forum_mode` tinyint(1) NOT NULL DEFAULT '0',
+  `mention` tinyint(1) NOT NULL DEFAULT '0',
   `last-child` tinyint(1) unsigned NOT NULL DEFAULT '1',
   PRIMARY KEY (`id`),
   KEY `uri` (`uri`),
@@ -589,6 +590,8 @@ CREATE TABLE IF NOT EXISTS `item` (
   KEY `uid_commented` (`uid`, `commented`),
   KEY `uid_created` (`uid`, `created`),
   KEY `uid_unseen` (`uid`, `unseen`),
+  KEY `mention` (`mention`),
+  KEY `resource-id` (`resource-id`),
   KEY `event_id` (`event-id`),
   FULLTEXT KEY `title` (`title`),
   FULLTEXT KEY `body` (`body`),
@@ -1030,17 +1033,21 @@ CREATE TABLE IF NOT EXISTS `spam` (
 --
 
 CREATE TABLE IF NOT EXISTS `term` (
-  `tid` INT UNSIGNED NOT NULL AUTO_INCREMENT,
+  `tid` int(10) unsigned NOT NULL AUTO_INCREMENT,
-  `oid` INT UNSIGNED NOT NULL ,
+  `aid` int(10) unsigned NOT NULL DEFAULT '0',
-  `otype` TINYINT( 3 ) UNSIGNED NOT NULL ,
+  `uid` int(10) unsigned NOT NULL DEFAULT '0',
-  `type` TINYINT( 3 ) UNSIGNED NOT NULL ,
+  `oid` int(10) unsigned NOT NULL,
-  `term` CHAR( 255 ) NOT NULL ,
+  `otype` tinyint(3) unsigned NOT NULL,
-  `url` CHAR( 255 ) NOT NULL, 
+  `type` tinyint(3) unsigned NOT NULL,
+  `term` char(255) NOT NULL,
+  `url` char(255) NOT NULL,
   PRIMARY KEY (`tid`),
   KEY `oid` (`oid`),
   KEY `otype` (`otype`),
   KEY `type` (`type`),
-  KEY `term`  ( `term` )
+  KEY `term` (`term`),
+  KEY `uid` (`uid`),
+  KEY `aid` (`aid`)
 ) ENGINE=MyISAM  DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------
@@ -1133,3 +1140,17 @@ CREATE TABLE IF NOT EXISTS `userd` (
   `username` char(255) NOT NULL,
   PRIMARY KEY (`id`)
 ) ENGINE=MyISAM DEFAULT CHARSET=utf8;
+
+-- --------------------------------------------------------
+
+--
+-- Table structure for table `tag`
+--
+
+CREATE TABLE IF NOT EXISTS `tag` (
+  `iid` int(11) NOT NULL,
+  `tag` char(255) NOT NULL,
+  `link` char(255) NOT NULL,
+  PRIMARY KEY (`iid`, `tag`),
+  KEY `tag` (`tag`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8;

include/api.php

@@ -644,7 +644,7 @@
 				$in_reply_to_screen_name = $lastwall['reply_author'];
 			}
 			$status_info = array(
-				'text' => html2plain(bbcode($lastwall['body']), 0),
+				'text' => html2plain(bbcode($lastwall['body'], false, false, true), 0),
 				'truncated' => false,
 				'created_at' => api_date($lastwall['created']),
 				'in_reply_to_status_id' => $in_reply_to_status_id,
@@ -703,7 +703,7 @@
 			$user_info['status'] = array(
 				'created_at' => api_date($lastwall['created']),
 				'id' => $lastwall['contact-id'],
-				'text' => html2plain(bbcode($lastwall['body']), 0),
+				'text' => html2plain(bbcode($lastwall['body'], false, false, true), 0),
 				'source' => (($lastwall['app']) ? $lastwall['app'] : 'web'),
 				'truncated' => false,
 				'in_reply_to_status_id' => $in_reply_to_status_id,
@@ -1042,10 +1042,11 @@
 
 		$myurl = $a->get_baseurl() . '/profile/'. $a->user['nickname'];
 		$myurl = substr($myurl,strpos($myurl,'://')+3);
-		$myurl = str_replace(array('www.','.'),array('','\\.'),$myurl);
+		//$myurl = str_replace(array('www.','.'),array('','\\.'),$myurl);
+		$myurl = str_replace('www.','',$myurl);
 		$diasp_url = str_replace('/profile/','/u/',$myurl);
 
-		if (get_config('system','use_fulltext_engine'))
+/*		if (get_config('system','use_fulltext_engine'))
 			$sql_extra .= sprintf(" AND `item`.`parent` IN (SELECT distinct(`parent`) from item where (MATCH(`author-link`) AGAINST ('".'"%s"'."' in boolean mode) or MATCH(`tag`) AGAINST ('".'"%s"'."' in boolean mode) or MATCH(tag) AGAINST ('".'"%s"'."' in boolean mode))) ",
 				dbesc(protect_sprintf($myurl)),
 				dbesc(protect_sprintf($myurl)),
@@ -1057,6 +1058,11 @@
 				dbesc(protect_sprintf('%' . $myurl . ']%')),
 				dbesc(protect_sprintf('%' . $diasp_url . ']%'))
 			);
+*/
+		$sql_extra .= sprintf(" AND `item`.`parent` IN (SELECT distinct(`parent`) from item where `author-link` IN ('https://%s', 'http://%s') OR `mention`)",
+			dbesc(protect_sprintf($myurl)),
+			dbesc(protect_sprintf($myurl))
+		);
 
 		if ($max_id > 0)
 			$sql_extra .= ' AND `item`.`id` <= '.intval($max_id);
@@ -1299,11 +1305,11 @@
 				$ret['text'] = bbcode($item['body']);
 			}
 			elseif ($_GET["getText"] == "plain") {
-				$ret['text'] = html2plain(bbcode($item['body']), 0);
+				$ret['text'] = html2plain(bbcode($item['body'], false, false, true), 0);
 			}
 		}
 		else {
-			$ret['text'] = $item['title']."\n".html2plain(bbcode($item['body']), 0);
+			$ret['text'] = $item['title']."\n".html2plain(bbcode($item['body'], false, false, true), 0);
 		}
 		if (isset($_GET["getUserObjects"]) && $_GET["getUserObjects"] == "false") {
 			unset($ret['sender']);
@@ -1347,7 +1353,7 @@
 			}
 
 			// Workaround for ostatus messages where the title is identically to the body
-			$statusbody = trim(html2plain(bbcode($item['body']), 0));
+			$statusbody = trim(html2plain(bbcode($item['body'], false, false, true), 0));
 			$statustitle = trim($item['title']);
 
 			if (($statustitle != '') and (strpos($statusbody, $statustitle) !== false))

include/bbcode.php

@@ -278,10 +278,39 @@ function bb_ShareAttributes($match) {
         return($text);
 }
 
+function bb_ShareAttributesSimple($match) {
+
+        $attributes = $match[1];
+
+        $author = "";
+        preg_match("/author='(.*?)'/ism", $attributes, $matches);
+        if ($matches[1] != "")
+                $author = html_entity_decode($matches[1],ENT_QUOTES,'UTF-8');
+
+        preg_match('/author="(.*?)"/ism', $attributes, $matches);
+        if ($matches[1] != "")
+                $author = $matches[1];
+
+        $profile = "";
+        preg_match("/profile='(.*?)'/ism", $attributes, $matches);
+        if ($matches[1] != "")
+                $profile = $matches[1];
+
+        preg_match('/profile="(.*?)"/ism', $attributes, $matches);
+        if ($matches[1] != "")
+                $profile = $matches[1];
+
+        $text = "<br />".html_entity_decode("&#x2672; ", ENT_QUOTES, 'UTF-8').' <a href="'.$profile.'">'.$author."</a>: <br />»".$match[2]."«";
+
+        return($text);
+}
+
 	// BBcode 2 HTML was written by WAY2WEB.net
 	// extended to work with Mistpark/Friendica - Mike Macgirvin
 
-function bbcode($Text,$preserve_nl = false, $tryoembed = true) {
+function bbcode($Text,$preserve_nl = false, $tryoembed = true, $simplehtml = false) {
+
+	$stamp1 = microtime(true);
 
 	$a = get_app();
 
@@ -340,13 +369,14 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true) {
 	$Text = str_replace("\r\n","\n", $Text);
 
 	// removing multiplicated newlines
-//	$search = array("\n\n\n", "\n ", " \n", "[/quote]\n\n", "\n[/quote]");
+	if (get_config("system", "remove_multiplicated_lines")) {
-//	$replace = array("\n\n", "\n", "\n", "[/quote]\n", "[/quote]");
+		$search = array("\n\n\n", "\n ", " \n", "[/quote]\n\n", "\n[/quote]");
-//	do {
+		$replace = array("\n\n", "\n", "\n", "[/quote]\n", "[/quote]");
-//		$oldtext = $Text;
+		do {
-//		$Text = str_replace($search, $replace, $Text);
+			$oldtext = $Text;
-//	} while ($oldtext != $Text);
+			$Text = str_replace($search, $replace, $Text);
-
+		} while ($oldtext != $Text);
+	}
 
 	$Text = str_replace(array("\r","\n"), array('<br />','<br />'), $Text);
 
@@ -515,7 +545,10 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true) {
 	$Text = preg_replace("/\[img\](.*?)\[\/img\]/ism", '<img src="$1" alt="' . t('Image/photo') . '" />', $Text);
 
 	// Shared content
+	if (!$simplehtml)
 		$Text = preg_replace_callback("/\[share(.*?)\](.*?)\[\/share\]/ism","bb_ShareAttributes",$Text);
+	else
+		$Text = preg_replace_callback("/\[share(.*?)\](.*?)\[\/share\]/ism","bb_ShareAttributesSimple",$Text);
 
 	$Text = preg_replace("/\[crypt\](.*?)\[\/crypt\]/ism",'<br/><img src="' .$a->get_baseurl() . '/images/lock_icon.gif" alt="' . t('Encrypted content') . '" title="' . t('Encrypted content') . '" /><br />', $Text);
 	$Text = preg_replace("/\[crypt=(.*?)\](.*?)\[\/crypt\]/ism",'<br/><img src="' .$a->get_baseurl() . '/images/lock_icon.gif" alt="' . t('Encrypted content') . '" title="' . '$1' . ' ' . t('Encrypted content') . '" /><br />', $Text);
@@ -617,7 +650,8 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true) {
 
 	// Clean up the HTML by loading and saving the HTML with the DOM
 	// Only do it when it has to be done - for performance reasons
-	if (!$tryoembed) {
+	// Update: Now it is done every time - since bad structured html can break a whole page
+	//if (!$tryoembed) {
 		$doc = new DOMDocument();
 		$doc->preserveWhiteSpace = false;
 
@@ -632,10 +666,12 @@ function bbcode($Text,$preserve_nl = false, $tryoembed = true) {
 		$Text = str_replace('<br></li>','</li>', $Text);
 
 		$Text = mb_convert_encoding($Text, "UTF-8", 'HTML-ENTITIES');
-	}
+	//}
 
 	call_hooks('bbcode',$Text);
 
+	$a->save_timestamp($stamp1, "parser");
+
 	return $Text;
 }
 

include/conversation.php

@@ -369,7 +369,6 @@ function visible_activity($item) {
 if(!function_exists('conversation')) {
 function conversation(&$a, $items, $mode, $update, $preview = false) {
 
-
 	require_once('include/bbcode.php');
 
 	$ssl_state = ((local_user()) ? true : false);
@@ -524,7 +523,26 @@ function conversation(&$a, $items, $mode, $update, $preview = false) {
 				$tags=array();
 				$hashtags = array();
 				$mentions = array();
-				foreach(explode(',',$item['tag']) as $tag){
+
+				$taglist = q("SELECT `type`, `term`, `url` FROM `term` WHERE `otype` = %d AND `oid` = %d AND `type` IN (%d, %d) ORDER BY `tid`",
+						intval(TERM_OBJ_POST), intval($item['id']), intval(TERM_HASHTAG), intval(TERM_MENTION));
+
+				foreach($taglist as $tag) {
+
+					if ($tag["url"] == "")
+						$tag["url"] = $searchpath.strtolower($tag["term"]);
+
+					if ($tag["type"] == TERM_HASHTAG) {
+						$hashtags[] = "#<a href=\"".$tag["url"]."\" target=\"external-link\">".$tag["term"]."</a>";
+						$prefix = "#";
+					} elseif ($tag["type"] == TERM_MENTION) {
+						$mentions[] = "@<a href=\"".$tag["url"]."\" target=\"external-link\">".$tag["term"]."</a>";
+						$prefix = "@";
+					}
+					$tags[] = $prefix."<a href=\"".$tag["url"]."\" target=\"external-link\">".$tag["term"]."</a>";
+				}
+
+				/*foreach(explode(',',$item['tag']) as $tag){
 					$tag = trim($tag);
 					if ($tag!="") {
 						$t = bbcode($tag);
@@ -534,7 +552,7 @@ function conversation(&$a, $items, $mode, $update, $preview = false) {
 						elseif($t[0] == '@')
 							$mentions[] = $t;
 					}
-				}
+				}*/
 
 				$sp = false;
 				$profile_link = best_link_url($item,$sp);

include/dba.php

@@ -23,6 +23,9 @@ class dba {
 	public  $error = false;
 
 	function __construct($server,$user,$pass,$db,$install = false) {
+		global $a;
+
+		$stamp1 = microtime(true);
 
 		$server = trim($server);
 		$user = trim($user);
@@ -64,6 +67,8 @@ class dba {
 			if(! $install)
 				system_unavailable();
 		}
+
+		$a->save_timestamp($stamp1, "network");
 	}
 
 	public function getdb() {
@@ -78,7 +83,6 @@ class dba {
 
 		$this->error = '';
 
-		if(x($a->config,'system') && x($a->config['system'],'db_log'))
 		$stamp1 = microtime(true);
 
 		if($this->mysqli)
@@ -86,10 +90,12 @@ class dba {
 		else
 			$result = @mysql_query($sql,$this->db);
 
-		if(x($a->config,'system') && x($a->config['system'],'db_log')) {
 		$stamp2 = microtime(true);
-			$duration = round($stamp2-$stamp1, 3);
+		$duration = (float)($stamp2-$stamp1);
-			if ($duration > $a->config["system"]["db_loglimit"]) {
+
+		if(x($a->config,'system') && x($a->config['system'],'db_log')) {
+			if (($duration > $a->config["system"]["db_loglimit"])) {
+				$duration = round($duration, 3);
 				$backtrace = debug_backtrace(DEBUG_BACKTRACE_IGNORE_ARGS);
 				@file_put_contents($a->config["system"]["db_log"], $duration."\t".
 						basename($backtrace[1]["file"])."\t".
@@ -162,6 +168,7 @@ class dba {
 			}
 		}
 
+		$a->save_timestamp($stamp1, "database");
 
 		if($this->debug)
 			logger('dba: ' . printable(print_r($r, true)));

include/diaspora.php

@@ -754,6 +754,20 @@ function diaspora_request($importer,$xml) {
 }
 
 function diaspora_post_allow($importer,$contact) {
+
+	// perhaps we were already sharing with this person. Now they're sharing with us.
+	// That makes us friends.
+	// Normally this should have handled by getting a request - but this could get lost
+	if($contact['rel'] == CONTACT_IS_FOLLOWER && $importer['page-flags'] != PAGE_COMMUNITY) {
+		q("UPDATE `contact` SET `rel` = %d, `writable` = 1 WHERE `id` = %d AND `uid` = %d LIMIT 1",
+			intval(CONTACT_IS_FRIEND),
+			intval($contact['id']),
+			intval($importer['uid'])
+		);
+		$contact['rel'] = CONTACT_IS_FRIEND;
+		logger('diaspora_post_allow: defining user '.$contact["nick"].' as friend');
+	}
+
 	if(($contact['blocked']) || ($contact['readonly']) || ($contact['archive']))
 		return false;
 	if($contact['rel'] == CONTACT_IS_SHARING || $contact['rel'] == CONTACT_IS_FRIEND)

include/items.php

@@ -5,7 +5,10 @@ require_once('include/oembed.php');
 require_once('include/salmon.php');
 require_once('include/crypto.php');
 require_once('include/Photo.php');
+require_once('include/tags.php');
+require_once('include/text.php');
 require_once('include/email.php');
+require_once('include/ostatus_conversation.php');
 
 function get_feed_for(&$a, $dfrn_id, $owner_nick, $last_update, $direction = 0) {
 
@@ -829,15 +832,30 @@ function get_atom_elements($feed,$item) {
 		}
 	}
 
+	// Search for ostatus conversation url
+	$links = $item->feed->data["child"][SIMPLEPIE_NAMESPACE_ATOM_10]["feed"][0]["child"][SIMPLEPIE_NAMESPACE_ATOM_10]["entry"][0]["child"]["http://www.w3.org/2005/Atom"]["link"];
+
+	if (is_array($links)) {
+		foreach ($links as $link) {
+			$conversation = array_shift($link["attribs"]);
+
+			if ($conversation["rel"] == "ostatus:conversation") {
+				$res["ostatus_conversation"] = $conversation["href"];
+				logger('get_atom_elements: found conversation url '.$res["ostatus_conversation"]);
+			}
+		};
+	}
+
 	$arr = array('feed' => $feed, 'item' => $item, 'result' => $res);
 
 	call_hooks('parse_atom', $arr);
 
 	//if (($res["title"] != "") or (strpos($res["body"], "RT @") > 0)) {
 	//if (strpos($res["body"], "RT @") !== false) {
-	//	$debugfile = tempnam("/home/ike/log", "item-res2-");
+	/*if (strpos($res["body"], "@") !== false) {
-	//	file_put_contents($debugfile, serialize($arr));
+		$debugfile = tempnam("/var/www/virtual/pirati.ca/phptmp/", "item-res2-");
-	//}
+		file_put_contents($debugfile, serialize($arr));
+	}*/
 
 	return $res;
 }
@@ -876,6 +894,15 @@ function item_store($arr,$force_parent = false) {
 		unset($arr['dsprsig']);
 	}
 
+	// if an OStatus conversation url was passed in, it is stored and then
+	// removed from the array.
+	$ostatus_conversation = null;
+
+	if (isset($arr["ostatus_conversation"])) {
+		$ostatus_conversation = $arr["ostatus_conversation"];
+	        unset($arr["ostatus_conversation"]);
+	}
+
 	if(x($arr, 'gravity'))
 		$arr['gravity'] = intval($arr['gravity']);
 	elseif($arr['parent-uri'] === $arr['uri'])
@@ -898,8 +925,23 @@ function item_store($arr,$force_parent = false) {
 		require_once('library/langdet/Text/LanguageDetect.php');
 		$naked_body = preg_replace('/\[(.+?)\]/','',$arr['body']);
 		$l = new Text_LanguageDetect;
-		$lng = $l->detectConfidence($naked_body);
+		//$lng = $l->detectConfidence($naked_body);
-		$arr['postopts'] = (($lng['language']) ? 'lang=' . $lng['language'] . ';' . $lng['confidence'] : '');
+		//$arr['postopts'] = (($lng['language']) ? 'lang=' . $lng['language'] . ';' . $lng['confidence'] : '');
+		$lng = $l->detect($naked_body, 3);
+
+		if (sizeof($lng) > 0) {
+			$postopts = "";
+
+			foreach ($lng as $language => $score) {
+				if ($postopts == "")
+					$postopts = "lang=";
+				else
+					$postopts .= ":";
+
+				$postopts .= $language.";".$score;
+			}
+			$arr['postopts'] = $postopts;
+		}
 	}
 
 	$arr['wall']          = ((x($arr,'wall'))          ? intval($arr['wall'])                : 0);
@@ -954,7 +996,7 @@ function item_store($arr,$force_parent = false) {
 	}
 	else {
 
-		// find the parent and snarf the item id and ACL's
+		// find the parent and snarf the item id and ACLs
 		// and anything else we need to inherit
 
 		$r = q("SELECT * FROM `item` WHERE `uri` = '%s' AND `uid` = %d ORDER BY `id` ASC LIMIT 1",
@@ -1058,8 +1100,8 @@ function item_store($arr,$force_parent = false) {
 	if(count($r)) {
 		$current_post = $r[0]['id'];
 		logger('item_store: created item ' . $current_post);
-	}
+		create_tags_from_item($r[0]['id']);
-	else {
+	} else {
 		logger('item_store: could not locate created item');
 		return 0;
 	}
@@ -1093,6 +1135,11 @@ function item_store($arr,$force_parent = false) {
 		intval($parent_deleted),
 		intval($current_post)
 	);
+	create_tags_from_item($current_post);
+
+	// Complete ostatus threads
+	if ($ostatus_conversation)
+		complete_conversation($current_post, $ostatus_conversation);
 
         $arr['id'] = $current_post;
         $arr['parent'] = $parent_id;
@@ -1136,6 +1183,18 @@ function item_store($arr,$force_parent = false) {
 
 	tag_deliver($arr['uid'],$current_post);
 
+	// Store the fresh generated item into the cache
+	$cachefile = get_cachefile($arr["guid"]."-".hash("md5", $arr['body']));
+
+	if (($cachefile != '') AND !file_exists($cachefile)) {
+		$s = prepare_text($arr['body']);
+		$a = get_app();
+		$stamp1 = microtime(true);
+		file_put_contents($cachefile, $s);
+		$a->save_timestamp($stamp1, "file");
+		logger('item_store: put item '.$current_post.' into cachefile '.$cachefile);
+	}
+
 	return $current_post;
 }
 
@@ -1810,6 +1869,7 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 										dbesc(implode(',',$newtags)),
 										intval($i[0]['id'])
 									);
+									create_tags_from_item($i[0]['id']);
 								}
 							}
 						}
@@ -1824,6 +1884,7 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 							dbesc($item['uri']),
 							intval($importer['uid'])
 						);
+						create_tags_from_itemuri($item['uri'], $importer['uid']);
 					}
 					else {
 						$r = q("UPDATE `item` SET `deleted` = 1, `edited` = '%s', `changed` = '%s',
@@ -1834,6 +1895,7 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 							dbesc($uri),
 							intval($importer['uid'])
 						);
+						create_tags_from_itemuri($uri, $importer['uid']);
 						if($item['last-child']) {
 							// ensure that last-child is set in case the comment that had it just got wiped.
 							q("UPDATE `item` SET `last-child` = 0, `changed` = '%s' WHERE `parent-uri` = '%s' AND `uid` = %d ",
@@ -1947,6 +2009,7 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 							dbesc($item_id),
 							intval($importer['uid'])
 						);
+						create_tags_from_itemuri($item_id, $importer['uid']);
 					}
 
 					// update last-child if it changes
@@ -2011,6 +2074,7 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 									dbesc($r[0]['tag'] . (strlen($r[0]['tag']) ? ',' : '') . $newtag),
 									intval($r[0]['id'])
 								);
+								create_tags_from_item($r[0]['id']);
 							}
 						}
 					}
@@ -2094,6 +2158,7 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 							dbesc($item_id),
 							intval($importer['uid'])
 						);
+						create_tags_from_itemuri($item_id, $importer['uid']);
 					}
 
 					// update last-child if it changes
@@ -2696,6 +2761,7 @@ function local_delivery($importer,$data) {
 										dbesc(implode(',',$newtags)),
 										intval($i[0]['id'])
 									);
+									create_tags_from_item($i[0]['id']);
 								}
 							}
 						}
@@ -2710,6 +2776,7 @@ function local_delivery($importer,$data) {
 							dbesc($item['uri']),
 							intval($importer['importer_uid'])
 						);
+						create_tags_from_itemuri($item['uri'], $importer['importer_uid']);
 					}
 					else {
 						$r = q("UPDATE `item` SET `deleted` = 1, `edited` = '%s', `changed` = '%s',
@@ -2720,6 +2787,7 @@ function local_delivery($importer,$data) {
 							dbesc($uri),
 							intval($importer['importer_uid'])
 						);
+						create_tags_from_itemuri($uri, $importer['importer_uid']);
 						if($item['last-child']) {
 							// ensure that last-child is set in case the comment that had it just got wiped.
 							q("UPDATE `item` SET `last-child` = 0, `changed` = '%s' WHERE `parent-uri` = '%s' AND `uid` = %d ",
@@ -2845,6 +2913,7 @@ function local_delivery($importer,$data) {
 							dbesc($item_id),
 							intval($importer['importer_uid'])
 						);
+						create_tags_from_itemuri($item_id, $importer['importer_uid']);
 
 						proc_run('php',"include/notifier.php","comment-import",$iid);
 
@@ -2917,6 +2986,7 @@ function local_delivery($importer,$data) {
 										intval($tagp[0]['id']),
 										dbesc(datetime_convert())
 									);
+									create_tags_from_item($tagp[0]['id']);
 								}
 							}
 						}
@@ -3017,6 +3087,7 @@ function local_delivery($importer,$data) {
 							dbesc($item_id),
 							intval($importer['importer_uid'])
 						);
+						create_tags_from_itemuri($item_id, $importer['importer_uid']);
 					}
 
 					// update last-child if it changes
@@ -3077,6 +3148,7 @@ function local_delivery($importer,$data) {
 									dbesc($r[0]['tag'] . (strlen($r[0]['tag']) ? ',' : '') . '#[url=' . $xo->id . ']'. $xo->content . '[/url]'),
 									intval($r[0]['id'])
 								);
+								create_tags_from_item($r[0]['id']);
 							}
 						}
 					}
@@ -3191,6 +3263,7 @@ function local_delivery($importer,$data) {
 						dbesc($item_id),
 						intval($importer['importer_uid'])
 					);
+					create_tags_from_itemuri($item_id, $importer['importer_uid']);
 				}
 
 				// update last-child if it changes
@@ -3910,6 +3983,7 @@ function drop_item($id,$interactive = true) {
 			dbesc(datetime_convert()),
 			intval($item['id'])
 		);
+		create_tags_from_item($item['id']);
 
 		// clean up categories and tags so they don't end up as orphans
 
@@ -3975,6 +4049,7 @@ function drop_item($id,$interactive = true) {
 				dbesc($item['parent-uri']),
 				intval($item['uid'])
 			);
+			create_tags_from_item($item['parent-uri'], $item['uid']);
 			// ignore the result
 		}
 		else {

include/network.php

@@ -7,6 +7,8 @@
 if(! function_exists('fetch_url')) {
 function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_content=Null) {
 
+	$stamp1 = microtime(true);
+
 	$a = get_app();
 
 	$ch = @curl_init($url);
@@ -76,9 +78,17 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
 	}
 
 	if($http_code == 301 || $http_code == 302 || $http_code == 303 || $http_code == 307) {
-		$matches = array();
+		$new_location_info = @parse_url($curl_info["redirect_url"]);
-		preg_match('/(Location:|URI:)(.*?)\n/', $header, $matches);
+		$old_location_info = @parse_url($curl_info["url"]);
-		$newurl = trim(array_pop($matches));
+
+		$newurl = $curl_info["redirect_url"];
+
+		if (($new_location_info["path"] == "") AND ($new_location_info["host"] != ""))
+			$newurl = $new_location_info["scheme"]."://".$new_location_info["host"].$old_location_info["path"];
+
+		//$matches = array();
+		//preg_match('/(Location:|URI:)(.*?)\n/', $header, $matches);
+		//$newurl = trim(array_pop($matches));
 		if(strpos($newurl,'/') === 0)
 			$newurl = $url . $newurl;
 		$url_parsed = @parse_url($newurl);
@@ -93,6 +103,9 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
 	$body = substr($s,strlen($header));
 	$a->set_curl_headers($header);
 	@curl_close($ch);
+
+	$a->save_timestamp($stamp1, "network");
+
 	return($body);
 }}
 
@@ -100,6 +113,9 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
 
 if(! function_exists('post_url')) {
 function post_url($url,$params, $headers = null, &$redirects = 0, $timeout = 0) {
+
+	$stamp1 = microtime(true);
+
 	$a = get_app();
 	$ch = curl_init($url);
 	if(($redirects > 8) || (! $ch))
@@ -182,6 +198,9 @@ function post_url($url,$params, $headers = null, &$redirects = 0, $timeout = 0)
 	$a->set_curl_headers($header);
 
 	curl_close($ch);
+
+	$a->save_timestamp($stamp1, "network");
+
 	return($body);
 }}
 
@@ -359,6 +378,7 @@ function lrdd($uri, $debug = false) {
 	logger('lrdd: constructed url: ' . $url);
 
 	$xml = fetch_url($url);
+
 	$headers = $a->get_curl_headers();
 
 	if (! $xml)
@@ -827,8 +847,11 @@ function scale_external_images($s, $include_link = true, $scale_replace = false)
 			$i = fetch_url($scaled);
 
 			$cachefile = get_cachefile(hash("md5", $scaled));
-			if ($cachefile != '')
+			if ($cachefile != '') {
+				$stamp1 = microtime(true);
 				file_put_contents($cachefile, $i);
+				$a->save_timestamp($stamp1, "file");
+			}
 
 			// guess mimetype from headers or filename
 			$type = guess_image_type($mtch[1],true);

include/ostatus_conversation.php

@@ -0,0 +1,173 @@
+<?php
+define('OSTATUS_DEFAULT_POLL_INTERVAL', 30); // given in minutes
+
+function check_conversations() {
+        $last = get_config('system','ostatus_last_poll');
+
+        $poll_interval = intval(get_config('system','ostatus_poll_interval'));
+        if(! $poll_interval)
+                $poll_interval = OSTATUS_DEFAULT_POLL_INTERVAL;
+
+        if($last) {
+                $next = $last + ($poll_interval * 60);
+                if($next > time()) {
+                        logger('complete_conversation: poll intervall not reached');
+                        return;
+                }
+        }
+
+        logger('complete_conversation: cron_start');
+
+        $start = date("Y-m-d H:i:s", time() - 86400);
+        $conversations = q("SELECT * FROM `term` WHERE `type` = 7 AND `term` > '%s'", 
+                                dbesc($start));
+        foreach ($conversations AS $conversation) {
+                $id = $conversation['oid'];
+                $url = $conversation['url'];
+                complete_conversation($id, $url);
+        }
+
+        logger('complete_conversation: cron_end');
+
+        set_config('system','ostatus_last_poll', time());
+}
+
+function complete_conversation($itemid, $conversation_url, $only_add_conversation = false) {
+	global $a;
+
+	//logger('complete_conversation: completing conversation url '.$conversation_url.' for id '.$itemid);
+
+	$messages = q("SELECT `uid`, `parent` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid));
+	if (!$messages)
+		return;
+	$message = $messages[0];
+
+	// Store conversation url if not done before
+	$conversation = q("SELECT `url` FROM `term` WHERE `uid` = %d AND `oid` = %d AND `otype` = %d AND `type` = %d",
+		intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION));
+
+	if (!$conversation) {
+		$r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`) VALUES (%d, %d, %d, %d, '%s', '%s')",
+			intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval(TERM_CONVERSATION), dbesc(datetime_convert()), dbesc($conversation_url));
+		logger('complete_conversation: Storing conversation url '.$conversation_url.' for id '.$itemid);
+	}
+
+	if ($only_add_conversation)
+		return;
+
+	// Get the parent
+	$parents = q("SELECT `id`, `uri`, `contact-id`, `type`, `verb`, `visible` FROM `item` WHERE `uid` = %d AND `id` = %d LIMIT 1",
+			intval($message["uid"]), intval($message["parent"]));
+	if (!$parents)
+		return;
+	$parent = $parents[0];
+
+	require_once('include/html2bbcode.php');
+	require_once('include/items.php');
+
+	$conv = str_replace("/conversation/", "/api/statusnet/conversation/", $conversation_url).".as";
+
+	logger('complete_conversation: fetching conversation url '.$conv.' for '.$itemid);
+	$conv_as = fetch_url($conv);
+
+	if ($conv_as) {
+		$conv_as = str_replace(',"statusnet:notice_info":', ',"statusnet_notice_info":', $conv_as);
+		$conv_as = json_decode($conv_as);
+
+		$first_id = "";
+		$items = array_reverse($conv_as->items);
+
+		foreach ($items as $single_conv) {
+			if ($first_id == "") {
+				$first_id = $single_conv->id;
+
+				$new_parents = q("SELECT `id`, `uri`, `contact-id`, `type`, `verb`, `visible` FROM `item` WHERE `uid` = %d AND `uri` = '%s' LIMIT 1",
+					intval($message["uid"]), dbesc($first_id));
+				if ($new_parents) {
+					$parent = $new_parents[0];
+					logger('complete_conversation: adopting new parent '.$parent["id"].' for '.$itemid);
+				} else {
+					$parent["id"] = 0;
+					$parent["uri"] = $first_id;
+				}
+			}
+
+			if (isset($single_conv->context->inReplyTo->id))
+				$parent_uri = $single_conv->context->inReplyTo->id;
+			else
+				$parent_uri = $parent["uri"];
+
+			if ($parent["id"] != 0) {
+				$message_exists = q("SELECT `id` FROM `item` WHERE `uid` = %d AND `uri` = '%s' LIMIT 1",
+							intval($message["uid"]), dbesc($single_conv->id));
+				if ($message_exists) {
+					$existing_message = $message_exists[0];
+					$r = q("UPDATE `item` SET `parent` = %d, `parent-uri` = '%s', `thr-parent` = '%s' WHERE `id` = %d LIMIT 1",
+						intval($parent["id"]),
+						dbesc($parent["uri"]),
+						dbesc($parent_uri),
+						intval($existing_message["id"]));
+					continue;
+				}
+			}
+
+			$arr = array();
+			$arr["uri"] = $single_conv->id;
+			$arr["plink"] = $single_conv->id;
+			$arr["uid"] = $message["uid"];
+			$arr["contact-id"] = $parent["contact-id"]; // To-Do
+			if ($parent["id"] != 0)
+				$arr["parent"] = $parent["id"];
+			$arr["parent-uri"] = $parent["uri"];
+			$arr["thr-parent"] = $parent_uri;
+			$arr["created"] = $single_conv->published;
+			$arr["edited"] = $single_conv->published;
+			//$arr["owner-name"] = $single_conv->actor->contact->displayName;
+			$arr["owner-name"] = $single_conv->actor->contact->preferredUsername;
+			$arr["owner-link"] = $single_conv->actor->id;
+			$arr["owner-avatar"] = $single_conv->actor->image->url;
+			//$arr["author-name"] = $single_conv->actor->contact->displayName;
+			$arr["author-name"] = $single_conv->actor->contact->preferredUsername;
+			$arr["author-link"] = $single_conv->actor->id;
+			$arr["author-avatar"] = $single_conv->actor->image->url;
+			$arr["body"] = html2bbcode($single_conv->content);
+			$arr["app"] = strip_tags($single_conv->statusnet_notice_info->source);
+			if ($arr["app"] == "")
+				$arr["app"] = $single_conv->provider->displayName;
+			$arr["verb"] = $parent["verb"];
+			$arr["visible"] = $parent["visible"];
+			$arr["location"] = $single_conv->location->displayName;
+			$arr["coord"] = trim($single_conv->location->lat." ".$single_conv->location->lon);
+
+			if ($arr["location"] == "")
+				unset($arr["location"]);
+
+			if ($arr["coord"] == "")
+				unset($arr["coord"]);
+
+			$newitem = item_store($arr);
+
+			// Add the conversation entry (but don't fetch the whole conversation)
+			complete_conversation($newitem, $conversation_url, true);
+
+			// If the newly created item is the top item then change the parent settings of the thread
+			if ($newitem AND ($arr["uri"] == $first_id)) {
+				logger('complete_conversation: setting new parent to id '.$newitem);
+				$new_parents = q("SELECT `id`, `uri`, `contact-id`, `type`, `verb`, `visible` FROM `item` WHERE `uid` = %d AND `id` = %d LIMIT 1",
+					intval($message["uid"]), intval($newitem));
+				if ($new_parents) {
+					$parent = $new_parents[0];
+					logger('complete_conversation: done changing parents to parent '.$newitem);
+				}
+
+				/*logger('complete_conversation: changing parents to parent '.$newitem.' old parent: '.$parent["id"].' new uri: '.$arr["uri"]);
+				$r = q("UPDATE `item` SET `parent` = %d, `parent-uri` = '%s' WHERE `parent` = %d",
+					intval($newitem),
+					dbesc($arr["uri"]),
+					intval($parent["id"]));
+				logger('complete_conversation: done changing parents to parent '.$newitem.' '.print_r($r, true));*/
+			}
+		}
+	}
+}
+?>

include/plugin.php

@@ -158,6 +158,8 @@ function load_hooks() {
 
 if(! function_exists('call_hooks')) {
 function call_hooks($name, &$data = null) {
+	$stamp1 = microtime(true);
+
 	$a = get_app();
 
 	if((is_array($a->hooks)) && (array_key_exists($name,$a->hooks))) {
@@ -173,7 +175,7 @@ function call_hooks($name, &$data = null) {
 			}
 			else {
 				// remove orphan hooks
-				q("delete from hook where hook = '%s' and file = '$s' and function = '%s' limit 1",
+				q("delete from hook where hook = '%s' and file = '%s' and function = '%s' limit 1",
 					dbesc($name),
 					dbesc($hook[0]),
 					dbesc($hook[1])
@@ -181,7 +183,6 @@ function call_hooks($name, &$data = null) {
 			}
 		}
 	}
-
 }}
 
 
@@ -199,6 +200,9 @@ function call_hooks($name, &$data = null) {
 
 if (! function_exists('get_plugin_info')){
 function get_plugin_info($plugin){
+
+	$a = get_app();
+
 	$info=Array(
 		'name' => $plugin,
 		'description' => "",
@@ -208,7 +212,10 @@ function get_plugin_info($plugin){
 
 	if (!is_file("addon/$plugin/$plugin.php")) return $info;
 
+	$stamp1 = microtime(true);
 	$f = file_get_contents("addon/$plugin/$plugin.php");
+	$a->save_timestamp($stamp1, "file");
+
 	$r = preg_match("|/\*.*\*/|msU", $f, $m);
 
 	if ($r){
@@ -271,9 +278,12 @@ function get_theme_info($theme){
 
 	if (!is_file("view/theme/$theme/theme.php")) return $info;
 
+	$a = get_app();
+	$stamp1 = microtime(true);
 	$f = file_get_contents("view/theme/$theme/theme.php");
-	$r = preg_match("|/\*.*\*/|msU", $f, $m);
+	$a->save_timestamp($stamp1, "file");
 
+	$r = preg_match("|/\*.*\*/|msU", $f, $m);
 
 	if ($r){
 		$ll = explode("\n", $m[0]);

include/poller.php

@@ -86,7 +86,8 @@ function poller_run(&$argv, &$argc){
 	if($abandon_days < 1)
 		$abandon_days = 0;
 
-	
+	// Check OStatus conversations
+	check_conversations();
 
 	// once daily run birthday_updates and then expire in background
 

include/tags.php

@@ -0,0 +1,127 @@
+<?php
+/*
+require_once("boot.php");
+if(@is_null($a)) {
+        $a = new App;
+}
+
+if(is_null($db)) {
+        @include(".htconfig.php");
+        require_once("dba.php");
+        $db = new dba($db_host, $db_user, $db_pass, $db_data);
+        unset($db_host, $db_user, $db_pass, $db_data);
+};
+
+$a->set_baseurl("https://pirati.ca");
+*/
+
+function create_tags_from_item($itemid) {
+	global $a;
+
+	$profile_base = $a->get_baseurl();
+	$profile_data = parse_url($profile_base);
+	$profile_base_friendica = $profile_data['host'].$profile_data['path']."/profile/";
+	$profile_base_diaspora = $profile_data['host'].$profile_data['path']."/u/";
+
+	$searchpath = $a->get_baseurl()."/search?tag=";
+
+	$messages = q("SELECT `guid`, `uid`, `id`, `edited`, `deleted`, `title`, `body`, `tag` FROM `item` WHERE `id` = %d LIMIT 1", intval($itemid));
+
+	if (!$messages)
+		return;
+
+	$message = $messages[0];
+
+	// Clean up all tags
+	q("DELETE FROM `term` WHERE `otype` = %d AND `oid` = %d AND `type` IN (%d, %d)",
+		intval(TERM_OBJ_POST),
+		intval($itemid),
+		intval(TERM_HASHTAG),
+		intval(TERM_MENTION));
+
+	if ($message["deleted"])
+		return;
+
+	$cachefile = get_cachefile($message["guid"]."-".hash("md5", $message['body']));
+
+	if (($cachefile != '') AND !file_exists($cachefile)) {
+		$s = prepare_text($message['body']);
+		$stamp1 = microtime(true);
+		file_put_contents($cachefile, $s);
+		$a->save_timestamp($stamp1, "file");
+		logger('create_tags_from_item: put item '.$message["id"].' into cachefile '.$cachefile);
+	}
+
+	$taglist = explode(",", $message["tag"]);
+
+	$tags = "";
+	foreach ($taglist as $tag)
+		if ((substr(trim($tag), 0, 1) == "#") OR (substr(trim($tag), 0, 1) == "@"))
+			$tags .= " ".trim($tag);
+		else
+			$tags .= " #".trim($tag);
+
+	$data = " ".$message["title"]." ".$message["body"]." ".$tags." ";
+
+	$tags = array();
+
+	$pattern = "/\W\#([^\[].*?)[\s'\".,:;\?!\[\]\/]/ism";
+	if (preg_match_all($pattern, $data, $matches))
+		foreach ($matches[1] as $match)
+			$tags["#".strtolower($match)] = ""; // $searchpath.strtolower($match);
+
+	$pattern = "/\W([\#@])\[url\=(.*?)\](.*?)\[\/url\]/ism";
+	if (preg_match_all($pattern, $data, $matches, PREG_SET_ORDER)) {
+		foreach ($matches as $match)
+			$tags[$match[1].strtolower(trim($match[3], ',.:;[]/\"?!'))] = $match[2];
+	}
+
+	foreach ($tags as $tag=>$link) {
+
+		if (substr(trim($tag), 0, 1) == "#") {
+			$type = TERM_HASHTAG;
+			$term = substr($tag, 1);
+		} elseif (substr(trim($tag), 0, 1) == "@") {
+			$type = TERM_MENTION;
+			$term = substr($tag, 1);
+		} else { // This shouldn't happen
+			$type = TERM_HASHTAG;
+			$term = $tag;
+		}
+
+		$r = q("INSERT INTO `term` (`uid`, `oid`, `otype`, `type`, `term`, `url`) VALUES (%d, %d, %d, %d, '%s', '%s')",
+			intval($message["uid"]), intval($itemid), intval(TERM_OBJ_POST), intval($type), dbesc($term), dbesc($link));
+
+		// Search for mentions
+		if ((substr($tag, 0, 1) == '@') AND (strpos($link, $profile_base_friendica) OR strpos($link, $profile_base_diaspora))) {
+			$users = q("SELECT `uid` FROM `contact` WHERE self AND (`url` = '%s' OR `nurl` = '%s')", $link, $link);
+			foreach ($users AS $user) {
+				if ($user["uid"] == $message["uid"])
+					q("UPDATE `item` SET `mention` = 1 WHERE `id` = %d", intval($itemid));
+			}
+		}
+	}
+}
+
+function create_tags_from_itemuri($itemuri, $uid) {
+	$messages = q("SELECT `id` FROM `item` WHERE uri ='%s' AND uid=%d", dbesc($itemuri), intval($uid));
+
+	foreach ($messages as $message)
+		create_tags_from_item($message["id"]);
+}
+
+function update_items() {
+	//$messages = q("SELECT `id` FROM `item` where tag !='' ORDER BY `created` DESC limit 10");
+	$messages = q("SELECT `id` FROM `item` where tag !=''");
+
+	foreach ($messages as $message)
+		create_tags_from_item($message["id"]);
+}
+
+//print_r($tags);
+//print_r($hashtags);
+//print_r($mentions);
+//update_items();
+//create_tags_from_item(265194);
+//create_tags_from_itemuri("infoagent@diasp.org:cce94abd104c06e8", 2);
+?>

include/text.php

@@ -15,7 +15,8 @@ if(! function_exists('replace_macros')) {
 function replace_macros($s,$r) {
 	global $t;
 
-//	$ts = microtime();
+	$stamp1 = microtime(true);
+
 	$a = get_app();
 
 	if($a->theme['template_engine'] === 'smarty3') {
@@ -37,9 +38,9 @@ function replace_macros($s,$r) {
 
 		$output = template_unescape($r);
 	}
-//	$tt = microtime() - $ts;
+	$a = get_app();
-//	$a = get_app();
+	$a->save_timestamp($stamp1, "rendering");
-//	$a->page['debug'] .= "$tt <br>\n";
+
 	return $output;
 }}
 
@@ -438,15 +439,26 @@ function load_view_file($s) {
 		$lang = 'en';
 	$b = basename($s);
 	$d = dirname($s);
-	if(file_exists("$d/$lang/$b"))
+	if(file_exists("$d/$lang/$b")) {
-		return file_get_contents("$d/$lang/$b");
+		$stamp1 = microtime(true);
+		$content = file_get_contents("$d/$lang/$b");
+		$a->save_timestamp($stamp1, "file");
+		return $content;
+	}
 
 	$theme = current_theme();
 
-	if(file_exists("$d/theme/$theme/$b"))
+	if(file_exists("$d/theme/$theme/$b")) {
-		return file_get_contents("$d/theme/$theme/$b");
+		$stamp1 = microtime(true);
+		$content = file_get_contents("$d/theme/$theme/$b");
+		$a->save_timestamp($stamp1, "file");
+		return $content;
+	}
 
-	return file_get_contents($s);
+	$stamp1 = microtime(true);
+	$content = file_get_contents($s);
+	$a->save_timestamp($stamp1, "file");
+	return $content;
 }}
 
 if(! function_exists('get_intltext_template')) {
@@ -461,17 +473,28 @@ function get_intltext_template($s) {
 	if(! isset($lang))
 		$lang = 'en';
 
-	if(file_exists("view/$lang$engine/$s"))
+	if(file_exists("view/$lang$engine/$s")) {
-		return file_get_contents("view/$lang$engine/$s");
+		$stamp1 = microtime(true);
-	elseif(file_exists("view/en$engine/$s"))
+		$content = file_get_contents("view/$lang$engine/$s");
-		return file_get_contents("view/en$engine/$s");
+		$a->save_timestamp($stamp1, "file");
-	else
+		return $content;
-		return file_get_contents("view$engine/$s");
+	} elseif(file_exists("view/en$engine/$s")) {
+		$stamp1 = microtime(true);
+		$content = file_get_contents("view/en$engine/$s");
+		$a->save_timestamp($stamp1, "file");
+		return $content;
+	} else {
+		$stamp1 = microtime(true);
+		$content = file_get_contents("view$engine/$s");
+		$a->save_timestamp($stamp1, "file");
+		return $content;
+	}
 }}
 
 if(! function_exists('get_markup_template')) {
 function get_markup_template($s, $root = '') {
-//	$ts = microtime();
+	$stamp1 = microtime(true);
+
 	$a = get_app();
 
 	if($a->theme['template_engine'] === 'smarty3') {
@@ -479,18 +502,19 @@ function get_markup_template($s, $root = '') {
 
 		$template = new FriendicaSmarty();
 		$template->filename = $template_file;
+		$a->save_timestamp($stamp1, "rendering");
 
-//		$tt = microtime() - $ts;
-//		$a->page['debug'] .= "$tt <br>\n";
 		return $template;
 	}
 	else {
 		$template_file = get_template_file($a, $s, $root);
-//		$file_contents = file_get_contents($template_file);
+		$a->save_timestamp($stamp1, "rendering");
-//		$tt = microtime() - $ts;
+
-//		$a->page['debug'] .= "$tt <br>\n";
+		$stamp1 = microtime(true);
-//		return $file_contents;
+		$content = file_get_contents($template_file);
-		return file_get_contents($template_file);
+		$a->save_timestamp($stamp1, "file");
+		return $content;
+
 	}
 }}
 
@@ -548,7 +572,9 @@ function logger($msg,$level = 0) {
 	if((! $debugging) || (! $logfile) || ($level > $loglevel))
 		return;
 
+	$stamp1 = microtime(true);
 	@file_put_contents($logfile, datetime_convert() . ':' . session_id() . ' ' . $msg . "\n", FILE_APPEND);
+	$a->save_timestamp($stamp1, "file");
 	return;
 }}
 
@@ -574,11 +600,12 @@ function get_tags($s) {
 	$ret = array();
 
 	// ignore anything in a code block
-
 	$s = preg_replace('/\[code\](.*?)\[\/code\]/sm','',$s);
 
-	// ignore anything in a bbtag
+	// Force line feeds at bbtags
+	$s = str_replace(array("[", "]"), array("\n[", "]\n"), $s);
 
+	// ignore anything in a bbtag
 	$s = preg_replace('/\[(.*?)\]/sm','',$s);
 
 	// Match full names against @tags including the space between first and last
@@ -1030,14 +1057,20 @@ function prepare_body($item,$attach = false) {
 	$a = get_app();
 	call_hooks('prepare_body_init', $item);
 
-	$cachefile = get_cachefile($item["guid"]."-".strtotime($item["edited"])."-".hash("crc32", $item['body']));
+	//$cachefile = get_cachefile($item["guid"]."-".strtotime($item["edited"])."-".hash("crc32", $item['body']));
+	$cachefile = get_cachefile($item["guid"]."-".hash("md5", $item['body']));
 
 	if (($cachefile != '')) {
-		if (file_exists($cachefile))
+		if (file_exists($cachefile)) {
+			$stamp1 = microtime(true);
 			$s = file_get_contents($cachefile);
-		else {
+			$a->save_timestamp($stamp1, "file");
+		} else {
 			$s = prepare_text($item['body']);
+			$stamp1 = microtime(true);
 			file_put_contents($cachefile, $s);
+			$a->save_timestamp($stamp1, "file");
+			logger('prepare_body: put item '.$item["id"].' into cachefile '.$cachefile);
 		}
 	} else
 		$s = prepare_text($item['body']);

mod/admin.php

@@ -130,7 +130,6 @@ function admin_content(&$a) {
 	 * Page content
 	 */
 	$o = '';
-	
 	// urls
 	if ($a->argc > 1){
 		switch ($a->argv[1]){
@@ -834,6 +833,7 @@ function admin_page_plugins(&$a){
 		}
 
 		$t = get_markup_template("admin_plugins_details.tpl");
+
 		return replace_macros($t, array(
 			'$title' => t('Administration'),
 			'$page' => t('Plugins'),
@@ -1037,7 +1037,6 @@ function admin_page_themes(&$a){
 
 		}
 
-
 		$screenshot = array( get_theme_screenshot($theme), t('Screenshot'));
 		if(! stristr($screenshot[0],$theme))
 			$screenshot = null;
@@ -1065,8 +1064,6 @@ function admin_page_themes(&$a){
 		));
 	}
 
-	 
-	
 	/**
 	 * List themes
 	 */

mod/community.php

@@ -64,23 +64,25 @@ function community_content(&$a, $update = 0) {
 
 	}
 
-	$r = q("SELECT distinct(`item`.`uri`), `item`.*, `item`.`id` AS `item_id`, 
+	//$r = q("SELECT distinct(`item`.`uri`)
+	$r = q("SELECT `item`.`uri`, `item`.*, `item`.`id` AS `item_id`, 
 		`contact`.`name`, `contact`.`photo`, `contact`.`url`, `contact`.`alias`, `contact`.`rel`,
 		`contact`.`network`, `contact`.`thumb`, `contact`.`self`, `contact`.`writable`, 
 		`contact`.`id` AS `cid`, `contact`.`uid` AS `contact-uid`,
 		`user`.`nickname`, `user`.`hidewall`
-		FROM `item` LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
+		FROM `item` FORCE INDEX (`received`) LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
 		LEFT JOIN `user` ON `user`.`uid` = `item`.`uid`
 		WHERE `item`.`visible` = 1 AND `item`.`deleted` = 0 and `item`.`moderated` = 0
 		AND `item`.`allow_cid` = ''  AND `item`.`allow_gid` = ''
 		AND `item`.`deny_cid`  = '' AND `item`.`deny_gid`  = '' 
 		AND `item`.`private` = 0 AND `item`.`wall` = 1 AND `user`.`hidewall` = 0
-		AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0 group by `item`.`uri`
+		AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0 AND `contact`.`self`
 		ORDER BY `received` DESC LIMIT %d, %d ",
 		intval($a->pager['start']),
 		intval($a->pager['itemspage'])
 
 	);
+//		group by `item`.`uri`
 
 	if(! count($r)) {
 		info( t('No results.') . EOL);

mod/content.php

@@ -160,16 +160,26 @@ function content_content(&$a, $update = 0) {
 
 	$sql_extra2 = (($nouveau) ? '' : " AND `item`.`parent` = `item`.`id` ");
 	$sql_extra3 = (($nouveau) ? '' : $sql_extra3);
+	$sql_table = "`item`";
 
 	if(x($_GET,'search')) {
 		$search = escape_tags($_GET['search']);
-		if (get_config('system','use_fulltext_engine')) {
+
+		if(strpos($search,'#') === 0) {
+			$tag = true;
+			$search = substr($search,1);
+		}
+
+		if (get_config('system','only_tag_search'))
+			$tag = true;
+
+		/*if (get_config('system','use_fulltext_engine')) {
 			if(strpos($search,'#') === 0)
-				$sql_extra .= sprintf(" AND (MATCH(tag) AGAINST ('".'"%s"'."' in boolean mode)) ",
+				$sql_extra .= sprintf(" AND (MATCH(tag) AGAINST ('%s' in boolean mode)) ",
 					dbesc(protect_sprintf($search))
 				);
 			else
-				$sql_extra .= sprintf(" AND (MATCH(`item`.`body`) AGAINST ('".'"%s"'."' in boolean mode) or MATCH(tag) AGAINST ('".'"%s"'."' in boolean mode)) ",
+				$sql_extra .= sprintf(" AND (MATCH(`item`.`body`, `item`.`title`) AGAINST ('%s' in boolean mode)) ",
 					dbesc(protect_sprintf($search)),
 					dbesc(protect_sprintf($search))
 				);
@@ -178,7 +188,19 @@ function content_content(&$a, $update = 0) {
 					dbesc(protect_sprintf('%' . $search . '%')),
 					dbesc(protect_sprintf('%]' . $search . '[%'))
 			);
+		}*/
+
+		if($tag) {
+			$sql_extra = sprintf(" AND `term`.`term` = '%s' AND `term`.`otype` = %d AND `term`.`type` = %d ",
+				dbesc(protect_sprintf($search)), intval(TERM_OBJ_POST), intval(TERM_HASHTAG));
+			$sql_table = "`term` LEFT JOIN `item` ON `item`.`id` = `term`.`oid` AND `item`.`uid` = `term`.`uid` ";
+		} else {
+			if (get_config('system','use_fulltext_engine'))
+				$sql_extra = sprintf(" AND MATCH (`item`.`body`, `item`.`title`) AGAINST ('%s' in boolean mode) ", dbesc(protect_sprintf($search)));
+			else
+				$sql_extra = sprintf(" AND `item`.`body` REGEXP '%s' ", dbesc(protect_sprintf(preg_quote($search))));
 		}
+
 	}
 	if(strlen($file)) {
 		$sql_extra .= file_tag_file_query('item',unxmlify($file));
@@ -189,26 +211,28 @@ function content_content(&$a, $update = 0) {
 		$myurl = substr($myurl,strpos($myurl,'://')+3);
 		$myurl = str_replace('www.','',$myurl);
 		$diasp_url = str_replace('/profile/','/u/',$myurl);
-		if (get_config('system','use_fulltext_engine'))
+		/*if (get_config('system','use_fulltext_engine'))
-			$sql_extra .= sprintf(" AND `item`.`parent` IN (SELECT distinct(`parent`) from item where (MATCH(`author-link`) AGAINST ('".'"%s"'."' in boolean mode) or MATCH(`tag`) AGAINST ('".'"%s"'."' in boolean mode) or MATCH(tag) AGAINST ('".'"%s"'."' in boolean mode))) ",
+			$sql_extra .= sprintf(" AND `item`.`parent` IN (SELECT distinct(`parent`) from $sql_table where (MATCH(`author-link`, `tag`) AGAINST ('%s' in boolean mode) or MATCH(tag) AGAINST ('%s' in boolean mode))) ",
 				dbesc(protect_sprintf($myurl)),
 				dbesc(protect_sprintf($myurl)),
 				dbesc(protect_sprintf($diasp_url))
 			);
 		else
-			$sql_extra .= sprintf(" AND `item`.`parent` IN (SELECT distinct(`parent`) from item where ( `author-link` like '%s' or `tag` like '%s' or tag like '%s' )) ",
+			$sql_extra .= sprintf(" AND `item`.`parent` IN (SELECT distinct(`parent`) from $sql_table where ( `author-link` like '%s' or `tag` like '%s' or tag like '%s' )) ",
 				dbesc(protect_sprintf('%' . $myurl)),
 				dbesc(protect_sprintf('%' . $myurl . ']%')),
 				dbesc(protect_sprintf('%' . $diasp_url . ']%'))
-			);
+			);*/
 
+		$sql_extra .= sprintf(" AND `item`.`parent` IN (SELECT distinct(`parent`) from item where `author-link` IN ('https://%s', 'http://%s') OR `mention`)",
+			dbesc(protect_sprintf($myurl)),
+			dbesc(protect_sprintf($myurl))
+		);
 	}
 
 	$pager_sql = sprintf(" LIMIT %d, %d ",intval($a->pager['start']), intval($a->pager['itemspage']));
 
 
-
-
 	if($nouveau) {
 		// "New Item View" - show all items unthreaded in reverse created date order
 
@@ -216,11 +240,10 @@ function content_content(&$a, $update = 0) {
 			`contact`.`name`, `contact`.`photo`, `contact`.`url`, `contact`.`rel`, `contact`.`writable`,
 			`contact`.`network`, `contact`.`thumb`, `contact`.`dfrn-id`, `contact`.`self`,
 			`contact`.`id` AS `cid`, `contact`.`uid` AS `contact-uid`
-			FROM `item`, `contact`
+			FROM $sql_table LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
 			WHERE `item`.`uid` = %d AND `item`.`visible` = 1
 			AND `item`.`deleted` = 0 and `item`.`moderated` = 0
 			$simple_update
-			AND `contact`.`id` = `item`.`contact-id`
 			AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
 			$sql_extra $sql_nets
 			ORDER BY `item`.`received` DESC $pager_sql ",
@@ -241,7 +264,7 @@ function content_content(&$a, $update = 0) {
 		$start = dba_timer();
 
 		$r = q("SELECT `item`.`id` AS `item_id`, `contact`.`uid` AS `contact_uid`
-			FROM `item` LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
+			FROM $sql_table LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
 			WHERE `item`.`uid` = %d AND `item`.`visible` = 1 AND `item`.`deleted` = 0
 			AND `item`.`moderated` = 0 AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
 			AND `item`.`parent` = `item`.`id`
@@ -268,9 +291,9 @@ function content_content(&$a, $update = 0) {
 				`contact`.`name`, `contact`.`photo`, `contact`.`url`, `contact`.`alias`, `contact`.`rel`, `contact`.`writable`,
 				`contact`.`network`, `contact`.`thumb`, `contact`.`dfrn-id`, `contact`.`self`,
 				`contact`.`id` AS `cid`, `contact`.`uid` AS `contact-uid`
-				FROM `item`, `contact`
+				FROM $sql_table LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
 				WHERE `item`.`uid` = %d AND `item`.`visible` = 1 AND `item`.`deleted` = 0
-				AND `item`.`moderated` = 0 AND `contact`.`id` = `item`.`contact-id`
+				AND `item`.`moderated` = 0
 				AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
 				AND `item`.`parent` IN ( %s )
 				$sql_extra ",

mod/item.php

@@ -19,6 +19,7 @@ require_once('include/crypto.php');
 require_once('include/enotify.php');
 require_once('include/email.php');
 require_once('library/langdet/Text/LanguageDetect.php');
+require_once('include/tags.php');
 
 function item_post(&$a) {
 
@@ -236,9 +237,23 @@ function item_post(&$a) {
 
 		if (version_compare(PHP_VERSION, '5.3.0', '>=')) {
 			$l = new Text_LanguageDetect;
-			$lng = $l->detectConfidence($naked_body);
+			//$lng = $l->detectConfidence($naked_body);
+			//$postopts = (($lng['language']) ? 'lang=' . $lng['language'] . ';' . $lng['confidence'] : '');
 
-			$postopts = (($lng['language']) ? 'lang=' . $lng['language'] . ';' . $lng['confidence'] : '');
+			$lng = $l->detect($naked_body, 3);
+
+			if (sizeof($lng) > 0) {
+				$postopts = "";
+
+				foreach ($lng as $language => $score) {
+					if ($postopts == "")
+						$postopts = "lang=";
+					else
+						$postopts .= ":";
+
+					$postopts .= $language.";".$score;
+				}
+			}
 
 			logger('mod_item: detect language' . print_r($lng,true) . $naked_body, LOGGER_DATA);
 		}
@@ -675,6 +690,7 @@ function item_post(&$a) {
 			intval($post_id),
 			intval($profile_uid)
 		);
+		create_tags_from_itemuri($post_id, $profile_uid);
 
 		// update filetags in pconfig
                 file_tag_update_pconfig($uid,$categories_old,$categories_new,'category');
@@ -740,10 +756,22 @@ function item_post(&$a) {
 	if(count($r)) {
 		$post_id = $r[0]['id'];
 		logger('mod_item: saved item ' . $post_id);
+		create_tags_from_item($post_id);
 
 		// update filetags in pconfig
                 file_tag_update_pconfig($uid,$categories_old,$categories_new,'category');
 
+		// Store the fresh generated item into the cache
+		$cachefile = get_cachefile($datarray["guid"]."-".hash("md5", $datarray['body']));
+
+		if (($cachefile != '') AND !file_exists($cachefile)) {
+			$s = prepare_text($datarray['body']);
+			$stamp1 = microtime(true);
+			file_put_contents($cachefile, $s);
+			$a->save_timestamp($stamp1, "file");
+			logger('mod_item: put item '.$r[0]['id'].' into cachefile '.$cachefile);
+		}
+
 		if($parent) {
 
 			// This item is the last leaf and gets the comment box, clear any ancestors

mod/network.php

@@ -573,7 +573,7 @@ function network_content(&$a, $update = 0) {
 	}
 
 
-	// We don't have to deal with ACL's on this page. You're looking at everything
+	// We don't have to deal with ACLs on this page. You're looking at everything
 	// that belongs to you, hence you can see all of it. We will filter by group if
 	// desired. 
 
@@ -647,16 +647,26 @@ function network_content(&$a, $update = 0) {
 
 	$sql_extra2 = (($nouveau) ? '' : " AND `item`.`parent` = `item`.`id` ");
 	$sql_extra3 = (($nouveau) ? '' : $sql_extra3);
+	$sql_table = "`item`";
 
 	if(x($_GET,'search')) {
 		$search = escape_tags($_GET['search']);
-		if (get_config('system','use_fulltext_engine')) {
+
+		if(strpos($search,'#') === 0) {
+                	$tag = true;
+			$search = substr($search,1);
+		}
+
+		if (get_config('system','only_tag_search'))
+			$tag = true;
+
+		/*if (get_config('system','use_fulltext_engine')) {
 			if(strpos($search,'#') === 0)
 				$sql_extra .= sprintf(" AND (MATCH(tag) AGAINST ('".'"%s"'."' in boolean mode)) ",
 					dbesc(protect_sprintf($search))
 				);
 			else
-				$sql_extra .= sprintf(" AND (MATCH(`item`.`body`) AGAINST ('".'"%s"'."' in boolean mode) or MATCH(tag) AGAINST ('".'"%s"'."' in boolean mode)) ",
+				$sql_extra .= sprintf(" AND (MATCH(`item`.`body`, `item`.`title`) AGAINST ('%s' in boolean mode)) ",
 					dbesc(protect_sprintf($search)),
 					dbesc(protect_sprintf($search))
 				);
@@ -665,6 +675,17 @@ function network_content(&$a, $update = 0) {
 					dbesc(protect_sprintf('%' . $search . '%')),
 					dbesc(protect_sprintf('%]' . $search . '[%'))
 			);
+		}*/
+
+		if($tag) {
+			$sql_extra = sprintf(" AND `term`.`term` = '%s' AND `term`.`otype` = %d AND `term`.`type` = %d ",
+					dbesc(protect_sprintf($search)), intval(TERM_OBJ_POST), intval(TERM_HASHTAG));
+			$sql_table = "`term` LEFT JOIN `item` ON `item`.`id` = `term`.`oid` AND `item`.`uid` = `term`.`uid` ";
+		} else {
+			if (get_config('system','use_fulltext_engine'))
+				$sql_extra = sprintf(" AND MATCH (`item`.`body`, `item`.`title`) AGAINST ('%s' in boolean mode) ", dbesc(protect_sprintf($search)));
+			else
+				$sql_extra = sprintf(" AND `item`.`body` REGEXP '%s' ", dbesc(protect_sprintf(preg_quote($search))));
 		}
 	}
 	if(strlen($file)) {
@@ -676,8 +697,10 @@ function network_content(&$a, $update = 0) {
 		$myurl = substr($myurl,strpos($myurl,'://')+3);
 		$myurl = str_replace('www.','',$myurl);
 		$diasp_url = str_replace('/profile/','/u/',$myurl);
-		if (get_config('system','use_fulltext_engine'))
+		/*if (get_config('system','use_fulltext_engine'))
 			$sql_extra .= sprintf(" AND `item`.`parent` IN (SELECT distinct(`parent`) from item where (MATCH(`author-link`) AGAINST ('".'"%s"'."' in boolean mode) or MATCH(`tag`) AGAINST ('".'"%s"'."' in boolean mode) or MATCH(tag) AGAINST ('".'"%s"'."' in boolean mode))) ",
+			//$sql_extra .= sprintf(" AND `item`.`parent` IN (SELECT distinct(`parent`) from item where (`author-link` IN ('https://%s', 'http://%s') OR MATCH(`tag`) AGAINST ('".'"%s"'."' in boolean mode) or MATCH(tag) AGAINST ('".'"%s"'."' in boolean mode))) ",
+			//$sql_extra .= sprintf(" AND `item`.`parent` IN (SELECT distinct(`parent`) from item where `author-link` IN ('https://%s', 'http://%s') OR `mention`)",
 				dbesc(protect_sprintf($myurl)),
 				dbesc(protect_sprintf($myurl)),
 				dbesc(protect_sprintf($diasp_url))
@@ -687,8 +710,12 @@ function network_content(&$a, $update = 0) {
 				dbesc(protect_sprintf('%' . $myurl)),
 				dbesc(protect_sprintf('%' . $myurl . ']%')),
 				dbesc(protect_sprintf('%' . $diasp_url . ']%'))
-			);
+			);*/
 
+		$sql_extra .= sprintf(" AND `item`.`parent` IN (SELECT distinct(`parent`) from item where `author-link` IN ('https://%s', 'http://%s') OR `mention`)",
+			dbesc(protect_sprintf($myurl)),
+			dbesc(protect_sprintf($myurl))
+		);
 	}
 
 	if($update) {
@@ -700,7 +727,7 @@ function network_content(&$a, $update = 0) {
 	else {
 		if( (! get_config('alt_pager', 'global')) && (! get_pconfig(local_user(),'system','alt_pager')) ) {
 		        $r = q("SELECT COUNT(*) AS `total`
-			        FROM `item` LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
+			        FROM $sql_table LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
 			        WHERE `item`.`uid` = %d AND `item`.`visible` = 1 AND `item`.`deleted` = 0
 			        AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
 			        $sql_extra2 $sql_extra3
@@ -731,11 +758,10 @@ function network_content(&$a, $update = 0) {
 			`contact`.`name`, `contact`.`photo`, `contact`.`url`, `contact`.`rel`, `contact`.`writable`,
 			`contact`.`network`, `contact`.`thumb`, `contact`.`dfrn-id`, `contact`.`self`,
 			`contact`.`id` AS `cid`, `contact`.`uid` AS `contact-uid`
-			FROM `item`, `contact`
+			FROM $sql_table LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
 			WHERE `item`.`uid` = %d AND `item`.`visible` = 1 
 			AND `item`.`deleted` = 0 and `item`.`moderated` = 0
 			$simple_update
-			AND `contact`.`id` = `item`.`contact-id`
 			AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
 			$sql_extra $sql_nets
 			ORDER BY `item`.`received` DESC $pager_sql ",
@@ -758,7 +784,7 @@ function network_content(&$a, $update = 0) {
 
 		if($update) {
 			$r = q("SELECT `parent` AS `item_id`, `contact`.`uid` AS `contact_uid`
-				FROM `item` LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
+				FROM $sql_table LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
 				WHERE `item`.`uid` = %d AND `item`.`visible` = 1 AND
 				(`item`.`deleted` = 0 OR item.verb = '" . ACTIVITY_LIKE ."' OR item.verb = '" . ACTIVITY_DISLIKE . "')
 				and `item`.`moderated` = 0 and `item`.`unseen` = 1
@@ -769,7 +795,7 @@ function network_content(&$a, $update = 0) {
 		}
 		else {
 			$r = q("SELECT `item`.`id` AS `item_id`, `contact`.`uid` AS `contact_uid`
-				FROM `item` LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
+				FROM $sql_table LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
 				WHERE `item`.`uid` = %d AND `item`.`visible` = 1 AND `item`.`deleted` = 0
 				AND `item`.`moderated` = 0 AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
 				AND `item`.`parent` = `item`.`id`
@@ -794,9 +820,9 @@ function network_content(&$a, $update = 0) {
 				`contact`.`name`, `contact`.`photo`, `contact`.`url`, `contact`.`alias`, `contact`.`rel`, `contact`.`writable`,
 				`contact`.`network`, `contact`.`thumb`, `contact`.`dfrn-id`, `contact`.`self`,
 				`contact`.`id` AS `cid`, `contact`.`uid` AS `contact-uid`
-				FROM `item`, `contact`
+				FROM $sql_table LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
 				WHERE `item`.`uid` = %d AND `item`.`visible` = 1 AND `item`.`deleted` = 0
-				AND `item`.`moderated` = 0 AND `contact`.`id` = `item`.`contact-id`
+				AND `item`.`moderated` = 0
 				AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
 				AND `item`.`parent` IN ( %s )
 				$sql_extra ",

mod/photo.php

@@ -169,10 +169,13 @@ function photo_init(&$a) {
 		}
 	}
 
-	if(isset($customres) && $customres > 0 && $customres < 500) {
+	// Resize only if its not a GIF
+	if ($mime != "image/gif") {
 		$ph = new Photo($data, $mimetype);
 		if($ph->is_valid()) {
+			if(isset($customres) && $customres > 0 && $customres < 500) {
 				$ph->scaleImageSquare($customres);
+			}
 			$data = $ph->imageString();
 			$mimetype = $ph->getType();
 		}

mod/photos.php

@@ -5,7 +5,7 @@ require_once('include/acl_selectors.php');
 require_once('include/bbcode.php');
 require_once('include/security.php');
 require_once('include/redir.php');
-
+require_once('include/tags.php');
 
 function photos_init(&$a) {
 
@@ -252,6 +252,7 @@ function photos_post(&$a) {
 						dbesc($rr['parent-uri']),
 						intval($page_owner_uid)
 					);
+					create_tags_from_itemuri($rr['parent-uri'], $page_owner_uid);
 
 					$drop_id = intval($rr['id']);
 
@@ -321,6 +322,7 @@ function photos_post(&$a) {
 					dbesc($i[0]['uri']),
 					intval($page_owner_uid)
 				);
+				create_tags_from_itemuri($i[0]['uri'], $page_owner_uid);
 
 				$url = $a->get_baseurl();
 				$drop_id = intval($i[0]['id']);
@@ -605,6 +607,7 @@ function photos_post(&$a) {
 				intval($item_id),
 				intval($page_owner_uid)
 			);
+			create_tags_from_item($item_id);
 
 			$best = 0;
 			foreach($p as $scales) {

mod/search.php

@@ -124,7 +124,10 @@ function search_content(&$a) {
 	if(! $search)
 		return $o;
 
-	if (get_config('system','use_fulltext_engine')) {
+	if (get_config('system','only_tag_search'))
+		$tag = true;
+
+	/*if (get_config('system','use_fulltext_engine')) {
 		if($tag)
 			$sql_extra = sprintf(" AND MATCH (`item`.`tag`) AGAINST ('".'"%s"'."' in boolean mode) ", '#'.dbesc(protect_sprintf($search)));
 		else
@@ -134,10 +137,24 @@ function search_content(&$a) {
 			$sql_extra = sprintf(" AND `item`.`tag` REGEXP '%s' ", 	dbesc('\\]' . protect_sprintf(preg_quote($search)) . '\\['));
 		else
 			$sql_extra = sprintf(" AND `item`.`body` REGEXP '%s' ", dbesc(protect_sprintf(preg_quote($search))));
+	}*/
+
+	if($tag) {
+		//$sql_extra = sprintf(" AND `term`.`term` = '%s' AND `term`.`otype` = %d AND `term`.`type` = %d",
+		//			dbesc(protect_sprintf($search)), intval(TERM_OBJ_POST), intval(TERM_HASHTAG));
+		//$sql_table = "`term` LEFT JOIN `item` ON `item`.`id` = `term`.`oid` AND `item`.`uid` = `term`.`uid` ";
+
+		$sql_extra = sprintf(" AND EXISTS (SELECT * FROM `term` WHERE `item`.`id` = `term`.`oid` AND `item`.`uid` = `term`.`uid` AND `term`.`term` = '%s' AND `term`.`otype` = %d AND `term`.`type` = %d) GROUP BY `item`.`uri` ",
+					dbesc(protect_sprintf($search)), intval(TERM_OBJ_POST), intval(TERM_HASHTAG));
+		$sql_table = "`item` FORCE INDEX (`uri`) ";
+	} else {
+		if (get_config('system','use_fulltext_engine')) {
+			$sql_extra = sprintf(" AND MATCH (`item`.`body`, `item`.`title`) AGAINST ('%s' in boolean mode) ", dbesc(protect_sprintf($search)));
+		} else {
+			$sql_extra = sprintf(" AND `item`.`body` REGEXP '%s' ", dbesc(protect_sprintf(preg_quote($search))));
+		}
+		$sql_table = "`item`";
 	}
-
-
-
 
 	// Here is the way permissions work in the search module...
 	// Only public posts can be shown
@@ -146,14 +163,15 @@ function search_content(&$a) {
 
 	if( (! get_config('alt_pager', 'global')) && (! get_pconfig(local_user(),'system','alt_pager')) ) {
 	        $r = q("SELECT distinct(`item`.`uri`) as `total`
-		        FROM `item` LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id` LEFT JOIN `user` ON `user`.`uid` = `item`.`uid`
+		        FROM $sql_table LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id` LEFT JOIN `user` ON `user`.`uid` = `item`.`uid`
 		        WHERE `item`.`visible` = 1 AND `item`.`deleted` = 0 and `item`.`moderated` = 0
 		        AND (( `item`.`allow_cid` = ''  AND `item`.`allow_gid` = '' AND `item`.`deny_cid`  = '' AND `item`.`deny_gid`  = '' AND `item`.`private` = 0 AND `user`.`hidewall` = 0) 
 			        OR ( `item`.`uid` = %d ))
 		        AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
-		        $sql_extra group by `item`.`uri` ", 
+		        $sql_extra ",
 		        intval(local_user())
 	        );
+//		        $sql_extra group by `item`.`uri` ",
 
 	        if(count($r))
 		        $a->set_pager_total(count($r));
@@ -164,25 +182,25 @@ function search_content(&$a) {
 	        }
 	}
 
-	$r = q("SELECT distinct(`item`.`uri`), `item`.*, `item`.`id` AS `item_id`, 
+	$r = q("SELECT `item`.`uri`, `item`.*, `item`.`id` AS `item_id`, 
 		`contact`.`name`, `contact`.`photo`, `contact`.`url`, `contact`.`alias`, `contact`.`rel`,
 		`contact`.`network`, `contact`.`thumb`, `contact`.`self`, `contact`.`writable`, 
 		`contact`.`id` AS `cid`, `contact`.`uid` AS `contact-uid`,
 		`user`.`nickname`, `user`.`uid`, `user`.`hidewall`
-		FROM `item` LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
+		FROM $sql_table LEFT JOIN `contact` ON `contact`.`id` = `item`.`contact-id`
 		LEFT JOIN `user` ON `user`.`uid` = `item`.`uid`
 		WHERE `item`.`visible` = 1 AND `item`.`deleted` = 0 and `item`.`moderated` = 0
 		AND (( `item`.`allow_cid` = ''  AND `item`.`allow_gid` = '' AND `item`.`deny_cid`  = '' AND `item`.`deny_gid`  = '' AND `item`.`private` = 0 AND `user`.`hidewall` = 0 ) 
 			OR ( `item`.`uid` = %d ))
 		AND `contact`.`blocked` = 0 AND `contact`.`pending` = 0
 		$sql_extra
-		group by `item`.`uri`	
 		ORDER BY `received` DESC LIMIT %d , %d ",
 		intval(local_user()),
 		intval($a->pager['start']),
 		intval($a->pager['itemspage'])
 
 	);
+//		group by `item`.`uri`
 
 	if(! count($r)) {
 		info( t('No results.') . EOL);

mods/readme.txt

@@ -3,3 +3,4 @@ Site speed can be improved when the following indexes are set. They cannot be se
 CREATE INDEX `uid_commented` ON `item` (`uid`, `commented`);
 CREATE INDEX `uid_created` ON `item` (`uid`, `created`);
 CREATE INDEX `uid_unseen` ON `item` (`uid`, `unseen`);
+CREATE INDEX `resource-id` ON `item` (`resource-id`);

object/Conversation.php

@@ -127,6 +127,7 @@ class Conversation extends BaseObject {
 	 *      _ false on failure
 	 */
 	public function get_template_data($alike, $dlike) {
+		global $a;
 		$result = array();
 
 		foreach($this->threads as $item) {
@@ -140,6 +141,7 @@ class Conversation extends BaseObject {
 			$result[] = $item_data;
 		}
 
+		//$a->mark_timestamp();
 		return $result;
 	}
 

object/Item.php

@@ -146,10 +146,30 @@ class Item extends BaseObject {
 		call_hooks('render_location',$locate);
 		$location = ((strlen($locate['html'])) ? $locate['html'] : render_location_google($locate));
 
+		$searchpath = $a->get_baseurl()."/search?tag=";
 		$tags=array();
 		$hashtags = array();
 		$mentions = array();
-		foreach(explode(',',$item['tag']) as $tag){
+
+		$taglist = q("SELECT `type`, `term`, `url` FROM `term` WHERE `otype` = %d AND `oid` = %d AND `type` IN (%d, %d) ORDER BY `tid`",
+				intval(TERM_OBJ_POST), intval($item['id']), intval(TERM_HASHTAG), intval(TERM_MENTION));
+
+		foreach($taglist as $tag) {
+
+			if ($tag["url"] == "")
+				$tag["url"] = $searchpath.strtolower($tag["term"]);
+
+			if ($tag["type"] == TERM_HASHTAG) {
+				$hashtags[] = "#<a href=\"".$tag["url"]."\" target=\"external-link\">".$tag["term"]."</a>";
+				$prefix = "#";
+			} elseif ($tag["type"] == TERM_MENTION) {
+				$mentions[] = "@<a href=\"".$tag["url"]."\" target=\"external-link\">".$tag["term"]."</a>";
+				$prefix = "@";
+			}
+			$tags[] = $prefix."<a href=\"".$tag["url"]."\" target=\"external-link\">".$tag["term"]."</a>";
+		}
+
+		/*foreach(explode(',',$item['tag']) as $tag){
 			$tag = trim($tag);
 			if ($tag!="") {
 				$t = bbcode($tag);
@@ -159,8 +179,7 @@ class Item extends BaseObject {
 				elseif($t[0] == '@')
 					$mentions[] = $t;
 			}
-
+		}*/
-		}        
 
 		$like    = ((x($alike,$item['uri'])) ? format_like($alike[$item['uri']],$alike[$item['uri'] . '-l'],'like',$item['uri']) : '');
 		$dislike = ((x($dlike,$item['uri'])) ? format_like($dlike[$item['uri']],$dlike[$item['uri'] . '-l'],'dislike',$item['uri']) : '');
@@ -214,8 +233,28 @@ class Item extends BaseObject {
 		localize_item($item);
 
 		if ($item["postopts"]) {
-			$langdata = explode(";", $item["postopts"]);
+			//$langdata = explode(";", $item["postopts"]);
-			$langstr = substr($langdata[0], 5)." (".round($langdata[1]*100, 1)."%)";
+			//$langstr = substr($langdata[0], 5)." (".round($langdata[1]*100, 1)."%)";
+			$langstr = "";
+			if (substr($item["postopts"], 0, 5) == "lang=") {
+				$postopts = substr($item["postopts"], 5);
+
+				$languages = explode(":", $postopts);
+
+				if (sizeof($languages) == 1) {
+					$languages = array();
+					$languages[] = $postopts;
+				}
+
+				foreach ($languages as $language) {
+					$langdata = explode(";", $language);
+					if ($langstr != "")
+						$langstr .= ", ";
+
+					//$langstr .= $langdata[0]." (".round($langdata[1]*100, 1)."%)";
+					$langstr .= round($langdata[1]*100, 1)."% ".$langdata[0];
+				}
+			}
 		}
 
 		$body = prepare_body($item,true);

update.php

@@ -1,6 +1,6 @@
 <?php
 
-define( 'UPDATE_VERSION' , 1159 );
+define( 'UPDATE_VERSION' , 1162 );
 
 /**
  *
@@ -1382,8 +1382,6 @@ function update_1157() {
 
 	if($r)
 		return UPDATE_SUCCESS;
-
-	return UPDATE_FAILED;
 }
 
 function update_1158() {
@@ -1402,3 +1400,42 @@ function update_1158() {
 	return UPDATE_FAILED;
 }
 
+function update_1159() {
+	$r = q("ALTER TABLE `term` ADD `aid` int(10) unsigned NOT NULL DEFAULT '0',
+		ADD `uid` int(10) unsigned NOT NULL DEFAULT '0',
+		ADD INDEX (`uid`),
+		ADD INDEX (`aid`)");
+
+	if(!$r)
+		return UPDATE_FAILED;
+
+	require_once('include/tags.php');
+	update_items();
+
+	return UPDATE_SUCCESS;
+}
+
+function update_1160() {
+	set_config('system', 'maintenance', 1);
+
+	// Wait for 15 seconds for current requests to
+	// clear before locking up the database
+	sleep(15);
+
+	$r = q("ALTER TABLE `item` ADD `mention` TINYINT(1) NOT NULL DEFAULT '0', ADD INDEX (`mention`)");
+	set_config('system', 'maintenance', 0);
+
+	if(!$r)
+		return UPDATE_FAILED;
+
+	return UPDATE_SUCCESS;
+}
+
+function update_1161() {
+	$r = q("ALTER TABLE `pconfig` ADD INDEX (`cat`)");
+
+	if(!$r)
+		return UPDATE_FAILED;
+
+	return UPDATE_SUCCESS;
+}

view/theme/vier/profile_vcard.tpl

@@ -18,7 +18,7 @@
 			</ul>
 			</div>
 		{{ else }}
-			<div class="profile-edit-side-div"><a class="profile-edit-side-link icon edit" title="$editprofile" href="profiles/$profid" ></a></div>
+			<div class="profile-edit-side-div"><a class="profile-edit-side-link icon edit" title="$editprofile" href="profiles" ></a></div>
 		{{ endif }}
 	</div>
 

view/theme/vier/style.css

@@ -1020,6 +1020,9 @@ section {
   max-width: 766px;
   padding: 10px 10px 10px 10px;
   background-color: white;
+  border-bottom: 1px solid lightgray;
+  border-right: 1px solid lightgray;
+  border-left: 1px solid lightgray;
 }
 /* wall item */
 .tread-wrapper {
@@ -1091,6 +1094,7 @@ border-bottom: 1px solid #D2D2D2;
   max-width: 660px;
   word-wrap: break-word;
   line-height: 1.36;
+  padding-bottom: 6px;
 }
 
 .wall-item-container .wall-item-content img {
@@ -1304,14 +1308,16 @@ border-bottom: 1px solid #D2D2D2;
 .wall-item-tags {
   padding-top: 1px;
   padding-bottom: 2px;
+  /*display: none;*/
 }
+
 .tag {
   color: rgb(153,153,153);
-  padding-left: 3px;
+  /*padding-left: 3px;
-  font-size: 10px;
+  font-size: 10px;*/
 }
 .tag a {
-  padding-right: 8px;
+  /*padding-right: 8px;*/
   color: rgb(153,153,153);
 }
 .wwto {
@@ -1724,7 +1730,7 @@ div.pager, .birthday-notice, ul.tabs a, #jot-preview-link, .comment-edit-submit-
     background: linear-gradient(top, #ffffff 0%,#ececf2 100%);*/
 }
 
-#event-notice:hover, ul.tabs li .active, ul.tabs a:hover, #jot-preview-link:hover, .comment-edit-submit-wrapper .fakelink:hover {
+#event-notice:hover, #birthday-notice:hover, ul.tabs li .active, ul.tabs a:hover, #jot-preview-link:hover, .comment-edit-submit-wrapper .fakelink:hover {
     color: black;
     background-color: #e5e5e5;
     text-decoration: none;