Merge remote-tracking branch 'upstream/develop' into 1502-contacts-in-scrape

CHANGELOG

@@ -1,3 +1,32 @@
+Version 3.3.3
+    
+	More separation between php and html in photo album (issue #1258) (rabuzarus)
+	Enhanced community page shows public posts from public contacts of public profiles (annando)
+	Support for IndieAuth/Web-sign-in (hauke)
+	New hooks "emailer_send_prepare" and "emailer_send" (fabrixxm)
+	New hook "oembed_fetch_url" (annando)
+	Add un/ignore function to quattro theme (tobiasd)
+	Enhanced POCO data (annando)
+	Use HTML5 features to validate inputs in install wizard and in some settings fields (tobiasd)
+	Option to receive text-only notification emails (fabrixxm)
+	Better OStatus support (annando)
+	Share-it button support (annando)
+	More reliable reshare from Diaspora (annando)
+	Load more images via proxy (annando)
+	util/typo.php uses "php -l" insead of "eval()" to validate code (fabrixxm)
+	Use $_SERVER array in cli script instead of $argv/$argc (issue #1218) (annando)
+	Updated vagrant setup script (silke)
+	API: support to star/unstar items (fabrixxm)
+	API: attachments for better AndStatus support (annando)
+	Fix missing spaces in photo URLs (issue #920) (annando)
+	Fix avatar for "remote-self" items (annando)
+	Fix encodings issues with scrape functionality (annando)
+	Fix site info scraping when URL points to big file (annando)
+	Fix tools for translations (ddorian1)
+	Fix API login via LDAP (issue #1286) (fabrixxm)
+	Fix to link URL in tabs, pager (issues #1341, #1190) (ddorian1)
+	Better documentation for developers (silke)
+
 Version 3.3.2
 
 	Set default value for all not-null fields (fixes SQL warinigs) (annando)

boot.php

@@ -18,7 +18,7 @@ define ( 'FRIENDICA_PLATFORM',     'Friendica');
 define ( 'FRIENDICA_CODENAME',     'Ginger');
 define ( 'FRIENDICA_VERSION',      '3.3.3-RC' );
 define ( 'DFRN_PROTOCOL_VERSION',  '2.23'    );
-define ( 'DB_UPDATE_VERSION',      1179      );
+define ( 'DB_UPDATE_VERSION',      1180      );
 define ( 'EOL',                    "<br />\r\n"     );
 define ( 'ATOM_TIME',              'Y-m-d\TH:i:s\Z' );
 

include/Emailer.php

@@ -51,7 +51,7 @@ class Emailer {
 			"Content-Transfer-Encoding: base64\n\n" .
 			$textBody . "\n";
 
-		if (!$email_textonly && !is_null($htmlBody)){
+		if (!$email_textonly && !is_null($params['htmlVersion'])){
 			$multipartMessageBody .=
 				"--" . $mimeBoundary . "\n" .				// text/html section
 				"Content-Type: text/html; charset=UTF-8\n" .

include/Scrape.php

@@ -374,6 +374,7 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 		$network = NETWORK_APPNET;
 	}
 
+
 	// Twitter is deactivated since twitter closed its old API
 	//$twitter = ((strpos($url,'twitter.com') !== false) ? true : false);
 	$lastfm  = ((strpos($url,'last.fm/user') !== false) ? true : false);
@@ -569,6 +570,10 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 			$network = NETWORK_DIASPORA;
 		elseif($has_lrdd)
 			$network  = NETWORK_OSTATUS;
+
+		if(strpos($url,'@'))
+			$addr = str_replace('acct:', '', $url);
+
 		$priority = 0;
 
 		if($hcard && ! $vcard) {
@@ -762,6 +767,22 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 	if(($network === NETWORK_FEED) && ($poll) && (! x($vcard,'fn')))
 		$vcard['fn'] = $url;
 
+	if (($notify != "") AND ($poll != "")) {
+		$baseurl = matching($notify, $poll);
+
+		$baseurl2 = matching($baseurl, $profile);
+		if ($baseurl2 != "")
+			$baseurl = $baseurl2;
+	}
+
+	if (($baseurl == "") AND ($notify != ""))
+		$baseurl = matching($profile, $notify);
+
+	if (($baseurl == "") AND ($poll != ""))
+		$baseurl = matching($profile, $poll);
+
+	$baseurl = rtrim($baseurl, "/");
+
 	$vcard['fn'] = notags($vcard['fn']);
 	$vcard['nick'] = str_replace(' ','',notags($vcard['nick']));
 
@@ -780,11 +801,12 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 	$result['network'] = $network;
 	$result['alias'] = $alias;
 	$result['pubkey'] = $pubkey;
+	$result['baseurl'] = $baseurl;
 
 	logger('probe_url: ' . print_r($result,true), LOGGER_DEBUG);
 
 	// Trying if it maybe a diaspora account
-	if ($result['network'] == NETWORK_FEED) {
+	if (($result['network'] == NETWORK_FEED) OR ($result['addr'] == "")) {
 		require_once('include/bbcode.php');
 		$address = GetProfileUsername($url, "", true);
 		$result2 = probe_url($address, $mode);
@@ -796,3 +818,20 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 
 	return $result;
 }
+
+function matching($part1, $part2) {
+	$len = min(strlen($part1), strlen($part2));
+
+	$match = "";
+	$matching = true;
+	$i = 0;
+	while (($i <= $len) AND $matching) {
+		if (substr($part1, $i, 1) == substr($part2, $i, 1))
+			$match .= substr($part1, $i, 1);
+		else
+			$matching = false;
+
+		$i++;
+	}
+	return($match);
+}

include/conversation.php

@@ -203,12 +203,11 @@ function localize_item(&$item){
 
 		// we can't have a translation string with three positions but no distinguishable text
 		// So here is the translate string.
-
 		$txt = t('%1$s poked %2$s');
 		
 		// now translate the verb
-
+		$poked_t = trim(sprintf($txt, "",""));
-		$txt = str_replace( t('poked'), t($verb), $txt);
+		$txt = str_replace( $poked_t, t($verb), $txt);
 
 		// then do the sprintf on the translation string
 

include/dbstructure.php

@@ -626,6 +626,7 @@ function db_definition() {
 					"keywords" => array("type" => "text", "not null" => "1"),
 					"gender" => array("type" => "varchar(32)", "not null" => "1", "default" => ""),
 					"network" => array("type" => "varchar(255)", "not null" => "1", "default" => ""),
+					"generation" => array("type" => "tinyint(3)", "not null" => "1", "default" => "0"),
 					),
 			"indexes" => array(
 					"PRIMARY" => array("id"),

include/diaspora.php

@@ -2398,7 +2398,7 @@ function diaspora_profile($importer,$xml,$msg) {
 	if (unxmlify($xml->searchable) == "true") {
 		require_once('include/socgraph.php');
 		poco_check($contact['url'], $name, NETWORK_DIASPORA, $images[0], $about, $location, $gender, $keywords, "",
-			datetime_convert(), $contact['id'], $importer['uid']);
+			datetime_convert(), 2, $contact['id'], $importer['uid']);
 	}
 
 	$profileurl = "";

include/items.php

@@ -1376,16 +1376,6 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
 		$current_post = $r[0]['id'];
 		logger('item_store: created item ' . $current_post);
 
-		// Add every contact to the global contact table
-		// Contacts from the statusnet connector are also added since you could add them in OStatus as well.
-		if (!$arr['private'] AND in_array($arr["network"],
-			array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, NETWORK_STATUSNET, ""))) {
-			poco_check($arr["author-link"], $arr["author-name"], $arr["network"], $arr["author-avatar"], "", "", "", "", "", $arr["received"], $arr["contact-id"], $arr["uid"]);
-
-			// Maybe its a body with a shared item? Then extract a global contact from it.
-			poco_contact_from_body($arr["body"], $arr["received"], $arr["contact-id"], $arr["uid"]);
-		}
-
 		// Set "success_update" to the date of the last time we heard from this contact
 		// This can be used to filter for inactive contacts and poco.
 		// Only do this for public postings to avoid privacy problems, since poco data is public.
@@ -1395,56 +1385,12 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
 				dbesc($arr['received']),
 				intval($arr['contact-id'])
 			);
-
-		// Only check for notifications on start posts
-		if ($arr['parent-uri'] === $arr['uri']) {
-			add_thread($r[0]['id']);
-			logger('item_store: Check notification for contact '.$arr['contact-id'].' and post '.$current_post, LOGGER_DEBUG);
-
-			// Send a notification for every new post?
-			$r = q("SELECT `notify_new_posts` FROM `contact` WHERE `id` = %d AND `uid` = %d AND `notify_new_posts` LIMIT 1",
-				intval($arr['contact-id']),
-				intval($arr['uid'])
-			);
-
-			if(count($r)) {
-				logger('item_store: Send notification for contact '.$arr['contact-id'].' and post '.$current_post, LOGGER_DEBUG);
-				$u = q("SELECT * FROM user WHERE uid = %d LIMIT 1",
-					intval($arr['uid']));
-
-				$item = q("SELECT * FROM `item` WHERE `id` = %d AND `uid` = %d",
-					intval($current_post),
-					intval($arr['uid'])
-				);
-
-				$a = get_app();
-
-				require_once('include/enotify.php');
-				notification(array(
-					'type'         => NOTIFY_SHARE,
-					'notify_flags' => $u[0]['notify-flags'],
-					'language'     => $u[0]['language'],
-					'to_name'      => $u[0]['username'],
-					'to_email'     => $u[0]['email'],
-					'uid'          => $u[0]['uid'],
-					'item'         => $item[0],
-					'link'         => $a->get_baseurl().'/display/'.urlencode($arr['guid']),
-					'source_name'  => $item[0]['author-name'],
-					'source_link'  => $item[0]['author-link'],
-					'source_photo' => $item[0]['author-avatar'],
-					'verb'         => ACTIVITY_TAG,
-					'otype'        => 'item'
-				));
-				logger('item_store: Notification sent for contact '.$arr['contact-id'].' and post '.$current_post, LOGGER_DEBUG);
-			}
-		}
-
 	} else {
 		logger('item_store: could not locate created item');
 		return 0;
 	}
 	if(count($r) > 1) {
-		logger('item_store: duplicated post occurred. Removing duplicates.');
+		logger('item_store: duplicated post occurred. Removing duplicates. uri = '.$arr['uri'].' uid = '.$arr['uid']);
 		q("DELETE FROM `item` WHERE `uri` = '%s' AND `uid` = %d AND `id` != %d ",
 			dbesc($arr['uri']),
 			intval($arr['uid']),
@@ -1494,7 +1440,6 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
 		dbesc(datetime_convert()),
 		intval($parent_id)
 	);
-	update_thread($parent_id);
 
 	if($dsprsig) {
 		q("insert into sign (`iid`,`signed_text`,`signature`,`signer`) values (%d,'%s','%s','%s') ",
@@ -1520,32 +1465,84 @@ function item_store($arr,$force_parent = false, $notify = false, $dontcache = fa
 
 	$deleted = tag_deliver($arr['uid'],$current_post);
 
-	// current post can be deleted if is for a communuty page and no mention are
+	// current post can be deleted if is for a community page and no mention are
 	// in it.
 	if (!$deleted AND !$dontcache) {
 
 		// Store the fresh generated item into the cache
-		$cachefile = get_cachefile(urlencode($arr["guid"])."-".hash("md5", $arr['body']));
+		put_item_in_cache($arr);
-
-		if (($cachefile != '') AND !file_exists($cachefile)) {
-			$s = prepare_text($arr['body']);
-			$a = get_app();
-			$stamp1 = microtime(true);
-			file_put_contents($cachefile, $s);
-			$a->save_timestamp($stamp1, "file");
-			logger('item_store: put item '.$current_post.' into cachefile '.$cachefile);
-		}
 
 		$r = q('SELECT * FROM `item` WHERE id = %d', intval($current_post));
 		if (count($r) == 1) {
 			call_hooks('post_remote_end', $r[0]);
-		} else {
+		} else
 			logger('item_store: new item not found in DB, id ' . $current_post);
 	}
+
+	// Add every contact of the post to the global contact table
+	poco_store($arr);
+
+	create_tags_from_item($current_post);
+	create_files_from_item($current_post);
+
+	// Only check for notifications on start posts
+	if ($arr['parent-uri'] === $arr['uri']) {
+		add_thread($current_post);
+		logger('item_store: Check notification for contact '.$arr['contact-id'].' and post '.$current_post, LOGGER_DEBUG);
+
+		// Send a notification for every new post?
+		$r = q("SELECT `notify_new_posts` FROM `contact` WHERE `id` = %d AND `uid` = %d AND `notify_new_posts` LIMIT 1",
+			intval($arr['contact-id']),
+			intval($arr['uid'])
+		);
+		$send_notification = count($r);
+
+		if (!$send_notification) {
+			$tags = q("SELECT `url` FROM `term` WHERE `otype` = %d AND `oid` = %d AND `type` = %d AND `uid` = %d",
+				intval(TERM_OBJ_POST), intval($current_post), intval(TERM_MENTION), intval($arr['uid']));
+
+			if (count($tags)) {
+				foreach ($tags AS $tag) {
+					$r = q("SELECT `id` FROM `contact` WHERE `nurl` = '%s' AND `uid` = %d AND `notify_new_posts`",
+						normalise_link($tag["url"]), intval($arr['uid']));
+					if (count($r))
+						$send_notification = true;
+				}
+			}
 		}
 
-	create_tags_from_item($current_post, $dontcache);
+		if ($send_notification) {
-	create_files_from_item($current_post);
+			logger('item_store: Send notification for contact '.$arr['contact-id'].' and post '.$current_post, LOGGER_DEBUG);
+			$u = q("SELECT * FROM user WHERE uid = %d LIMIT 1",
+				intval($arr['uid']));
+
+			$item = q("SELECT * FROM `item` WHERE `id` = %d AND `uid` = %d",
+				intval($current_post),
+				intval($arr['uid'])
+			);
+
+			$a = get_app();
+
+			require_once('include/enotify.php');
+			notification(array(
+				'type'         => NOTIFY_SHARE,
+				'notify_flags' => $u[0]['notify-flags'],
+				'language'     => $u[0]['language'],
+				'to_name'      => $u[0]['username'],
+				'to_email'     => $u[0]['email'],
+				'uid'          => $u[0]['uid'],
+				'item'         => $item[0],
+				'link'         => $a->get_baseurl().'/display/'.urlencode($arr['guid']),
+				'source_name'  => $item[0]['author-name'],
+				'source_link'  => $item[0]['author-link'],
+				'source_photo' => $item[0]['author-avatar'],
+				'verb'         => ACTIVITY_TAG,
+				'otype'        => 'item'
+			));
+			logger('item_store: Notification sent for contact '.$arr['contact-id'].' and post '.$current_post, LOGGER_DEBUG);
+		}
+	} else
+		update_thread($parent_id);
 
 	if ($notify)
 		proc_run('php', "include/notifier.php", $notify_type, $current_post);
@@ -2078,6 +2075,7 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 	$photo_timestamp = '';
 	$photo_url = '';
 	$birthday = '';
+	$contact_updated = '';
 
 	$hubs = $feed->get_links('hub');
 	logger('consume_feed: hubs: ' . print_r($hubs,true), LOGGER_DATA);
@@ -2113,6 +2111,9 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 
 	if((is_array($contact)) && ($photo_timestamp) && (strlen($photo_url)) && ($photo_timestamp > $contact['avatar-date'])) {
 		logger('consume_feed: Updating photo for '.$contact['name'].' from '.$photo_url.' uid: '.$contact['uid']);
+
+		$contact_updated = $photo_timestamp;
+
 		require_once("include/Photo.php");
 		$photo_failure = false;
 		$have_photo = false;
@@ -2170,6 +2171,9 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 	}
 
 	if((is_array($contact)) && ($name_updated) && (strlen($new_name)) && ($name_updated > $contact['name-date'])) {
+		if ($name_updated > $contact_updated)
+			$contact_updated = $name_updated;
+
 		$r = q("select * from contact where uid = %d and id = %d limit 1",
 			intval($contact['uid']),
 			intval($contact['id'])
@@ -2194,6 +2198,9 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 		}
 	}
 
+	if ($contact_updated AND $new_name AND $photo_url)
+		poco_check($contact['url'], $new_name, NETWORK_DFRN, $photo_url, "", "", "", "", "", $contact_updated, 2, $contact['id'], $contact['uid']);
+
 	if(strlen($birthday)) {
 		if(substr($birthday,0,4) != $contact['bdyear']) {
 			logger('consume_feed: updating birthday: ' . $birthday);
@@ -2240,7 +2247,6 @@ function consume_feed($xml,$importer,&$contact, &$hub, $datedir = 0, $pass = 0)
 
 			$contact['bdyear'] = substr($birthday,0,4);
 		}
-
 	}
 
 	$community_page = 0;
@@ -2806,6 +2812,7 @@ function local_delivery($importer,$data) {
 	$new_name = '';
 	$photo_timestamp = '';
 	$photo_url = '';
+	$contact_updated = '';
 
 
 	$rawtags = $feed->get_feed_tags( NAMESPACE_DFRN, 'owner');
@@ -2834,6 +2841,9 @@ function local_delivery($importer,$data) {
 	}
 
 	if(($photo_timestamp) && (strlen($photo_url)) && ($photo_timestamp > $importer['avatar-date'])) {
+
+		$contact_updated = $photo_timestamp;
+
 		logger('local_delivery: Updating photo for ' . $importer['name']);
 		require_once("include/Photo.php");
 		$photo_failure = false;
@@ -2892,6 +2902,9 @@ function local_delivery($importer,$data) {
 	}
 
 	if(($name_updated) && (strlen($new_name)) && ($name_updated > $importer['name-date'])) {
+		if ($name_updated > $contact_updated)
+			$contact_updated = $name_updated;
+
 		$r = q("select * from contact where uid = %d and id = %d limit 1",
 			intval($importer['importer_uid']),
 			intval($importer['id'])
@@ -2916,7 +2929,8 @@ function local_delivery($importer,$data) {
 		}
 	}
 
-
+	if ($contact_updated AND $new_name AND $photo_url)
+		poco_check($importer['url'], $new_name, NETWORK_DFRN, $photo_url, "", "", "", "", "", $contact_updated, 2, $importer['id'], $importer['importer_uid']);
 
 	// Currently unsupported - needs a lot of work
 	$reloc = $feed->get_feed_tags( NAMESPACE_DFRN, 'relocate' );
@@ -4669,8 +4683,8 @@ function drop_item($id,$interactive = true) {
 				dbesc($item['parent-uri']),
 				intval($item['uid'])
 			);
-			create_tags_from_item($item['parent-uri'], $item['uid']);
+			create_tags_from_itemuri($item['parent-uri'], $item['uid']);
-			create_files_from_item($item['parent-uri'], $item['uid']);
+			create_files_from_itemuri($item['parent-uri'], $item['uid']);
 			delete_thread_uri($item['parent-uri'], $item['uid']);
 			// ignore the result
 		}

include/network.php

@@ -71,6 +71,7 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
 
 	$base = $s;
 	$curl_info = @curl_getinfo($ch);
+	@curl_close($ch);
 	$http_code = $curl_info['http_code'];
 	logger('fetch_url '.$url.': '.$http_code." ".$s, LOGGER_DATA);
 	$header = '';
@@ -94,7 +95,7 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
 			$newurl = $new_location_info["scheme"]."://".$new_location_info["host"].$old_location_info["path"];
 
 		$matches = array();
-		if (preg_match('/(Location:|URI:)(.*?)\n/', $header, $matches)) {
+		if (preg_match('/(Location:|URI:)(.*?)\n/i', $header, $matches)) {
 			$newurl = trim(array_pop($matches));
 		}
 		if(strpos($newurl,'/') === 0)
@@ -110,7 +111,6 @@ function fetch_url($url,$binary = false, &$redirects = 0, $timeout = 0, $accept_
 
 	$body = substr($s,strlen($header));
 	$a->set_curl_headers($header);
-	@curl_close($ch);
 
 	$a->save_timestamp($stamp1, "network");
 

include/onepoll.php

@@ -176,7 +176,7 @@ function onepoll_run(&$argv, &$argc){
 			return;
 		}
 
-		if(! strstr($handshake_xml,'<?xml')) {
+		if(! strstr($handshake_xml,'<')) {
 			logger('poller: response from ' . $url . ' did not contain XML.');
 
 			mark_for_death($contact);
@@ -284,13 +284,13 @@ function onepoll_run(&$argv, &$argc){
 	}
 	elseif($contact['network'] === NETWORK_MAIL || $contact['network'] === NETWORK_MAIL2) {
 
-		logger("onepoll: mail: Fetching", LOGGER_DEBUG);
+		logger("Mail: Fetching", LOGGER_DEBUG);
 
 		$mail_disabled = ((function_exists('imap_open') && (! get_config('system','imap_disabled'))) ? 0 : 1);
 		if($mail_disabled)
 			return;
 
-		logger("onepoll: Mail: Enabled", LOGGER_DEBUG);
+		logger("Mail: Enabled", LOGGER_DEBUG);
 
 		$mbox = null;
 		$x = q("SELECT `prvkey` FROM `user` WHERE `uid` = %d LIMIT 1",
@@ -312,7 +312,9 @@ function onepoll_run(&$argv, &$argc){
 					intval($mailconf[0]['id']),
 					intval($importer_uid)
 				);
-			}
+				logger("Mail: Connected to " . $mailconf[0]['user']);
+			} else
+				logger("Mail: Connection error ".$mailconf[0]['user']." ".print_r(imap_errors()));
 		}
 		if($mbox) {
 
@@ -523,7 +525,10 @@ function onepoll_run(&$argv, &$argc){
 						}
 					}
 				}
-			}
+			} else
+				logger("Mail: no mails for ".$mailconf[0]['user']);
+
+			logger("Mail: closing connection for ".$mailconf[0]['user']);
 			imap_close($mbox);
 		}
 	}
@@ -537,7 +542,7 @@ function onepoll_run(&$argv, &$argc){
 
 	if($xml) {
 		logger('poller: received xml : ' . $xml, LOGGER_DATA);
-		if((! strstr($xml,'<?xml')) && (! strstr($xml,'<rss'))) {
+		if(! strstr($xml,'<')) {
 			logger('poller: post_handshake: response from ' . $url . ' did not contain XML.');
 			$r = q("UPDATE `contact` SET `last-update` = '%s' WHERE `id` = %d",
 				dbesc(datetime_convert()),

include/socgraph.php

@@ -42,7 +42,7 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) {
 	if(! $url)
 		return;
 
-	$url = $url . (($uid) ? '/@me/@all?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender' : '?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender') ;
+	$url = $url . (($uid) ? '/@me/@all?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation' : '?fields=displayName,urls,photos,updated,network,aboutMe,currentLocation,tags,gender,generation') ;
 
 	logger('poco_load: ' . $url, LOGGER_DEBUG);
 
@@ -76,6 +76,7 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) {
 		$about = '';
 		$keywords = '';
 		$gender = '';
+		$generation = 0;
 
 		$name = $entry->displayName;
 
@@ -115,11 +116,18 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) {
 		if(isset($entry->gender))
 			$gender = $entry->gender;
 
+		if(isset($entry->generation) AND ($entry->generation > 0))
+			$generation = ++$entry->generation;
+
 		if(isset($entry->tags))
 			foreach($entry->tags as $tag)
 				$keywords = implode(", ", $tag);
 
-		poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $cid, $uid, $zcid);
+		// If you query a Friendica server for its profiles, the network has to be Friendica
+		if ($uid == 0)
+			$network = NETWORK_DFRN;
+
+		poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $generation, $cid, $uid, $zcid);
 
 		// Update the Friendica contacts. Diaspora is doing it via a message. (See include/diaspora.php)
 		if (($location != "") OR ($about != "") OR ($keywords != "") OR ($gender != ""))
@@ -142,24 +150,60 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) {
 
 }
 
-function poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $cid = 0, $uid = 0, $zcid = 0) {
+function poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $generation, $cid = 0, $uid = 0, $zcid = 0) {
+
+	$a = get_app();
+
+	// Generation:
+	//  0: No definition
+	//  1: Profiles on this server
+	//  2: Contacts of profiles on this server
+	//  3: Contacts of contacts of profiles on this server
+	//  4: ...
+
 	$gcid = "";
 
 	if ($profile_url == "")
 		return $gcid;
 
+	// Don't store the statusnet connector as network
+	// We can't simply set this to NETWORK_OSTATUS since the connector could have fetched posts from friendica as well
+	if ($network == NETWORK_STATUSNET)
+		$network = "";
+
+	// The global contacts should contain the original picture, not the cached one
+	if (($generation != 1) AND stristr(normalise_link($profile_photo), normalise_link($a->get_baseurl()."/photo/")))
+		$profile_photo = "";
+
+	$r = q("SELECT `network` FROM `contact` WHERE `nurl` = '%s' AND `network` != '' AND `network` != '%s' LIMIT 1",
+		dbesc(normalise_link($profile_url)), dbesc(NETWORK_STATUSNET)
+	);
+	if(count($r))
+		$network = $r[0]["network"];
+
+	if (($network == "") OR ($network == NETWORK_OSTATUS)) {
+		$r = q("SELECT `network`, `url` FROM `contact` WHERE `alias` IN ('%s', '%s') AND `network` != '' AND `network` != '%s' LIMIT 1",
+			dbesc($profile_url), dbesc(normalise_link($profile_url)), dbesc(NETWORK_STATUSNET)
+		);
+		if(count($r)) {
+			$network = $r[0]["network"];
+			$profile_url = $r[0]["url"];
+		}
+	}
+
 	$x = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1",
 		dbesc(normalise_link($profile_url))
 	);
-	if(count($x))
+	if(count($x) AND ($network == "") AND ($x[0]["network"] != NETWORK_STATUSNET))
 		$network = $x[0]["network"];
 
 	if (($network == "") OR ($name == "") OR ($profile_photo == "")) {
 		require_once("include/Scrape.php");
 
-		$data = probe_url($profile_url, PROBE_DIASPORA);
+		$data = probe_url($profile_url);
 		$network = $data["network"];
 		$name = $data["name"];
+		$profile_url = $data["url"];
 		$profile_photo = $data["photo"];
 	}
 
@@ -173,10 +217,10 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	if (($name == "") OR ($profile_photo == ""))
 		return $gcid;
 
-	if (!in_array($network, array(NETWORK_DFRN, NETWORK_OSTATUS, NETWORK_DIASPORA, NETWORK_STATUSNET)))
+	if (!in_array($network, array(NETWORK_DFRN, NETWORK_OSTATUS, NETWORK_DIASPORA)))
 		return $gcid;
 
-	logger("profile-check URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG);
+	logger("profile-check generation: ".$generation." Network: ".$network." URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG);
 
 	if(count($x)) {
 		$gcid = $x[0]['id'];
@@ -193,10 +237,13 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 		if (($keywords == "") AND ($x[0]['keywords'] != ""))
 			$keywords = $x[0]['keywords'];
 
+		if (($generation == 0) AND ($x[0]['generation'] > 0))
+			$generation = $x[0]['generation'];
+
 		if($x[0]['name'] != $name || $x[0]['photo'] != $profile_photo || $x[0]['updated'] < $updated) {
-			q("update gcontact set `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s',
+			q("UPDATE `gcontact` SET `name` = '%s', `network` = '%s', `photo` = '%s', `connect` = '%s', `url` = '%s',
-				`updated` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s'
+				`updated` = '%s', `location` = '%s', `about` = '%s', `keywords` = '%s', `gender` = '%s', `generation` = %d
-				where `nurl` = '%s'",
+				WHERE (`generation` >= %d OR `generation` = 0) AND `nurl` = '%s'",
 				dbesc($name),
 				dbesc($network),
 				dbesc($profile_photo),
@@ -207,12 +254,14 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 				dbesc($about),
 				dbesc($keywords),
 				dbesc($gender),
+				intval($generation),
+				intval($generation),
 				dbesc(normalise_link($profile_url))
 			);
 		}
 	} else {
-		q("insert into `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `updated`, `location`, `about`, `keywords`, `gender`)
+		q("INSERT INTO `gcontact` (`name`,`network`, `url`,`nurl`,`photo`,`connect`, `updated`, `location`, `about`, `keywords`, `gender`, `generation`)
-			values ('%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s', '%s')",
+			VALUES ('%s', '%s', '%s', '%s', '%s','%s', '%s', '%s', '%s', '%s', '%s', %d)",
 			dbesc($name),
 			dbesc($network),
 			dbesc($profile_url),
@@ -223,7 +272,8 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 			dbesc($location),
 			dbesc($about),
 			dbesc($keywords),
-			dbesc($gender)
+			dbesc($gender),
+			intval($generation)
 		);
 		$x = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1",
 			dbesc(normalise_link($profile_url))
@@ -290,7 +340,65 @@ function sub_poco_from_share($share, $created, $cid, $uid) {
 		return;
 
 	logger("prepare poco_check for profile ".$profile, LOGGER_DEBUG);
-        poco_check($profile, "", "", "", "", "", "", "", "", $created, $cid, $uid);
+        poco_check($profile, "", "", "", "", "", "", "", "", $created, 3, $cid, $uid);
+}
+
+function poco_store($item) {
+
+	// Isn't it public?
+	if ($item['private'])
+		return;
+
+	// Or is it from a network where we don't store the global contacts?
+	if (!in_array($item["network"], array(NETWORK_DFRN, NETWORK_DIASPORA, NETWORK_OSTATUS, NETWORK_STATUSNET, "")))
+		return;
+
+	// Is it a global copy?
+	$store_gcontact = ($item["uid"] == 0);
+
+	// Is it a comment on a global copy?
+	if (!$store_gcontact AND ($item["uri"] != $item["parent-uri"])) {
+		$q = q("SELECT `id` FROM `item` WHERE `uri`='%s' AND `uid` = 0", $item["parent-uri"]);
+		$store_gcontact = count($q);
+	}
+
+	if (!$store_gcontact)
+		return;
+
+	// "3" means: We don't know this contact directly (Maybe a reshared item)
+	$generation = 3;
+	$network = "";
+	$profile_url = $item["author-link"];
+
+	// Is it a user from our server?
+	$q = q("SELECT `id` FROM `contact` WHERE `self` AND `nurl` = '%s' LIMIT 1",
+		dbesc(normalise_link($item["author-link"])));
+	if (count($q)) {
+		logger("Our user (generation 1): ".$item["author-link"], LOGGER_DEBUG);
+		$generation = 1;
+		$network = NETWORK_DFRN;
+	} else { // Is it a contact from a user on our server?
+		$q = q("SELECT `network`, `url` FROM `contact` WHERE `uid` != 0 AND `network` != ''
+			AND (`nurl` = '%s' OR `alias` IN ('%s', '%s')) AND `network` != '%s' LIMIT 1",
+			dbesc(normalise_link($item["author-link"])),
+			dbesc(normalise_link($item["author-link"])),
+			dbesc($item["author-link"]),
+			dbesc(NETWORK_STATUSNET));
+		if (count($q)) {
+			$generation = 2;
+			$network = $q[0]["network"];
+			$profile_url = $q[0]["url"];
+			logger("Known contact (generation 2): ".$profile_url, LOGGER_DEBUG);
+		}
+	}
+
+	if ($generation == 3)
+		logger("Unknown contact (generation 3): ".$item["author-link"], LOGGER_DEBUG);
+
+	poco_check($profile_url, $item["author-name"], $network, $item["author-avatar"], "", "", "", "", "", $item["received"], $generation, $item["contact-id"], $item["uid"]);
+
+	// Maybe its a body with a shared item? Then extract a global contact from it.
+	poco_contact_from_body($item["body"], $item["received"], $item["contact-id"], $item["uid"]);
 }
 
 function count_common_friends($uid,$cid) {

include/tags.php

@@ -1,5 +1,5 @@
 <?php
-function create_tags_from_item($itemid, $dontcache = false) {
+function create_tags_from_item($itemid) {
 	global $a;
 
 	$profile_base = $a->get_baseurl();
@@ -26,18 +26,6 @@ function create_tags_from_item($itemid, $dontcache = false) {
 	if ($message["deleted"])
 		return;
 
-	if (!$dontcache) {
-		$cachefile = get_cachefile(urlencode($message["guid"])."-".hash("md5", $message['body']));
-
-		if (($cachefile != '') AND !file_exists($cachefile)) {
-			$s = prepare_text($message['body']);
-			$stamp1 = microtime(true);
-			file_put_contents($cachefile, $s);
-			$a->save_timestamp($stamp1, "file");
-			logger('create_tags_from_item: put item '.$message["id"].' into cachefile '.$cachefile);
-		}
-	}
-
 	$taglist = explode(",", $message["tag"]);
 
 	$tags = "";

include/text.php

@@ -385,6 +385,18 @@ function alt_pager(&$a, $i) {
 
 }}
 
+if(! function_exists('scroll_loader')) {
+/**
+ * Loader for infinite scrolling
+ * @return string html for loader
+ */
+function scroll_loader() {
+	$tpl = get_markup_template("scroll_loader.tpl");
+	return replace_macros($tpl, array(
+		'wait' => t('Loading more entries...'),
+		'end' => t('The end')
+	));
+}}
 
 if(! function_exists('expand_acl')) {
 /**
@@ -1281,6 +1293,18 @@ function redir_private_images($a, &$item) {
 
 }}
 
+function put_item_in_cache($item) {
+	$cachefile = get_cachefile(urlencode($item["guid"])."-".hash("md5", $item['body']));
+
+	if (($cachefile != '') AND !file_exists($cachefile)) {
+		$s = prepare_text($item['body']);
+		$a = get_app();
+		$stamp1 = microtime(true);
+		file_put_contents($cachefile, $s);
+		$a->save_timestamp($stamp1, "file");
+		logger('put item '.$item["guid"].' into cachefile '.$cachefile);
+	}
+}
 
 // Given an item array, convert the body element from bbcode to html and add smilie icons.
 // If attach is true, also add icons for item attachments

index.php

@@ -498,30 +498,38 @@ $(document).ready(function() {
 });
 
 function loadcontent() {
-	//$("div.loader").show();
+	if (lockLoadContent) return;
+	lockLoadContent = true;
+
+	$("#scroll-loader").fadeIn('normal');
 
 	num+=1;
 
 	console.log('Loading page ' + num);
 
 	$.get('/network?mode=raw$reload_uri&page=' + num, function(data) {
+		$("#scroll-loader").hide();
+		if ($(data).length > 0) {
 			$(data).insertBefore('#conversation-end');
+			lockLoadContent = false;
+		} else {
+			$("#scroll-end").fadeIn('normal');
+		}
 	});
-
-	//$("div.loader").fadeOut('normal');
 }
 
 var num = $pageno;
+var lockLoadContent = false;
 
 $(window).scroll(function(e){
 
 	if ($(document).height() != $(window).height()) {
 		// First method that is expected to work - but has problems with Chrome
-		if ($(window).scrollTop() == $(document).height() - $(window).height())
+		if ($(window).scrollTop() > ($(document).height() - $(window).height() * 1.5))
 			loadcontent();
 	} else {
 		// This method works with Chrome - but seems to be much slower in Firefox
-		if ($(window).scrollTop() > (($("section").height() + $("header").height() + $("footer").height()) - $(window).height()))
+		if ($(window).scrollTop() > (($("section").height() + $("header").height() + $("footer").height()) - $(window).height() * 1.5))
 			loadcontent();
 	}
 });

mod/item.php

@@ -22,6 +22,7 @@ require_once('library/langdet/Text/LanguageDetect.php');
 require_once('include/tags.php');
 require_once('include/files.php');
 require_once('include/threads.php');
+require_once('include/text.php');
 
 function item_post(&$a) {
 
@@ -824,21 +825,12 @@ function item_post(&$a) {
 	if(count($r)) {
 		$post_id = $r[0]['id'];
 		logger('mod_item: saved item ' . $post_id);
-		add_thread($post_id);
 
 		// update filetags in pconfig
 		file_tag_update_pconfig($uid,$categories_old,$categories_new,'category');
 
 		// Store the fresh generated item into the cache
-		$cachefile = get_cachefile(urlencode($datarray["guid"])."-".hash("md5", $datarray['body']));
+		put_item_in_cache($datarray);
-
-		if (($cachefile != '') AND !file_exists($cachefile)) {
-			$s = prepare_text($datarray['body']);
-			$stamp1 = microtime(true);
-			file_put_contents($cachefile, $s);
-			$a->save_timestamp($stamp1, "file");
-			logger('mod_item: put item '.$r[0]['id'].' into cachefile '.$cachefile);
-		}
 
 		if($parent) {
 
@@ -947,6 +939,7 @@ function item_post(&$a) {
 		dbesc(datetime_convert()),
 		intval($parent)
 	);
+	if ($post_id != $parent)
 		update_thread($parent);
 
 	$datarray['id']    = $post_id;
@@ -990,7 +983,9 @@ function item_post(&$a) {
 
 	create_tags_from_item($post_id);
 	create_files_from_item($post_id);
-	update_thread($post_id);
+
+	if ($post_id == $parent)
+		add_thread($post_id);
 
 	// This is a real juggling act on shared hosting services which kill your processes
 	// e.g. dreamhost. We used to start delivery to our native delivery agents in the background

mod/network.php

@@ -815,7 +815,9 @@ die("ss");
 	$o .= conversation($a,$items,$mode,$update);
 
 	if(!$update) {
-		if(!get_config('system', 'old_pager')) {
+		if(get_pconfig(local_user(),'system','infinite_scroll')) {
+				$o .= scroll_loader();
+		} elseif(!get_config('system', 'old_pager')) {
 		        $o .= alt_pager($a,count($items));
 		} else {
 		        $o .= paginate($a);

mod/poco.php

@@ -135,9 +135,9 @@ function poco_init(&$a) {
 	if(x($_GET,'updatedSince') AND !$global)
 		$ret['updatedSince'] = false;
 
-	$ret['startIndex']   = (string) $startIndex;
+	$ret['startIndex']   = (int) $startIndex;
-	$ret['itemsPerPage'] = (string) $itemsPerPage;
+	$ret['itemsPerPage'] = (int) $itemsPerPage;
-	$ret['totalResults'] = (string) $totalResults;
+	$ret['totalResults'] = (int) $totalResults;
 	$ret['entry']        = array();
 
 
@@ -153,7 +153,8 @@ function poco_init(&$a) {
 		'network' => false,
 		'gender' => false,
 		'tags' => false,
-		'address' => false
+		'address' => false,
+		'generation' => false
 	);
 
 	if((! x($_GET,'fields')) || ($_GET['fields'] === '@all'))
@@ -168,6 +169,15 @@ function poco_init(&$a) {
 	if(is_array($r)) {
 		if(count($r)) {
 			foreach($r as $rr) {
+				if (!isset($rr['generation'])) {
+					if ($global)
+						$rr['generation'] = 3;
+					elseif ($system_mode)
+						$rr['generation'] = 1;
+					else
+						$rr['generation'] = 2;
+				}
+
 				if (($rr['about'] == "") AND isset($rr['pabout']))
 					$rr['about'] = $rr['pabout'];
 
@@ -198,7 +208,7 @@ function poco_init(&$a) {
 
 				$entry = array();
 				if($fields_ret['id'])
-					$entry['id'] = $rr['id'];
+					$entry['id'] = (int)$rr['id'];
 				if($fields_ret['displayName'])
 					$entry['displayName'] = $rr['name'];
 				if($fields_ret['aboutMe'])
@@ -207,6 +217,8 @@ function poco_init(&$a) {
 					$entry['currentLocation'] = $rr['location'];
 				if($fields_ret['gender'])
 					$entry['gender'] = $rr['gender'];
+				if($fields_ret['generation'])
+					$entry['generation'] = (int)$rr['generation'];
 				if($fields_ret['urls']) {
 					$entry['urls'] = array(array('value' => $rr['url'], 'type' => 'profile'));
 					if($rr['addr'] && ($rr['network'] !== NETWORK_MAIL))

update.php

@@ -1,6 +1,6 @@
 <?php
 
-define( 'UPDATE_VERSION' , 1179 );
+define( 'UPDATE_VERSION' , 1180 );
 
 /**
  *

view/templates/profile_vcard.tpl

@@ -8,9 +8,9 @@
 	{{if $pdesc}}<div class="title">{{$profile.pdesc}}</div>{{/if}}
 
 	{{if $profile.picdate}}
-		<div id="profile-photo-wrapper"><img class="photo u-photo" width="175" height="175" src="{{$profile.photo}}?rev={{$profile.picdate}}" alt="{{$profile.name}}"></div>
+		<div id="profile-photo-wrapper"><a href="{{$profile.url}}"><img class="photo u-photo" width="175" height="175" src="{{$profile.photo}}?rev={{$profile.picdate}}" alt="{{$profile.name}}"></a></div>
         {{else}}
-		<div id="profile-photo-wrapper"><img class="photo u-photo" width="175" height="175" src="{{$profile.photo}}" alt="{{$profile.name}}"></div>
+		<div id="profile-photo-wrapper"><a href="{{$profile.url}}"><img class="photo u-photo" width="175" height="175" src="{{$profile.photo}}" alt="{{$profile.name}}"></a></div>
         {{/if}}
 	{{if $profile.network_name}}<dl class="network"><dt class="network-label">{{$network}}</dt><dd class="x-network">{{$profile.network_name}}</dd></dl>{{/if}}
 	{{if $location}}

view/templates/scroll_loader.tpl

@@ -0,0 +1,8 @@
+<div id="scroll-loader" class="pager" style="display: none;">
+	<img class="scroll_loader_image" src="images/rotator.gif" />
+	<span class="scroll_loader_text">{{$wait}}</span>
+</div>
+
+<div id="scroll-end" class="pager" style="display: none;">
+	<span class="scroll_loader_text">{{$end}}</span>
+</div>

view/theme/duepuntozero/deriv/darkzero.css

@@ -95,7 +95,7 @@ input#dfrn-url {
 	background-color: #222222;
 	color: #FFFFFF !important;
 }
-.pager_first a, .pager_last a, .pager_prev a, .pager_next a, .pager_n a, .pager_current {
+.pager_first a, .pager_last a, .pager_prev a, .pager_next a, .pager_n a, .pager_current, .scroll_loader_text {
     color: #000088;
 }
 

view/theme/duepuntozero/style.css

@@ -1476,7 +1476,8 @@ blockquote.shared_content {
 .pager_last,
 .pager_prev,
 .pager_next,
-.pager_n {
+.pager_n,
+.scroll_loader_text {
 	border: 1px solid black;
 	background: #EEE;
 	padding: 4px;

view/theme/frost-mobile/style.css

@@ -1886,7 +1886,8 @@ input#profile-jot-email {
 }*/
 
 .pager_prev a,
-.pager_next a {
+.pager_next a,
+.scroll_loader_text {
 	font-size: 1.5em;
 	padding: 0.2em 1em;
 	border: 1px solid #aaa;

view/theme/frost/style.css

@@ -1782,7 +1782,8 @@ input#dfrn-url {
 .pager_last,
 .pager_prev,
 .pager_next,
-.pager_n {
+.pager_n,
+.scroll_loader_text {
 /*	background: #EEE;*/
 }
 

view/theme/frost/templates/head.tpl

@@ -8,6 +8,8 @@
 
 <link rel="stylesheet" type="text/css" href="{{$stylesheet}}" media="all" />
 
+<script type="text/javascript" src="{{$baseurl}}/js/jquery.js" ></script>
+
 <link rel="shortcut icon" href="{{$baseurl}}/images/friendica-32.png" />
 <link rel="search"
          href="{{$baseurl}}/opensearch" 

view/theme/vier/style.css

@@ -203,7 +203,8 @@ div.pager a {
 }
  
 span.pager_first a, span.pager_n a, 
-span.pager_last a, span.pager_prev a, span.pager_next a {
+span.pager_last a, span.pager_prev a, span.pager_next a,
+span.scroll_loader_text {
   color: darkgray;
 }
 

view/theme/vier/templates/profile_vcard.tpl

@@ -15,9 +15,9 @@
 
 
 	{{if $profile.picdate}}
-		<div id="profile-photo-wrapper"><img class="photo" src="{{$profile.photo}}?rev={{$profile.picdate}}" alt="{{$profile.name}}" /></div>
+		<div id="profile-photo-wrapper"><a href="{{$profile.url}}"><img class="photo" src="{{$profile.photo}}?rev={{$profile.picdate}}" alt="{{$profile.name}}" /></a></div>
 	{{else}}
-		<div id="profile-photo-wrapper"><img class="photo" src="{{$profile.photo}}" alt="{{$profile.name}}" /></div>
+		<div id="profile-photo-wrapper"><a href="{{$profile.url}}"><img class="photo" src="{{$profile.photo}}" alt="{{$profile.name}}" /></a></div>
 	{{/if}}
 	{{if $pdesc}}<div class="title">{{$profile.pdesc}}</div>{{/if}}