Scrape now doesn't fetch page data when its twitter and Co./Update script that updates the contacts from the profile

include/Scrape.php

@@ -356,6 +356,23 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 	$diaspora_key = '';
 	$has_lrdd = false;
 	$email_conversant = false;
+	$connectornetworks = false;
+	$appnet = false;
+
+	if (strpos($url,'twitter.com')) {
+		$connectornetworks = true;
+		$network = NETWORK_TWITTER;
+	}
+
+	if (strpos($url,'www.facebook.com')) {
+		$connectornetworks = true;
+		$network = NETWORK_FACEBOOK;
+	}
+
+	if (strpos($url,'alpha.app.net')) {
+		$appnet = true;
+		$network = NETWORK_APPNET;
+	}
 
 	// Twitter is deactivated since twitter closed its old API
 	//$twitter = ((strpos($url,'twitter.com') !== false) ? true : false);
@@ -363,7 +380,7 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 
 	$at_addr = ((strpos($url,'@') !== false) ? true : false);
 
-	if((! $twitter) && (! $lastfm)) {
+	if((!$appnet) && (!$lastfm) && !$connectornetworks) {
 
 		if(strpos($url,'mailto:') !== false && $at_addr) {
 			$url = str_replace('mailto:','',$url);
@@ -606,13 +623,16 @@ function probe_url($url, $mode = PROBE_NORMAL) {
 			// Will leave it to others to figure out how to grab the avatar, which is on the $url page in the open graph meta links
 		}
 
-		if($twitter || ! $poll)
+		if($appnet || ! $poll)
 			$check_feed = true;
 		if((! isset($vcard)) || (! x($vcard,'fn')) || (! $profile))
 			$check_feed = true;
 		if(($at_addr) && (! count($links)))
 			$check_feed = false;
 
+		if ($connectornetworks)
+			$check_feed = false;
+
 		if($check_feed) {
 
 			$feedret = scrape_feed(($poll) ? $poll : $url);

include/socgraph.php

@@ -116,7 +116,6 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) {
 			foreach($entry->tags as $tag)
 				$keywords = implode(", ", $tag);
 
-		if ($network != "") {
 		poco_check($profile_url, $name, $network, $profile_photo, $about, $location, $gender, $keywords, $connect_url, $updated, $cid, $uid, $zcid);
 
 		// Update the Friendica contacts. Diaspora is doing it via a message. (See include/diaspora.php)
@@ -130,7 +129,6 @@ function poco_load($cid,$uid = 0,$zcid = 0,$url = null) {
 				dbesc(normalise_link($profile_url)),
 				dbesc(NETWORK_DFRN));
 	}
-	}
 	logger("poco_load: loaded $total entries",LOGGER_DEBUG);
 
 	q("DELETE FROM `glink` WHERE `cid` = %d AND `uid` = %d AND `zcid` = %d AND `updated` < UTC_TIMESTAMP - INTERVAL 2 DAY",
@@ -147,6 +145,12 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 	if ($profile_url == "")
 		return $gcid;
 
+	$x = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1",
+		dbesc(normalise_link($profile_url))
+	);
+	if(count($x))
+		$network = $x[0]["network"];
+
 	if (($network == "") OR ($name == "") OR ($profile_photo == "")) {
 		require_once("include/Scrape.php");
 
@@ -156,17 +160,20 @@ function poco_check($profile_url, $name, $network, $profile_photo, $about, $loca
 		$profile_photo = $data["photo"];
 	}
 
-	if (!in_array($network, array(NETWORK_DFRN, NETWORK_OSTATUS, NETWORK_DIASPORA, NETWORK_STATUSNET)))
+	if (count($x) AND ($x[0]["network"] == "") AND ($network != "")) {
-		return $gcid;
+		q("UPDATE `gcontact` SET `network` = '%s' WHERE `nurl` = '%s'",
+			dbesc($network),
+			dbesc(normalise_link($profile_url))
+		);
+	}
 
 	if (($name == "") OR ($profile_photo == ""))
 		return $gcid;
 
-	logger("profile-check URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG);
+	if (!in_array($network, array(NETWORK_DFRN, NETWORK_OSTATUS, NETWORK_DIASPORA, NETWORK_STATUSNET)))
+		return $gcid;
 
-	$x = q("SELECT * FROM `gcontact` WHERE `nurl` = '%s' LIMIT 1",
+	logger("profile-check URL: ".$profile_url." name: ".$name." avatar: ".$profile_photo, LOGGER_DEBUG);
-		dbesc(normalise_link($profile_url))
-	);
 
 	if(count($x)) {
 		$gcid = $x[0]['id'];

mod/profiles.php

@@ -149,6 +149,7 @@ function profile_clean_keywords($keywords) {
 	$cleaned = array();
 	foreach ($keywords as $keyword) {
 		$keyword = trim(strtolower($keyword));
+		$keyword = trim($keyword, "#");
 		if ($keyword != "")
 			$cleaned[] = $keyword;
 	}

update.php

@@ -1609,3 +1609,24 @@ All following update functions are ONLY for jobs that need to run AFTER the data
 
 Database changes are ONLY applied in the file include/dbstructure.php.
 */
+
+function update_1177() {
+	require_once("mod/profiles.php");
+
+	$profiles = q("SELECT `uid`, `about`, `locality`, `pub_keywords`, `gender` FROM `profile` WHERE `is-default`");
+
+	foreach ($profiles AS $profile) {
+		if ($profile["about"].$profile["locality"].$profile["pub_keywords"].$profile["gender"] == "")
+			continue;
+
+		$profile["pub_keywords"] = profile_clean_keywords($profile["pub_keywords"]);
+
+		$r = q("UPDATE `contact` SET `about` = '%s', `location` = '%s', `keywords` = '%s', `gender` = '%s' WHERE `self` AND `uid` = %d",
+				dbesc($profile["about"]),
+				dbesc($profile["locality"]),
+				dbesc($profile["pub_keywords"]),
+				dbesc($profile["gender"]),
+				intval($profile["uid"])
+			);
+	}
+}