Merge pull request #82 from annando/master

New Twitter/Statusnet-shortening - cache improvements with "privacy_image_cache"

privacy_image_cache/privacy_image_cache.php

@@ -32,7 +32,21 @@ function privacy_image_cache_module() {}
 
 
 function privacy_image_cache_init() {
-	global $a;
+	global $a, $_SERVER;
+
+	if (isset($_SERVER['HTTP_IF_MODIFIED_SINCE'])) {
+		header('HTTP/1.1 304 Not Modified');
+		header("Last-Modified: " . gmdate("D, d M Y H:i:s", time()) . " GMT");
+		header('Etag: '.$_SERVER['HTTP_IF_NONE_MATCH']);
+		header("Expires: " . gmdate("D, d M Y H:i:s", time() + (31536000)) . " GMT");
+		header("Cache-Control: max-age=31536000");
+		if(function_exists('header_remove')) {
+			header_remove('Last-Modified');
+			header_remove('Expires');
+			header_remove('Cache-Control');
+		}
+		exit;
+	}
 
 	if ($a->config["system"]["db_log"] != "")
 		$stamp1 = microtime(true);
@@ -46,17 +60,18 @@ function privacy_image_cache_init() {
 	// Double encoded url - happens with Diaspora
 	$urlhash2 = 'pic:' . sha1(urldecode($_REQUEST['url']));
 
-	$cache = get_config('system','itemcache');
+	$cachefile = get_cachefile(hash("md5", $_REQUEST['url']));
-	if (($cache != '') and is_dir($cache)) {
+	if ($cachefile != '') {
-		$cachefile = $cache."/".hash("md5", $_REQUEST['url']);
 		if (file_exists($cachefile)) {
 			$img_str = file_get_contents($cachefile);
 
 			$mime = image_type_to_mime_type(exif_imagetype($cachefile));
 
 			header("Content-type: $mime");
-			header("Expires: " . gmdate("D, d M Y H:i:s", time() + (3600*24)) . " GMT");
+			header("Last-Modified: " . gmdate("D, d M Y H:i:s", time()) . " GMT");
-			header("Cache-Control: max-age=" . (3600*24));
+			header('Etag: "'.md5($img_str).'"');
+			header("Expires: " . gmdate("D, d M Y H:i:s", time() + (31536000)) . " GMT");
+			header("Cache-Control: max-age=31536000");
 
 			echo $img_str;
 
@@ -79,14 +94,6 @@ function privacy_image_cache_init() {
 		$mime = $r[0]["desc"];
 		if ($mime == "") $mime = "image/jpeg";
 
-		// Test
-		//if ($mime == "image/jpeg") {
-		//	$img = new Photo($img_str);
-		//	if($img->is_valid()) {
-		//		$img->scaleImage(1000);
-		//		$img_str = $img->imageString();
-		//	}
-		//}
 	} else {
 		// It shouldn't happen but it does - spaces in URL
 		$_REQUEST['url'] = str_replace(" ", "+", $_REQUEST['url']);
@@ -102,6 +109,12 @@ function privacy_image_cache_init() {
 		if ((substr($a->get_curl_code(), 0, 1) == "4") or (!$img_str)) {
 			$img_str = file_get_contents("images/blank.png");
 			$mime = "image/png";
+			$cachefile = ""; // Clear the cachefile so that the dummy isn't stored
+			$img = new Photo($img_str);
+			if($img->is_valid()) {
+				$img->scaleImage(1);
+				$img_str = $img->imageString();
+			}
 		//} else if (substr($img_str, 0, 6) == "GIF89a") {
 		} else if ($mime != "image/jpeg") {
 			$image = @imagecreatefromstring($img_str);
@@ -137,12 +150,15 @@ function privacy_image_cache_init() {
 	}
 
 	// Writing in cachefile
-	if (isset($cachefile) && ($cachefile != '') and (file_exists($cachefile)) and (exif_imagetype($cachefile) > 0))
+	// and (file_exists($cachefile)) and (exif_imagetype($cachefile) > 0))
+	if ($cachefile != '')
 		file_put_contents($cachefile, $img_str);
 
 	header("Content-type: $mime");
-	header("Expires: " . gmdate("D, d M Y H:i:s", time() + (3600*24)) . " GMT");
+	header("Last-Modified: " . gmdate("D, d M Y H:i:s", time()) . " GMT");
-	header("Cache-Control: max-age=" . (3600*24));
+	header('Etag: "'.md5($img_str).'"');
+	header("Expires: " . gmdate("D, d M Y H:i:s", time() + (31536000)) . " GMT");
+	header("Cache-Control: max-age=31536000");
 
 	echo $img_str;
 
@@ -210,6 +226,8 @@ function privacy_image_cache_display_item_hook(&$a, &$o) {
             $o["output"]["thumb"] = $a->get_baseurl() . "/privacy_image_cache/?url=" . escape_tags(addslashes(rawurlencode($o["output"]["thumb"])));
         if (isset($o["output"]["author-avatar"]) && !privacy_image_cache_is_local_image($o["output"]["author-avatar"]))
             $o["output"]["author-avatar"] = $a->get_baseurl() . "/privacy_image_cache/?url=" . escape_tags(addslashes(rawurlencode($o["output"]["author-avatar"])));
+        if (isset($o["output"]["owner-avatar"]) && !privacy_image_cache_is_local_image($o["output"]["owner-avatar"]))
+            $o["output"]["owner-avatar"] = $a->get_baseurl() . "/privacy_image_cache/?url=" . escape_tags(addslashes(rawurlencode($o["output"]["owner-avatar"])));
     }
 }
 

statusnet/statusnet.php

@@ -404,6 +404,110 @@ function short_link($url) {
     return $slinky->short();
 } };
 
+function statusnet_shortenmsg($b, $max_char) {
+	require_once("include/bbcode.php");
+	require_once("include/html2plain.php");
+
+	// Looking for the first image
+	$image = '';
+	if(preg_match("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/is",$b['body'],$matches))
+		$image = $matches[3];
+
+	if ($image == '')
+		if(preg_match("/\[img\](.*?)\[\/img\]/is",$b['body'],$matches))
+			$image = $matches[1];
+
+	$multipleimages = (strpos($b['body'], "[img") != strrpos($b['body'], "[img"));
+
+	// When saved into the database the content is sent through htmlspecialchars
+	// That means that we have to decode all image-urls
+	$image = htmlspecialchars_decode($image);
+
+	$body = $b["body"];
+	if ($b["title"] != "")
+		$body = $b["title"]."\n\n".$body;
+
+	// remove the recycle signs and the names since they aren't helpful on twitter
+	// recycle 1
+	$recycle = html_entity_decode("♲ ", ENT_QUOTES, 'UTF-8');
+	$body = preg_replace( '/'.$recycle.'\[url\=(\w+.*?)\](\w+.*?)\[\/url\]/i', "\n", $body);
+	// recycle 2 (Test)
+	$recycle = html_entity_decode("◌ ", ENT_QUOTES, 'UTF-8');
+	$body = preg_replace( '/'.$recycle.'\[url\=(\w+.*?)\](\w+.*?)\[\/url\]/i', "\n", $body);
+
+	// At first convert the text to html
+	$html = bbcode($body, false, false);
+
+	// Then convert it to plain text
+	//$msg = trim($b['title']." \n\n".html2plain($html, 0, true));
+	$msg = trim(html2plain($html, 0, true));
+	$msg = html_entity_decode($msg,ENT_QUOTES,'UTF-8');
+
+	// Removing multiple newlines
+	while (strpos($msg, "\n\n\n") !== false)
+		$msg = str_replace("\n\n\n", "\n\n", $msg);
+
+	// Removing multiple spaces
+	while (strpos($msg, "  ") !== false)
+		$msg = str_replace("  ", " ", $msg);
+
+	// Removing URLs
+	$msg = preg_replace('/(https?\:\/\/[a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/i', "", $msg);
+
+	$msg = trim($msg);
+
+	$link = '';
+	// look for bookmark-bbcode and handle it with priority
+	if(preg_match("/\[bookmark\=([^\]]*)\](.*?)\[\/bookmark\]/is",$b['body'],$matches))
+		$link = $matches[1];
+
+	$multiplelinks = (strpos($b['body'], "[bookmark") != strrpos($b['body'], "[bookmark"));
+
+	// If there is no bookmark element then take the first link
+	if ($link == '') {
+		$links = collecturls($html);
+		if (sizeof($links) > 0) {
+			reset($links);
+			$link = current($links);
+		}
+		$multiplelinks = (sizeof($links) > 1);
+	}
+
+	$msglink = "";
+	if ($multiplelinks)
+		$msglink = $b["plink"];
+	else if ($link != "")
+		$msglink = $link;
+	else if ($multipleimages)
+		$msglink = $b["plink"];
+	else if ($image != "")
+		$msglink = $image;
+
+	if (($msglink == "") and strlen($msg) > $max_char)
+		$msglink = $b["plink"];
+
+	if (strlen($msglink) > 20)
+		$msglink = short_link($msglink);
+
+	if (strlen(trim($msg." ".$msglink)) > $max_char) {
+		$msg = substr($msg, 0, $max_char - (strlen($msglink)));
+		$lastchar = substr($msg, -1);
+		$msg = substr($msg, 0, -1);
+		$pos = strrpos($msg, "\n");
+		if ($pos > 0)
+			$msg = substr($msg, 0, $pos-1);
+		else if ($lastchar != "\n")
+			$msg = substr($msg, 0, -3)."...";
+	}
+	$msg = str_replace("\n", " ", $msg);
+
+	// Removing multiple spaces - again
+	while (strpos($msg, "  ") !== false)
+		$msg = str_replace("  ", " ", $msg);
+
+	return(trim($msg." ".$msglink));
+}
+
 function statusnet_post_hook(&$a,&$b) {
 
 	/**
@@ -433,9 +537,12 @@ function statusnet_post_hook(&$a,&$b) {
                 // we can later send to StatusNet. This way we can "gain" some
                 // information during shortening of potential links but do not
                 // shorten all the links in a 200000 character long essay.
+
+		$intelligent_shortening = get_config('statusnet','intelligent_shortening');
+		if (!$intelligent_shortening) {
 	                if (! $b['title']=='') {
 				$tmp = $b['title'].": \n".$b['body'];
-//                    $tmp = substr($tmp, 0, 4*$max_char);
+	//                    $tmp = substr($tmp, 0, 4*$max_char);
         	        } else {
                 	    $tmp = $b['body']; // substr($b['body'], 0, 3*$max_char);
 	                }
@@ -505,6 +612,8 @@ function statusnet_post_hook(&$a,&$b) {
 			}
 
 			$msg = trim($msg);
+		} else
+			$msg = statusnet_shortenmsg($b, $max_char);
 
 		// and now dent it :-)
 		if(strlen($msg)) {

twitter/twitter.php

@@ -261,6 +261,112 @@ function short_link ($url) {
     return $slinky->short();
 } };
 
+function twitter_shortenmsg($b) {
+	require_once("include/bbcode.php");
+	require_once("include/html2plain.php");
+
+	$max_char = 140;
+
+	// Looking for the first image
+	$image = '';
+	if(preg_match("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/is",$b['body'],$matches))
+		$image = $matches[3];
+
+	if ($image == '')
+		if(preg_match("/\[img\](.*?)\[\/img\]/is",$b['body'],$matches))
+			$image = $matches[1];
+
+	$multipleimages = (strpos($b['body'], "[img") != strrpos($b['body'], "[img"));
+
+	// When saved into the database the content is sent through htmlspecialchars
+	// That means that we have to decode all image-urls
+	$image = htmlspecialchars_decode($image);
+
+	$body = $b["body"];
+	if ($b["title"] != "")
+		$body = $b["title"]."\n\n".$body;
+
+	// remove the recycle signs and the names since they aren't helpful on twitter
+	// recycle 1
+	$recycle = html_entity_decode("♲ ", ENT_QUOTES, 'UTF-8');
+	$body = preg_replace( '/'.$recycle.'\[url\=(\w+.*?)\](\w+.*?)\[\/url\]/i', "\n", $body);
+	// recycle 2 (Test)
+	$recycle = html_entity_decode("◌ ", ENT_QUOTES, 'UTF-8');
+	$body = preg_replace( '/'.$recycle.'\[url\=(\w+.*?)\](\w+.*?)\[\/url\]/i', "\n", $body);
+
+	// At first convert the text to html
+	$html = bbcode($body, false, false);
+
+	// Then convert it to plain text
+	//$msg = trim($b['title']." \n\n".html2plain($html, 0, true));
+	$msg = trim(html2plain($html, 0, true));
+	$msg = html_entity_decode($msg,ENT_QUOTES,'UTF-8');
+
+	// Removing multiple newlines
+	while (strpos($msg, "\n\n\n") !== false)
+		$msg = str_replace("\n\n\n", "\n\n", $msg);
+
+	// Removing multiple spaces
+	while (strpos($msg, "  ") !== false)
+		$msg = str_replace("  ", " ", $msg);
+
+	// Removing URLs
+	$msg = preg_replace('/(https?\:\/\/[a-zA-Z0-9\:\/\-\?\&\;\.\=\_\~\#\%\$\!\+\,]+)/i', "", $msg);
+
+	$msg = trim($msg);
+
+	$link = '';
+	// look for bookmark-bbcode and handle it with priority
+	if(preg_match("/\[bookmark\=([^\]]*)\](.*?)\[\/bookmark\]/is",$b['body'],$matches))
+		$link = $matches[1];
+
+	$multiplelinks = (strpos($b['body'], "[bookmark") != strrpos($b['body'], "[bookmark"));
+
+	// If there is no bookmark element then take the first link
+	if ($link == '') {
+		$links = collecturls($html);
+		if (sizeof($links) > 0) {
+			reset($links);
+			$link = current($links);
+		}
+		$multiplelinks = (sizeof($links) > 1);
+	}
+
+	$msglink = "";
+	if ($multiplelinks)
+		$msglink = $b["plink"];
+	else if ($link != "")
+		$msglink = $link;
+	else if ($multipleimages)
+		$msglink = $b["plink"];
+	else if ($image != "")
+		$msglink = $image;
+
+	if (($msglink == "") and strlen($msg) > $max_char)
+		$msglink = $b["plink"];
+
+	if (strlen($msglink) > 20)
+		$msglink = short_link($msglink);
+
+	if (strlen(trim($msg." ".$msglink)) > $max_char) {
+		$msg = substr($msg, 0, $max_char - (strlen($msglink)));
+		$lastchar = substr($msg, -1);
+		$msg = substr($msg, 0, -1);
+		$pos = strrpos($msg, "\n");
+		if ($pos > 0)
+			$msg = substr($msg, 0, $pos-1);
+		else if ($lastchar != "\n")
+			$msg = substr($msg, 0, -3)."...";
+	}
+	$msg = str_replace("\n", " ", $msg);
+
+	// Removing multiple spaces - again
+	while (strpos($msg, "  ") !== false)
+		$msg = str_replace("  ", " ", $msg);
+
+	return(trim($msg." ".$msglink));
+}
+
 function twitter_post_hook(&$a,&$b) {
 
 	/**
@@ -294,6 +400,10 @@ function twitter_post_hook(&$a,&$b) {
 		$tweet = new TwitterOAuth($ckey,$csecret,$otoken,$osecret);
                 // in theory max char is 140 but T. uses t.co to make links 
                 // longer so we give them 10 characters extra
+
+                $intelligent_shortening = get_config('twitter','intelligent_shortening');
+
+		if (!$intelligent_shortening) {
 			$max_char = 130; // max. length for a tweet
 	                // we will only work with up to two times the length of the dent 
 	                // we can later send to Twitter. This way we can "gain" some 
@@ -301,7 +411,7 @@ function twitter_post_hook(&$a,&$b) {
 	                // shorten all the links in a 200000 character long essay.
 	                if (! $b['title']=='') {
 	                    $tmp = $b['title'] . ' : '. $b['body'];
-//                    $tmp = substr($tmp, 0, 4*$max_char);
+	//                    $tmp = substr($tmp, 0, 4*$max_char);
 	                } else {
 	                    $tmp = $b['body']; // substr($b['body'], 0, 3*$max_char);
 	                }
@@ -369,6 +479,8 @@ function twitter_post_hook(&$a,&$b) {
 			}
 
 			$msg = trim($msg);
+		} else
+			$msg = twitter_shortenmsg($b);
 
 		// and now tweet it :-)
 		if(strlen($msg)) {