Merge remote-tracking branch 'upstream/develop' into more-temp-stuff

INSTALL.txt

@@ -64,7 +64,7 @@ you wish to communicate with the Diaspora network.
 password, database name).
 
     - Friendica needs the permission to create and delete fields and tables in its own database.
-
+    - Please check the additional notes if running on MySQ 5.7.17 or newer
 
 4. If you know in advance that it will be impossible for the web server to 
 write or create files in your web directory, create an empty file called 
@@ -291,3 +291,21 @@ This is obvious as soon as you notice that the friendica-cron uses proc_open to
 execute php-scripts that also use proc_open, but it took me quite some time to
 find that out. I hope this saves some time for other people using suhosin with
 function blacklists.
+
+########################################################################
+Unable to create all mysql tables on MySQL 5.7.17 or newer
+#######################################################################
+
+If the setup fails to create all the database tables and/or manual
+creation from the command line fails, with this error: 
+
+ERROR 1067 (42000) at line XX: Invalid default value for 'created' 
+
+You need to adjust your my.cnf and add the following setting under 
+the [mysqld] section : 
+
+sql_mode = '';
+
+After that, restart mysql and try again. 
+
+

include/ParseUrl.php

@@ -130,7 +130,7 @@ class ParseUrl {
 		$url = trim($url, "'");
 		$url = trim($url, '"');
 
-		$url = original_url($url);
+		$url = strip_tracking_query_params($url);
 
 		$siteinfo["url"] = $url;
 		$siteinfo["type"] = "link";
@@ -142,8 +142,7 @@ class ParseUrl {
 		$ch = curl_init();
 		curl_setopt($ch, CURLOPT_URL, $url);
 		curl_setopt($ch, CURLOPT_HEADER, 1);
-		curl_setopt($ch, CURLOPT_NOBODY, 1);
-		curl_setopt($ch, CURLOPT_TIMEOUT, 3);
+		curl_setopt($ch, CURLOPT_TIMEOUT, 10);
 		curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
 		curl_setopt($ch, CURLOPT_USERAGENT, $a->get_useragent());
 		curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false));
@@ -151,7 +150,6 @@ class ParseUrl {
 
 		$header = curl_exec($ch);
 		$curl_info = @curl_getinfo($ch);
-		$http_code = $curl_info["http_code"];
 		curl_close($ch);
 
 		$a->save_timestamp($stamp1, "network");
@@ -197,26 +195,6 @@ class ParseUrl {
 			}
 		}
 
-		$stamp1 = microtime(true);
-
-		// Now fetch the body as well
-		$ch = curl_init();
-		curl_setopt($ch, CURLOPT_URL, $url);
-		curl_setopt($ch, CURLOPT_HEADER, 1);
-		curl_setopt($ch, CURLOPT_NOBODY, 0);
-		curl_setopt($ch, CURLOPT_TIMEOUT, 10);
-		curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($ch, CURLOPT_USERAGENT, $a->get_useragent());
-		curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (($check_cert) ? true : false));
-		curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, (($check_cert) ? 2 : false));
-
-		$header = curl_exec($ch);
-		$curl_info = @curl_getinfo($ch);
-		$http_code = $curl_info["http_code"];
-		curl_close($ch);
-
-		$a->save_timestamp($stamp1, "network");
-
 		// Fetch the first mentioned charset. Can be in body or header
 		$charset = "";
 		if (preg_match('/charset=(.*?)['."'".'"\s\n]/', $header, $matches)) {

include/network.php

@@ -670,18 +670,21 @@ function fix_contact_ssl_policy(&$contact,$new_policy) {
 	}
 }
 
-function original_url($url, $depth=1, $fetchbody = false) {
-
-	$a = get_app();
-
-	// Remove Analytics Data from Google and other tracking platforms
+/**
+ * @brief Remove Google Analytics and other tracking platforms params from URL
+ *
+ * @param string $url Any user-submitted URL that may contain tracking params
+ * @return string The same URL stripped of tracking parameters
+ */
+function strip_tracking_query_params($url)
+{
 	$urldata = parse_url($url);
 	if (is_string($urldata["query"])) {
 		$query = $urldata["query"];
 		parse_str($query, $querydata);
 
-		if (is_array($querydata))
-			foreach ($querydata AS $param=>$value)
+		if (is_array($querydata)) {
+			foreach ($querydata AS $param => $value) {
 				if (in_array($param, array("utm_source", "utm_medium", "utm_term", "utm_content", "utm_campaign",
 							"wt_mc", "pk_campaign", "pk_kwd", "mc_cid", "mc_eid",
 							"fb_action_ids", "fb_action_types", "fb_ref",
@@ -701,10 +704,36 @@ function original_url($url, $depth=1, $fetchbody = false) {
 
 					$url = str_replace(array("?&", "&&"), array("?", ""), $url);
 				}
+			}
+		}
 
-		if (substr($url, -1, 1) == "?")
+		if (substr($url, -1, 1) == "?") {
 			$url = substr($url, 0, -1);
 		}
+	}
+
+	return $url;
+}
+
+/**
+ * @brief Returns the original URL of the provided URL
+ *
+ * This function strips tracking query params and follows redirections, either
+ * through HTTP code or meta refresh tags. Stops after 10 redirections.
+ *
+ * @todo Remove the $fetchbody parameter that generates an extraneous HEAD request
+ *
+ * @see ParseUrl::getSiteinfo
+ *
+ * @param string $url A user-submitted URL
+ * @param int $depth The current redirection recursion level (internal)
+ * @param bool $fetchbody Wether to fetch the body or not after the HEAD requests
+ * @return string A canonical URL
+ */
+function original_url($url, $depth = 1, $fetchbody = false) {
+	$a = get_app();
+
+	$url = strip_tracking_query_params($url);
 
 	if ($depth > 10)
 		return($url);

util/vagrant_provision.sh

@@ -51,7 +51,7 @@ if [ $( lsb_release -c | cut -f 2 ) == "trusty" ]; then
     sudo service apache2 restart
 elif [ $( lsb_release -c | cut -f 2 ) == "xenial" ]; then
     echo ">>> Installing PHP7"
-    sudo apt-get install -y php libapache2-mod-php php-cli php-mysql php-curl php-gd
+    sudo apt-get install -y php libapache2-mod-php php-cli php-mysql php-curl php-gd php-mbstring
     sudo apt-get install -y imagemagick
     sudo apt-get install -y php-imagick
     sudo systemctl restart apache2