Merge branch 'feature/better-graphs' into develop

.htconfig.php

@@ -13,6 +13,14 @@ $default_timezone = 'Europe/Amsterdam';
 // What is your site name?
 $a->config['sitename'] = "EXPERIMENTAL Friendica public directory";
 
+//Statistic display settings.
+$a->config['stats'] = array(
+  
+  //For site health, the max age for which to display data.
+  'maxDataAge' => 3600*24*30*4 //120 days = ~4 months
+  
+);
+
 //Settings related to the syncing feature.
 $a->config['syncing'] = array(
   
@@ -46,7 +54,7 @@ $a->config['site-health'] = array(
   //Wait for at least ... before probing a site again.
   //The longer this value, the more "stable" site-healths will be over time.
   //Note: If a bad (negative) health site submits something, a probe will be performed regardless.
-  'min_probe_delay' => 3*24*3600, // 3 days
+  'min_probe_delay' => 24*3600, // 1 day
   
   //Probes get a simple /friendica/json file from the server.
   //Feel free to set this timeout to a very tight value.

dfrndir.sql

@@ -154,6 +154,7 @@ CREATE TABLE IF NOT EXISTS `user` (
 CREATE TABLE IF NOT EXISTS `site-health` (
   `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
   `base_url` varchar(255) NOT NULL,
+  `effective_base_url` varchar(255) NULL DEFAULT NULL,
   `health_score` int(11) NOT NULL DEFAULT 0,
   `no_scrape_url` varchar(255) NULL DEFAULT NULL,
   `dt_first_noticed` datetime NOT NULL,

include/site-health.php

@@ -29,7 +29,7 @@ function notice_site($url, $check_health=false)
     $entry = $result[0];
     
     //If we are allowed to do health checks...
-    if(!!$check_health){
+    if($check_health){
       
       //And the site is in bad health currently, do a check now.
       //This is because you have a high certainty the site may perform better now.
@@ -57,7 +57,7 @@ function notice_site($url, $check_health=false)
     );
     
     //And in case we should probe now, do so.
-    if(!!$check_health){
+    if($check_health){
       
       $result = q(
         "SELECT * FROM `site-health` WHERE `base_url`= '%s' ORDER BY `id` ASC LIMIT 1",
@@ -134,7 +134,7 @@ function run_site_probe($id, &$entry_out)
     CURLOPT_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS,
     
     //Basic request
-    CURLOPT_USERAGENT => 'friendica-directory-probe-0.1',
+    CURLOPT_USERAGENT => 'friendica-directory-probe-1.0',
     CURLOPT_RETURNTRANSFER => true,
     CURLOPT_URL => $probe_location
     
@@ -159,7 +159,7 @@ function run_site_probe($id, &$entry_out)
     //Probe again, without strict SSL.
     $options[CURLOPT_SSL_VERIFYPEER] = false;
     
-    //Replace the handler.
+    //Replace the handle.
     curl_close($handle);
     $handle = curl_init();
     curl_setopt_array($handle, $options);
@@ -178,13 +178,14 @@ function run_site_probe($id, &$entry_out)
   $time = round(($probe_end - $probe_start) * 1000);
   $status = curl_getinfo($handle, CURLINFO_HTTP_CODE);
   $type = curl_getinfo($handle, CURLINFO_CONTENT_TYPE);
-  $effective_url = curl_getinfo($handle, CURLINFO_EFFECTIVE_URL);
+  $info = curl_getinfo($handle);
   
   //Done with CURL now.
   curl_close($handle);
   
   #TODO: if the site redirects elsewhere, notice this site and record an issue.
-  $wrong_base_url = parse_site_from_url($effective_url) !== $entry['base_url'];
+  $effective_base_url = parse_site_from_url($info['url']);
+  $wrong_base_url = $effective_base_url !== $entry['base_url'];
   
   try{
     $data = json_decode($probe_data);
@@ -195,6 +196,18 @@ function run_site_probe($id, &$entry_out)
   $parse_failed = !$data;
   
   $parsedDataQuery = '';
+  
+  logger('Effective Base URL: ' . $effective_base_url);
+  
+  if($wrong_base_url){
+    $parsedDataQuery .= sprintf(
+      "`effective_base_url` = '%s',",
+      dbesc($effective_base_url)
+    );
+  }else{
+    $parsedDataQuery .= "`effective_base_url` = NULL,";
+  }
+  
   if(!$parse_failed){
     
     $given_base_url_match = $data->url == $base_url;
@@ -208,7 +221,7 @@ function run_site_probe($id, &$entry_out)
     );
     
     //Update any health calculations or otherwise processed data.
-    $parsedDataQuery = sprintf(
+    $parsedDataQuery .= sprintf(
       "`dt_last_seen` = NOW(),
        `name` = '%s',
        `version` = '%s',
@@ -307,8 +320,8 @@ function health_score_after_probe($current, $probe_success, $time=null, $version
       $current = min($current, 30);
     }
     
-    //Older than 3.2.x?
+    //Older than 3.3.x?
-    elseif(intval($versionParts[1] < 2)){
+    elseif(intval($versionParts[1] < 3)){
       $current -= 5; //Somewhat outdated.
     }
     

include/smoothing.js

@@ -0,0 +1,81 @@
+(function(){
+    
+    window.Smoothing = {
+        
+        /**
+         * Applies both a moving average bracket and and exponential smoothing.
+         * @param  {array}  raw     The raw Y values.
+         * @param  {float}  factor  The exponential smoothing factor to apply (between o and 1).
+         * @param  {int}    bracket The amount of datapoints to add to the backet on each side! (2 = 5 data points)
+         * @return {array}          The smoothed Y values.
+         */
+        exponentialMovingAverage: function(raw, factor, bracket){
+            
+            var output = [];
+            var smoother = new ExponentialSmoother(factor);
+            
+            //Transform each data point with the smoother.
+            for (var i = 0; i < raw.length; i++){
+                
+                var input = raw[i];
+                
+                //See if we should bracket.
+                if(bracket > 0){
+                    
+                    //Cap our start and end so it doesn't go out of bounds.
+                    var start = Math.max(i-bracket, 0);
+                    var end = Math.min(i+bracket, raw.length);
+                    
+                    //Push the range to our input.
+                    input = [];
+                    for(var j = start; j < end; j++){
+                        input.push(raw[j]);
+                    }
+                    
+                }
+                
+                output.push(
+                    smoother.transform(input)
+                );
+            };
+            
+            return output;
+            
+        }
+        
+    };
+    
+    // Exponential Smoother class.
+    var ExponentialSmoother = function(factor){
+        this.currentValue = null;
+        this.smoothingFactor = factor || 1;
+    };
+    
+    ExponentialSmoother.prototype.transform = function(input){
+        
+        // In case our input is a bracket, first average it.
+        if(input.length){
+            var len = input.length;
+            var sum = 0;
+            for (var i = input.length - 1; i >= 0; i--)
+                sum += input[i]
+            input = sum/len;
+        }
+        
+        // Start with our initial value.
+        if(this.currentValue === null){
+            this.currentValue = input;
+        }
+        
+        // Our output is basically an updated value.
+        return this.currentValue = 
+            
+            // Weigh our current value with the smoothing factor.
+            (this.currentValue * this.smoothingFactor) +
+            
+            // Add the input to it with the inverse value of the smoothing factor.
+            ( (1-this.smoothingFactor) * input );
+        
+    };
+    
+})();

mod/health.php

@@ -45,6 +45,7 @@ function health_search(&$a, $search)
 					'<span class="health '.health_score_to_name($site['health_score']).'">&hearts;</span> '.
 					'<a href="/health/'.$site['id'].'">' . $site['base_url'] . '</a> '.
 					'(' . $site['users'] . ')'.
+					($site['effective_base_url'] ? ' -&gt; <abbr title="Redirects to this domain.">'.$site['effective_base_url'].'</abbr>' : '').
 					"<br />\r\n";
 			}
 			
@@ -70,7 +71,7 @@ function health_summary(&$a){
 	$sites = array();
 	
 	//Find the user count per site.
-	$r = q("SELECT `homepage` FROM `profile` WHERE 1");
+	$r = q("SELECT `homepage` FROM `profile`");
 	if(count($r)) {
 		foreach($r as $rr) {
 			$site = parse_site_from_url($rr['homepage']);
@@ -82,11 +83,11 @@ function health_summary(&$a){
 		}
 	}
 	
-	//See if we have a health for them.
+	//See if we have a health for them AND they provide SSL.
 	$sites_with_health = array();
 	$site_healths = array();
 	
-	$r = q("SELECT * FROM `site-health` WHERE `reg_policy`='REGISTER_OPEN'");
+	$r = q("SELECT * FROM `site-health` WHERE `reg_policy`='REGISTER_OPEN' AND `ssl_state` = 1");
 	if(count($r)) {
 		foreach($r as $rr) {
 			$sites_with_health[$rr['base_url']] = (($sites[$rr['base_url']] / 100) + 10) * intval($rr['health_score']);
@@ -106,7 +107,7 @@ function health_summary(&$a){
 		
 		//Skip small sites.
 		$users = $sites[$k];
-		if($users < 10) continue;
+		if($users < 5) continue;
 		
 		$public_sites .=
 			'<span class="health '.health_score_to_name($site['health_score']).'">&hearts;</span> '.
@@ -129,6 +130,42 @@ function health_summary(&$a){
 function health_details($a, $id)
 {
 	
+	//Max data age in MySQL date.
+	$maxDate = date('Y-m-d H:i:s', time()-($a->config['stats']['maxDataAge']));
+	
+	//Include graphael line charts.
+	$a->page['htmlhead'] .= '<script type="text/javascript" src="'.$a->get_baseurl().'/include/smoothing.js"></script>'.PHP_EOL;
+	$a->page['htmlhead'] .= '<script type="text/javascript" src="'.$a->get_baseurl().'/include/raphael.js"></script>'.PHP_EOL;
+	$a->page['htmlhead'] .= '<script type="text/javascript" src="'.$a->get_baseurl().'/include/g.raphael.js"></script>'.PHP_EOL;
+	$a->page['htmlhead'] .= '<script type="text/javascript" src="'.$a->get_baseurl().'/include/g.line-min.js"></script>'.PHP_EOL;
+	$a->page['htmlhead'] .= '<script type="text/javascript">
+		window.smoothingFactor = 0.3;
+		window.smoothingBracket = 2;
+		window.availableCharts = [];
+		window.drawRaw = false;
+		window.drawCharts = function(){
+			for (var i = availableCharts.length - 1; i >= 0; i--) {
+				availableCharts[i](jQuery);
+			};
+		};
+		window.onHoverPoint = function(r){ return function(){
+            this.tags = r.set();
+            var i = this.y.length-1;
+            this.tags.push(r.popup(this.x, this.y[i], Math.round(this.values[i])+"ms", "right", 5).insertBefore(this));
+        }};
+        window.onUnHoverPoint = function(r){ return function(){
+            this.tags && this.tags.remove();
+        }};
+		jQuery(function($){
+			drawCharts();
+			$(".js-toggle-raw").click(function(e){
+				e.preventDefault();
+				drawRaw=!drawRaw;
+				drawCharts();
+			});
+		});
+	</script>';
+	
 	//The overall health status.
 	$r = q(
 		"SELECT * FROM `site-health`
@@ -142,6 +179,22 @@ function health_details($a, $id)
 	
 	$site = $r[0];
 	
+	//Does it redirect to a known site?
+	$redirectStatement = '';
+	if($site['effective_base_url']){
+		
+		//The effective health status.
+		$r = q(
+			"SELECT * FROM `site-health`
+			WHERE `base_url`= '%s'",
+			dbesc($site['effective_base_url'])
+		);
+		if(count($r)){
+			$redirectStatement = '<a href="/health/'.$r[0]['id'].'">Redirects to '.$site['effective_base_url'].'</a>';
+		}
+		
+	}
+	
 	//Figure out SSL state.
 	$urlMeta = parse_url($site['base_url']);
 	if($urlMeta['scheme'] !== 'https'){
@@ -169,8 +222,10 @@ function health_details($a, $id)
 	//Get avg probe speed.
 	$r = q(
 		"SELECT AVG(`request_time`) as `avg_probe_time` FROM `site-probe`
-		WHERE `site_health_id` = %u",
+		WHERE `site_health_id` = %u
-		intval($site['id'])
+		AND `dt_performed` > '%s'",
+		intval($site['id']),
+		$maxDate
 	);
 	if(count($r)){
 		$site['avg_probe_time'] = $r[0]['avg_probe_time'];
@@ -184,8 +239,10 @@ function health_details($a, $id)
 			AVG(`photo_time`) as `avg_photo_time`,
 			AVG(`total_time`) as `avg_submit_time`
 		FROM `site-scrape`
-		WHERE `site_health_id` = %u",
+		WHERE `site_health_id` = %u
-		intval($site['id'])
+		AND `dt_performed` > '%s'",
+		intval($site['id']),
+		$maxDate
 	);
 	if(count($r)){
 		$site['avg_profile_time'] = $r[0]['avg_profile_time'];
@@ -196,62 +253,14 @@ function health_details($a, $id)
 	
 	//Get probe speed data.
 	$r = q(
-		"SELECT `request_time`, `dt_performed` FROM `site-probe`
+		"SELECT AVG(`request_time`) as `avg_time`, date(`dt_performed`) as `date` FROM `site-probe`
-		WHERE `site_health_id` = %u",
+		WHERE `site_health_id` = %u
-		intval($site['id'])
+		AND `dt_performed` > '%s'
+		GROUP BY `date`",
+		intval($site['id']),
+		$maxDate
 	);
 	if(count($r)){
-		//Include graphael line charts.
-		$a->page['htmlhead'] .= '<script type="text/javascript" src="'.$a->get_baseurl().'/include/raphael.js"></script>'.PHP_EOL;
-		$a->page['htmlhead'] .= '<script type="text/javascript" src="'.$a->get_baseurl().'/include/g.raphael.js"></script>'.PHP_EOL;
-		$a->page['htmlhead'] .= '<script type="text/javascript" src="'.$a->get_baseurl().'/include/g.line-min.js"></script>';
-		$speeds = array();
-		$times = array();
-		$mintime = time();
-		foreach($r as $row){
-			$speeds[] = $row['request_time'];
-			$time = strtotime($row['dt_performed']);
-			$times[] = $time;
-			if($mintime > $time) $mintime = $time;
-		}
-		for($i=0; $i < count($times); $i++){
-			$times[$i] -= $mintime;
-			$times[$i] = floor($times[$i] / (24*3600));
-		}
-		$a->page['htmlhead'] .=
-			'<script type="text/javascript">
-				jQuery(function($){
-					
-					var r = Raphael("probe-chart")
-						, x = ['.implode(',', $times).']
-						, y = ['.implode(',', $speeds).']
-					;
-					
-					r.linechart(30, 15, 400, 300, x, [y], {symbol:"circle", axis:"0 0 0 1", shade:true, width:1.5}).hoverColumn(function () {
-            this.tags = r.set();
-            for (var i = 0, ii = this.y.length; i < ii; i++) {
-              this.tags.push(r.popup(this.x, this.y[i], this.values[i]+"ms", "right", 5).insertBefore(this).attr([{ fill: "#eee" }, { fill: this.symbols[i].attr("fill") }]));
-            }
-	        }, function () {
-            this.tags && this.tags.remove();
-	        });
-					
-				});
-			</script>';
-	}
-	
-	//Get scrape speed data.
-	$r = q(
-		"SELECT AVG(`total_time`) as `avg_time`, date(`dt_performed`) as `date` FROM `site-scrape`
-		WHERE `site_health_id` = %u GROUP BY `date`",
-		intval($site['id'])
-		// date('Y-m-d H:i:s', time()-(3*24*3600)) //Max 3 days old.
-	);
-	if($r && count($r)){
-		//Include graphael line charts.
-		$a->page['htmlhead'] .= '<script type="text/javascript" src="'.$a->get_baseurl().'/include/raphael.js"></script>'.PHP_EOL;
-		$a->page['htmlhead'] .= '<script type="text/javascript" src="'.$a->get_baseurl().'/include/g.raphael.js"></script>'.PHP_EOL;
-		$a->page['htmlhead'] .= '<script type="text/javascript" src="'.$a->get_baseurl().'/include/g.line-min.js"></script>';
 		$speeds = array();
 		$times = array();
 		$mintime = time();
@@ -267,23 +276,78 @@ function health_details($a, $id)
 		}
 		$a->page['htmlhead'] .=
 			'<script type="text/javascript">
-				jQuery(function($){
+				(function(){
 					
-					var r = Raphael("scrape-chart")
+					var x = ['.implode(',', $times).'];
-						, x = ['.implode(',', $times).']
+					var y = ['.implode(',', $speeds).'];
-						, y = ['.implode(',', $speeds).']
+					var smoothY = Smoothing.exponentialMovingAverage(y, smoothingFactor, smoothingBracket);
-					;
 					
-					r.linechart(30, 15, 400, 300, x, [y], {shade:true, axis:"0 0 0 1", width:1}).hoverColumn(function () {
+					availableCharts.push(function($){
-            this.tags = r.set();
+						
-            for (var i = 0, ii = this.y.length; i < ii; i++) {
+						var id = "probe-chart";
-              this.tags.push(r.popup(this.x, this.y[i], Math.round(this.values[i])+"ms", "right", 5).insertBefore(this));
+						$("#"+id+" svg").remove();
-            }
+						var r = Raphael(id);
-	        }, function () {
+						var values = [smoothY];
-            this.tags && this.tags.remove();
+						if(drawRaw){
-	        });
+							values.push(y);
+						}
+						
+						r.linechart(30, 15, 400, 295, x, values, {axis:"0 0 1 1", shade:true, width:0.8, axisxstep:6})
+							.hoverColumn(onHoverPoint(r), onUnHoverPoint(r));
+						
+					});
 					
-				});
+				})();
+			</script>';
+	}
+	
+	//Get scrape speed data.
+	$r = q(
+		"SELECT AVG(`total_time`) as `avg_time`, date(`dt_performed`) as `date` FROM `site-scrape`
+		WHERE `site_health_id` = %u
+		AND `dt_performed` > '%s'
+		GROUP BY `date`",
+		intval($site['id']),
+		$maxDate
+	);
+	if($r && count($r)){
+		$speeds = array();
+		$times = array();
+		$mintime = time();
+		foreach($r as $row){
+			$speeds[] = $row['avg_time'];
+			$time = strtotime($row['date']);
+			$times[] = $time;
+			if($mintime > $time) $mintime = $time;
+		}
+		for($i=0; $i < count($times); $i++){
+			$times[$i] -= $mintime;
+			$times[$i] = floor($times[$i] / (24*3600));
+		}
+		$a->page['htmlhead'] .=
+			'<script type="text/javascript">
+				(function(){
+					
+					var x = ['.implode(',', $times).'];
+					var y = ['.implode(',', $speeds).'];
+					var smoothY = Smoothing.exponentialMovingAverage(y, smoothingFactor, smoothingBracket);
+					
+					availableCharts.push(function($){
+						
+						var id = "scrape-chart";
+						$("#"+id+" svg").remove();
+						var r = Raphael(id);
+						var values = [smoothY];
+						if(drawRaw){
+							values.push(y);
+						}
+						
+						r.linechart(30, 15, 400, 295, x, values, {shade:true, axis:"0 0 1 1", width:0.8, axisxstep:6})
+							.hoverColumn(onHoverPoint(r), onUnHoverPoint(r));
+						
+					});
+					
+				})();
 			</script>';
 	}
 	
@@ -298,6 +362,7 @@ function health_details($a, $id)
 	$tpl .= file_get_contents('view/health_details.tpl');
 	return replace_macros($tpl, array(
 		'$name' => $site['name'],
+		'$redirectStatement' => $redirectStatement,
 		'$policy' => $policy,
 		'$site_info' => $site['info'],
 		'$base_url' => $site['base_url'],

view/health_details.tpl

@@ -1,12 +1,13 @@
 <h1>
   <span class="health $health_name">&hearts;</span> $name<br>
-  <sup><a href="$base_url">$base_url</a></sup>
+  <sup><a href="$base_url">$base_url</a></sup><br>
 </h1>
 
 <p><a href="/health">&laquo; Back to index</a></p>
 
 <div class="meta">
   <h3>General information</h3>
+  <div class="redirect">$redirectStatement</div>
   <div class="users">$users users</div>
   <div class="policy">$policy registration policy</div>
   <div class="version">Friendica $version</div>
@@ -23,13 +24,15 @@
 <div class="performance">
   <h3>Performance information</h3>
   <div style="float:left;margin-right:30px;padding-top:20px;">
+    <p><em>Based on the last 120 days.</em></p>
     <div class="probe_speed">Probe speed: $avg_probe_timems</div>
     <div class="photo_speed">Photo speed: $avg_photo_timems</div>
     <div class="profile_speed">Profile speed: $avg_profile_timems</div>
     <div class="scrape_speed">Scrape speed: $avg_scrape_timems</div>
     <div class="submit_speed">Submit speed: $avg_submit_timems</div>
-    <span class="health perfect">$no_scrape_support</span>
+    <span class="health perfect">$no_scrape_support</span><br>
+    <a href="javascript:;" class="js-toggle-raw">Toggle raw data</a>
   </div>
-  <div id="probe-chart" class="speed-chart">Probe speed</div>
+  <div id="probe-chart" class="speed-chart">&lt;- older &nbsp; Probe speed in ms &nbsp; newer -&gt;</div>
-  <div id="scrape-chart" class="speed-chart">Submit speed</div>
+  <div id="scrape-chart" class="speed-chart">&lt;- older &nbsp; Submit speed in ms &nbsp; newer -&gt;</div>
 </div>