Improved site-health detection.

* Detects redirects.
* Better descriptions in site-health details.
* Changed healthy site requirements to require HTTPS and >= 5 users.
* Cleaned graph javascript a little.

dfrndir.sql

@@ -154,6 +154,7 @@ CREATE TABLE IF NOT EXISTS `user` (
 CREATE TABLE IF NOT EXISTS `site-health` (
   `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
   `base_url` varchar(255) NOT NULL,
+  `effective_base_url` varchar(255) NULL DEFAULT NULL,
   `health_score` int(11) NOT NULL DEFAULT 0,
   `no_scrape_url` varchar(255) NULL DEFAULT NULL,
   `dt_first_noticed` datetime NOT NULL,

include/site-health.php

@@ -29,7 +29,7 @@ function notice_site($url, $check_health=false)
     $entry = $result[0];
     
     //If we are allowed to do health checks...
-    if(!!$check_health){
+    if($check_health){
       
       //And the site is in bad health currently, do a check now.
       //This is because you have a high certainty the site may perform better now.
@@ -57,7 +57,7 @@ function notice_site($url, $check_health=false)
     );
     
     //And in case we should probe now, do so.
-    if(!!$check_health){
+    if($check_health){
       
       $result = q(
         "SELECT * FROM `site-health` WHERE `base_url`= '%s' ORDER BY `id` ASC LIMIT 1",
@@ -134,7 +134,7 @@ function run_site_probe($id, &$entry_out)
     CURLOPT_PROTOCOLS => CURLPROTO_HTTP | CURLPROTO_HTTPS,
     
     //Basic request
-    CURLOPT_USERAGENT => 'friendica-directory-probe-0.1',
+    CURLOPT_USERAGENT => 'friendica-directory-probe-1.0',
     CURLOPT_RETURNTRANSFER => true,
     CURLOPT_URL => $probe_location
     
@@ -159,7 +159,7 @@ function run_site_probe($id, &$entry_out)
     //Probe again, without strict SSL.
     $options[CURLOPT_SSL_VERIFYPEER] = false;
     
-    //Replace the handler.
+    //Replace the handle.
     curl_close($handle);
     $handle = curl_init();
     curl_setopt_array($handle, $options);
@@ -178,13 +178,14 @@ function run_site_probe($id, &$entry_out)
   $time = round(($probe_end - $probe_start) * 1000);
   $status = curl_getinfo($handle, CURLINFO_HTTP_CODE);
   $type = curl_getinfo($handle, CURLINFO_CONTENT_TYPE);
-  $effective_url = curl_getinfo($handle, CURLINFO_EFFECTIVE_URL);
+  $info = curl_getinfo($handle);
   
   //Done with CURL now.
   curl_close($handle);
   
   #TODO: if the site redirects elsewhere, notice this site and record an issue.
-  $wrong_base_url = parse_site_from_url($effective_url) !== $entry['base_url'];
+  $effective_base_url = parse_site_from_url($info['url']);
+  $wrong_base_url = $effective_base_url !== $entry['base_url'];
   
   try{
     $data = json_decode($probe_data);
@@ -195,6 +196,18 @@ function run_site_probe($id, &$entry_out)
   $parse_failed = !$data;
   
   $parsedDataQuery = '';
+  
+  logger('Effective Base URL: ' . $effective_base_url);
+  
+  if($wrong_base_url){
+    $parsedDataQuery .= sprintf(
+      "`effective_base_url` = '%s',",
+      dbesc($effective_base_url)
+    );
+  }else{
+    $parsedDataQuery .= "`effective_base_url` = NULL,";
+  }
+  
   if(!$parse_failed){
     
     $given_base_url_match = $data->url == $base_url;
@@ -208,7 +221,7 @@ function run_site_probe($id, &$entry_out)
     );
     
     //Update any health calculations or otherwise processed data.
-    $parsedDataQuery = sprintf(
+    $parsedDataQuery .= sprintf(
       "`dt_last_seen` = NOW(),
        `name` = '%s',
        `version` = '%s',
@@ -307,8 +320,8 @@ function health_score_after_probe($current, $probe_success, $time=null, $version
       $current = min($current, 30);
     }
     
-    //Older than 3.2.x?
+    //Older than 3.3.x?
-    elseif(intval($versionParts[1] < 2)){
+    elseif(intval($versionParts[1] < 3)){
       $current -= 5; //Somewhat outdated.
     }
     

mod/health.php

@@ -45,6 +45,7 @@ function health_search(&$a, $search)
 					'<span class="health '.health_score_to_name($site['health_score']).'">&hearts;</span> '.
 					'<a href="/health/'.$site['id'].'">' . $site['base_url'] . '</a> '.
 					'(' . $site['users'] . ')'.
+					($site['effective_base_url'] ? ' -&gt; <abbr title="Redirects to this domain.">'.$site['effective_base_url'].'</abbr>' : '').
 					"<br />\r\n";
 			}
 			
@@ -70,7 +71,7 @@ function health_summary(&$a){
 	$sites = array();
 	
 	//Find the user count per site.
-	$r = q("SELECT `homepage` FROM `profile` WHERE 1");
+	$r = q("SELECT `homepage` FROM `profile`");
 	if(count($r)) {
 		foreach($r as $rr) {
 			$site = parse_site_from_url($rr['homepage']);
@@ -82,11 +83,11 @@ function health_summary(&$a){
 		}
 	}
 	
-	//See if we have a health for them.
+	//See if we have a health for them AND they provide SSL.
 	$sites_with_health = array();
 	$site_healths = array();
 	
-	$r = q("SELECT * FROM `site-health` WHERE `reg_policy`='REGISTER_OPEN'");
+	$r = q("SELECT * FROM `site-health` WHERE `reg_policy`='REGISTER_OPEN' AND `ssl_state` = 1");
 	if(count($r)) {
 		foreach($r as $rr) {
 			$sites_with_health[$rr['base_url']] = (($sites[$rr['base_url']] / 100) + 10) * intval($rr['health_score']);
@@ -106,7 +107,7 @@ function health_summary(&$a){
 		
 		//Skip small sites.
 		$users = $sites[$k];
-		if($users < 10) continue;
+		if($users < 5) continue;
 		
 		$public_sites .=
 			'<span class="health '.health_score_to_name($site['health_score']).'">&hearts;</span> '.
@@ -178,6 +179,22 @@ function health_details($a, $id)
 	
 	$site = $r[0];
 	
+	//Does it redirect to a known site?
+	$redirectStatement = '';
+	if($site['effective_base_url']){
+		
+		//The effective health status.
+		$r = q(
+			"SELECT * FROM `site-health`
+			WHERE `base_url`= '%s'",
+			dbesc($site['effective_base_url'])
+		);
+		if(count($r)){
+			$redirectStatement = '<a href="/health/'.$r[0]['id'].'">Redirects to '.$site['effective_base_url'].'</a>';
+		}
+		
+	}
+	
 	//Figure out SSL state.
 	$urlMeta = parse_url($site['base_url']);
 	if($urlMeta['scheme'] !== 'https'){
@@ -205,8 +222,10 @@ function health_details($a, $id)
 	//Get avg probe speed.
 	$r = q(
 		"SELECT AVG(`request_time`) as `avg_probe_time` FROM `site-probe`
-		WHERE `site_health_id` = %u",
+		WHERE `site_health_id` = %u
-		intval($site['id'])
+		AND `dt_performed` > '%s'",
+		intval($site['id']),
+		$maxDate
 	);
 	if(count($r)){
 		$site['avg_probe_time'] = $r[0]['avg_probe_time'];
@@ -220,8 +239,10 @@ function health_details($a, $id)
 			AVG(`photo_time`) as `avg_photo_time`,
 			AVG(`total_time`) as `avg_submit_time`
 		FROM `site-scrape`
-		WHERE `site_health_id` = %u",
+		WHERE `site_health_id` = %u
-		intval($site['id'])
+		AND `dt_performed` > '%s'",
+		intval($site['id']),
+		$maxDate
 	);
 	if(count($r)){
 		$site['avg_profile_time'] = $r[0]['avg_profile_time'];
@@ -255,23 +276,28 @@ function health_details($a, $id)
 		}
 		$a->page['htmlhead'] .=
 			'<script type="text/javascript">
-				availableCharts.push(function($){
+				(function(){
 					
-					var id = "probe-chart";
-					$("#"+id+" svg").remove();
-					var r = Raphael(id);
 					var x = ['.implode(',', $times).'];
 					var y = ['.implode(',', $speeds).'];
 					var smoothY = Smoothing.exponentialMovingAverage(y, smoothingFactor, smoothingBracket);
-					var values = [smoothY];
-					if(drawRaw){
-						values.push(y);
-					}
 					
-					r.linechart(30, 15, 400, 300, x, values, {axis:"0 0 0 1", shade:true, width:0.8})
+					availableCharts.push(function($){
-						.hoverColumn(onHoverPoint(r), onUnHoverPoint(r));
+						
+						var id = "probe-chart";
+						$("#"+id+" svg").remove();
+						var r = Raphael(id);
+						var values = [smoothY];
+						if(drawRaw){
+							values.push(y);
+						}
+						
+						r.linechart(30, 15, 400, 295, x, values, {axis:"0 0 1 1", shade:true, width:0.8, axisxstep:6})
+							.hoverColumn(onHoverPoint(r), onUnHoverPoint(r));
+						
+					});
 					
-				});
+				})();
 			</script>';
 	}
 	
@@ -300,23 +326,28 @@ function health_details($a, $id)
 		}
 		$a->page['htmlhead'] .=
 			'<script type="text/javascript">
-				availableCharts.push(function($){
+				(function(){
 					
-					var id = "scrape-chart";
-					$("#"+id+" svg").remove();
-					var r = Raphael(id);
 					var x = ['.implode(',', $times).'];
 					var y = ['.implode(',', $speeds).'];
 					var smoothY = Smoothing.exponentialMovingAverage(y, smoothingFactor, smoothingBracket);
-					var values = [smoothY];
-					if(drawRaw){
-						values.push(y);
-					}
 					
-					r.linechart(30, 15, 400, 300, x, values, {shade:true, axis:"0 0 0 1", width:0.8})
+					availableCharts.push(function($){
-						.hoverColumn(onHoverPoint(r), onUnHoverPoint(r));
+						
+						var id = "scrape-chart";
+						$("#"+id+" svg").remove();
+						var r = Raphael(id);
+						var values = [smoothY];
+						if(drawRaw){
+							values.push(y);
+						}
+						
+						r.linechart(30, 15, 400, 295, x, values, {shade:true, axis:"0 0 1 1", width:0.8, axisxstep:6})
+							.hoverColumn(onHoverPoint(r), onUnHoverPoint(r));
+						
+					});
 					
-				});
+				})();
 			</script>';
 	}
 	
@@ -331,6 +362,7 @@ function health_details($a, $id)
 	$tpl .= file_get_contents('view/health_details.tpl');
 	return replace_macros($tpl, array(
 		'$name' => $site['name'],
+		'$redirectStatement' => $redirectStatement,
 		'$policy' => $policy,
 		'$site_info' => $site['info'],
 		'$base_url' => $site['base_url'],

view/health_details.tpl

@@ -1,12 +1,13 @@
 <h1>
   <span class="health $health_name">&hearts;</span> $name<br>
-  <sup><a href="$base_url">$base_url</a></sup>
+  <sup><a href="$base_url">$base_url</a></sup><br>
 </h1>
 
 <p><a href="/health">&laquo; Back to index</a></p>
 
 <div class="meta">
   <h3>General information</h3>
+  <div class="redirect">$redirectStatement</div>
   <div class="users">$users users</div>
   <div class="policy">$policy registration policy</div>
   <div class="version">Friendica $version</div>
@@ -23,6 +24,7 @@
 <div class="performance">
   <h3>Performance information</h3>
   <div style="float:left;margin-right:30px;padding-top:20px;">
+    <p><em>Based on the last 120 days.</em></p>
     <div class="probe_speed">Probe speed: $avg_probe_timems</div>
     <div class="photo_speed">Photo speed: $avg_photo_timems</div>
     <div class="profile_speed">Profile speed: $avg_profile_timems</div>
@@ -31,6 +33,6 @@
     <span class="health perfect">$no_scrape_support</span><br>
     <a href="javascript:;" class="js-toggle-raw">Toggle raw data</a>
   </div>
-  <div id="probe-chart" class="speed-chart">Probe speed in ms<br>(last 120 days)</div>
+  <div id="probe-chart" class="speed-chart">&lt;- older &nbsp; Probe speed in ms &nbsp; newer -&gt;</div>
-  <div id="scrape-chart" class="speed-chart">Submit speed in ms<br>(last 120 days)</div>
+  <div id="scrape-chart" class="speed-chart">&lt;- older &nbsp; Submit speed in ms &nbsp; newer -&gt;</div>
 </div>