Merge pull request #31 from MrPetovan/bug/fix-maintenance-cron

Fix maintenance cron

boot.php

@@ -79,7 +79,7 @@ if (!function_exists('logger')) {
 		}
 		require_once('include/datetime.php');
 
-		@file_put_contents($logfile, datetime_convert() . ':' . ' ' . $msg . "\n", FILE_APPEND);
+		@file_put_contents($logfile, datetime_convert() . ' [#' . getmypid() . '] ' . $msg . "\n", FILE_APPEND);
 		return;
 	}
 }

include/cron_maintain.php

@@ -21,12 +21,23 @@ require_once '.htconfig.php';
 require_once 'dba.php';
 $db = new dba($db_host, $db_user, $db_pass, $db_data, $install);
 
-//Get our set of items. Youngest items first, after the threshold.
+
-//This may be counter-intuitive, but is to prevent items that fail to update from blocking the rest.
+//Get the maintenance backlog size.
+$res = q("SELECT count(*) as `count`
+FROM `profile`
+WHERE `updated` < '%s'",
+	dbesc(date('Y-m-d H:i:s', time() - $a->config['maintenance']['min_scrape_delay']))
+);
+$maintenance_backlog = 'unknown';
+if (count($res)) {
+	$maintenance_backlog = $res[0]['count'] . ' entries left';
+}
+
+//Get our set of items. Oldest items first, after the threshold.
 $res = q("SELECT `id`, `homepage`, `censored`
 FROM `profile`
 WHERE `updated` < '%s'
-ORDER BY `updated` DESC
+ORDER BY `updated` ASC
 LIMIT %u",
 	dbesc(date('Y-m-d H:i:s', time() - $a->config['maintenance']['min_scrape_delay'])),
 	intval($a->config['maintenance']['max_scrapes'])
@@ -56,9 +67,9 @@ $threads = array();
 
 //Debug...
 if ($verbose) {
-	echo "Creating $threadc maintainer threads for $items profiles." . PHP_EOL;
+	echo "Creating $threadc maintainer threads for $items profiles, $maintenance_backlog"  . PHP_EOL;
 }
-logger("Creating $threadc maintainer threads for $items profiles.");
+logger("Creating $threadc maintainer threads for $items profiles. $maintenance_backlog");
 
 for ($i = 0; $i < $threadc; $i++) {
 

include/submit.php

@@ -29,6 +29,15 @@ function run_submit($url) {
 	if(count($r)) {
 		$profile_exists = true;
 		$profile_id = $r[0]['id'];
+
+		$r = q("UPDATE `profile` SET
+			`available` = 0,
+			`updated` = '%s'
+			WHERE `id` = %d LIMIT 1",
+
+			dbesc(datetime_convert()),
+			intval($profile_id)
+		);
 	}
 
 	//Remove duplicates.
@@ -48,6 +57,7 @@ function run_submit($url) {
 
 	//Skip the scrape? :D
 	$noscrape = $site_health && $site_health['no_scrape_url'];
+
 	if($noscrape){
 
 		//Find out who to look up.
@@ -67,7 +77,7 @@ function run_submit($url) {
 	}
 
 	//Empty result is due to an offline site.
-	if(!count($parms)){
+	if(!count($parms) > 1){
 
 		//For large sites this could lower the health too quickly, so don't track health.
 		//But for sites that are already in bad status. Do a cleanup now.
@@ -87,18 +97,20 @@ function run_submit($url) {
 		return true; //This is a good update.
 	}
 
-	//This is most likely a problem with the site configuration. Ignore.
-	elseif(validate_dfrn($parms)) {
-		return false;
-	}
-	
 	if((x($parms,'hide')) || (! (x($parms,'fn')) && (x($parms,'photo')))) {
 		if($profile_exists) {
+			logger('Profile inferred to be opted out of the directory.');
 			nuke_record($url);
 		}
 		return true; //This is a good update.
 	}
 
+	//This is most likely a problem with the site configuration. Ignore.
+	if(validate_dfrn($parms)) {
+		logger('Site is unavailable');
+		return false;
+	}
+
 	$photo = $parms['photo'];
 
 	dbesc_array($parms);
@@ -118,6 +130,7 @@ function run_submit($url) {
 			`nurl` = '%s',
 			`comm` = %d,
 			`tags` = '%s',
+			`available` = 1,
 			`updated` = '%s'
 			WHERE `id` = %d LIMIT 1",
 

mod/directory.php

@@ -46,7 +46,7 @@ function directory_content(App $a)
 
 	$sql_extra = str_replace('%', '%%', $sql_extra);
 
-	$r = q("SELECT COUNT(*) AS `total` FROM `profile` WHERE `censored` = 0 $sql_extra ");
+	$r = q("SELECT COUNT(*) AS `total` FROM `profile` WHERE `censored` = 0 AND `available` = 1 $sql_extra ");
 	if (count($r)) {
 		$total = $r[0]['total'];
 		$a->set_pager_total($total);
@@ -58,7 +58,7 @@ function directory_content(App $a)
 		$order = ' ORDER BY `updated` DESC, `id` DESC ';
 	}
 
-	$r = q("SELECT * FROM `profile` WHERE `censored` = 0 $sql_extra $order LIMIT %d , %d ",
+	$r = q("SELECT * FROM `profile` WHERE `censored` = 0 AND `available` = 1 $sql_extra $order LIMIT %d , %d ",
 		intval($a->pager['start']),
 		intval($a->pager['itemspage'])
 	);

mod/search.php

@@ -60,7 +60,7 @@ function search_content(App $a)
 	$sql_extra = str_replace('%', '%%', $sql_extra);
 
 	$total = 0;
-	$r = q("SELECT COUNT(*) AS `total` FROM `profile` WHERE `censored` = 0 $sql_extra ");
+	$r = q("SELECT COUNT(*) AS `total` FROM `profile` WHERE `censored` = 0 AND `available` = 1 $sql_extra ");
 	if (count($r)) {
 		$total = $r[0]['total'];
 		$a->set_pager_total($total);
@@ -72,7 +72,7 @@ function search_content(App $a)
 		$order = ' ORDER BY `updated` DESC, `id` DESC ';
 	}
 
-	$r = q("SELECT * FROM `profile` WHERE `censored` = 0 $sql_extra $order LIMIT %d , %d ",
+	$r = q("SELECT * FROM `profile` WHERE `censored` = 0 AND `available` = 1 $sql_extra $order LIMIT %d , %d ",
 		intval($a->pager['start']),
 		intval($a->pager['itemspage'])
 	);