Check Diaspora contacts, don't poll unreachable contacts too often

src/Worker/Cron.php

@@ -153,17 +153,20 @@ Class Cron {
 			: ''
 		);
 
-		$contacts = q("SELECT `contact`.`id` FROM `user`
+		$contacts = q("SELECT `contact`.`id`, `contact`.`nick`, `contact`.`name`, `contact`.`network`,
+					`contact`.`last-update`, `contact`.`priority`, `contact`.`subhub`
+				FROM `user`
 				STRAIGHT_JOIN `contact`
 				ON `contact`.`uid` = `user`.`uid` AND `contact`.`rel` IN (%d, %d) AND `contact`.`poll` != ''
-					AND `contact`.`network` IN ('%s', '%s', '%s', '%s', '%s') $sql_extra
+					AND `contact`.`network` IN ('%s', '%s', '%s', '%s', '%s', '%s') $sql_extra
 					AND NOT `contact`.`self` AND NOT `contact`.`blocked` AND NOT `contact`.`readonly`
 					AND NOT `contact`.`archive`
-				WHERE NOT `user`.`account_expired` AND NOT `user`.`account_removed` $abandon_sql ORDER BY RAND()",
+				WHERE NOT `user`.`account_expired` AND NOT `user`.`account_removed` $abandon_sql",
 			intval(CONTACT_IS_SHARING),
 			intval(CONTACT_IS_FRIEND),
 			dbesc(NETWORK_DFRN),
 			dbesc(NETWORK_OSTATUS),
+			dbesc(NETWORK_DIASPORA),
 			dbesc(NETWORK_FEED),
 			dbesc(NETWORK_MAIL),
 			dbesc(NETWORK_MAIL2)
@@ -173,94 +176,81 @@ Class Cron {
 			return;
 		}
 
-		foreach ($contacts as $c) {
+		foreach ($contacts as $contact) {
 
-			$res = q("SELECT * FROM `contact` WHERE `id` = %d LIMIT 1",
+			if ($manual_id) {
-				intval($c['id'])
+				$contact['last-update'] = NULL_DATE;
-			);
-
-			if (!DBM::is_result($res)) {
-				continue;
 			}
 
-			foreach ($res as $contact) {
+			if (in_array($contact['network'], array(NETWORK_DFRN, NETWORK_OSTATUS))) {
-
+				$contact['priority'] = 2;
-				$xml = false;
-
-				if ($manual_id) {
-					$contact['last-update'] = NULL_DATE;
-				}
-
-				if (in_array($contact['network'], array(NETWORK_DFRN, NETWORK_OSTATUS))) {
-					$contact['priority'] = 2;
-				}
-
-				if ($contact['subhub'] && in_array($contact['network'], array(NETWORK_DFRN, NETWORK_OSTATUS))) {
-					/*
-					 * We should be getting everything via a hub. But just to be sure, let's check once a day.
-					 * (You can make this more or less frequent if desired by setting 'pushpoll_frequency' appropriately)
-					 * This also lets us update our subscription to the hub, and add or replace hubs in case it
-					 * changed. We will only update hubs once a day, regardless of 'pushpoll_frequency'.
-					 */
-					$poll_interval = Config::get('system', 'pushpoll_frequency');
-					$contact['priority'] = (($poll_interval !== false) ? intval($poll_interval) : 3);
-				}
-
-				if (($contact['priority'] >= 0) && !$force) {
-					$update = false;
-
-					$t = $contact['last-update'];
-
-					/*
-					 * Based on $contact['priority'], should we poll this site now? Or later?
-					 */
-					switch ($contact['priority']) {
-						case 5:
-							if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 month")) {
-								$update = true;
-							}
-							break;
-						case 4:
-							if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 week")) {
-								$update = true;
-							}
-							break;
-						case 3:
-							if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 day")) {
-								$update = true;
-							}
-							break;
-						case 2:
-							if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 12 hour")) {
-								$update = true;
-							}
-							break;
-						case 1:
-							if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 hour")) {
-								$update = true;
-							}
-							break;
-						case 0:
-						default:
-							if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + ".$min_poll_interval." minute")) {
-								$update = true;
-							}
-							break;
-					}
-					if (!$update) {
-						continue;
-					}
-				}
-
-				logger("Polling " . $contact["network"] . " " . $contact["id"] . " " . $contact["nick"] . " " . $contact["name"]);
-
-				if (($contact['network'] == NETWORK_FEED) && ($contact['priority'] <= 3)) {
-					$priority = PRIORITY_MEDIUM;
-				} else {
-					$priority = PRIORITY_LOW;
-				}
-				Worker::add(array('priority' => $priority, 'dont_fork' => true), 'OnePoll', (int)$contact['id']);
 			}
+
+			if ($contact['subhub'] && in_array($contact['network'], array(NETWORK_DFRN, NETWORK_OSTATUS))) {
+				/*
+				 * We should be getting everything via a hub. But just to be sure, let's check once a day.
+				 * (You can make this more or less frequent if desired by setting 'pushpoll_frequency' appropriately)
+				 * This also lets us update our subscription to the hub, and add or replace hubs in case it
+				 * changed. We will only update hubs once a day, regardless of 'pushpoll_frequency'.
+				 */
+				$poll_interval = Config::get('system', 'pushpoll_frequency');
+				$contact['priority'] = (($poll_interval !== false) ? intval($poll_interval) : 3);
+			}
+
+			if (($contact['priority'] >= 0) && !$force) {
+				$update = false;
+
+				$t = $contact['last-update'];
+
+				/*
+				 * Based on $contact['priority'], should we poll this site now? Or later?
+				 */
+				switch ($contact['priority']) {
+					case 5:
+						if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 month")) {
+							$update = true;
+						}
+						break;
+					case 4:
+						if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 week")) {
+							$update = true;
+						}
+						break;
+					case 3:
+						if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 day")) {
+							$update = true;
+						}
+						break;
+					case 2:
+						if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 12 hour")) {
+							$update = true;
+						}
+						break;
+					case 1:
+						if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + 1 hour")) {
+							$update = true;
+						}
+						break;
+					case 0:
+					default:
+						if (datetime_convert('UTC', 'UTC', 'now') > datetime_convert('UTC', 'UTC', $t . " + ".$min_poll_interval." minute")) {
+							$update = true;
+						}
+						break;
+				}
+				if (!$update) {
+					continue;
+				}
+			}
+
+			logger("Polling " . $contact["network"] . " " . $contact["id"] . " " . $contact["nick"] . " " . $contact["name"]);
+
+			if (($contact['network'] == NETWORK_FEED) && ($contact['priority'] <= 3)) {
+				$priority = PRIORITY_MEDIUM;
+			} else {
+				$priority = PRIORITY_LOW;
+			}
+			Worker::add(array('priority' => $priority, 'dont_fork' => true), 'OnePoll', (int)$contact['id']);
 		}
 	}
 }

src/Worker/OnePoll.php

@@ -23,7 +23,7 @@ Class OnePoll
 		require_once 'include/items.php';
 		require_once 'include/queue_fn.php';
 
-		logger('onepoll: start');
+		logger('start');
 
 		$manual_id  = 0;
 		$generation = 0;
@@ -36,7 +36,7 @@ Class OnePoll
 		}
 
 		if (!$contact_id) {
-			logger('onepoll: no contact');
+			logger('no contact');
 			return;
 		}
 
@@ -88,10 +88,20 @@ Class OnePoll
 				$last_updated = PortableContact::lastUpdated($contact["url"]);
 				$updated = datetime_convert();
 				if ($last_updated) {
+					logger('Diaspora contact '.$contact['id'].' had last update on '.$last_updated, LOGGER_DEBUG);
+
+					// The last public item can be older than the last item we got
+					if ($last_updated < $contact['last-item']) {
+						$last_updated = $contact['last-item'];
+					}
+
 					$fields = array('last-item' => $last_updated, 'last-update' => $updated, 'success_update' => $updated);
 					dba::update('contact', $fields, array('id' => $contact['id']));
+					Contact::unmarkForArchival($contact);
 				} else {
 					dba::update('contact', array('last-update' => $updated, 'failure_update' => $updated), array('id' => $contact['id']));
+					Contact::markForArchival($contact);
+					logger('Diaspora contact '.$contact['id'].' is marked for archival', LOGGER_DEBUG);
 				}
 			}
 			return;
@@ -122,12 +132,18 @@ Class OnePoll
 		if (($contact['network'] === NETWORK_OSTATUS) || ($contact['network'] === NETWORK_DIASPORA) || ($contact['network'] === NETWORK_DFRN)) {
 			if (!PortableContact::reachable($contact['url'])) {
 				logger("Skipping probably dead contact ".$contact['url']);
+
+				// set the last-update so we don't keep polling
+				dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
 				return;
 			}
 
 			if (!update_contact($contact["id"])) {
 				Contact::markForArchival($contact);
 				logger('Contact is marked dead');
+
+				// set the last-update so we don't keep polling
+				dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
 				return;
 			} else {
 				Contact::unmarkForArchival($contact);
@@ -136,6 +152,9 @@ Class OnePoll
 
 		if ($importer_uid == 0) {
 			logger('Ignore public contacts');
+
+			// set the last-update so we don't keep polling
+			dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
 			return;
 		}
 
@@ -145,12 +164,15 @@ Class OnePoll
 
 		if (!DBM::is_result($r)) {
 			logger('No self contact for user '.$importer_uid);
+
+			// set the last-update so we don't keep polling
+			dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
 			return;
 		}
 
 		$importer = $r[0];
 
-		logger("onepoll: poll: ({$contact['id']}) IMPORTER: {$importer['name']}, CONTACT: {$contact['name']}");
+		logger("poll: ({$contact['network']}-{$contact['id']}) IMPORTER: {$importer['name']}, CONTACT: {$contact['name']}");
 
 		if ($contact['network'] === NETWORK_DFRN) {
 			$idtosend = $orig_id = (($contact['dfrn-id']) ? $contact['dfrn-id'] : $contact['issued-id']);
@@ -179,6 +201,9 @@ Class OnePoll
 			$ret = z_fetch_url($url);
 
 			if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) {
+				// set the last-update so we don't keep polling
+				dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
+				Contact::markForArchival($contact);
 				return;
 			}
 
@@ -186,7 +211,7 @@ Class OnePoll
 
 			$html_code = $a->get_curl_code();
 
-			logger('onepoll: handshake with url ' . $url . ' returns xml: ' . $handshake_xml, LOGGER_DATA);
+			logger('handshake with url ' . $url . ' returns xml: ' . $handshake_xml, LOGGER_DATA);
 
 
 			if (!strlen($handshake_xml) || ($html_code >= 400) || !$html_code) {
@@ -201,7 +226,6 @@ Class OnePoll
 				// set the last-update so we don't keep polling
 				$fields = array('last-update' => datetime_convert(), 'failure_update' => datetime_convert());
 				dba::update('contact', $fields, array('id' => $contact['id']));
-
 				return;
 			}
 
@@ -212,7 +236,6 @@ Class OnePoll
 
 				$fields = array('last-update' => datetime_convert(), 'failure_update' => datetime_convert());
 				dba::update('contact', $fields, array('id' => $contact['id']));
-
 				return;
 			}
 
@@ -234,6 +257,8 @@ Class OnePoll
 			}
 
 			if ((intval($res->status) != 0) || !strlen($res->challenge) || !strlen($res->dfrn_id)) {
+				// set the last-update so we don't keep polling
+				dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
 				return;
 			}
 
@@ -264,8 +289,12 @@ Class OnePoll
 			}
 
 			if ($final_dfrn_id != $orig_id) {
-				logger('ID did not decode: ' . $contact['id'] . ' orig: ' . $orig_id . ' final: ' . $final_dfrn_id);
 				// did not decode properly - cannot trust this site
+				logger('ID did not decode: ' . $contact['id'] . ' orig: ' . $orig_id . ' final: ' . $final_dfrn_id);
+
+				// set the last-update so we don't keep polling
+				dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
+				Contact::markForArchival($contact);
 				return;
 			}
 
@@ -298,6 +327,8 @@ Class OnePoll
 			// Are we allowed to import from this person?
 
 			if ($contact['rel'] == CONTACT_IS_FOLLOWER || $contact['blocked'] || $contact['readonly']) {
+				// set the last-update so we don't keep polling
+				dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
 				return;
 			}
 
@@ -305,6 +336,9 @@ Class OnePoll
 			$ret = z_fetch_url($contact['poll'], false, $redirects, array('cookiejar' => $cookiejar));
 
 			if ($ret['errno'] == CURLE_OPERATION_TIMEDOUT) {
+				// set the last-update so we don't keep polling
+				dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
+				Contact::markForArchival($contact);
 				return;
 			}
 
@@ -317,6 +351,9 @@ Class OnePoll
 
 			$mail_disabled = ((function_exists('imap_open') && (! Config::get('system', 'imap_disabled'))) ? 0 : 1);
 			if ($mail_disabled) {
+				// set the last-update so we don't keep polling
+				dba::update('contact', ['last-update' => datetime_convert()], ['id' => $contact['id']]);
+				Contact::markForArchival($contact);
 				return;
 			}
 
@@ -351,7 +388,7 @@ Class OnePoll
 
 					$metas = Email::messageMeta($mbox, implode(',', $msgs));
 					if (count($metas) != count($msgs)) {
-						logger("onepoll: for " . $mailconf['user'] . " there are ". count($msgs) . " messages but received " . count($metas) . " metas", LOGGER_DEBUG);
+						logger("for " . $mailconf['user'] . " there are ". count($msgs) . " messages but received " . count($metas) . " metas", LOGGER_DEBUG);
 					} else {
 						$msgs = array_combine($msgs, $metas);
 
@@ -560,7 +597,7 @@ Class OnePoll
 
 				$fields = array('last-update' => datetime_convert(), 'failure_update' => datetime_convert());
 				dba::update('contact', $fields, array('id' => $contact['id']));
-
+				Contact::markForArchival($contact);
 				return;
 			}