Merge pull request #7323 from annando/contact-discovery

Removing functionality from OnePoll.php

src/Model/Contact.php

@@ -1328,7 +1328,7 @@ class Contact extends BaseObject
 
 			// Update the contact in the background if needed but it is called by the frontend
 			if ($update_contact && $no_update && in_array($contact['network'], Protocol::NATIVE_SUPPORT)) {
-				Worker::add(PRIORITY_LOW, "UpdateContact", $contact_id);
+				Worker::add(PRIORITY_LOW, "UpdateContact", $contact_id, ($uid == 0 ? 'force' : ''));
 			}
 
 			if (!$update_contact || $no_update) {
@@ -1425,7 +1425,7 @@ class Contact extends BaseObject
 
 			// Update in the background when we fetched the data solely from the database
 			if ($background_update) {
-				Worker::add(PRIORITY_LOW, "UpdateContact", $contact_id);
+				Worker::add(PRIORITY_LOW, "UpdateContact", $contact_id, ($uid == 0 ? 'force' : ''));
 			}
 
 			// Update the newly created contact from data in the gcontact table
@@ -1452,7 +1452,7 @@ class Contact extends BaseObject
 			}
 		}
 
-		if (!empty($data['photo'])) {
+		if (!empty($data['photo']) && ($data['network'] != Protocol::FEED)) {
 			self::updateAvatar($data['photo'], $uid, $contact_id);
 		}
 
@@ -1755,6 +1755,51 @@ class Contact extends BaseObject
 		return $data;
 	}
 
+        /**
+	 * @brief Helper function for "updateFromProbe". Updates personal and public contact
+	 *
+	 * @param array $contact The personal contact entry
+	 * @param array $fields  The fields that are updated
+	 * @throws \Exception
+	 */
+	private static function updateContact($id, $uid, $url, array $fields)
+	{
+		DBA::update('contact', $fields, ['id' => $id]);
+
+		if ($uid != 0) {
+			return;
+		}
+
+		// Archive or unarchive the contact. We only need to do this for the public contact.
+		// The archive/unarchive function will update the personal contacts by themselves.
+		$contact = DBA::selectFirst('contact', [], ['id' => $id]);
+		if (!empty($fields['success_update'])) {
+			self::unmarkForArchival($contact);
+		} elseif (!empty($fields['failure_update'])) {
+			self::markForArchival($contact);
+		}
+
+		$condition = ['self' => false, 'nurl' => Strings::normaliseLink($url),
+			'network' => [Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS]];
+
+		// These contacts are sharing with us, we don't poll them.
+		// This means that we don't set the update fields in "OnePoll.php".
+		$condition['rel'] = self::SHARING;
+		DBA::update('contact', $fields, $condition);
+
+		unset($fields['last-update']);
+		unset($fields['success_update']);
+		unset($fields['failure_update']);
+
+		if (empty($fields)) {
+			return;
+		}
+
+		// We are polling these contacts, so we mustn't set the update fields here.
+		$condition['rel'] = [self::FOLLOWER, self::FRIEND];
+		DBA::update('contact', $fields, $condition);
+	}
+
 	/**
 	 * @param integer $id      contact id
 	 * @param string  $network Optional network we are probing for
@@ -1767,7 +1812,7 @@ class Contact extends BaseObject
 	{
 		/*
 		  Warning: Never ever fetch the public key via Probe::uri and write it into the contacts.
-		  This will reliably kill your communication with Friendica contacts.
+		  This will reliably kill your communication with old Friendica contacts.
 		 */
 
 		$fields = ['avatar', 'uid', 'name', 'nick', 'url', 'addr', 'batch', 'notify',
@@ -1785,12 +1830,14 @@ class Contact extends BaseObject
 
 		$ret = Probe::uri($contact['url'], $network, $uid, !$force);
 
-		// If Probe::uri fails the network code will be different (mostly "feed" or "unkn")
+		$updated = DateTimeFormat::utcNow();
-		if (in_array($ret['network'], [Protocol::FEED, Protocol::PHANTOM]) && ($ret['network'] != $contact['network'])) {
-			return false;
-		}
 
-		if (!in_array($ret['network'], Protocol::NATIVE_SUPPORT)) {
+		// If Probe::uri fails the network code will be different (mostly "feed" or "unkn")
+		if (!in_array($ret['network'], Protocol::NATIVE_SUPPORT) ||
+			(in_array($ret['network'], [Protocol::FEED, Protocol::PHANTOM]) && ($ret['network'] != $contact['network']))) {
+			if ($force && ($uid == 0)) {
+				self::updateContact($id, $uid, $ret['url'], ['last-update' => $updated, 'failure_update' => $updated]);
+			}
 			return false;
 		}
 
@@ -1807,17 +1854,28 @@ class Contact extends BaseObject
 			}
 		}
 
+		if ($ret['network'] != Protocol::FEED) {
+			self::updateAvatar($ret['photo'], $uid, $id, $update || $force);
+		}
+
 		if (!$update) {
+			if ($force && ($uid == 0)) {
+				self::updateContact($id, $uid, $ret['url'], ['last-update' => $updated, 'success_update' => $updated]);
+			}
 			return true;
 		}
 
 		$ret['nurl'] = Strings::normaliseLink($ret['url']);
-		$ret['updated'] = DateTimeFormat::utcNow();
+		$ret['updated'] = $updated;
 
-		self::updateAvatar($ret['photo'], $uid, $id, true);
+		if ($force && ($uid == 0)) {
+			$ret['last-update'] = $updated;
+			$ret['success_update'] = $updated;
+		}
 
 		unset($ret['photo']);
-		DBA::update('contact', $ret, ['id' => $id]);
+
+		self::updateContact($id, $uid, $ret['url'], $ret);
 
 		// Update the corresponding gcontact entry
 		PortableContact::lastUpdated($ret["url"]);

src/Worker/Cron.php

@@ -17,16 +17,10 @@ use Friendica\Util\DateTimeFormat;
 
 class Cron
 {
-	public static function execute($parameter = '', $generation = 0)
+	public static function execute()
 	{
 		$a = BaseObject::getApp();
 
-		// Poll contacts with specific parameters
-		if (!empty($parameter)) {
-			self::pollContacts($parameter, $generation);
-			return;
-		}
-
 		$last = Config::get('system', 'last_cron');
 
 		$poll_interval = intval(Config::get('system', 'cron_interval'));
@@ -115,7 +109,10 @@ class Cron
 		}
 
 		// Poll contacts
-		self::pollContacts($parameter, $generation);
+		self::pollContacts();
+
+		// Update contact information
+		self::updatePublicContacts();
 
 		Logger::log('cron: end');
 
@@ -125,98 +122,76 @@ class Cron
 	}
 
 	/**
-	 * @brief Poll contacts for unreceived messages
+	 * @brief Update public contacts
-	 *
-	 * @todo  Currently it seems as if the following parameter aren't used at all ...
-	 *
-	 * @param string  $parameter Parameter (force, restart, ...) for the contact polling
-	 * @param integer $generation
 	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
 	 */
-	private static function pollContacts($parameter, $generation) {
+	private static function updatePublicContacts() {
-		$manual_id  = 0;
+		$count = 0;
-		$generation = 0;
+		$last_updated = DateTimeFormat::utc('now - 1 week');
-		$force      = false;
+		$condition = ["`network` IN (?, ?, ?, ?) AND `uid` = ? AND NOT `self` AND `last-update` < ?",
+			Protocol::ACTIVITYPUB, Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, 0, $last_updated];
 
-		if ($parameter == 'force') {
+		$total = DBA::count('contact', $condition);
-			$force = true;
+		$oldest_date = '';
+		$oldest_id = '';
+		$contacts = DBA::select('contact', ['id', 'last-update'], $condition, ['limit' => 100, 'order' => ['last-update']]);
+		while ($contact = DBA::fetch($contacts)) {
+			if (empty($oldest_id)) {
+				$oldest_id = $contact['id'];
+				$oldest_date = $contact['last-update'];
 			}
-		if ($parameter == 'restart') {
+			Worker::add(PRIORITY_LOW, "UpdateContact", $contact['id'], 'force');
-			$generation = intval($generation);
+			++$count;
-			if (!$generation) {
-				exit();
 		}
+		Logger::info('Initiated update for public contacts', ['interval' => $count, 'total' => $total, 'id' => $oldest_id, 'oldest' => $oldest_date]);
+		DBA::close($contacts);
 	}
 
-		if (intval($parameter)) {
+	/**
-			$manual_id = intval($parameter);
+	 * @brief Poll contacts for unreceived messages
-			$force     = true;
+	 *
-		}
+	 * @throws \Friendica\Network\HTTPException\InternalServerErrorException
-
+	 */
+	private static function pollContacts() {
 		$min_poll_interval = Config::get('system', 'min_poll_interval', 1);
 
-		$sql_extra = (($manual_id) ? " AND `id` = $manual_id " : "");
-
 		Addon::reload();
 
-		// Only poll from those with suitable relationships,
+		$sql = "SELECT `contact`.`id`, `contact`.`nick`, `contact`.`name`, `contact`.`network`, `contact`.`archive`,
-		// and which have a polling address and ignore Diaspora since
-		// we are unable to match those posts with a Diaspora GUID and prevent duplicates.
-
-		$abandon_days = intval(Config::get('system', 'account_abandon_days'));
-		if ($abandon_days < 1) {
-			$abandon_days = 0;
-		}
-		$abandon_sql = (($abandon_days)
-			? sprintf(" AND `user`.`login_date` > UTC_TIMESTAMP() - INTERVAL %d DAY ", intval($abandon_days))
-			: ''
-		);
-
-		$contacts = q("SELECT `contact`.`id`, `contact`.`nick`, `contact`.`name`, `contact`.`network`, `contact`.`archive`,
 					`contact`.`last-update`, `contact`.`priority`, `contact`.`rel`, `contact`.`subhub`
 				FROM `user`
 				STRAIGHT_JOIN `contact`
 				ON `contact`.`uid` = `user`.`uid` AND `contact`.`poll` != ''
-					AND `contact`.`network` IN ('%s', '%s', '%s', '%s', '%s', '%s') $sql_extra
+					AND `contact`.`network` IN (?, ?, ?, ?)
 					AND NOT `contact`.`self` AND NOT `contact`.`blocked`
-				WHERE NOT `user`.`account_expired` AND NOT `user`.`account_removed` $abandon_sql",
+					AND `contact`.`rel` != ?
-			DBA::escape(Protocol::ACTIVITYPUB),
+				WHERE NOT `user`.`account_expired` AND NOT `user`.`account_removed`";
-			DBA::escape(Protocol::DFRN),
+
-			DBA::escape(Protocol::OSTATUS),
+		$parameters = [Protocol::DFRN, Protocol::OSTATUS, Protocol::FEED, Protocol::MAIL, Contact::FOLLOWER];
-			DBA::escape(Protocol::DIASPORA),
+
-			DBA::escape(Protocol::FEED),
+		// Only poll from those with suitable relationships,
-			DBA::escape(Protocol::MAIL)
+		// and which have a polling address and ignore Diaspora since
-		);
+		// we are unable to match those posts with a Diaspora GUID and prevent duplicates.
+		$abandon_days = intval(Config::get('system', 'account_abandon_days'));
+		if ($abandon_days < 1) {
+			$abandon_days = 0;
+		}
+
+		if (!empty($abandon_days)) {
+			$sql .= " AND `user`.`login_date` > UTC_TIMESTAMP() - INTERVAL ? DAY";
+			$parameters[] = $abandon_days;
+		}
+
+		$contacts = DBA::p($sql, $parameters);
 
 		if (!DBA::isResult($contacts)) {
 			return;
 		}
 
-		foreach ($contacts as $contact) {
+		while ($contact = DBA::fetch($contacts)) {
-
-			if ($manual_id) {
-				$contact['last-update'] = DBA::NULL_DATETIME;
-			}
-
 			// Friendica and OStatus are checked once a day
 			if (in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) {
-				$contact['priority'] = 2;
+				$contact['priority'] = 3;
-			}
-
-			if ($contact['subhub'] && in_array($contact['network'], [Protocol::DFRN, Protocol::OSTATUS])) {
-				/*
-				 * We should be getting everything via a hub. But just to be sure, let's check once a day.
-				 * (You can make this more or less frequent if desired by setting 'pushpoll_frequency' appropriately)
-				 * This also lets us update our subscription to the hub, and add or replace hubs in case it
-				 * changed. We will only update hubs once a day, regardless of 'pushpoll_frequency'.
-				 */
-				$poll_interval = Config::get('system', 'pushpoll_frequency');
-				$contact['priority'] = (!is_null($poll_interval) ? intval($poll_interval) : 3);
-			}
-
-			// Check ActivityPub and Diaspora contacts or followers once a week
-			if (in_array($contact["network"], [Protocol::ACTIVITYPUB, Protocol::DIASPORA]) || ($contact["rel"] == Contact::FOLLOWER)) {
-				$contact['priority'] = 4;
 			}
 
 			// Check archived contacts once a month
@@ -224,7 +199,7 @@ class Cron
 				$contact['priority'] = 5;
 			}
 
-			if (($contact['priority'] >= 0) && !$force) {
+			if ($contact['priority'] >= 0) {
 				$update = false;
 
 				$t = $contact['last-update'];
@@ -260,7 +235,7 @@ class Cron
 						break;
 					case 0:
 					default:
-						if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + ".$min_poll_interval." minute")) {
+						if (DateTimeFormat::utcNow() > DateTimeFormat::utc($t . " + " . $min_poll_interval . " minute")) {
 							$update = true;
 						}
 						break;
@@ -282,5 +257,6 @@ class Cron
 
 			Worker::add(['priority' => $priority, 'dont_fork' => true, 'force_priority' => true], 'OnePoll', (int)$contact['id']);
 		}
+		DBA::close($contacts);
 	}
 }

src/Worker/OnePoll.php

@@ -11,9 +11,9 @@ use Friendica\Core\Logger;
 use Friendica\Core\PConfig;
 use Friendica\Core\Protocol;
 use Friendica\Database\DBA;
-use Friendica\Model\APContact;
 use Friendica\Model\Contact;
 use Friendica\Model\Item;
+use Friendica\Model\User;
 use Friendica\Protocol\ActivityPub;
 use Friendica\Protocol\Email;
 use Friendica\Protocol\PortableContact;
@@ -41,6 +41,9 @@ class OnePoll
 			return;
 		}
 
+		if ($force) {
+			Contact::updateFromProbe($contact_id, true);
+		}
 
 		$contact = DBA::selectFirst('contact', [], ['id' => $contact_id]);
 		if (!DBA::isResult($contact)) {
@@ -56,98 +59,28 @@ class OnePoll
 
 		$importer_uid = $contact['uid'];
 
+		$updated = DateTimeFormat::utcNow();
+
+		if ($importer_uid == 0) {
+			Logger::log('Ignore public contacts');
+
+			// set the last-update so we don't keep polling
+			DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]);
+			return;
+		}
+
 		// Possibly switch the remote contact to AP
 		if ($protocol === Protocol::OSTATUS) {
 			ActivityPub\Receiver::switchContact($contact['id'], $importer_uid, $contact['url']);
 			$contact = DBA::selectFirst('contact', [], ['id' => $contact_id]);
 		}
 
-		$updated = DateTimeFormat::utcNow();
-
-		// These three networks can be able to speak AP, so we are trying to fetch AP profile data here
-		if (in_array($protocol, [Protocol::ACTIVITYPUB, Protocol::DIASPORA, Protocol::DFRN])) {
-			$apcontact = APContact::getByURL($contact['url'], true);
-
-			if (($protocol === Protocol::ACTIVITYPUB) && empty($apcontact)) {
-				self::updateContact($contact, ['last-update' => $updated, 'failure_update' => $updated]);
-				Contact::markForArchival($contact);
-				Logger::log('Contact archived');
-				return;
-			} elseif (!empty($apcontact)) {
-				$fields = ['last-update' => $updated, 'success_update' => $updated];
-				self::updateContact($contact, $fields);
-				Contact::unmarkForArchival($contact);
-			}
-		}
-
-		// Diaspora users, archived users and followers are only checked if they still exist.
-		if (($protocol != Protocol::ACTIVITYPUB) && ($contact['archive'] || ($contact["network"] == Protocol::DIASPORA) || ($contact["rel"] == Contact::FOLLOWER))) {
-			$last_updated = PortableContact::lastUpdated($contact["url"], true);
-
-			if ($last_updated) {
-				Logger::log('Contact '.$contact['id'].' had last update on '.$last_updated, Logger::DEBUG);
-
-				// The last public item can be older than the last item we got
-				if ($last_updated < $contact['last-item']) {
-					$last_updated = $contact['last-item'];
-				}
-
-				$fields = ['last-item' => DateTimeFormat::utc($last_updated), 'last-update' => $updated, 'success_update' => $updated];
-				self::updateContact($contact, $fields);
-				Contact::unmarkForArchival($contact);
-			} else {
-				self::updateContact($contact, ['last-update' => $updated, 'failure_update' => $updated]);
-				Contact::markForArchival($contact);
-				Logger::log('Contact archived');
-				return;
-			}
-		}
-
-		// Update the contact entry
-		if (in_array($protocol, [Protocol::ACTIVITYPUB, Protocol::OSTATUS, Protocol::DIASPORA, Protocol::DFRN])) {
-			// Currently we can't check every AP implementation, so we don't do it at all
-			if (($protocol != Protocol::ACTIVITYPUB) && !PortableContact::reachable($contact['url'])) {
-				Logger::log("Skipping probably dead contact ".$contact['url']);
-
-				// set the last-update so we don't keep polling
-				self::updateContact($contact, ['last-update' => $updated]);
-				return;
-			}
-
-			if (!Contact::updateFromProbe($contact["id"])) {
-				// set the last-update so we don't keep polling
-				self::updateContact($contact, ['last-update' => $updated]);
-				Contact::markForArchival($contact);
-				Logger::log('Contact archived');
-				return;
-			} else {
-				$fields = ['last-update' => $updated, 'success_update' => $updated];
-				self::updateContact($contact, $fields);
-				Contact::unmarkForArchival($contact);
-			}
-		}
-
 		// load current friends if possible.
 		if (!empty($contact['poco']) && ($contact['success_update'] > $contact['failure_update'])) {
-			$r = q("SELECT count(*) AS total FROM glink
+			if (!DBA::exists('glink', ["`cid` = ? AND updated > UTC_TIMESTAMP() - INTERVAL 1 DAY", $contact['id']])) {
-				WHERE `cid` = %d AND updated > UTC_TIMESTAMP() - INTERVAL 1 DAY",
-				intval($contact['id'])
-			);
-			if (DBA::isResult($r)) {
-				if (!$r[0]['total']) {
 				PortableContact::loadWorker($contact['id'], $importer_uid, 0, $contact['poco']);
 			}
 		}
-		}
-
-		// We don't poll our followers
-		if ($contact["rel"] == Contact::FOLLOWER) {
-			Logger::log("Don't poll follower");
-
-			// set the last-update so we don't keep polling
-			DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]);
-			return;
-		}
 
 		// Don't poll if polling is deactivated (But we poll feeds and mails anyway)
 		if (!in_array($protocol, [Protocol::FEED, Protocol::MAIL]) && Config::get('system', 'disable_polling')) {
@@ -167,19 +100,9 @@ class OnePoll
 			return;
 		}
 
-		if ($importer_uid == 0) {
+		$importer = User::getOwnerDataById($importer_uid);
-			Logger::log('Ignore public contacts');
 
-			// set the last-update so we don't keep polling
+		if (empty($importer)) {
-			DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]);
-			return;
-		}
-
-		$r = q("SELECT `contact`.*, `user`.`page-flags` FROM `contact` INNER JOIN `user` on `contact`.`uid` = `user`.`uid` WHERE `user`.`uid` = %d AND `contact`.`self` = 1 LIMIT 1",
-			intval($importer_uid)
-		);
-
-		if (!DBA::isResult($r)) {
 			Logger::log('No self contact for user '.$importer_uid);
 
 			// set the last-update so we don't keep polling
@@ -187,7 +110,6 @@ class OnePoll
 			return;
 		}
 
-		$importer = $r[0];
 		$url = '';
 		$xml = false;
 
@@ -203,14 +125,120 @@ class OnePoll
 			$hub_update = false;
 		}
 
-		$last_update = (($contact['last-update'] <= DBA::NULL_DATETIME)
-			? DateTimeFormat::utc('now - 7 days', DateTimeFormat::ATOM)
-			: DateTimeFormat::utc($contact['last-update'], DateTimeFormat::ATOM)
-		);
-
 		Logger::log("poll: ({$protocol}-{$contact['id']}) IMPORTER: {$importer['name']}, CONTACT: {$contact['name']}");
 
+		$xml = '';
+
 		if ($protocol === Protocol::DFRN) {
+			$xml = self::pollDFRN($contact, $updated);
+		} elseif (($protocol === Protocol::OSTATUS)
+			|| ($protocol === Protocol::DIASPORA)
+			|| ($protocol === Protocol::FEED)) {
+			$xml = self::pollFeed($contact, $protocol, $updated);
+		} elseif ($protocol === Protocol::MAIL) {
+			self::pollMail($contact, $importer_uid, $updated);
+		}
+
+		if (!empty($xml)) {
+			Logger::log('received xml : ' . $xml, Logger::DATA);
+			if (!strstr($xml, '<')) {
+				Logger::log('post_handshake: response from ' . $url . ' did not contain XML.');
+
+				$fields = ['last-update' => $updated, 'failure_update' => $updated];
+				self::updateContact($contact, $fields);
+				Contact::markForArchival($contact);
+				return;
+			}
+
+
+			Logger::log("Consume feed of contact ".$contact['id']);
+
+			consume_feed($xml, $importer, $contact, $hub);
+
+			// do it a second time for DFRN so that any children find their parents.
+			if ($protocol === Protocol::DFRN) {
+				consume_feed($xml, $importer, $contact, $hub);
+			}
+
+			$hubmode = 'subscribe';
+			if ($protocol === Protocol::DFRN || $contact['blocked']) {
+				$hubmode = 'unsubscribe';
+			}
+
+			if (($protocol === Protocol::OSTATUS ||  $protocol == Protocol::FEED) && (! $contact['hub-verify'])) {
+				$hub_update = true;
+			}
+
+			if ($force) {
+				$hub_update = true;
+			}
+
+			Logger::log("Contact ".$contact['id']." returned hub: ".$hub." Network: ".$protocol." Relation: ".$contact['rel']." Update: ".$hub_update);
+
+			if (strlen($hub) && $hub_update && (($contact['rel'] != Contact::FOLLOWER) || $protocol == Protocol::FEED)) {
+				Logger::log('hub ' . $hubmode . ' : ' . $hub . ' contact name : ' . $contact['name'] . ' local user : ' . $importer['name']);
+				$hubs = explode(',', $hub);
+
+				if (count($hubs)) {
+					foreach ($hubs as $h) {
+						$h = trim($h);
+
+						if (!strlen($h)) {
+							continue;
+						}
+
+						subscribe_to_hub($h, $importer, $contact, $hubmode);
+					}
+				}
+			}
+
+			self::updateContact($contact, ['last-update' => $updated, 'success_update' => $updated]);
+			DBA::update('gcontact', ['last_contact' => $updated], ['nurl' => $contact['nurl']]);
+			Contact::unmarkForArchival($contact);
+		} elseif (in_array($contact["network"], [Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::FEED])) {
+			self::updateContact($contact, ['last-update' => $updated, 'failure_update' => $updated]);
+			DBA::update('gcontact', ['last_failure' => $updated], ['nurl' => $contact['nurl']]);
+			Contact::markForArchival($contact);
+		} else {
+			self::updateContact($contact, ['last-update' => $updated]);
+		}
+
+		Logger::log('End');
+		return;
+	}
+
+	private static function RemoveReply($subject)
+	{
+		while (in_array(strtolower(substr($subject, 0, 3)), ["re:", "aw:"])) {
+			$subject = trim(substr($subject, 4));
+		}
+
+		return $subject;
+	}
+
+	/**
+	 * @brief Updates a personal contact entry and the public contact entry
+	 *
+	 * @param array $contact The personal contact entry
+	 * @param array $fields  The fields that are updated
+	 * @throws \Exception
+	 */
+	private static function updateContact(array $contact, array $fields)
+	{
+		DBA::update('contact', $fields, ['id' => $contact['id']]);
+		DBA::update('contact', $fields, ['uid' => 0, 'nurl' => $contact['nurl']]);
+	}
+
+	/**
+	 * @brief Poll DFRN contacts
+	 *
+	 * @param  array  $contact The personal contact entry
+	 * @param  string $updated The updated date
+	 * @return string polled XML
+	 * @throws \Exception
+	 */
+	private static function pollDFRN(array $contact, $updated)
+	{
 		$idtosend = $orig_id = (($contact['dfrn-id']) ? $contact['dfrn-id'] : $contact['issued-id']);
 		if (intval($contact['duplex']) && $contact['dfrn-id']) {
 			$idtosend = '0:' . $orig_id;
@@ -223,12 +251,16 @@ class OnePoll
 		$perm = 'rw';
 
 		// But this may be our first communication, so set the writable flag if it isn't set already.
-
 		if (!intval($contact['writable'])) {
 			$fields = ['writable' => true];
 			DBA::update('contact', $fields, ['id' => $contact['id']]);
 		}
 
+		$last_update = (($contact['last-update'] <= DBA::NULL_DATETIME)
+			? DateTimeFormat::utc('now - 7 days', DateTimeFormat::ATOM)
+			: DateTimeFormat::utc($contact['last-update'], DateTimeFormat::ATOM)
+		);
+
 		$url = $contact['poll'] . '?dfrn_id=' . $idtosend
 			. '&dfrn_version=' . DFRN_PROTOCOL_VERSION
 			. '&type=data&last_update=' . $last_update
@@ -241,7 +273,7 @@ class OnePoll
 			self::updateContact($contact, ['last-update' => $updated]);
 			Contact::markForArchival($contact);
 			Logger::log('Contact archived');
-				return;
+			return false;
 		}
 
 		$handshake_xml = $curlResult->getBody();
@@ -259,7 +291,7 @@ class OnePoll
 			$fields = ['last-update' => $updated, 'failure_update' => $updated];
 			self::updateContact($contact, $fields);
 			Contact::markForArchival($contact);
-				return;
+			return false;
 		}
 
 		if (!strstr($handshake_xml, '<')) {
@@ -268,10 +300,9 @@ class OnePoll
 			$fields = ['last-update' => $updated, 'failure_update' => $updated];
 			self::updateContact($contact, $fields);
 			Contact::markForArchival($contact);
-				return;
+			return false;
 		}
 
-
 		$res = XML::parseString($handshake_xml);
 
 		if (intval($res->status) == 1) {
@@ -290,7 +321,7 @@ class OnePoll
 			// set the last-update so we don't keep polling
 			DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]);
 			Logger::log('Contact status is ' . $res->status);
-				return;
+			return false;
 		}
 
 		if (((float)$res->dfrn_version > 2.21) && ($contact['poco'] == '')) {
@@ -323,7 +354,7 @@ class OnePoll
 		// Since we mostly don't use it anyway, we won't dig into it deeper, but simply ignore it.
 		if (empty($final_dfrn_id) || empty($orig_id)) {
 			Logger::log('Contact has got no ID - quitting');
-				return;
+			return false;
 		}
 
 		if ($final_dfrn_id != $orig_id) {
@@ -333,19 +364,27 @@ class OnePoll
 			// set the last-update so we don't keep polling
 			DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]);
 			Contact::markForArchival($contact);
-				return;
+			return false;
 		}
 
 		$postvars['dfrn_id'] = $idtosend;
 		$postvars['dfrn_version'] = DFRN_PROTOCOL_VERSION;
 		$postvars['perm'] = 'rw';
 
-			$xml = Network::post($contact['poll'], $postvars)->getBody();
+		return Network::post($contact['poll'], $postvars)->getBody();
-
+	}
-		} elseif (($protocol === Protocol::OSTATUS)
-			|| ($protocol === Protocol::DIASPORA)
-			|| ($protocol === Protocol::FEED)) {
 
+	/**
+	 * @brief Poll Feed/OStatus contacts
+	 *
+	 * @param  array  $contact The personal contact entry
+	 * @param  string $protocol The used protocol of the contact
+	 * @param  string $updated The updated date
+	 * @return string polled XML
+	 * @throws \Exception
+	 */
+	private static function pollFeed(array $contact, $protocol, $updated)
+	{
 		// Upgrading DB fields from an older Friendica version
 		// Will only do this once per notify-enabled OStatus contact
 		// or if relationship changes
@@ -367,7 +406,7 @@ class OnePoll
 			// set the last-update so we don't keep polling
 			DBA::update('contact', ['last-update' => $updated], ['id' => $contact['id']]);
 			Logger::log('Contact is blocked or only a follower');
-				return;
+			return false;
 		}
 
 		$cookiejar = tempnam(get_temppath(), 'cookiejar-onepoll-');
@@ -379,12 +418,22 @@ class OnePoll
 			self::updateContact($contact, ['last-update' => $updated]);
 			Contact::markForArchival($contact);
 			Logger::log('Contact archived');
-				return;
+			return false;
 		}
 
-			$xml = $curlResult->getBody();
+		return $curlResult->getBody();
+	}
 
-		} elseif ($protocol === Protocol::MAIL) {
+	/**
+	 * @brief Poll Mail contacts
+	 *
+	 * @param  array   $contact      The personal contact entry
+	 * @param  integer $importer_uid The UID of the importer
+	 * @param  string  $updated      The updated date
+	 * @throws \Exception
+	 */
+	private static function pollMail(array $contact, $importer_uid, $updated)
+	{
 		Logger::log("Mail: Fetching for ".$contact['addr'], Logger::DEBUG);
 
 		$mail_disabled = ((function_exists('imap_open') && !Config::get('system', 'imap_disabled')) ? 0 : 1);
@@ -419,7 +468,10 @@ class OnePoll
 			}
 		}
 
-			if ($mbox) {
+		if (!$mbox) {
+			return;
+		}
+
 		$msgs = Email::poll($mbox, $contact['addr']);
 
 		if (count($msgs)) {
@@ -480,7 +532,6 @@ class OnePoll
 						continue;
 					}
 
-
 					// look for a 'references' or an 'in-reply-to' header and try to match with a parent item we have locally.
 					$raw_refs = ((property_exists($meta, 'references')) ? str_replace("\t", '', $meta->references) : '');
 					if (!trim($raw_refs)) {
@@ -584,7 +635,7 @@ class OnePoll
 					if ($datarray['parent-uri'] === $datarray['uri']) {
 						$datarray['private'] = 1;
 					}
-							if (($protocol === Protocol::MAIL) && !PConfig::get($importer_uid, 'system', 'allow_public_email_replies')) {
+					if (!PConfig::get($importer_uid, 'system', 'allow_public_email_replies')) {
 						$datarray['private'] = 1;
 						$datarray['allow_cid'] = '<' . $contact['id'] . '>';
 					}
@@ -620,95 +671,4 @@ class OnePoll
 		Logger::log("Mail: closing connection for ".$mailconf['user']);
 		imap_close($mbox);
 	}
-		}
-
-		if ($xml) {
-			Logger::log('received xml : ' . $xml, Logger::DATA);
-			if (!strstr($xml, '<')) {
-				Logger::log('post_handshake: response from ' . $url . ' did not contain XML.');
-
-				$fields = ['last-update' => $updated, 'failure_update' => $updated];
-				self::updateContact($contact, $fields);
-				Contact::markForArchival($contact);
-				return;
-			}
-
-
-			Logger::log("Consume feed of contact ".$contact['id']);
-
-			consume_feed($xml, $importer, $contact, $hub);
-
-			// do it a second time for DFRN so that any children find their parents.
-			if ($protocol === Protocol::DFRN) {
-				consume_feed($xml, $importer, $contact, $hub);
-			}
-
-			$hubmode = 'subscribe';
-			if ($protocol === Protocol::DFRN || $contact['blocked']) {
-				$hubmode = 'unsubscribe';
-			}
-
-			if (($protocol === Protocol::OSTATUS ||  $protocol == Protocol::FEED) && (! $contact['hub-verify'])) {
-				$hub_update = true;
-			}
-
-			if ($force) {
-				$hub_update = true;
-			}
-
-			Logger::log("Contact ".$contact['id']." returned hub: ".$hub." Network: ".$protocol." Relation: ".$contact['rel']." Update: ".$hub_update);
-
-			if (strlen($hub) && $hub_update && (($contact['rel'] != Contact::FOLLOWER) || $protocol == Protocol::FEED)) {
-				Logger::log('hub ' . $hubmode . ' : ' . $hub . ' contact name : ' . $contact['name'] . ' local user : ' . $importer['name']);
-				$hubs = explode(',', $hub);
-
-				if (count($hubs)) {
-					foreach ($hubs as $h) {
-						$h = trim($h);
-
-						if (!strlen($h)) {
-							continue;
-						}
-
-						subscribe_to_hub($h, $importer, $contact, $hubmode);
-					}
-				}
-			}
-
-			self::updateContact($contact, ['last-update' => $updated, 'success_update' => $updated]);
-			DBA::update('gcontact', ['last_contact' => $updated], ['nurl' => $contact['nurl']]);
-			Contact::unmarkForArchival($contact);
-		} elseif (in_array($contact["network"], [Protocol::DFRN, Protocol::DIASPORA, Protocol::OSTATUS, Protocol::FEED])) {
-			self::updateContact($contact, ['last-update' => $updated, 'failure_update' => $updated]);
-			DBA::update('gcontact', ['last_failure' => $updated], ['nurl' => $contact['nurl']]);
-			Contact::markForArchival($contact);
-		} else {
-			self::updateContact($contact, ['last-update' => $updated]);
-		}
-
-		Logger::log('End');
-		return;
-	}
-
-	private static function RemoveReply($subject)
-	{
-		while (in_array(strtolower(substr($subject, 0, 3)), ["re:", "aw:"])) {
-			$subject = trim(substr($subject, 4));
-		}
-
-		return $subject;
-	}
-
-	/**
-	 * @brief Updates a personal contact entry and the public contact entry
-	 *
-	 * @param array $contact The personal contact entry
-	 * @param array $fields  The fields that are updated
-	 * @throws \Exception
-	 */
-	private static function updateContact(array $contact, array $fields)
-	{
-		DBA::update('contact', $fields, ['id' => $contact['id']]);
-		DBA::update('contact', $fields, ['uid' => 0, 'nurl' => $contact['nurl']]);
-	}
 }

src/Worker/UpdateContact.php

@@ -13,16 +13,12 @@ use Friendica\Database\DBA;
 
 class UpdateContact
 {
-	public static function execute($contact_id)
+	public static function execute($contact_id, $command = '')
 	{
-		$success = Contact::updateFromProbe($contact_id);
+		$force = ($command == "force");
-		// Update the "updated" field if the contact could be probed.
-		// We don't do this in the function above, since we don't want to
-		// update the contact whenever that function is called from anywhere.
-		if ($success) {
-			DBA::update('contact', ['updated' => DateTimeFormat::utcNow()], ['id' => $contact_id]);
-		}
 
-		Logger::info('Updated from probe', ['id' => $contact_id, 'success' => $success]);
+		$success = Contact::updateFromProbe($contact_id, '', $force);
+
+		Logger::info('Updated from probe', ['id' => $contact_id, 'force' => $force, 'success' => $success]);
 	}
 }