use post-content table instead of item-content

This reverts commit 0f5ba218f6.

ifttt/ifttt.php

@@ -179,5 +179,5 @@ function ifttt_message($uid, $item)
 		$link = hash('ripemd128', $item['msg']);
 	}
 
-	Post\Delayed::add($link, $post, Worker::PRIORITY_MEDIUM, Post\Delayed::PREPARED);
+	Post\Delayed::add($link, $post, Worker::PRIORITY_MEDIUM, Post\Delayed::UNPREPARED);
 }

mailstream/mailstream.php

@@ -8,7 +8,6 @@
 
 use Friendica\Content\Text\BBCode;
 use Friendica\Core\Hook;
-use Friendica\Core\Logger;
 use Friendica\Core\Renderer;
 use Friendica\Core\System;
 use Friendica\Core\Worker;
@@ -32,7 +31,7 @@ function mailstream_install()
 	Hook::register('post_remote_end', 'addon/mailstream/mailstream.php', 'mailstream_post_hook');
 	Hook::register('mailstream_send_hook', 'addon/mailstream/mailstream.php', 'mailstream_send_hook');
 
-	Logger::info("installed mailstream");
+	DI::logger()->info("installed mailstream");
 }
 
 /**
@@ -86,7 +85,7 @@ function mailstream_generate_id(string $uri): string
 	$host = DI::baseUrl()->getHost();
 	$resource = hash('md5', $uri);
 	$message_id = "<" . $resource . "@" . $host . ">";
-	Logger::debug('generated message ID', ['id' => $message_id, 'uri' => $uri]);
+	DI::logger()->debug('generated message ID', ['id' => $message_id, 'uri' => $uri]);
 	return $message_id;
 }
 
@@ -95,20 +94,53 @@ function mailstream_send_hook(array $data)
 	$criteria = array('uid' => $data['uid'], 'contact-id' => $data['contact-id'], 'uri' => $data['uri']);
 	$item = Post::selectFirst([], $criteria);
 	if (empty($item)) {
-		Logger::error('could not find item');
+		DI::logger()->error('could not find item');
+		return;
+	}
+
+	if ($item['deleted']) {
+		DI::logger()->debug('mailstream_send_hook skipping deleted item', ['guid' => $item['guid']]);
 		return;
 	}
 
 	$user = User::getById($item['uid']);
 	if (empty($user)) {
-		Logger::error('could not find user', ['uid' => $item['uid']]);
+		DI::logger()->error('mailstream_send_hook could not find user', ['uid' => $item['uid']]);
 		return;
 	}
 
-	if (!mailstream_send($data['message_id'], $item, $user)) {
-		Logger::debug('send failed, will retry', $data);
+	$author = DBA::selectFirst('contact', ['nick', 'blocked', 'uri-id'], ['id' => $data['author-id'], 'self' => false]);
+	if (!DBA::isResult($author)) {
+		DI::logger()->error('mailstream_send_hook could not find author', ['guid' => $item['guid'], 'author-id' => $data['author-id']]);
+		return;
+	}
+	if ($author['blocked']) {
+		DI::logger()->info('mailstream_send_hook author is blocked', ['guid' => $item['guid'], 'author-id' => $data['author-id']]);
+		return;
+	}
+	$collapsed = false;
+	$user_contact = DBA::selectFirst('user-contact', ['cid', 'blocked', 'ignored', 'collapsed'], ['uid' => $item['uid'], 'uri-id' => $item['author-uri-id']]);
+	if (!DBA::isResult($user_contact)) {
+		$user_contact = DBA::selectFirst('user-contact', ['cid', 'blocked', 'ignored', 'collapsed'], ['uid' => $item['uid'], 'cid' => $item['author-id']]);
+	}
+	if (DBA::isResult($user_contact)) {
+		if ($user_contact['blocked']) {
+			DI::logger()->info('mailstream_send_hook author is blocked', ['guid' => $item['guid'], 'cid' => $user_contact['cid']]);
+			return;
+		}
+		if ($user_contact['ignored']) {
+			DI::logger()->info('mailstream_send_hook author is ignored', ['guid' => $item['guid'], 'cid' => $user_contact['cid']]);
+			return;
+		}
+		if ($user_contact['collapsed']) {
+			$collapsed = true;
+		}
+	}
+
+	if (!mailstream_send($data['message_id'], $item, $user, $collapsed)) {
+		DI::logger()->debug('mailstream_send_hook send failed, will retry', $data);
 		if (!Worker::defer()) {
-			Logger::error('failed and could not defer', $data);
+			DI::logger()->error('failed and could not defer', $data);
 		}
 	}
 }
@@ -124,32 +156,32 @@ function mailstream_send_hook(array $data)
 function mailstream_post_hook(array &$item)
 {
 	if ($item['uid'] === 0) {
-		Logger::debug('mailstream: root user, skipping item ' . $item['id']);
+		DI::logger()->debug('mailstream: root user, skipping item ' . $item['id']);
 		return;
 	}
 	if (!DI::pConfig()->get($item['uid'], 'mailstream', 'enabled')) {
-		Logger::debug('mailstream: not enabled.', ['item' => $item['id'], ' uid ' => $item['uid']]);
+		DI::logger()->debug('mailstream: not enabled.', ['item' => $item['id'], ' uid ' => $item['uid']]);
 		return;
 	}
 	if (!$item['contact-id']) {
-		Logger::debug('no contact-id', ['item' => $item['id']]);
+		DI::logger()->debug('no contact-id', ['item' => $item['id']]);
 		return;
 	}
 	if (!$item['uri']) {
-		Logger::debug('no uri', ['item' => $item['id']]);
+		DI::logger()->debug('no uri', ['item' => $item['id']]);
 		return;
 	}
 	if ($item['verb'] == Activity::ANNOUNCE) {
-		Logger::debug('ignoring announce', ['item' => $item['id']]);
+		DI::logger()->debug('ignoring announce', ['item' => $item['id']]);
 		return;
 	}
 	if (DI::pConfig()->get($item['uid'], 'mailstream', 'nolikes')) {
 		if ($item['verb'] == Activity::LIKE) {
-			Logger::debug('ignoring like', ['item' => $item['id']]);
+			DI::logger()->debug('ignoring like', ['item' => $item['id']]);
 			return;
 		}
 		if ($item['verb'] == Activity::DISLIKE) {
-			Logger::debug('ignoring dislike', ['item' => $item['id']]);
+			DI::logger()->debug('ignoring dislike', ['item' => $item['id']]);
 			return;
 		}
 	}
@@ -159,6 +191,7 @@ function mailstream_post_hook(array &$item)
 	$send_hook_data = [
 		'uid' => $item['uid'],
 		'contact-id' => $item['contact-id'],
+		'author-id' => $item['author-id'],
 		'uri' => $item['uri'],
 		'message_id' => $message_id,
 		'tries' => 0,
@@ -200,7 +233,7 @@ function mailstream_do_images(array &$item, array &$attachments)
 		try {
 			$curlResult = DI::httpClient()->get($url, HttpClientAccept::DEFAULT, [HttpClientOptions::COOKIEJAR => $cookiejar]);
 			if (!$curlResult->isSuccess()) {
-				Logger::debug('mailstream: fetch image url failed', [
+				DI::logger()->debug('mailstream: fetch image url failed', [
 					'url' => $url,
 					'item_id' => $item['id'],
 					'return_code' => $curlResult->getReturnCode()
@@ -208,7 +241,7 @@ function mailstream_do_images(array &$item, array &$attachments)
 				continue;
 			}
 		} catch (InvalidArgumentException $e) {
-			Logger::error('exception fetching url', ['url' => $url, 'item_id' => $item['id']]);
+			DI::logger()->error('exception fetching url', ['url' => $url, 'item_id' => $item['id']]);
 			continue;
 		}
 		$attachments[$url] = [
@@ -309,7 +342,7 @@ function mailstream_subject(array $item): string
 	}
 	$contact = Contact::selectFirst([], ['id' => $item['contact-id'], 'uid' => $item['uid']]);
 	if (!DBA::isResult($contact)) {
-		Logger::error('no contact', [
+		DI::logger()->error('no contact', [
 			'item' => $item['id'],
 			'plink' => $item['plink'],
 			'contact id' => $item['contact-id'],
@@ -345,31 +378,38 @@ function mailstream_subject(array $item): string
  * @param string $message_id ID of the message (RFC 1036)
  * @param array  $item       content of the item
  * @param array  $user       results from the user table
+ * @param bool   $collapsed  true if the content should be hidden
  *
  * @return bool True if this message has been completed.  False if it should be retried.
  */
-function mailstream_send(string $message_id, array $item, array $user): bool
+function mailstream_send(string $message_id, array $item, array $user, bool $collapsed): bool
 {
 	if (!is_array($item)) {
-		Logger::error('item is empty', ['message_id' => $message_id]);
+		DI::logger()->error('item is empty', ['message_id' => $message_id]);
 		return false;
 	}
 
 	if (!$item['visible']) {
-		Logger::debug('item not yet visible', ['item uri' => $item['uri']]);
+		DI::logger()->debug('item not yet visible', ['item uri' => $item['uri']]);
 		return false;
 	}
 	if (!$message_id) {
-		Logger::error('no message ID supplied', ['item uri' => $item['uri'], 'user email' => $user['email']]);
+		DI::logger()->error('no message ID supplied', ['item uri' => $item['uri'], 'user email' => $user['email']]);
 		return true;
 	}
 
 	require_once(dirname(__file__) . '/phpmailer/class.phpmailer.php');
 
-	$item['body'] = Post\Media::addAttachmentsToBody($item['uri-id'], $item['body']);
+	if ($collapsed) {
+		$item['body'] = DI::l10n()->t('Content from %s is collapsed', $item['author-name']);
+	} else {
+		$item['body'] = Post\Media::addAttachmentsToBody($item['uri-id'], $item['body']);
+	}
 
 	$attachments = [];
-	mailstream_do_images($item, $attachments);
+	if (!$collapsed) {
+		mailstream_do_images($item, $attachments);
+	}
 	$frommail = DI::config()->get('mailstream', 'frommail');
 	if ($frommail == '') {
 		$frommail = 'friendica@localhost.local';
@@ -418,15 +458,15 @@ function mailstream_send(string $message_id, array $item, array $user): bool
 		if (!$mail->Send()) {
 			throw new Exception($mail->ErrorInfo);
 		}
-		Logger::debug('sent message', [
+		DI::logger()->debug('sent message', [
 			'message ID' => $mail->MessageID,
 			'subject' => $mail->Subject,
 			'address' => $address
 		]);
 	} catch (phpmailerException $e) {
-		Logger::debug('PHPMailer exception sending message', ['id' => $message_id, 'error' => $e->errorMessage()]);
+		DI::logger()->debug('mailstream_send PHPMailer exception sending message', ['item uri' => $item['uri'], 'message_id' => $message_id, 'error' => $e->errorMessage()]);
 	} catch (Exception $e) {
-		Logger::debug('exception sending message', ['id' => $message_id, 'error' => $e->getMessage()]);
+		DI::logger()->debug('mailstream_send exception sending message', ['item uri' => $item['uri'], 'message_id' => $message_id, 'error' => $e->errorMessage()]);
 	}
 
 	return true;

phototrack/database.sql

@@ -0,0 +1,23 @@
+CREATE TABLE IF NOT EXISTS `phototrack_photo_use` (
+       `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+       `resource-id` char(64) NOT NULL,
+       `table` char(64) NOT NULL,
+       `field` char(64) NOT NULL,
+       `row-id` int(11) NOT NULL,
+       `checked` timestamp NOT NULL DEFAULT now(),
+       PRIMARY KEY (`id`),
+       INDEX `resource-id` (`resource-id`),
+       INDEX `row` (`table`,`field`,`row-id`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+CREATE TABLE IF NOT EXISTS `phototrack_row_check` (
+       `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+       `table` char(64) NOT NULL,
+       `row-id` int(11) NOT NULL,
+       `checked` timestamp NOT NULL DEFAULT now(),
+       PRIMARY KEY (`id`),
+       INDEX `row` (`table`,`row-id`),
+       INDEX `checked` (`checked`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+SELECT TRUE

phototrack/phototrack.php

@@ -0,0 +1,272 @@
+<?php
+/**
+ * Name: Photo Track
+ * Description: Track which photos are actually being used and delete any others
+ * Version: 1.0
+ * Author: Matthew Exon <http://mat.exon.name>
+ */
+
+/*
+ * List of tables and the fields that are checked:
+ * 
+ * contact: photo thumb micro about
+ * fcontact: photo
+ * fsuggest: photo
+ * gcontact: photo about
+ * item: body
+ * item-content: body
+ * mail: from-photo
+ * notify: photo
+ * profile: photo thumb about
+ */
+
+use Friendica\Core\Addon;
+use Friendica\Core\Logger;
+use Friendica\Object\Image;
+use Friendica\Database\DBA;
+use Friendica\Util\Images;
+use Friendica\Util\DateTimeFormat;
+use Friendica\DI;
+
+if (!defined('PHOTOTRACK_DEFAULT_BATCH_SIZE')) {
+    define('PHOTOTRACK_DEFAULT_BATCH_SIZE', 1000);
+}
+// Time in *minutes* between searching for photo uses
+if (!defined('PHOTOTRACK_DEFAULT_SEARCH_INTERVAL')) {
+    define('PHOTOTRACK_DEFAULT_SEARCH_INTERVAL', 10);
+}
+
+function phototrack_install() {
+    global $db;
+
+    Addon::registerHook('post_local_end', 'addon/phototrack/phototrack.php', 'phototrack_post_local_end');
+    Addon::registerHook('post_remote_end', 'addon/phototrack/phototrack.php', 'phototrack_post_remote_end');
+    Addon::registerHook('notifier_end', 'addon/phototrack/phototrack.php', 'phototrack_notifier_end');
+    Addon::registerHook('cron', 'addon/phototrack/phototrack.php', 'phototrack_cron');
+
+    if (DI::config()->get('phototrack', 'dbversion') != '0.1') {
+        $schema = file_get_contents(dirname(__file__).'/database.sql');
+        $arr = explode(';', $schema);
+        foreach ($arr as $a) {
+            if (!DBA::e($a)) {
+                Logger::warning('Unable to create database table: ' . DBA::errorMessage());
+                return;
+            }
+        }
+        DI::config()->set('phototrack', 'dbversion', '0.1');
+    }
+}
+
+function phototrack_uninstall() {
+    Addon::unregisterHook('post_local_end', 'addon/phototrack/phototrack.php', 'phototrack_post_local_end');
+    Addon::unregisterHook('post_remote_end', 'addon/phototrack/phototrack.php', 'phototrack_post_remote_end');
+    Addon::unregisterHook('notifier_end', 'addon/phototrack/phototrack.php', 'phototrack_notifier_end');
+    Addon::unregisterHook('cron', 'addon/phototrack/phototrack.php', 'phototrack_cron');
+}
+
+function phototrack_module() {}
+
+function phototrack_finished_row($table, $id) {
+    $existing = DBA::selectFirst('phototrack_row_check', ['id'], ['table' => $table, 'row-id' => $id]);
+    if (!is_bool($existing)) {
+        DBA::update('phototrack_row_check', ['checked' => DateTimeFormat::utcNow()], ['table' => $table, 'row-id' => $id]);
+    }
+    else {
+        DBA::insert('phototrack_row_check', ['table' => $table, 'row-id' => $id, 'checked' => DateTimeFormat::utcNow()]);
+    }
+}
+
+function phototrack_photo_use($photo, $table, $field, $id) {
+    Logger::debug('@@@ phototrack_photo_use ' . $photo);
+    foreach (Images::supportedTypes() as $m => $e) {
+        $photo = str_replace(".$e", '', $photo);
+    }
+    if (substr($photo, -2, 1) == '-') {
+        $resolution = intval(substr($photo,-1,1));
+        $photo = substr($photo,0,-2);
+    }
+    if (strlen($photo) != 32) {
+        return;
+    }
+    $r = DBA::selectFirst('photo', ['resource-id'], ['resource-id' => $photo]);
+    if (!DBA::isResult($r)) {
+        return;
+    }
+    $rid = $r['resource-id'];
+    $existing = DBA::selectFirst('phototrack_photo_use', ['id'], ['resource-id' => $rid, 'table' => $table, 'field' => $field, 'row-id' => $id]);
+    if (DBA::isResult($existing)) {
+        DBA::update('phototrack_photo_use', ['checked' => DateTimeFormat::utcNow()], ['resource-id' => $rid, 'table' => $table, 'field' => $field, 'row-id' => $id]);
+    }
+    else {
+        DBA::insert('phototrack_photo_use', ['resource-id' => $rid, 'table' => $table, 'field' => $field, 'row-id' => $id, 'checked' => DateTimeFormat::utcNow()]);
+    }
+}
+
+function phototrack_check_field_url($a, $table, $id_field, $field, $id, $url) {
+    Logger::info('@@@ phototrack_check_field_url table ' . $table . ' id_field ' . $id_field . ' field ' . $field . ' id ' . $id . ' url ' . $url);
+    $baseurl = DI::baseUrl()->get(true);
+    if (strpos($url, $baseurl) === FALSE) {
+        return;
+    }
+    else {
+        $url = substr($url, strlen($baseurl));
+        Logger::info('@@@ phototrack_check_field_url funny url stuff ' . $url . ' base ' . $baseurl);
+    }
+    if (strpos($url, '/photo/') === FALSE) {
+        return;
+    }
+    else {
+        $url = substr($url, strlen('/photo/'));
+        Logger::info('@@@ phototrack_check_field_url more url stuff ' . $url);
+    }
+    if (preg_match('/([0-9a-z]{32})/', $url, $matches)) {
+        $rid = $matches[0];
+        Logger::info('@@@ phototrack_check_field_url rid ' . $rid);
+        phototrack_photo_use($rid, $table, $field, $id);
+    }
+}
+
+function phototrack_check_field_bbcode($a, $table, $id_field, $field, $id, $value) {
+    Logger::info('@@@ phototrack_check_field_url table ' . $table . ' id_field ' . $id_field . ' field ' . $field . ' id ' . $id . ' value ' . $value);
+    $baseurl = DI::baseUrl()->get(true);
+    $matches = array();
+    preg_match_all("/\[img(\=([0-9]*)x([0-9]*))?\](.*?)\[\/img\]/ism", $value, $matches);
+    foreach ($matches[4] as $url) {
+        phototrack_check_field_url($a, $table, $id_field, $field, $id, $url);
+    }
+}
+
+function phototrack_post_local_end(&$a, &$item) {
+    phototrack_check_row($a, 'item', 'id', $item);
+    phototrack_check_row($a, 'item-content', 'id', $item);
+}
+
+function phototrack_post_remote_end(&$a, &$item) {
+    phototrack_check_row($a, 'item', 'id', $item);
+    phototrack_check_row($a, 'item-content', 'id', $item);
+}
+
+function phototrack_notifier_end($item) {
+}
+
+function phototrack_check_row($a, $table, $id_field, $row) {
+    switch ($table) {
+        case 'post-content':
+            $fields = array(
+                'body' => 'bbcode');
+            break;
+        case 'contact':
+            $fields = array(
+                'photo' => 'url',
+                'thumb' => 'url',
+                'micro' => 'url',
+                'about' => 'bbcode');
+            break;
+        case 'fcontact':
+            $fields = array(
+                'photo' => 'url');
+            break;
+        case 'fsuggest':
+            $fields = array(
+                'photo' => 'url');
+            break;
+        case 'gcontact':
+            $fields = array(
+                'photo' => 'url',
+                'about' => 'bbcode');
+            break;
+        default: $fields = array(); break;
+    }
+    foreach ($fields as $field => $type) {
+        switch ($type) {
+        case 'bbcode': phototrack_check_field_bbcode($a, $table, $id_field, $field, $row['id'], $row[$field]); break;
+        case 'url': phototrack_check_field_url($a, $table, $id_field, $field, $row['id'], $row[$field]); break;
+        }
+    }
+    phototrack_finished_row($table, $row['id']);
+}
+
+function phototrack_batch_size() {
+    $batch_size = DI::config()->get('phototrack', 'batch_size');
+    if ($batch_size > 0) {
+        return $batch_size;
+    }
+    return PHOTOTRACK_DEFAULT_BATCH_SIZE;
+}
+
+function phototrack_search_table($a, $table, $id_field) {
+    $batch_size = phototrack_batch_size();
+    $rows = DBA::p("SELECT `$table`.* FROM `$table` LEFT OUTER JOIN phototrack_row_check ON ( phototrack_row_check.`table` = '$table' AND phototrack_row_check.`row-id` = `$table`.$id_field ) WHERE ( ( phototrack_row_check.checked IS NULL ) OR ( phototrack_row_check.checked < DATE_SUB(NOW(), INTERVAL 1 MONTH) ) ) ORDER BY phototrack_row_check.checked LIMIT $batch_size");
+    if (DBA::isResult($rows)) {
+        while ($row = DBA::fetch($rows)) {
+            phototrack_check_row($a, $table, $id_field, $row);
+        }
+    }
+    $r = DBA::p("SELECT COUNT(*) FROM `$table` LEFT OUTER JOIN phototrack_row_check ON ( phototrack_row_check.`table` = '$table' AND phototrack_row_check.`row-id` = `$table`.$id_field ) WHERE ( ( phototrack_row_check.checked IS NULL ) OR ( phototrack_row_check.checked < DATE_SUB(NOW(), INTERVAL 1 MONTH) ) )");
+    Logger::info("@@@ phototrack_search_table " . print_r(DBA::fetch($r)));
+    $remaining = DBA::fetch($r)['count'];
+    Logger::info('phototrack: searched ' . DBA::numRows($rows) . ' rows in table ' . $table . ', ' . $remaining . ' still remaining to search');
+    return $remaining;
+}
+
+function phototrack_cron_time() {
+    $prev_remaining = DI::config()->get('phototrack', 'remaining_items');
+    if ($prev_remaining > 10 * phototrack_batch_size()) {
+        Logger::debug('phototrack: more than ' . (10 * phototrack_batch_size()) . ' items remaining');
+        return true;
+    }
+    $last = DI::config()->get('phototrack', 'last_search');
+    $search_interval = intval(DI::config()->get('phototrack', 'search_interval'));
+    if (!$search_interval) {
+        $search_interval = PHOTOTRACK_DEFAULT_SEARCH_INTERVAL;
+    }
+    if ($last) {
+        $next = $last + ($search_interval * 60);
+        if ($next > time()) {
+            Logger::debug('phototrack: search interval not reached');
+            return false;
+        }
+    }
+            Logger::debug('@@@ phototrack: search interval reached last ' . $last . ' search interval ' . $search_interval);
+    return true;
+}
+
+function phototrack_cron($a, $b) {
+    return; // @@@ something is broken
+    if (!phototrack_cron_time()) {
+        return;
+    }
+    DI::config()->set('phototrack', 'last_search', time());
+
+    $remaining = 0;
+    $remaining += phototrack_search_table($a, 'post-content', 'uri-id');
+    $remaining += phototrack_search_table($a, 'contact', 'id');
+    $remaining += phototrack_search_table($a, 'fcontact', 'id');
+    $remaining += phototrack_search_table($a, 'fsuggest', 'id');
+    $remaining += phototrack_search_table($a, 'gcontact', 'id');
+
+    DI::config()->set('phototrack', 'remaining_items', $remaining);
+    if ($remaining === 0) {
+        phototrack_tidy();
+    }
+}
+
+function phototrack_tidy() {
+    $batch_size = phototrack_batch_size();
+    DBA::e('CREATE TABLE IF NOT EXISTS `phototrack-temp` (`resource-id` char(255) not null)');
+    DBA::e('INSERT INTO `phototrack-temp` SELECT DISTINCT(`resource-id`) FROM photo WHERE photo.`created` < DATE_SUB(NOW(), INTERVAL 2 MONTH)');
+    $rows = DBA::p('SELECT `phototrack-temp`.`resource-id` FROM `phototrack-temp` LEFT OUTER JOIN phototrack_photo_use ON (`phototrack-temp`.`resource-id` = phototrack_photo_use.`resource-id`) WHERE phototrack_photo_use.id IS NULL limit ' . /*$batch_size*/1000);
+    if (DBA::isResult($rows)) {
+        foreach ($rows as $row) {
+            Logger::debug('phototrack: remove photo ' . $row['resource-id']);
+            DBA::e('DELETE FROM photo WHERE `resource-id` = "' . $row['resource-id'] . '"');
+        }
+        Logger::info('phototrack_tidy: deleted ' . DBA::numRows($rows) . ' photos');
+    }
+    DBA::e('DROP TABLE `phototrack-temp`');
+    $rows = DBA::p('SELECT id FROM phototrack_photo_use WHERE checked < DATE_SUB(NOW(), INTERVAL 2 MONTH)');
+    foreach ($rows as $row) {
+        DBA::e( 'DELETE FROM phototrack_photo_use WHERE id = ' . $row['id']);
+    }
+    Logger::info('phototrack_tidy: deleted ' . DBA::numRows($rows) . ' phototrack_photo_use rows');
+}

publicise/publicise.php

@@ -0,0 +1,11 @@
+"SELECT `uid` FROM `contact` WHERE `id` = %d AND `reason` = 'publicise'", intval($item['contact-id']));
+    if (!$r1) {
+        return;
+    }
+
+    Logger::debug('Publicise: moving to wall: ' . $item['uid'] . ' ' . $item['contact-id'] . ' ' . $item['uri']);
+    $item['type'] = 'wall';
+    $item['wall'] = 1;
+    $item['private'] = 0;
+}
+

publicise/templates/admin.tpl

@@ -0,0 +1,39 @@
+{{*
+ *	AUTOMATICALLY GENERATED TEMPLATE
+ *	DO NOT EDIT THIS FILE, CHANGES WILL BE OVERWRITTEN
+ *
+ *}}
+<form method="post">
+  <table>
+    <thead>
+      <tr>
+        <th>{{$feed_t}}</th>
+        <th>{{$publicised_t}}</th>
+        <th>{{$comments_t}}</th>
+        <th>{{$expire_t}}</th>
+      </tr>
+    </thead>
+    <tbody>
+{{foreach $feeds as $f}}
+      <tr>
+        <td>
+          <a href="{{$f.url}}">
+            <img style="vertical-align:middle" src='{{$f.micro}}'>
+            <span style="margin-left:1em">{{$f.name}}</span>
+          </a>
+        </td>
+        <td>
+{{include file="field_yesno.tpl" field=$f.enabled}}
+        </td>
+        <td>
+{{include file="field_yesno.tpl" field=$f.comments}}
+        </td>
+        <td>
+          <input name="publicise-expire-{{$f.id}}" value="{{$f.expire}}">
+        </td>
+      </tr>
+{{/foreach}}
+    </tbody>
+  </table>
+  <input type="submit" size="70" value="{{$submit_t}}">
+</form>

retriever/database.sql

@@ -0,0 +1,42 @@
+CREATE TABLE IF NOT EXISTS `retriever_rule` (
+       `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+       `uid` int(11) NOT NULL,
+       `contact-id` int(11) NOT NULL,
+       `data` mediumtext NULL DEFAULT NULL,
+       PRIMARY KEY (`id`),
+       KEY `uid` (`uid`),
+       KEY `contact-id` (`contact-id`)
+) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
+CREATE TABLE IF NOT EXISTS `retriever_item` (
+       `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+       `item-uri` varbinary(255) NOT NULL,
+       `item-uid` int(10) unsigned NOT NULL DEFAULT '0',
+       `contact-id` int(10) unsigned NOT NULL DEFAULT '0',
+       `resource` int(11) NOT NULL,
+       `finished` tinyint(1) unsigned NOT NULL DEFAULT '0',
+       KEY `resource` (`resource`),
+       KEY `finished` (`finished`),
+       KEY `item-uid` (`item-uid`),
+       KEY `all` (`item-uri`, `item-uid`, `contact-id`),
+       PRIMARY KEY (`id`)
+) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
+CREATE TABLE IF NOT EXISTS `retriever_resource` (
+       `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+       `item-uid` int(10) unsigned NOT NULL DEFAULT '0',
+       `contact-id` int(10) unsigned NOT NULL DEFAULT '0',
+       `type` char(255) NULL DEFAULT NULL,
+       `binary` int(1) NOT NULL DEFAULT 0,
+       `url` varbinary(700) NOT NULL,
+       `created` timestamp NOT NULL DEFAULT now(),
+       `completed` timestamp NULL DEFAULT NULL,
+       `last-try` timestamp NULL DEFAULT NULL,
+       `num-tries` int(11) NOT NULL DEFAULT 0,
+       `data` mediumblob NULL DEFAULT NULL,
+       `http-code` smallint(1) unsigned NULL DEFAULT NULL,
+       `redirect-url` varbinary(700) NOT NULL,
+       KEY `url` (`url`),
+       KEY `completed` (`completed`),
+       PRIMARY KEY (`id`)
+) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

retriever/retriever.php

@@ -0,0 +1,1074 @@
+<?php
+  /**
+   * Name: Retriever
+   * Description: Follow the permalink of RSS/Atom feed items and replace the summary with the full content.
+   * Version: 1.0
+   * Author: Matthew Exon <http://mat.exon.name>
+   */
+
+use Friendica\App;
+use Friendica\Core\Addon;
+use Friendica\Core\Hook;
+use Friendica\Core\Logger;
+use Friendica\Core\Renderer;
+use Friendica\Core\Session;
+use Friendica\Core\System;
+use Friendica\Content\Text\HTML;
+use Friendica\Content\Text\BBCode;
+use Friendica\Model\Photo;
+use Friendica\Network\HTTPClient\Client\HttpClientAccept;
+use Friendica\Network\HTTPClient\Client\HttpClientOptions;
+use Friendica\Object\Image;
+use Friendica\Util\Network;
+use Friendica\Database\DBA;
+use Friendica\Model\ItemURI;
+use Friendica\Model\Item;
+use Friendica\Model\Post;
+use Friendica\Util\DateTimeFormat;
+use Friendica\DI;
+
+/**
+ * @brief Installation hook for retriever plugin
+ */
+function retriever_install() {
+	DI::logger()->debug('Install retriever');
+
+	Hook::register('addon_settings', 'addon/retriever/retriever.php', 'retriever_addon_settings');
+	Hook::register('addon_settings_post', 'addon/retriever/retriever.php', 'retriever_addon_settings_post');
+	Hook::register('post_remote', 'addon/retriever/retriever.php', 'retriever_post_remote_hook');
+	Hook::register('contact_photo_menu', 'addon/retriever/retriever.php', 'retriever_contact_photo_menu');
+	Hook::register('retriever_mod_post', 'addon/retriever/retriever.php', 'retriever_content');
+	Hook::register('cron', 'addon/retriever/retriever.php', 'retriever_cron');
+
+	if (DI::config()->get('retriever', 'dbversion') == '0.14') {
+		if (!DBA::e("ALTER TABLE `retriever_rule` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci") ||
+		    !DBA::e("ALTER TABLE `retriever_item` MODIFY `item-uri` varbinary(255) NOT NULL") ||
+		    !DBA::e("ALTER TABLE `retriever_item` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci") ||
+		    !DBA::e("ALTER TABLE `retriever_resource` MODIFY `url` varbinary(700) NOT NULL") ||
+		    !DBA::e("ALTER TABLE `retriever_resource` MODIFY `redirect-url` varbinary(700) NOT NULL")) {
+		    !DBA::e("ALTER TABLE `retriever_resource` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci") ||
+			DI::logger()->warning('Unable to update database tables: ' . DBA::errorMessage());
+			return;
+		}
+		DI::config()->set('retriever', 'dbversion', '0.15');
+	}
+	if (DI::config()->get('retriever', 'dbversion') != '0.15') {
+		$schema = file_get_contents(dirname(__file__).'/database.sql');
+		$tables = explode(';', $schema);
+		foreach ($tables as $table) {
+			if (!DBA::e($table)) {
+				DI::logger()->warning('Unable to create database table: ' . DBA::errorMessage());
+				return;
+			}
+		}
+		DI::config()->set('retriever', 'downloads_per_cron', '100');
+		DI::config()->set('retriever', 'dbversion', '0.14');
+	}
+}
+
+/**
+ * @brief Uninstallation hook for retriever plugin
+ */
+function retriever_uninstall() {
+	DI::logger()->debug('Uninstall retriever');
+
+	Hook::unregister('addon_settings', 'addon/retriever/retriever.php', 'retriever_addon_settings');
+	Hook::unregister('addon_settings_post', 'addon/retriever/retriever.php', 'retriever_addon_settings_post');
+	Hook::unregister('post_remote', 'addon/retriever/retriever.php', 'retriever_post_remote_hook');
+	Hook::unregister('addon_settings', 'addon/retriever/retriever.php', 'retriever_addon_settings');
+	Hook::unregister('addon_settings_post', 'addon/retriever/retriever.php', 'retriever_addon_settings_post');
+	Hook::unregister('contact_photo_menu', 'addon/retriever/retriever.php', 'retriever_contact_photo_menu');
+	Hook::unregister('retriever_mod_post', 'addon/retriever/retriever.php', 'retriever_content');
+	Hook::unregister('cron', 'addon/retriever/retriever.php', 'retriever_cron');
+}
+
+/**
+ * This is a statement rather than an actual function definition. The simple
+ * existence of this method is checked to figure out if the addon offers a
+ * module.
+ */
+function retriever_module() {}
+
+/**
+ * @brief Admin page hook for retriever plugin
+ *
+ * @param string $o HTML to append content to (by ref)
+ */
+function retriever_addon_admin(string &$o) {
+	$template = Renderer::getMarkupTemplate('admin.tpl', 'addon/retriever/');
+
+	$downloads_per_cron = DI::config()->get('retriever', 'downloads_per_cron');
+	$downloads_per_cron_config = ['downloads_per_cron',
+				      DI::l10n()->t('Downloads per Cron'),
+				      $downloads_per_cron,
+				      DI::l10n()->t('Maximum number of downloads to attempt during each run of the cron job.')];
+
+	$allow_images = DI::config()->get('retriever', 'allow_images');
+	$allow_images_config = ['allow_images',
+				      DI::l10n()->t('Allow Retrieving Images'),
+				      $allow_images,
+				      DI::l10n()->t('Allow users to request images be downloaded as well as text.<br><b>Warning: the images are not automatically deleted and may fill up your database.</b>')];
+
+	$o .= Renderer::replaceMacros($template, [
+					      '$downloads_per_cron' => $downloads_per_cron_config,
+					      '$allow_images' => $allow_images_config,
+					      '$submit' => DI::l10n()->t('Save Settings')]);
+}
+
+/**
+ * @brief Admin page post hook for retriever plugin
+ */
+function retriever_addon_admin_post () {
+	if (!empty($_POST['downloads_per_cron'])) {
+		DI::config()->set('retriever', 'downloads_per_cron', $_POST['downloads_per_cron']);
+	}
+	DI::config()->set('retriever', 'allow_images', $_POST['allow_images']);
+}
+
+/**
+ * @brief Cron jobs for retriever plugin
+ */
+function retriever_cron() {
+	$downloads_per_cron = DI::config()->get('retriever', 'downloads_per_cron', '100');
+
+	// Do this first, otherwise it can interfere with retriever_retrieve_items
+	retriever_clean_up_completed_resources((int)$downloads_per_cron);
+
+	retriever_retrieve_items((int)$downloads_per_cron);
+	retriever_tidy();
+}
+
+// This global variable is used to track the number of items that have been retrieved during the course of this process
+$retriever_item_count = 0;
+
+/**
+ * @brief Searches for items in the retriever_items table that should be retrieved and attempts to retrieve them
+ *
+ * @param int $max_items Maximum number of items to retrieve in this call
+ */
+function retriever_retrieve_items(int $max_items) {
+	global $retriever_item_count;
+
+	$retriever_schedule = array(array(1,'minute'),
+				    array(10,'minute'),
+				    array(1,'hour'),
+				    array(1,'day'),
+				    array(2,'day'),
+				    array(1,'week'),
+				    array(1,'month'));
+
+	$schedule_clauses = array();
+	for ($i = 0; $i < count($retriever_schedule); $i++) {
+		$num = $retriever_schedule[$i][0];
+		$unit = $retriever_schedule[$i][1];
+		array_push($schedule_clauses,
+			   '(`num-tries` = ' . $i . ' AND TIMESTAMPADD(' . DBA::escape($unit) .
+			   ', ' . intval($num) . ', `last-try`) < now())');
+	}
+
+	$retrieve_items = $max_items - $retriever_item_count;
+	do {
+		DI::logger()->debug('retriever_retrieve_items: asked for maximum ' . $max_items . ', already retrieved ' . intval($retriever_item_count) . ', retrieve ' . $retrieve_items);
+		$retriever_resources = DBA::selectToArray('retriever_resource', [], ['`completed` IS NULL AND (`last-try` IS NULL OR ' . implode(' OR ', $schedule_clauses) . ')'], ['order' => ['last-try' => 0], 'limit' => $retrieve_items]);
+		if (!is_array($retriever_resources)) {
+			break;
+		}
+		if (count($retriever_resources) == 0) {
+			break;
+		}
+		DI::logger()->debug('retriever_retrieve_items: found ' . count($retriever_resources) . ' waiting resources in database');
+		foreach ($retriever_resources as $retriever_resource) {
+			retrieve_resource($retriever_resource);
+			$retriever_item_count++;
+		}
+		$retrieve_items = $max_items - $retriever_item_count;
+	}
+	while ($retrieve_items > 0);
+	DI::logger()->debug('retriever_retrieve_items: finished retrieving items');
+}
+
+/**
+ * @brief Looks for items that are waiting even though the resource has completed.  This shouldn't happen, but is worth cleaning up if it does.
+ *
+ * @param int $max_items Maximum number of items to retrieve in this call
+ */
+function retriever_clean_up_completed_resources(int $max_items) {
+	// TODO: figure out how to do this with DBA module
+	$r = DBA::p("SELECT retriever_resource.`id` as resource, retriever_item.`id` as item FROM retriever_resource, retriever_item, retriever_rule WHERE retriever_item.`finished` = 0 AND retriever_item.`resource` = retriever_resource.`id` AND retriever_resource.`completed` IS NOT NULL AND retriever_item.`contact-id` = retriever_rule.`contact-id` AND retriever_item.`item-uid` = retriever_rule.`uid` LIMIT $max_items");
+	if (!DBA::isResult($r)) {
+		return;
+	}
+	DI::logger()->debug('retriever_clean_up_completed_resources: items waiting even though resource has completed: ' . DBA::numRows($r));
+	while ($rr = DBA::fetch($r)) {
+		$retriever_item = DBA::selectFirst('retriever_item', [], ['id' => intval($rr['item'])]);
+		if (!DBA::isResult($retriever_item)) {
+			DI::logger()->warning('retriever_clean_up_completed_resources: no retriever item with id ' . $rr['item']);
+			continue;
+		}
+		$item = retriever_get_item($retriever_item);
+		if (!$item) {
+			DI::logger()->warning('retriever_clean_up_completed_resources: no item ' . $retriever_item['item-uri']);
+			continue;
+		}
+		$retriever_rule = get_retriever_rule($retriever_item['contact-id'], $item['uid'], false);
+		if (!$retriever_rule) {
+			DI::logger()->warning('retriever_clean_up_completed_resources: no retriever for uri ' . $retriever_item['item-uri'] . ' uid ' . $retriever_item['uid'] . ' ' . $retriever_item['contact-id']);
+			continue;
+		}
+		$resource = DBA::selectFirst('retriever_resource', [], ['id' => intval($rr['resource'])]);
+		retriever_apply_completed_resource_to_item($retriever_rule, $item, $resource);
+		// TODO: I don't really get how the $old_fields argument to DBA::update works
+		DBA::update('retriever_item', ['finished' => 1], ['id' => intval($retriever_item['id'])], ['finished' => 0]);
+		retriever_check_item_completed($item);
+	}
+}
+
+/**
+ * @brief Deletes old rows from the retriever_item and retriever_resource table that are unlikely to be needed
+ */
+function retriever_tidy() {
+	DBA::delete('retriever_resource', ['completed IS NOT NULL AND completed < DATE_SUB(now(), INTERVAL 1 WEEK)']);
+	DBA::delete('retriever_resource', ['completed IS NULL AND created < DATE_SUB(now(), INTERVAL 3 MONTH)']);
+
+	$r = DBA::p("SELECT retriever_item.id FROM retriever_item LEFT OUTER JOIN retriever_resource ON (retriever_item.resource = retriever_resource.id) WHERE retriever_resource.id is null");
+	if (!DBA::isResult($r)) {
+		return;
+	}
+	DI::logger()->info('retriever_tidy: found ' . DBA::numRows($r) . ' retriever_items with no retriever_resource');
+	while ($rr = DBA::fetch($r)) {
+		DBA::delete('retriever_item', ['id' => intval($rr['id'])]);
+	}
+}
+
+/**
+ * @brief Special case of retrieving a resource: if the URL is a data URL, do not use cURL, decode the URL directly
+ *
+ * @param array $resource The row from the retriever_resource table
+ */
+function retrieve_dataurl_resource(array $resource) {
+	if (!preg_match("/date:(.*);base64,(.*)/", $resource['url'], $matches)) {
+		DI::logger()->warning('retrieve_dataurl_resource: resource ' . $resource['id'] . ' does not match pattern');
+	} else {
+		$resource['type'] = $matches[1];
+		$resource['data'] = base64url_decode($matches[2]);
+	}
+
+	// Succeed or fail, there's no point retrying
+	DBA::update('retriever_resource', ['id' => intval($resource['id'])], ['last-try' => DateTimeFormat::utcNow(), 'num-tries' => intval($resource['num-tries']) + 1, 'completed' => DateTimeFormat::utcNow(), 'data' => $resource['data'], 'type' => $resource['type']], ['last-try' => false]);
+	retriever_resource_completed($resource);
+}
+
+/**
+ * @brief Makes an attempt to retrieve the supplied resource, and updates the row in the table with the results
+ *
+ * @param array $resource The row from the retriever_resource table
+ */
+function retrieve_resource(array $resource) {
+	$components = parse_url($resource['url']);
+	if (!$components) {
+		DI::logger()->warning('retrieve_resource: URL ' . $resource['url'] . ' could not be parsed');
+	}
+	if ($components['scheme'] == "data") {
+		return retrieve_dataurl_resource($resource);
+	}
+	if (($components['scheme'] != "http") && ($components['scheme'] != "https")) {
+		DI::logger()->warning('retrieve_resource: URL scheme not supported for ' . $resource['url']);
+		DBA::update('retriever_resource', ['completed' => DateTimeFormat::utcNow()], ['id' => intval($resource['id'])], ['completed' => false]);
+		retriever_resource_completed($resource);
+		return;
+	}
+
+	$retriever_rule = get_retriever_rule($resource['contact-id'], $resource['item-uid'], false);
+	if (!$retriever_rule) {
+		DI::logger()->warning('retrieve_resource: no rule found for resource id ' . $resource['id'] . ' contact ' . $resource['contact-id'] . ' item ' . $resource['item-uid']);
+		DBA::update('retriever_resource', ['completed' => DateTimeFormat::utcNow()], ['id' => intval($resource['id'])], ['completed' => false]);
+		retriever_resource_completed($resource);
+		return;
+	}
+	$rule_data = $retriever_rule['data'];
+	if (!$rule_data) {
+		DI::logger()->warning('retrieve_resource: no rule data found for resource id ' . $resource['id'] . ' contact ' . $resource['contact-id'] . ' item ' . $resource['item-uid']);
+		DBA::update('retriever_resource', ['completed' => DateTimeFormat::utcNow()], ['id' => intval($resource['id'])], ['completed' => false]);
+		retriever_resource_completed($resource);
+		return;
+	}
+
+	try {
+		DI::logger()->debug('retrieve_resource: ' . ($resource['num-tries'] + 1) . ' attempt at resource ' . $resource['id'] . ' ' . $resource['url']);
+		$cookiejar = '';
+		if (array_key_exists('storecookies', $rule_data) && $rule_data['storecookies']) {
+			$cookiejar = tempnam(System::getTempPath(), 'cookiejar-retriever-');
+			file_put_contents($cookiejar, $rule_data['cookiedata']);
+		}
+		$fetch_result = DI::httpClient()->get($resource['url'], HttpClientAccept::DEFAULT, [HttpClientOptions::COOKIEJAR => $cookiejar]);
+		if (array_key_exists('storecookies', $rule_data) && $rule_data['storecookies']) {
+			$retriever_rule['data']['cookiedata'] = file_get_contents($cookiejar);
+			DBA::update('retriever_rule', ['data' => json_encode($retriever_rule['data'])], ['id' => intval($retriever_rule["id"])], $retriever_rule);
+			unlink($cookiejar);
+		}
+		$resource['data'] = $fetch_result->getBodyString();
+		$resource['http-code'] = $fetch_result->getReturnCode();
+		$resource['type'] = $fetch_result->getContentType();
+		$resource['redirect-url'] = $fetch_result->getRedirectUrl();
+		DI::logger()->debug('retrieve_resource: got code ' . $resource['http-code'] . ' retrieving resource ' . $resource['id'] . ' final url ' . $resource['redirect-url']);
+	} catch (Exception $e) {
+		DI::logger()->info('retrieve_resource: unable to retrieve ' . $resource['url'] . ' - ' . $e->getMessage());
+	}
+	DBA::update('retriever_resource', ['last-try' => DateTimeFormat::utcNow(), 'num-tries' => intval($resource['num-tries']) + 1, 'http-code' => intval($resource['http-code']), 'redirect-url' => $resource['redirect-url']], ['id' => intval($resource['id'])], ['last-try' => false]);
+	if ($resource['data']) {
+		DBA::update('retriever_resource', ['completed' => DateTimeFormat::utcNow(), 'data' => $resource['data'], 'type' => $resource['type']], ['id' => intval($resource['id'])], ['completed' => false]);
+		retriever_resource_completed($resource);
+	}
+}
+
+/**
+ * @brief Gets the retriever configuration for a particular contact.  Optionally, will create a blank configuration.
+ *
+ * @param int $contact_id The Contact ID of the retriever configuration
+ * @param int $uid The User ID of the retriever configuration
+ * @param boolean $create Whether to create a new configuration if none exists already
+ * @return array The row from the retriever_rule database for this configuration
+ */
+function get_retriever_rule(string $contact_id, string $uid, bool $create) {
+	$retriever_rule = DBA::selectFirst('retriever_rule', [], ['contact-id' => intval($contact_id), 'uid' => intval($uid)]);
+	if ($retriever_rule) {
+		$retriever_rule['data'] = json_decode($retriever_rule['data'], true);
+		return $retriever_rule;
+	}
+	if ($create) {
+		DBA::insert('retriever_rule', ['uid' => intval($uid), 'contact-id' => intval($contact_id)]);
+		$retriever_rule = DBA::selectFirst('retriever_rule', [], ['contact-id' => intval($contact_id), 'uid' => intval($uid)]);
+		return $retriever_rule;
+	}
+}
+
+/**
+ * @brief Looks up the item from the database that corresponds to the retriever_item
+ *
+ * @param array $retriever_item Row from the retriever_item table
+ * @return array Item that was found, or undef if no item could be found
+ */
+function retriever_get_item(array $retriever_item) {
+	$item = Post::selectFirst([], ['uri' => $retriever_item['item-uri'], 'uid' => intval($retriever_item['item-uid']), 'contact-id' => intval($retriever_item['contact-id'])]);
+	if (!DBA::isResult($item)) {
+		DI::logger()->warning('retriever_get_item: no item found for uri ' . $retriever_item['item-uri']);
+		return;
+	}
+	return $item;
+}
+
+/**
+ * @brief This function should be called when a resource is completed to trigger all next steps, based on the corresponding retriever item
+ *
+ * @param int $retriever_item_id ID of the retriever item corresponding to this resource
+ * @param array $resource The full details of the completed resource
+ */
+function retriever_item_completed(string $retriever_item_id, array $resource) {
+	DI::logger()->debug('retriever_item_completed: id ' . $retriever_item_id . ' url ' . $resource['url']);
+
+	$retriever_item = DBA::selectFirst('retriever_item', [], ['id' => intval($retriever_item_id)]);
+	if (!DBA::isResult($retriever_item)) {
+		DI::logger()->info('retriever_item_completed: no retriever item with id ' . $retriever_item_id);
+		return;
+	}
+	$item = retriever_get_item($retriever_item);
+	if (!$item) {
+		DI::logger()->warning('retriever_item_completed: no item ' . $retriever_item['item-uri']);
+		return;
+	}
+	// Note: the retriever might be null.  Doesn't matter.
+	$retriever_rule = get_retriever_rule($retriever_item['contact-id'], $retriever_item['item-uid'], false);
+
+	if ($retriever_rule) {
+		retriever_apply_completed_resource_to_item($retriever_rule, $item, $resource);
+	}
+
+	DBA::update('retriever_item', ['finished' => 1], ['id' => intval($retriever_item['id'])], ['finished' => 0]);
+	retriever_check_item_completed($item);
+}
+
+/**
+ * @brief This function should be called when a resource is completed to trigger all next steps
+ *
+ * @param array $resource The full details of the completed resource
+ */
+function retriever_resource_completed(array $resource) {
+	DI::logger()->debug('retriever_resource_completed: id ' . $resource['id'] . ' url ' . $resource['url']);
+	foreach (DBA::selectToArray('retriever_item', ['id'], ['resource' => intval($resource['id'])]) as $retriever_item) {
+		retriever_item_completed($retriever_item['id'], $resource);
+	}
+}
+
+/**
+ * @brief For a retriever config for a particular contact, remove existing artifacts for a number of completed items and queue them to be tried again.  Will make the items invisible until they are again completed.  The items chosen will be the most recently received.
+ *
+ * @param array $retriever The row from the retriever_rule table for the contact
+ * @param int $num The number of existing items to queue for retrieval
+ */
+function apply_retrospective(array $retriever, int $num) {
+	foreach (Post::selectToArray([], ['contact-id' => intval($retriever['contact-id'])], ['order' => ['received' => true], 'limit' => $num]) as $item) {
+		Item::update(['visible' => 0], ['id' => intval($item['id'])]);
+		foreach (DBA::selectToArray('retriever_item', [], ['item-uri' => $item['uri'], 'item-uid' => $item['uid'], 'contact-id' => $item['contact-id']]) as $retriever_item) {
+			DBA::delete('retriever_resource', ['id' => $retriever_item['resource']]);
+			DBA::delete('retriever_item', ['id' => $retriever_item['id']]);
+		}
+		retriever_on_item_insert($retriever, $item);
+	}
+}
+
+/**
+ * @brief Queues an item for retrieval.  It does not actually perform the retrieval.
+ *
+ * @param array $retriever Retriever rule configuration for this contact
+ * @param array $item Item that should be retrieved.  This may or may not have been already stored in the database.
+ *
+ * TODO: This queries then inserts.  It should use some kind of lock to avoid requesting the same resource twice.
+ */
+function retriever_on_item_insert(array $retriever, array &$item) {
+	if (!$retriever || !$retriever['id']) {
+		DI::logger()->info('retriever_on_item_insert: No retriever supplied');
+		return;
+	}
+	if (!array_key_exists('enable', $retriever['data']) || !$retriever['data']['enable'] == "on") {
+		return;
+	}
+	if (array_key_exists('plink', $item) && strlen($item['plink'])) {
+		$url = $item['plink'];
+	}
+	else {
+		if (!array_key_exists('uri-id', $item)) {
+			DI::logger()->warning('retriever_on_item_insert: item ' . $item['id'] . ' has no plink and no uri-id');
+			return;
+		}
+		$content = DBA::selectFirst('post-content', [], ['uri-id' => $item['uri-id']]);
+		$url = $content['plink'];
+	}
+
+	if (array_key_exists('modurl', $retriever['data']) && $retriever['data']['modurl']) {
+		$orig_url = $url;
+		$url = preg_replace('/' . $retriever['data']['pattern'] . '/', $retriever['data']['replace'], $orig_url);
+		DI::logger()->debug('retriever_on_item_insert: Changed ' . $orig_url . ' to ' . $url);
+	}
+
+	$resource = add_retriever_resource($url, $item['uid'], $item['contact-id']);
+	if (is_array($resource)) {
+		$retriever_item_id = add_retriever_item($item, $resource);
+	}
+}
+
+/**
+ * @brief Creates a new resource to be downloaded from the supplied URL.  Unique resources are created for each URL, UID and contact ID, because different contact IDs may have different rules for how to retrieve them.  If the URL is actually a data URL, the resource is completed immediately.
+ *
+ * @param string $url URL of the resource to be downloaded
+ * @param int $uid User ID that this resource is being downloaded fore
+ * @param int $cid Contact ID of the item that triggered the downloading of this resource
+ * @param boolean $binary Specifies if this download should be done in binary mode
+ * @return array The created resource
+ */
+function add_retriever_resource(string $url, string $uid, string $cid, bool $binary = false) {
+	DI::logger()->debug('add_retriever_resource: url ' . $url . ' uid ' . $uid . ' contact-id ' . $cid);
+
+	$scheme = parse_url($url, PHP_URL_SCHEME);
+	if ($scheme == 'data') {
+		$fp = fopen($url, 'r');
+		$meta = stream_get_meta_data($fp);
+		$type = $meta['mediatype'];
+		$data = stream_get_contents($fp);
+		fclose($fp);
+
+		$url = 'md5://' . hash('md5', $url);
+		$resource = DBA::selectFirst('retriever_resource', [], ['url' => $url, 'item-uid' => intval($uid), 'contact-id' => intval($cid)]);
+		if ($resource) {
+			DI::logger()->debug('add_retriever_resource: Resource ' . $url . ' already requested');
+			return $resource;
+		}
+
+		DBA::insert('retriever_resource', ['item-uid' => intval($uid), 'contact-id' => intval($cid), 'type' => $type, 'binary' => ($binary ? 1 : 0), 'url' => $url, 'completed' => DateTimeFormat::utcNow(), 'data' => $data, 'redirect-url' => '']);
+		$resource = DBA::selectFirst('retriever_resource', [], ['url' => $url, 'item-uid' => intval($uid), 'contact-id' => intval($cid)]);
+		if ($resource) {
+			retriever_resource_completed($resource);
+		}
+		return $resource;
+	}
+
+	// 700 characters is the size of this field in the database
+	if (strlen($url) > 700) {
+		DI::logger()->warning('add_retriever_resource: URL is longer than 700 characters');
+	}
+
+	$resource = DBA::selectFirst('retriever_resource', [], ['url' => $url, 'item-uid' => intval($uid), 'contact-id' => intval($cid)]);
+	if ($resource) {
+		DI::logger()->debug('add_retriever_resource: Resource ' . $url . ' uid ' . $uid . ' cid ' . $cid . ' already requested');
+		return $resource;
+	}
+
+	DBA::insert('retriever_resource', ['item-uid' => intval($uid), 'contact-id' => intval($cid), 'binary' => ($binary ? 1 : 0), 'url' => $url, 'redirect-url' => '']);
+	return DBA::selectFirst('retriever_resource', [], ['url' => $url, 'item-uid' => intval($uid), 'contact-id' => intval($cid)]);
+}
+
+/**
+ * @brief Adds a retriever item for the supplied resource and item, to mark that this item should wait for the resource to be completed.  Does not create a retriever item if a matching one already exists.
+ *
+ * @param array $item Item that is waiting for the resource.  This may or may not have been already stored in the database.
+ * @param array $resource Resource that the item needs to wait for.  This must have already been stored in the database.
+ * @return int ID of the retriever item that was created, or the existing one if present
+ */
+function add_retriever_item(array $item, array $resource) {
+	DI::logger()->debug('add_retriever_item: ' . $resource['url'] . ' for ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id']);
+
+	if (!array_key_exists('id', $resource) || !$resource['id']) {
+		DI::logger()->warning('add_retriever_item: resource is empty');
+		return;
+	}
+	if (DBA::selectFirst('retriever_item', [], ['item-uri' => $item['uri'], 'item-uid' => intval($item['uid']), 'resource' => intval($resource['id'])])) {
+		DI::logger()->info("add_retriever_item: retriever item already present for " . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id']);
+		return;
+	}
+	DBA::insert('retriever_item', ['item-uri' => $item['uri'], 'item-uid' => intval($item['uid']), 'contact-id' => intval($item['contact-id']), 'resource' => intval($resource['id'])]);
+	$retriever_item = DBA::selectFirst('retriever_item', ['id'], ['item-uri' => $item['uri'], 'item-uid' => intval($item['uid']), 'resource' => intval($resource['id'])]);
+	if (!$retriever_item) {
+		DI::logger()->info("add_retriever_item: couldn't create retriever item for " . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id']);
+		return;
+	}
+	DI::logger()->debug('add_retriever_item: created retriever_item ' . $retriever_item['id'] . ' for item ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id']);
+	return $retriever_item['id'];
+}
+
+/**
+ * @brief Analyse a completed text resource (such as HTML) for the character encoding used
+ *
+ * @param array $resource The completed resource
+ * @return string Character encoding, e.g. "utf-8" or "iso-8859-1"
+ */
+function retriever_get_encoding(array $resource) {
+	$matches = array();
+	if (preg_match('/charset=(.*)/', $resource['type'], $matches)) {
+		return trim(array_pop($matches));
+	}
+	return 'utf-8';
+}
+
+/**
+ * @brief Apply the XSLT template to the DOM document
+ *
+ * @param string $xslt_text Text of the XSLT template
+ * @param DOMDocument $doc Input to the XSLT template
+ * @return DOMDocument Result of applying the template
+ */
+function retriever_apply_xslt_text(string $xslt_text, DOMDocument $doc) {
+	if (!$xslt_text) {
+		DI::logger()->info('retriever_apply_xslt_text: empty XSLT text');
+		return $doc;
+	}
+	$xslt_doc = new DOMDocument();
+	if (!$xslt_doc->loadXML($xslt_text)) {
+		DI::logger()->info('retriever_apply_xslt_text: could not load XML');
+		return $doc;
+	}
+	$xp = new XsltProcessor();
+	$xp->importStylesheet($xslt_doc);
+	$result = $xp->transformToDoc($doc);
+	return $result;
+}
+
+/**
+ * @brief Applies the retriever rules to the downloaded resource, and stores the results as the new body text of the item
+ *
+ * @param array $retriever Retriever rules as stored in the database, with the "data" element already decoded from JSON
+ * @param array &$item Item to be in which to store the new body (by ref).  This may or may not be already stored in the database.
+ * @param array $resource Newly completed resource, which should be text (HTML or XML)
+ */
+function retriever_apply_dom_filter(array $retriever, array &$item, array $resource) {
+	DI::logger()->debug('retriever_apply_dom_filter: applying XSLT to uri ' . $item['uri'] . ' uid ' . $item['uid'] . ' contact ' . $item['contact-id']);
+
+	if (!array_key_exists('include', $retriever['data']) && !array_key_exists('customxslt', $retriever['data'])) {
+		DI::logger()->info('retriever_apply_dom_filter: no include and no customxslt');
+		return;
+	}
+	if (!$resource['data']) {
+		DI::logger()->info('retriever_apply_dom_filter: no text to work with');
+		return;
+	}
+
+	$doc = retriever_load_into_dom($resource);
+
+	$doc = retriever_extract($doc, $retriever);
+	if (!$doc) {
+		DI::logger()->info('retriever_apply_dom_filter: failed to apply extract XSLT template');
+		return;
+	}
+
+	$doc = retriever_globalise_urls($doc, $resource);
+	if (!$doc) {
+		DI::logger()->info('retriever_apply_dom_filter: failed to apply fix urls XSLT template');
+		return;
+	}
+
+	$body = HTML::toBBCode($doc->saveHTML());
+	if (!strlen($body)) {
+		DI::logger()->info('retriever_apply_dom_filter retriever ' . $retriever['id'] . ' item ' . $item['id'] . ': output was empty');
+		return;
+	}
+	$body .= "\n\n" . DI::l10n()->t('Retrieved') . ' ' . date("Y-m-d") . ': [url=';
+	$body .=  $item['plink'];
+	$body .= ']' . $item['plink'] . '[/url]';
+
+	DI::logger()->debug('retriever_apply_dom_filter: XSLT result \"' . $body . '\"');
+	retriever_set_body($item, $body);
+}
+
+/**
+ * @brief Converts the completed resource, which must be HTML or XML, into a DOM document
+ *
+ * @param array $resource The resource containing the text content
+ */
+function retriever_load_into_dom(array $resource) {
+	$encoding = retriever_get_encoding($resource);
+	$content = mb_convert_encoding($resource['data'], 'HTML-ENTITIES', $encoding);
+	$doc = new DOMDocument('1.0', 'UTF-8');
+	if (strpos($resource['type'], 'html') !== false) {
+		@$doc->loadHTML($content);
+	}
+	else {
+		$doc->loadXML($content);
+	}
+	return $doc;
+}
+
+/**
+ * @brief Applies the retriever rules, including configuration for included and excluded portions, to the DOM document
+ *
+ * @param DOMDocument $doc The original DOM document downloaded from the link
+ * @param array $retriever The retriever configuration for this contact
+ * @return DOMDocument New DOM document containing only the desired content
+ */
+function retriever_extract(DOMDocument $doc, array $retriever) {
+	$params = array('$spec' => $retriever['data']);
+	$extract_template = Renderer::getMarkupTemplate('extract.tpl', 'addon/retriever/');
+	$extract_xslt = Renderer::replaceMacros($extract_template, $params);
+	if ($retriever['data']['include']) {
+		DI::logger()->debug('retriever_apply_dom_filter: applying include/exclude template \"' . $extract_xslt . '\"');
+		$doc = retriever_apply_xslt_text($extract_xslt, $doc);
+	}
+	if (array_key_exists('customxslt', $retriever['data']) && $retriever['data']['customxslt']) {
+		DI::logger()->debug('retriever_extract: applying custom XSLT \"' . $retriever['data']['customxslt'] . '\"');
+		$doc = retriever_apply_xslt_text($retriever['data']['customxslt'], $doc);
+	}
+	return $doc;
+}
+
+/**
+ * @brief Converts local URLs in the DOM document to global URLs
+ *
+ * @param DOMDocument $doc DOM document potentially containing links
+ * @param array $resource Completed resource which contains the text in the DOM document
+ * @return DOMDocument New DOM document with global URLs
+ */
+function retriever_globalise_urls(DOMDocument $doc, array $resource) {
+	$url = $resource['redirect-url'];
+	if ($url == "") {
+		$url = $resource['url'];
+	}
+	$components = parse_url($url);
+	if (!array_key_exists('scheme', $components) || !array_key_exists('host', $components) || !array_key_exists('path', $components)) {
+		return $doc;
+	}
+	$rooturl = $components['scheme'] . "://" . $components['host'];
+	$dirurl = $rooturl . dirname($components['path']);
+	$params = array('$dirurl' => $dirurl, '$rooturl' => $rooturl);
+	$fix_urls_template = Renderer::getMarkupTemplate('fix-urls.tpl', 'addon/retriever/');
+	$fix_urls_xslt = Renderer::replaceMacros($fix_urls_template, $params);
+	$doc = retriever_apply_xslt_text($fix_urls_xslt, $doc);
+	return $doc;
+}
+
+/**
+ * @brief Returns the body text for the supplied item.  If the item has already been stored in the database, this will fetch the content from the database rather than from the supplied array.
+ *
+ * @param array $item Row from the item table
+ */
+function retriever_get_body(array $item) {
+	if (!array_key_exists('uri-id', $item) || !$item['uri-id']) {
+		// item has not yet been stored in database
+		return $item['body'];
+	}
+
+	// item has been stored in database, body is stored in the post-content table
+	$content = DBA::selectFirst('post-content', ['body'], ['uri-id' => $item['uri-id']]);
+	if (!$content) {
+		DI::logger()->warning('retriever_get_body: post-content uri-id ' . $item['uri-id'] . ' has no content');
+		return $item['body'];
+	}
+	if (!$content['body']) {
+		DI::logger()->warning('retriever_get_body: post-content uri-id ' . $item['uri-id'] . ' has no body');
+		return $item['body'];
+	}
+	return $content['body'];
+}
+
+/**
+ * @brief Updates the item with the supplied body text.  If the item has already been stored in the database, this will update the database too.
+ *
+ * @param array &$item Item in which to set the body (by ref).  This may or may not be already stored in the database.
+ * @param string $body New body content
+ */
+function retriever_set_body(array &$item, string $body) {
+	$item['body'] = $body;
+	if (!array_key_exists('id', $item) || !$item['id']) {
+		// item has not yet been stored in database
+		return;
+	}
+	Item::update(['body' => $body], ['id' => intval($item['id'])]);
+}
+
+/**
+ * @brief Searches for images in the item and adds corresponding retriever_items.  If the images have already been downloaded, updates the body in the supplied item array.
+ *
+ * @param array &$item Item to be searched for images and updated (by ref).  This may or may not be already stored in the database.
+ */
+function retrieve_images(array &$item) {
+	if (!DI::config()->get('retriever', 'allow_images')) {
+		return;
+	}
+
+	$body = retriever_get_body($item);
+	if (!strlen($body)) {
+		DI::logger()->warning('retrieve_images: no body for item ' . $item['uri']);
+		return;
+	}
+
+	// I suspect that the first two are not used any more?
+	preg_match_all("/\[img\=([0-9]*)x([0-9]*)\](.*?)\[\/img\]/ism", $body, $matches1);
+	preg_match_all("/\[img\](.*?)\[\/img\]/ism", $body, $matches2);
+	preg_match_all("/\[img\=([^\]]*)\]([^[]*)\[\/img\]/ism", $body, $matches3);
+	$matches = array_merge($matches1[3], $matches2[1], $matches3[1]);
+	DI::logger()->debug('retrieve_images: found ' . count($matches) . ' images for item ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id']);
+	foreach ($matches as $url) {
+		if (!$url) {
+			continue;
+		}
+		if (strpos($url, (string)(DI::baseUrl())) === FALSE) {
+			$resource = add_retriever_resource($url, $item['uid'], $item['contact-id'], true);
+			if (!is_array($resource)) {
+				DI::logger()->error('retrieve_images: could not add resource', ['url' => $url, 'uid' => $item['uid'], 'contact-id' => $item['contact-id']]);
+				continue;
+			}
+			if (!$resource['completed']) {
+				add_retriever_item($item, $resource);
+				continue;
+			}
+			retriever_transform_images($item, $resource);
+		}
+	}
+}
+
+/**
+ * @brief Checks if an item has been completed, i.e. all its associated retriever_item rows have been retrieved.  If so, update the item to be visible again.
+ *
+ * @param array &$item Row from the item table (by ref)
+ */
+function retriever_check_item_completed(array &$item)
+{
+	$waiting = DBA::selectFirst('retriever_item', [], ['item-uri' => $item['uri'], 'item-uid' => intval($item['uid']), 'contact-id' => intval($item['contact-id']), 'finished' => 0]);
+	DI::logger()->debug('retriever_check_item_completed: item ' . $item['uri'] . ' ' . $item['uid'] . ' '. $item['contact-id'] . ' waiting for resources');
+	$old_visible = $item['visible'];
+	$item['visible'] = $waiting ? 0 : 1;
+	if (array_key_exists('id', $item) && ($item['id'] > 0) && ($old_visible != $item['visible'])) {
+		DI::logger()->debug('retriever_check_item_completed: changing visible flag to ' . $item['visible']);
+		Item::update(['visible' => $item['visible']], ['id' => intval($item['id'])]);
+	}
+}
+
+/**
+ * @brief Updates an item with a completed resource.  If the resource was text, update the body with the new content.  If the resource was an image, replace remote images in the body with a local version.
+ *
+ * @param array $retriever Rule configuration for this contact
+ * @param array &$item Row from the item table (by ref)
+ * @param array $resource The resource that has just been completed
+ */
+function retriever_apply_completed_resource_to_item(array $retriever, array &$item, array $resource) {
+	DI::logger()->debug('retriever_apply_completed_resource_to_item', ['retriever' => $retriever ? $retriever['id'] : 'none', 'resource' => $resource['url'], 'plink' => $item['plink']]);
+	if (strpos($resource['type'], 'image') !== false) {
+		retriever_transform_images($item, $resource);
+	}
+	if (!$retriever) {
+		DI::logger()->warning('retriever_apply_completed_resource_to_item: no retriever');
+		return;
+	}
+	if ((strpos($resource['type'], 'html') !== false) ||
+	    (strpos($resource['type'], 'xml') !== false)) {
+		retriever_apply_dom_filter($retriever, $item, $resource);
+		if ($retriever['data']['images'] ) {
+			retrieve_images($item);
+		}
+	}
+}
+
+/**
+ * @brief Stores the image downloaded in the supplied resource and updates the item body by replacing the remote URL with the local URL.  The body will be updated in the supplied item array.  If the item has already been stored, and therefore has an ID already, the row in the database will be updated too.
+ *
+ * @param array &$item Row from the item table (by ref)
+ * @param array $resource Row from the resource table containing successfully downloaded image
+ *
+ * TODO: split this into two functions, one to store the image, the other to change the item body
+ */
+function retriever_transform_images(array &$item, array $resource) {
+	if (!$resource['data']) {
+		DI::logger()->info('retriever_transform_images: no data available for ' . $resource['id'] . ' ' . $resource['url']);
+		return;
+	}
+
+	$data = $resource['data'];
+	$type = $resource['type'];
+	$uid = $item['uid'];
+	$cid = $item['contact-id'];
+	$rid = Photo::newResource();
+	$path = parse_url($resource['url'], PHP_URL_PATH);
+	$parts = pathinfo($path);
+	$filename = $parts['filename'] . (array_key_exists('extension', $parts) ? '.' . $parts['extension'] : '');
+	$album = 'Retriever';
+	$scale = 0;
+	$desc = ''; // TODO: store alt text with resource when it's requested so we can fill this in
+	DI::logger()->debug('retriever_transform_images storing ' . strlen($data) . ' bytes type ' . $type . ': uid ' . $uid . ' cid ' . $cid . ' rid ' . $rid . ' filename ' . $filename . ' album ' . $album . ' scale ' . $scale . ' desc ' . $desc);
+	$image = new Image($data, $type);
+	if (!$image->isValid()) {
+		DI::logger()->warning('retriever_transform_images: invalid image found at URL ' . $resource['url'] . ' for item ' . $item['id']);
+		return;
+	}
+	try {
+		$photo = Photo::store($image, $uid, $cid, $rid, $filename, $album, $scale, Photo::DEFAULT, '', '', '', '', $desc);
+	} catch (Exception $e) {
+		DI::logger()->error('retriever_transform_images: unable to store photo ' . $resource['url'] . ' error: ' . $e->getMessage());
+		return;
+	}
+	$new_url = DI::baseUrl() . '/photo/' . $rid . '-0' . $image->getExt();
+	if (!strlen($new_url)) {
+		DI::logger()->warning('retriever_transform_images: no replacement URL for image ' . $resource['url']);
+		return;
+	}
+
+	$body = retriever_get_body($item);
+
+	DI::logger()->debug('retriever_transform_images: replacing ' . $resource['url'] . ' with ' . $new_url . ' in item ' . $item['uri']);
+	$body = str_replace($resource["url"], $new_url, $body);
+	retriever_set_body($item, $body);
+}
+
+/**
+ * @brief Displays the retriever configuration page for a contact.  Alternatively, if the user clicked the "help" button, display the help content.
+ */
+function retriever_content() {
+    $e = new \Exception;
+	if (!DI::userSession()->getLocalUserId()) {
+		DI::page()['content'] .= "<p>Please log in</p>";
+		return;
+	}
+	if (DI::args()->get(1) === 'help') {
+		$feeds = DBA::selectToArray('contact', ['id', 'name', 'thumb'], ['uid' => DI::userSession()->getLocalUserId(), 'network' => 'feed']);
+		for ($i = 0; $i < count($feeds); ++$i) {
+			$feeds[$i]['url'] = DI::baseUrl() . '/retriever/' . $feeds[$i]['id'];
+		}
+		$template = Renderer::getMarkupTemplate('/help.tpl', 'addon/retriever/');
+		DI::page()['content'] .= Renderer::replaceMacros($template, array(
+								       '$config' => DI::baseUrl() . '/settings/addon',
+								       '$allow_images' => DI::config()->get('retriever', 'allow_images'),
+								       '$feeds' => $feeds));
+		return;
+	}
+	if (DI::args()->get(1)) {
+		$arg1 = DI::args()->get(1);
+		$retriever_rule = get_retriever_rule($arg1, DI::userSession()->getLocalUserId(), false);
+		if (!$retriever_rule) {
+			$retriever_rule = ['id' => 0, 'data' => ['enable' => 0, 'modurl' => '', 'pattern' => '', 'replace' => '', 'images' => 0, 'storecookies' => 0, 'cookiedata' => '', 'customxslt' => '', 'include' => '', 'exclude' => '']];
+		}
+
+		if (!empty($_POST["id"])) {
+			$retriever_rule = get_retriever_rule($arg1, DI::userSession()->getLocalUserId(), true);
+			$retriever_rule['data'] = array();
+			foreach (array('modurl', 'pattern', 'replace', 'enable', 'images', 'customxslt', 'storecookies', 'cookiedata') as $setting) {
+				if (empty($_POST['retriever_' . $setting])) {
+					$retriever_rule['data'][$setting] = NULL;
+				}
+				else {
+					$retriever_rule['data'][$setting] = $_POST['retriever_' . $setting];
+				}
+			}
+			foreach ($_POST as $k=>$v) {
+				if (preg_match("/retriever-(include|exclude)-(\d+)-(element|attribute|value)/", $k, $matches)) {
+					$retriever_rule['data'][$matches[1]][intval($matches[2])][$matches[3]] = $v;
+				}
+			}
+			// You've gotta have an element, even if it's just "*"
+			foreach ($retriever_rule['data']['include'] as $k=>$clause) {
+				if (!$clause['element']) {
+					unset($retriever_rule['data']['include'][$k]);
+				}
+			}
+			foreach ($retriever_rule['data']['exclude'] as $k=>$clause) {
+				if (!$clause['element']) {
+					unset($retriever_rule['data']['exclude'][$k]);
+				}
+			}
+			DBA::update('retriever_rule', ['data' => json_encode($retriever_rule['data'])], ['id' => intval($retriever_rule["id"])], ['data' => '']);
+			DI::page()['content'] .= "<p><b>Settings Updated";
+			if (!empty($_POST["retriever_retrospective"])) {
+				apply_retrospective($retriever_rule, $_POST["retriever_retrospective"]);
+				DI::page()['content'] .= " and retrospectively applied to " . $_POST["retriever_retrospective"] . " posts";
+			}
+			DI::page()['content'] .= ".</p></b>";
+		}
+
+		$template = Renderer::getMarkupTemplate('/rule-config.tpl', 'addon/retriever/');
+		DI::page()['content'] .= Renderer::replaceMacros($template, array(
+								       '$enable' => array(
+									       'retriever_enable',
+									       DI::l10n()->t('Enabled'),
+									       $retriever_rule['data']['enable']),
+								       '$modurl' => array(
+									       'retriever_modurl',
+									       DI::l10n()->t('Modify URL'),
+									       $retriever_rule['data']['modurl'],
+									       DI::l10n()->t("Modify each article's URL with regular expressions before retrieving.")),
+								       '$pattern' => array(
+									       'retriever_pattern',
+									       DI::l10n()->t('URL Pattern'),
+									       $retriever_rule['data']['pattern'],
+									       DI::l10n()->t('Regular expression matching part of the URL to replace')),
+								       '$replace' => array(
+									       'retriever_replace',
+									       DI::l10n()->t('URL Replace'),
+									       $retriever_rule['data']['replace'],
+									       DI::l10n()->t('Text to replace matching part of above regular expression')),
+								       '$allow_images' => DI::config()->get('retriever', 'allow_images'),
+								       '$images' => array(
+									       'retriever_images',
+									       DI::l10n()->t('Download Images'),
+									       $retriever_rule['data']['images']),
+								       '$retrospective' => array(
+									       'retriever_retrospective',
+									       DI::l10n()->t('Retrospectively Apply'),
+									       '0',
+									       DI::l10n()->t('Reapply the rules to this number of posts')),
+								       'storecookies' => array(
+									       'retriever_storecookies',
+									       DI::l10n()->t('Store cookies'),
+									       $retriever_rule['data']['storecookies'],
+									       DI::l10n()->t("Preserve cookie data across fetches.")),
+								       '$cookiedata' => array(
+									       'retriever_cookiedata',
+									       DI::l10n()->t('Cookie Data'),
+									       $retriever_rule['data']['cookiedata'],
+									       DI::l10n()->t("Latest cookie data for this feed.  Example: [{\"Name\":\"cookie-name\",\"Value\":\"cookie-value\",\"Domain\":\"example.com\",\"Path\":\"\\/path\\/\",\"Max-Age\":null,\"Expires\":1682450014,\"Secure\":true,\"Discard\":false,\"HttpOnly\":true}]")),
+								       '$customxslt' => array(
+									       'retriever_customxslt',
+									       DI::l10n()->t('Custom XSLT'),
+									       $retriever_rule['data']['customxslt'],
+									       DI::l10n()->t("When standard rules aren't enough, apply custom XSLT to the article")),
+								       '$title' => DI::l10n()->t('Retrieve Feed Content'),
+								       '$help' => DI::baseUrl() . '/retriever/help',
+								       '$help_t' => DI::l10n()->t('Get Help'),
+								       '$submit_t' => DI::l10n()->t('Submit'),
+								       '$submit' => DI::l10n()->t('Save Settings'),
+								       '$id' => ($retriever_rule["id"] ? $retriever_rule["id"] : "create"),
+								       '$tag_t' => DI::l10n()->t('Tag'),
+								       '$attribute_t' => DI::l10n()->t('Attribute'),
+								       '$value_t' => DI::l10n()->t('Value'),
+								       '$add_t' => DI::l10n()->t('Add'),
+								       '$remove_t' => DI::l10n()->t('Remove'),
+								       '$include_t' => DI::l10n()->t('Include'),
+								       '$include' => $retriever_rule['data']['include'],
+								       '$exclude_t' => DI::l10n()->t('Exclude'),
+								       '$exclude' => $retriever_rule['data']['exclude']));
+		return;
+	}
+}
+
+/**
+ * @brief Hook that adds the retriever option to the contact menu
+ *
+ * @param array $args Contact menu details to be filled in (by ref)
+ */
+function retriever_contact_photo_menu(array &$args) {
+	if (!$args) {
+		return;
+	}
+	if ($args["contact"]["network"] == "feed") {
+		$args["menu"]['retriever'] = array(DI::l10n()->t('Retriever'), DI::baseUrl() . '/retriever/' . $args["contact"]['id']);
+	}
+}
+
+/**
+ * @brief Hook for processing new incoming items
+ *
+ * @param array $item New item, which has not yet been inserted into database (by ref)
+ */
+function retriever_post_remote_hook(array &$item) {
+	DI::logger()->info('retriever_post_remote_hook: ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id']);
+
+	$retriever_rule = get_retriever_rule($item['contact-id'], $item["uid"], false);
+	if ($retriever_rule) {
+		retriever_on_item_insert($retriever_rule, $item);
+	}
+	else {
+		if (DI::config()->get($item["uid"], 'retriever', 'oembed')) {
+			// Convert to HTML and back to take advantage of bbcode's resolution of oembeds.
+			$body = retriever_get_body($item);
+			$body = HTML::toBBCode(BBCode::convert($body));
+			retriever_set_body($item, $body);
+		}
+		if (DI::config()->get($item["uid"], 'retriever', 'all_photos')) {
+			retrieve_images($item);
+		}
+	}
+	retriever_check_item_completed($item);
+}
+
+/**
+ * @brief Hook for adding per-user retriever settings to the user's settings page
+ *
+ * @param array $data Hook data array
+ */
+function retriever_addon_settings(array &$data) {
+	$all_photos = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'retriever', 'all_photos');
+	$oembed = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'retriever', 'oembed');
+	$template = Renderer::getMarkupTemplate('/settings.tpl', 'addon/retriever/');
+	$config = array('$submit' => DI::l10n()->t('Save Settings'),
+			'$title' => DI::l10n()->t('Retriever Settings'),
+			'$help' => DI::baseUrl() . '/retriever/help',
+			'$allow_images' => DI::config()->get('retriever', 'allow_images'));
+	$config['$allphotos'] = array('retriever_all_photos',
+				      DI::l10n()->t('All Photos'),
+				      $all_photos,
+				      DI::l10n()->t('Check this to retrieve photos for all posts'));
+	$config['$oembed'] = array('retriever_oembed',
+				   DI::l10n()->t('Resolve OEmbed'),
+				   $oembed,
+				   DI::l10n()->t('Check this to attempt to retrieve embedded content for all posts'));
+	$html = Renderer::replaceMacros($template, $config);
+    $data = [
+        'addon' => 'retriever',
+        'title' => DI::l10n()->t('Retriever Settings'),
+        'html' =>  $html,
+    ];
+}
+
+/**
+ * @brief Hook for processing post results from user's settings page
+ *
+ * @param array $post Posted content
+ * @return void
+ */
+function retriever_addon_settings_post(array $post) {
+	if ($post['retriever_all_photos']) {
+		DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'retriever', 'all_photos', $post['retriever_all_photos']);
+	}
+	else {
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'retriever', 'all_photos');
+	}
+	if ($post['retriever_oembed']) {
+		DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'retriever', 'oembed', $post['retriever_oembed']);
+	}
+	else {
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'retriever', 'oembed');
+	}
+}

retriever/templates/admin.tpl

@@ -0,0 +1,9 @@
+{{*
+ *	AUTOMATICALLY GENERATED TEMPLATE
+ *	DO NOT EDIT THIS FILE, CHANGES WILL BE OVERWRITTEN
+ *
+ *}}
+{{include file="field_input.tpl" field=$downloads_per_cron}}
+{{include file="field_checkbox.tpl" field=$allow_images}}
+<div class="submit"><input type="submit" name="page_site" value="{{$submit}}"></div>
+

retriever/templates/extract.tpl

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+  <xsl:output method="html" indent="yes" version="4.0"/>
+
+  <xsl:template match="text()"/>
+{{function clause_xpath}}{{if !$clause.attribute}}{{$clause.element}}{{elseif $clause.attribute == 'class'}}{{$clause.element}}[contains(concat(' ', normalize-space(@class), ' '), '{{$clause.value}}')]{{else}}{{$clause.element}}[@{{$clause.attribute}}='{{$clause.value}}']{{/if}}{{/function}}
+{{foreach $spec.include as $clause}}
+
+  <xsl:template match="{{clause_xpath clause=$clause}}">
+    <xsl:copy>
+      <xsl:apply-templates select="node()|@*" mode="remove"/>
+    </xsl:copy>
+  </xsl:template>{{/foreach}}
+{{foreach $spec.exclude as $clause}}
+
+  <xsl:template match="{{clause_xpath clause=$clause}}" mode="remove"/>{{/foreach}}
+
+  <xsl:template match="node()|@*" mode="remove">
+    <xsl:copy>
+      <xsl:apply-templates select="node()|@*" mode="remove"/>
+    </xsl:copy>
+  </xsl:template>
+
+</xsl:stylesheet>

retriever/templates/fix-urls.tpl

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!-- attempt to replace relative URLs with absolute URLs -->
+<!-- http://stackoverflow.com/questions/3824631/replace-href-value-in-anchor-tags-of-html-using-xslt -->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+  <xsl:output method="html" indent="yes" version="4.0"/>
+
+  <xsl:template match="node()|@*">
+    <xsl:copy>
+      <xsl:apply-templates select="node()|@*"/>
+    </xsl:copy>
+  </xsl:template>
+
+  <xsl:template match="*/@src[starts-with(.,'.')]">
+    <xsl:attribute name="src">
+      <xsl:value-of select="concat('{{$dirurl}}/',.)"/>
+    </xsl:attribute>
+  </xsl:template>
+  <xsl:template match="*/@src[starts-with(.,'/')]">
+    <xsl:attribute name="src">
+      <xsl:value-of select="concat('{{$rooturl}}',.)"/>
+    </xsl:attribute>
+  </xsl:template>
+  <xsl:template match="*/@src[not(starts-with(.,'/')) and not(contains(.,':'))]">
+    <xsl:attribute name="src">
+      <xsl:value-of select="concat('{{$dirurl}}',.)"/>
+    </xsl:attribute>
+  </xsl:template>
+
+</xsl:stylesheet>

retriever/templates/help.tpl

@@ -0,0 +1,163 @@
+<h2>Retriever Plugin Help</h2>
+<p>
+This plugin replaces the short excerpts you normally get in RSS feeds
+with the full content of the article from the source website.  You
+specify which part of the page you're interested in with a set of
+rules.  When each item arrives, the plugin downloads the full page
+from the website, extracts content using the rules, and replaces the
+original article.
+</p>
+<p>
+There's a few reasons you may want to do this.  The source website
+might be slow or overloaded.  The source website might be
+untrustworthy, in which case using Friendica to scrub the HTML is a
+good idea.  You might be on a LAN that blacklists certain websites.
+It also works neatly with the mailstream plugin, allowing you to read
+a news stream comfortably without needing continuous Internet
+connectivity.
+</p>
+<p>
+However, setting up retriever can be quite tricky since it depends on
+the internal design of the website.  That was designed to make life
+easy for the website's developers, not for you.  You'll need to have
+some familiarity with HTML, and be willing to adapt when the website
+suddenly changes everything without notice.
+</p>
+<h3>Configuring Retriever for a feed</h3>
+<p>
+To set up retriever for an RSS feed, go to the "Contacts" page and
+find your feed.  Then click on the drop-down menu on the contact.
+Select "Retriever" to get to the retriever configuration.
+</p>
+<p>
+The "Include" configuration section specifies parts of the page to
+include in the article.  Each row has three components:
+</p>
+<ul>
+<li>An HTML tag (e.g. "div", "span", "p")</li>
+<li>An attribute (usually "class" or "id")</li>
+<li>A value for the attribute</li>
+</ul>
+<p>
+A simple case is when the article is wrapped in a "div" element:
+</p>
+<pre>
+    ...
+    &lt;div class="ArticleWrapper"&gt;
+      &lt;h2&gt;Man Bites Dog&lt;/h2&gt;
+      &lt;img src="mbd.jpg"&gt;
+      &lt;p&gt;
+        Residents of the sleepy community of Nowheresville were
+        shocked yesterday by the sight of creepy local weirdo Jim
+        McOddman assaulting innocent local dog Snufflekins with his
+        false teeth.
+      &lt;/p&gt;
+      ...
+    &lt;/div&gt;
+    ...
+</pre>
+<p>
+You then specify the tag "div", attribute "class", and value
+"ArticleWrapper".  Everything else in the page, such as navigation
+panels and menus and footers and so on, will be discarded.  If there
+is more than one section of the page you want to include, specify each
+one on a separate row.  If the matching section contains some sections
+you want to remove, specify those in the "Exclude" section in the same
+way.
+</p>
+<p>
+Once you've got a configuration that you think will work, you can try
+it out on some existing articles.  Type a number into the
+"Retrospectively Apply" box and click "Submit".  After a while
+(exactly how long depends on your system's cron configuration) the new
+articles should be available.
+</p>
+<h3>Techniques</h3>
+<p>
+You can leave the attribute and value blank to include all the
+corresponding elements with the specified tag name.  You can also use
+a tag name of just an asterisk ("*"), which will match any element type with the
+specified attribute regardless of the tag.
+</p>
+<p>
+Note that the "class" attribute is a special case.  Many web page
+templates will put multiple different classes in the same element,
+separated by spaces.  If you specify an attribute of "class" it will
+match an element if any of its classes matches the specified value.
+For example:
+</p>
+<pre>
+    &lt;div class="article breaking-news"&gt;
+</pre>
+<p>
+In this case you can specify a value of "article", or "breaking-news".
+You can also specify "article breaking-news", but that won't match if
+the website suddenly changes to "breaking-news article", so that's not
+recommended.
+</p>
+<p>
+One useful trick you can try is using the website's "print" pages.
+Many news sites have print versions of all their articles.  These are
+usually drastically simplified compared to the live website page.
+Sometimes this is a good way to get the whole article when it's
+normally split across multiple pages.
+</p>
+<p>
+Hopefully the URL for the print page is a predictable variant of the
+normal article URL.  For example, an article URL like:
+</p>
+<pre>
+    http://www.newssite.com/article-8636.html
+</pre>
+<p>
+...might have a print version at:
+</p>
+<pre>
+    http://www.newssite.com/print/article-8636.html
+</pre>
+<p>
+To change the URL used to retrieve the page, use the "URL Pattern" and
+"URL Replace" fields.  The pattern is a regular expression matching
+part of the URL to replace.  In this case, you might use a pattern of
+"/article" and a replace string of "/print/article".  A common pattern
+is simply a dollar sign ("$"), used to add the replace string to the end of the URL.
+</p>
+<h3>Background Processing</h3>
+<p>
+Note that retrieving and processing the articles can take some time,
+so it's done in the background.  Incoming articles will be marked as
+invisible while they're in the process of being downloaded.  If a URL
+fails, the plugin will keep trying at progressively longer intervals
+for up to a month, in case the website is temporarily overloaded or
+the network is down.
+</p>
+{{if $allow_images}}
+<h3>Retrieving Images</h3>
+<p>
+Retriever can also optionally download images and store them in the
+local Friendica instance.  Just check the "Download Images" box.  You
+can also download images in every item from your network, whether it's
+an RSS feed or not.  Go to the "Settings" page and
+click <a href="$config">"Plugin settings"</a>.  Then check the "All
+Photos" box in the "Retriever Settings" section and click "Submit".
+</p>
+{{/if}}
+<h2>Configure Feeds:</h2>
+<div>
+{{foreach $feeds as $feed}}
+<div class="contact-entry-wrapper" id="contact-entry-wrapper-{{$feed.id}}">
+	<a href="{{$feed.url}} title="{{$feed.img_hover}}">
+		<div class="contact-entry-photo-wrapper">
+			<div class="contact-entry-photo mframe" id="contact-entry-photo-{{$feed.id}}">
+				<img src="{{$feed.thumb}}" {{$feed.sparkle}} alt="{{$feed.name}}"/>
+			</div>
+		</div>
+		<div class="contact-entry-desc">
+			<div class="contact-entry-name" id="contact-entry-name-{{$feed.id}}">
+				{{$feed.name}}
+			</div>
+		</div>
+	</a>
+</div>
+{{/foreach}}
+</div>

retriever/templates/rule-config.tpl

@@ -0,0 +1,154 @@
+<div class="settings-block">
+  <script language="javascript">
+function retriever_add_row(id)
+{
+    var tbody = document.getElementById(id);
+    var last = tbody.rows[tbody.childElementCount - 1];
+    var count = +last.id.replace(id + '-', '');
+    count++;
+    var row = document.createElement('tr');
+    row.id = id + '-' + count;
+    var cell1 = document.createElement('td');
+    var inptag = document.createElement('input');
+    inptag.name = row.id + '-element';
+    cell1.appendChild(inptag);
+    row.appendChild(cell1);
+    var cell2 = document.createElement('td');
+    var inpatt = document.createElement('input');
+    inpatt.name = row.id + '-attribute';
+    cell2.appendChild(inpatt);
+    row.appendChild(cell2);
+    var cell3 = document.createElement('td');
+    var inpval = document.createElement('input');
+    inpval.name = row.id + '-value';
+    cell3.appendChild(inpval);
+    row.appendChild(cell3);
+    var cell4 = document.createElement('td');
+    var butrem = document.createElement('input');
+    butrem.id = row.id + '-rem';
+    butrem.type = 'button';
+    butrem.onclick = function(){retriever_remove_row(id, count)};
+    butrem.value = '{{$remove_t}}';
+    cell4.appendChild(butrem);
+    row.appendChild(cell4);
+    tbody.appendChild(row);
+}
+
+function retriever_remove_row(id, number)
+{
+    var tbody = document.getElementById(id);
+    var row = document.getElementById(id + '-' + number);
+    tbody.removeChild(row);
+}
+
+function retriever_toggle_url_block()
+{
+    var pattern = document.querySelector("#id_retriever_pattern").parentNode;
+    if (document.querySelector("#id_retriever_modurl").checked) {
+        pattern.style.display = "block";
+    }
+    else {
+        pattern.style.display = "none";
+    }
+
+    var replace = document.querySelector("#id_retriever_replace").parentNode;
+    if (document.querySelector("#id_retriever_modurl").checked) {
+        replace.style.display = "block";
+    }
+    else {
+        replace.style.display = "none";
+    }
+}
+
+function retriever_toggle_cookiedata_block()
+{
+    var div = document.querySelector("#id_retriever_cookiedata").parentNode;
+    if (document.querySelector("#id_retriever_storecookies").checked) {
+        div.style.display = "block";
+    }
+    else {
+        div.style.display = "none";
+    }
+}
+
+document.addEventListener('DOMContentLoaded', function() {
+    retriever_toggle_url_block();
+    document.querySelector("#id_retriever_modurl").addEventListener('change', retriever_toggle_url_block, false);
+    retriever_toggle_cookiedata_block();
+    document.querySelector("#id_retriever_storecookies").addEventListener('change', retriever_toggle_cookiedata_block, false);
+}, false);
+  </script>
+  <h2>{{$title}}</h2>
+  <p><a href="{{$help}}">{{$help_t}}</a></p>
+  <form method="post">
+    <input type="hidden" name="id" value="{{$id}}">
+{{include file="field_checkbox.tpl" field=$enable}}
+    <h3>{{$include_t}}:</h3>
+    <div>
+      <table>
+        <thead>
+          <tr><th>{{$tag_t}}</th><th>{{$attribute_t}}</th><th>{{$value_t}}</th></tr>
+        </thead>
+        <tbody id="retriever-include">
+{{if $include}}
+  {{foreach $include as $k=>$m}}
+          <tr id="retriever-include-{{$k}}">
+            <td><input name="retriever-include-{{$k}}-element" value="{{$m.element}}"></td>
+            <td><input name="retriever-include-{{$k}}-attribute" value="{{$m.attribute}}"></td>
+            <td><input name="retriever-include-{{$k}}-value" value="{{$m.value}}"></td>
+            <td><input id="retrieve-include-{{$k}}-rem" type="button" onclick="retriever_remove_row('retriever-include', {{$k}})" value="{{$remove_t}}"></td>
+          </tr>
+  {{/foreach}}
+{{else}}
+          <tr id="retriever-include-0">
+            <td><input name="retriever-include-0-element"></td>
+            <td><input name="retriever-include-0-attribute"></td>
+            <td><input name="retriever-include-0-value"></td>
+            <td><input id="retrieve-include-0-rem" type="button" onclick="retriever_remove_row('retriever-include', 0)" value="{{$remove_t}}"></td>
+          </tr>
+{{/if}}
+        </tbody>
+      </table>
+      <input type="button" onclick="retriever_add_row('retriever-include')" value="{{$add_t}}">
+    </div>
+    <h3>{{$exclude_t}}:</h3>
+    <div>
+      <table>
+        <thead>
+          <tr><th>{{$tag_t}}</th><th>{{$attribute_t}}</th><th>{{$value_t}}</th></tr>
+        </thead>
+        <tbody id="retriever-exclude">
+{{if $exclude}}
+  {{foreach $exclude as $k=>$r}}
+          <tr id="retriever-exclude-{{$k}}">
+            <td><input name="retriever-exclude-{{$k}}-element" value="{{$r.element}}"></td>
+            <td><input name="retriever-exclude-{{$k}}-attribute" value="{{$r.attribute}}"></td>
+            <td><input name="retriever-exclude-{{$k}}-value" value="{{$r.value}}"></td>
+            <td><input id="retrieve-exclude-{{$k}}-rem" type="button" onclick="retriever_remove_row('retriever-exclude', {{$k}})" value="{{$remove_t}}"></td>
+          </tr>
+  {{/foreach}}
+{{else}}
+          <tr id="retriever-exclude-0">
+            <td><input name="retriever-exclude-0-element"></td>
+            <td><input name="retriever-exclude-0-attribute"></td>
+            <td><input name="retriever-exclude-0-value"></td>
+            <td><input id="retrieve-exclude-0-rem" type="button" onclick="retriever_remove_row('retriever-exclude', 0)" value="{{$remove_t}}"></td>
+          </tr>
+{{/if}}
+        </tbody>
+      </table>
+      <input type="button" onclick="retriever_add_row('retriever-exclude')" value="{{$add_t}}">
+    </div>
+{{include file="field_checkbox.tpl" field=$modurl}}
+{{include file="field_input.tpl" field=$pattern}}
+{{include file="field_input.tpl" field=$replace}}
+{{if $allow_images}}
+{{include file="field_checkbox.tpl" field=$images}}
+{{/if}}
+{{include file="field_textarea.tpl" field=$customxslt}}
+{{include file="field_checkbox.tpl" field=$storecookies}}
+{{include file="field_textarea.tpl" field=$cookiedata}}
+{{include file="field_input.tpl" field=$retrospective}}
+    <input type="submit" size="70" value="{{$submit_t}}">
+  </form>
+</div>

retriever/templates/settings.tpl

@@ -0,0 +1,5 @@
+<p><a href="{{$help}}">Get Help</a></p>
+{{if $allow_images}}
+{{include file="field_checkbox.tpl" field=$allphotos}}
+{{/if}}
+{{include file="field_checkbox.tpl" field=$oembed}}

twitter/twitter.php

@@ -1,14 +1,14 @@
 <?php
 /**
- * Name: Twitter Post Connector
- * Description: Post to Twitter
- * Version: 2.0
+ * Name: Twitter Connector
+ * Description: Bidirectional (posting, relaying and reading) connector for Twitter.
+ * Version: 1.1.0
  * Author: Tobias Diekershoff <https://f.diekershoff.de/profile/tobias>
  * Author: Michael Vogel <https://pirati.ca/profile/heluecht>
  * Maintainer: Hypolite Petovan <https://friendica.mrpetovan.com/profile/hypolite>
- * Maintainer: Michael Vogel <https://pirati.ca/profile/heluecht>
+ * Status: unsupported
  *
- * Copyright (c) 2011-2023 Tobias Diekershoff, Michael Vogel, Hypolite Petovan
+ * Copyright (c) 2011-2013 Tobias Diekershoff, Michael Vogel, Hypolite Petovan
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -34,42 +34,178 @@
  * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  */
+/*   Twitter Addon for Friendica
+ *
+ *   Author: Tobias Diekershoff
+ *           tobias.diekershoff@gmx.net
+ *
+ *   License:3-clause BSD license
+ *
+ *   Configuration:
+ *     To use this addon you need a OAuth Consumer key pair (key & secret)
+ *     you can get it from Twitter at https://twitter.com/apps
+ *
+ *     Register your Friendica site as "Client" application with "Read & Write" access
+ *     we do not need "Twitter as login". When you've registered the app you get the
+ *     OAuth Consumer key and secret pair for your application/site.
+ *
+ *     Add this key pair to your config/twitter.config.php file or use the admin panel.
+ *
+ *     	return [
+ *          'twitter' => [
+ * 		        'consumerkey' => '',
+ *  		    'consumersecret' => '',
+ *          ],
+ *      ];
+ *
+ *     To activate the addon itself add it to the system.addon
+ *     setting. After this, your user can configure their Twitter account settings
+ *     from "Settings -> Addon Settings".
+ *
+ *     Requirements: PHP5, curl
+ */
 
+use Abraham\TwitterOAuth\TwitterOAuth;
+use Abraham\TwitterOAuth\TwitterOAuthException;
+use Codebird\Codebird;
+use Friendica\App;
 use Friendica\Content\Text\BBCode;
 use Friendica\Content\Text\Plaintext;
 use Friendica\Core\Hook;
 use Friendica\Core\Logger;
+use Friendica\Core\Protocol;
 use Friendica\Core\Renderer;
 use Friendica\Core\Worker;
+use Friendica\Database\DBA;
 use Friendica\DI;
+use Friendica\Model\Contact;
+use Friendica\Model\Conversation;
+use Friendica\Model\Group;
 use Friendica\Model\Item;
+use Friendica\Model\ItemURI;
 use Friendica\Model\Post;
+use Friendica\Model\Tag;
+use Friendica\Model\User;
+use Friendica\Protocol\Activity;
 use Friendica\Core\Config\Util\ConfigFileManager;
+use Friendica\Core\System;
 use Friendica\Model\Photo;
-use Friendica\Object\Image;
-use GuzzleHttp\Client;
-use GuzzleHttp\Exception\RequestException;
-use GuzzleHttp\HandlerStack;
-use GuzzleHttp\Subscriber\Oauth\Oauth1;
+use Friendica\Util\DateTimeFormat;
+use Friendica\Util\Images;
+use Friendica\Util\Strings;
 
-const TWITTER_IMAGE_SIZE = [2000000, 1000000, 500000, 100000, 50000];
+require_once __DIR__ . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'autoload.php';
+
+define('TWITTER_DEFAULT_POLL_INTERVAL', 5); // given in minutes
 
 function twitter_install()
 {
-	Hook::register('load_config', __FILE__, 'twitter_load_config');
-	Hook::register('connector_settings', __FILE__, 'twitter_settings');
+	//  we need some hooks, for the configuration and for sending tweets
+	Hook::register('load_config'            , __FILE__, 'twitter_load_config');
+	Hook::register('connector_settings'     , __FILE__, 'twitter_settings');
 	Hook::register('connector_settings_post', __FILE__, 'twitter_settings_post');
-	Hook::register('hook_fork', __FILE__, 'twitter_hook_fork');
-	Hook::register('post_local', __FILE__, 'twitter_post_local');
-	Hook::register('notifier_normal', __FILE__, 'twitter_post_hook');
-	Hook::register('jot_networks', __FILE__, 'twitter_jot_nets');
+	Hook::register('hook_fork'              , __FILE__, 'twitter_hook_fork');
+	Hook::register('post_local'             , __FILE__, 'twitter_post_local');
+	Hook::register('notifier_normal'        , __FILE__, 'twitter_post_hook');
+	Hook::register('jot_networks'           , __FILE__, 'twitter_jot_nets');
+	Hook::register('cron'                   , __FILE__, 'twitter_cron');
+	Hook::register('support_follow'         , __FILE__, 'twitter_support_follow');
+	Hook::register('follow'                 , __FILE__, 'twitter_follow');
+	Hook::register('unfollow'               , __FILE__, 'twitter_unfollow');
+	Hook::register('block'                  , __FILE__, 'twitter_block');
+	Hook::register('unblock'                , __FILE__, 'twitter_unblock');
+	Hook::register('expire'                 , __FILE__, 'twitter_expire');
+	Hook::register('prepare_body'           , __FILE__, 'twitter_prepare_body');
+	Hook::register('check_item_notification', __FILE__, 'twitter_check_item_notification');
+	Hook::register('probe_detect'           , __FILE__, 'twitter_probe_detect');
+	Hook::register('item_by_link'           , __FILE__, 'twitter_item_by_link');
+	Hook::register('parse_link'             , __FILE__, 'twitter_parse_link');
+	Logger::info('installed twitter');
 }
 
+// Hook functions
+
 function twitter_load_config(ConfigFileManager $loader)
 {
 	DI::appHelper()->getConfigCache()->load($loader->loadAddonConfig('twitter'), \Friendica\Core\Config\ValueObject\Cache::SOURCE_STATIC);
 }
 
+function twitter_check_item_notification(array &$notification_data)
+{
+	$own_id = DI::pConfig()->get($notification_data['uid'], 'twitter', 'own_id');
+
+	$own_user = Contact::selectFirst(['url'], ['uid' => $notification_data['uid'], 'alias' => 'twitter::'.$own_id]);
+	if ($own_user) {
+		$notification_data['profiles'][] = $own_user['url'];
+	}
+}
+
+function twitter_support_follow(array &$data)
+{
+	if ($data['protocol'] == Protocol::TWITTER) {
+		$data['result'] = true;
+	}
+}
+
+function twitter_follow(array &$contact)
+{
+	Logger::info('Check if contact is twitter contact', ['url' => $contact['url']]);
+
+	if (!strstr($contact['url'], '://twitter.com') && !strstr($contact['url'], '@twitter.com')) {
+		return;
+	}
+
+	// contact seems to be a twitter contact, so continue
+	$nickname = preg_replace("=https?://twitter.com/(.*)=ism", "$1", $contact['url']);
+	$nickname = str_replace('@twitter.com', '', $nickname);
+
+	$uid = DI::userSession()->getLocalUserId();
+
+	if (!twitter_api_contact('friendships/create', ['network' => Protocol::TWITTER, 'nick' => $nickname], $uid)) {
+		$contact = null;
+		return;
+	}
+
+	$user = twitter_fetchuser($nickname);
+
+	$contact_id = twitter_fetch_contact($uid, $user, true);
+
+	$contact = Contact::getById($contact_id, ['name', 'nick', 'url', 'addr', 'batch', 'notify', 'poll', 'request', 'confirm', 'poco', 'photo', 'priority', 'network', 'alias', 'pubkey']);
+
+	if (DBA::isResult($contact)) {
+		$contact['contact'] = $contact;
+	}
+}
+
+function twitter_unfollow(array &$hook_data)
+{
+	$hook_data['result'] = twitter_api_contact('friendships/destroy', $hook_data['contact'], $hook_data['uid']);
+}
+
+function twitter_block(array &$hook_data)
+{
+	$hook_data['result'] = twitter_api_contact('blocks/create', $hook_data['contact'], $hook_data['uid']);
+
+	if ($hook_data['result'] === true) {
+		$cdata = Contact::getPublicAndUserContactID($hook_data['contact']['id'], $hook_data['uid']);
+		Contact::remove($cdata['user']);
+	}
+}
+
+function twitter_unblock(array &$hook_data)
+{
+	$hook_data['result'] = twitter_api_contact('blocks/destroy', $hook_data['contact'], $hook_data['uid']);
+}
+
+function twitter_api_contact(string $apiPath, array $contact, int $uid): ?bool
+{
+	if ($contact['network'] !== Protocol::TWITTER) {
+		return null;
+	}
+
+	return (bool)twitter_api_call($uid, $apiPath, ['screen_name' => $contact['nick']]);
+}
+
 function twitter_jot_nets(array &$jotnets_fields)
 {
 	if (!DI::userSession()->getLocalUserId()) {
@@ -88,30 +224,75 @@ function twitter_jot_nets(array &$jotnets_fields)
 	}
 }
 
+
 function twitter_settings_post()
 {
-	if (!DI::userSession()->getLocalUserId() || empty($_POST['twitter-submit'])) {
+	if (!DI::userSession()->getLocalUserId()) {
+		return;
+	}
+	// don't check twitter settings if twitter submit button is not clicked
+	if (empty($_POST['twitter-disconnect']) && empty($_POST['twitter-submit'])) {
 		return;
 	}
 
-	$api_key       = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'api_key');
-	$api_secret    = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'api_secret');
-	$access_token  = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'access_token');
-	$access_secret = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'access_secret');
+	if (!empty($_POST['twitter-disconnect'])) {
+		/*		 * *
+		 * if the twitter-disconnect checkbox is set, clear the OAuth key/secret pair
+		 * from the user configuration
+		 */
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'twitter', 'consumerkey');
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'twitter', 'consumersecret');
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'twitter', 'oauthtoken');
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'twitter', 'oauthsecret');
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'twitter', 'post');
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'twitter', 'post_by_default');
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'twitter', 'lastid');
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'twitter', 'thread');
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'twitter', 'mirror_posts');
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'twitter', 'import');
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'twitter', 'create_user');
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'twitter', 'auto_follow');
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'twitter', 'own_id');
+	} else {
+		if (isset($_POST['twitter-pin'])) {
+			//  if the user supplied us with a PIN from Twitter, let the magic of OAuth happen
+			Logger::notice('got a Twitter PIN');
+			$ckey    = DI::config()->get('twitter', 'consumerkey');
+			$csecret = DI::config()->get('twitter', 'consumersecret');
+			//  the token and secret for which the PIN was generated were hidden in the settings
+			//  form as token and token2, we need a new connection to Twitter using these token
+			//  and secret to request a Access Token with the PIN
+			try {
+				if (empty($_POST['twitter-pin'])) {
+					throw new Exception(DI::l10n()->t('You submitted an empty PIN, please Sign In with Twitter again to get a new one.'));
+				}
 
-	DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'post',            (bool)$_POST['twitter-enable']);
-	DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'post_by_default', (bool)$_POST['twitter-default']);
-	DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'api_key',         $_POST['twitter-api-key']);
-	DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'api_secret',      $_POST['twitter-api-secret']);
-	DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'access_token',    $_POST['twitter-access-token']);
-	DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'access_secret',   $_POST['twitter-access-secret']);
+				$connection = new TwitterOAuth($ckey, $csecret, $_POST['twitter-token'], $_POST['twitter-token2']);
+				$token = $connection->oauth('oauth/access_token', ['oauth_verifier' => $_POST['twitter-pin']]);
+				//  ok, now that we have the Access Token, save them in the user config
+				DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'oauthtoken', $token['oauth_token']);
+				DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'oauthsecret', $token['oauth_token_secret']);
+				DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'post', 1);
+			} catch(Exception $e) {
+				DI::sysmsg()->addNotice($e->getMessage());
+			} catch(TwitterOAuthException $e) {
+				DI::sysmsg()->addNotice($e->getMessage());
+			}
+		} else {
+			//  if no PIN is supplied in the POST variables, the user has changed the setting
+			//  to post a tweet for every new __public__ posting to the wall
+			DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'post', intval($_POST['twitter-enable']));
+			DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'post_by_default', intval($_POST['twitter-default']));
+			DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'thread', intval($_POST['twitter-thread']));
+			DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'mirror_posts', intval($_POST['twitter-mirror']));
+			DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'import', intval($_POST['twitter-import']));
+			DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'create_user', intval($_POST['twitter-create_user']));
+			DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'twitter', 'auto_follow', intval($_POST['twitter-auto_follow']));
 
-	if (
-		empty(DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'last_status')) ||
-		($api_key != $_POST['twitter-api-key']) || ($api_secret != $_POST['twitter-api-secret']) ||
-		($access_token != $_POST['twitter-access-token']) || ($access_secret != $_POST['twitter-access-secret'])
-	) {
-		twitter_test_connection(DI::userSession()->getLocalUserId());
+			if (!intval($_POST['twitter-mirror'])) {
+				DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'twitter', 'lastid');
+			}
+		}
 	}
 }
 
@@ -121,41 +302,116 @@ function twitter_settings(array &$data)
 		return;
 	}
 
-	$enabled      = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'post') ?? false;
-	$def_enabled  = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'post_by_default') ?? false;
+	$user = User::getById(DI::userSession()->getLocalUserId());
 
-	$api_key       = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'api_key');
-	$api_secret    = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'api_secret');
-	$access_token  = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'access_token');
-	$access_secret = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'access_secret');
+	DI::page()->registerStylesheet(__DIR__ . '/twitter.css', 'all');
 
-	$last_status = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'last_status');
-	if (!empty($last_status['code']) && !empty($last_status['reason'])) {
-		$status_title = sprintf('%d - %s', $last_status['code'], $last_status['reason']);
+	/*	 * *
+	 * 1) Check that we have global consumer key & secret
+	 * 2) If no OAuthtoken & stuff is present, generate button to get some
+	 * 3) Checkbox for "Send public notices (280 chars only)
+	 */
+	$ckey    = DI::config()->get('twitter', 'consumerkey');
+	$csecret = DI::config()->get('twitter', 'consumersecret');
+	$otoken  = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'oauthtoken');
+	$osecret = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'oauthsecret');
+
+	$enabled            = intval(DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'post'));
+	$defenabled         = intval(DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'post_by_default'));
+	$threadenabled      = intval(DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'thread'));
+	$mirrorenabled      = intval(DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'mirror_posts'));
+	$importenabled      = intval(DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'import'));
+	$create_userenabled = intval(DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'create_user'));
+	$auto_followenabled = intval(DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'auto_follow'));
+
+	// Hide the submit button by default
+	$submit = '';
+
+	if ((!$ckey) && (!$csecret)) {
+		/* no global consumer keys
+		 * display warning and skip personal config
+		 */
+		$html = '<p>' . DI::l10n()->t('No consumer key pair for Twitter found. Please contact your site administrator.') . '</p>';
 	} else {
-		$status_title = DI::l10n()->t('No status.');
-	}
-	$status_content = $last_status['content'] ?? '';
+		// ok we have a consumer key pair now look into the OAuth stuff
+		if ((!$otoken) && (!$osecret)) {
+			/* the user has not yet connected the account to twitter...
+			 * get a temporary OAuth key/secret pair and display a button with
+			 * which the user can request a PIN to connect the account to a
+			 * account at Twitter.
+			 */
+			$connection = new TwitterOAuth($ckey, $csecret);
+			try {
+				$result = $connection->oauth('oauth/request_token', ['oauth_callback' => 'oob']);
 
-	$t    = Renderer::getMarkupTemplate('connector_settings.tpl', 'addon/twitter/');
-	$html = Renderer::replaceMacros($t, [
-		'$enable'        => ['twitter-enable', DI::l10n()->t('Allow posting to Twitter'), $enabled, DI::l10n()->t('If enabled all your <strong>public</strong> postings can be posted to the associated Twitter account. You can choose to do so by default (here) or for every posting separately in the posting options when writing the entry.')],
-		'$default'       => ['twitter-default', DI::l10n()->t('Send public postings to Twitter by default'), $def_enabled],
-		'$api_key'       => ['twitter-api-key', DI::l10n()->t('API Key'), $api_key],
-		'$api_secret'    => ['twitter-api-secret', DI::l10n()->t('API Secret'), $api_secret],
-		'$access_token'  => ['twitter-access-token', DI::l10n()->t('Access Token'), $access_token],
-		'$access_secret' => ['twitter-access-secret', DI::l10n()->t('Access Secret'), $access_secret],
-		'$help'          => DI::l10n()->t('Each user needs to register their own app to be able to post to Twitter. Please visit https://developer.twitter.com/en/portal/projects-and-apps to register a project. Inside the project you then have to register an app. You will find the needed data for the connector on the page "Keys and token" in the app settings.'),
-		'$status_title'  => ['twitter-status-title', DI::l10n()->t('Last Status Summary'), $status_title, '', '', 'readonly'],
-		'$status'        => ['twitter-status', DI::l10n()->t('Last Status Content'), $status_content, '', '', 'readonly'],
-	]);
+				$html = '<p>' . DI::l10n()->t('At this Friendica instance the Twitter addon was enabled but you have not yet connected your account to your Twitter account. To do so click the button below to get a PIN from Twitter which you have to copy into the input box below and submit the form. Only your <strong>public</strong> posts will be posted to Twitter.') . '</p>';
+				$html .= '<a href="' . $connection->url('oauth/authorize', ['oauth_token' => $result['oauth_token']]) . '" target="_twitter"><img src="addon/twitter/lighter.png" alt="' . DI::l10n()->t('Log in with Twitter') . '"></a>';
+				$html .= '<div id="twitter-pin-wrapper">';
+				$html .= '<label id="twitter-pin-label" for="twitter-pin">' . DI::l10n()->t('Copy the PIN from Twitter here') . '</label>';
+				$html .= '<input id="twitter-pin" type="text" name="twitter-pin" />';
+				$html .= '<input id="twitter-token" type="hidden" name="twitter-token" value="' . $result['oauth_token'] . '" />';
+				$html .= '<input id="twitter-token2" type="hidden" name="twitter-token2" value="' . $result['oauth_token_secret'] . '" />';
+				$html .= '</div>';
+
+				$submit = null;
+			} catch (TwitterOAuthException $e) {
+				$html = '<p>' . DI::l10n()->t('An error occured: ') . $e->getMessage() . '</p>';
+			}
+		} else {
+			/*			 * *
+			 *  we have an OAuth key / secret pair for the user
+			 *  so let's give a chance to disable the postings to Twitter
+			 */
+			$connection = new TwitterOAuth($ckey, $csecret, $otoken, $osecret);
+			try {
+				$account = $connection->get('account/verify_credentials');
+				if (property_exists($account, 'screen_name') &&
+					property_exists($account, 'description') &&
+					property_exists($account, 'profile_image_url')
+				) {
+					$connected = DI::l10n()->t('Currently connected to: <a href="https://twitter.com/%1$s" target="_twitter">%1$s</a>', $account->screen_name);
+				} else {
+					Logger::notice('Invalid twitter info (verify credentials).', ['auth' => TwitterOAuth::class]);
+				}
+
+				if ($user['hidewall']) {
+					$privacy_warning = DI::l10n()->t('<strong>Note</strong>: Due to your privacy settings (<em>Hide your profile details from unknown viewers?</em>) the link potentially included in public postings relayed to Twitter will lead the visitor to a blank page informing the visitor that the access to your profile has been restricted.');
+				}
+
+				$t    = Renderer::getMarkupTemplate('connector_settings.tpl', 'addon/twitter/');
+				$html = Renderer::replaceMacros($t, [
+					'$l10n' => [
+						'connected'       => $connected ?? '',
+						'invalid'         => DI::l10n()->t('Invalid Twitter info'),
+						'disconnect'      => DI::l10n()->t('Disconnect'),
+						'privacy_warning' => $privacy_warning ?? '',
+					],
+
+					'$account'     => $account,
+					'$enable'      => ['twitter-enable', DI::l10n()->t('Allow posting to Twitter'), $enabled, DI::l10n()->t('If enabled all your <strong>public</strong> postings can be posted to the associated Twitter account. You can choose to do so by default (here) or for every posting separately in the posting options when writing the entry.')],
+					'$default'     => ['twitter-default', DI::l10n()->t('Send public postings to Twitter by default'), $defenabled],
+					'$thread'      => ['twitter-thread', DI::l10n()->t('Use threads instead of truncating the content'), $threadenabled],
+					'$mirror'      => ['twitter-mirror', DI::l10n()->t('Mirror all posts from twitter that are no replies'), $mirrorenabled],
+					'$import'      => ['twitter-import', DI::l10n()->t('Import the remote timeline'), $importenabled],
+					'$create_user' => ['twitter-create_user', DI::l10n()->t('Automatically create contacts'), $create_userenabled, DI::l10n()->t('This will automatically create a contact in Friendica as soon as you receive a message from an existing contact via the Twitter network. If you do not enable this, you need to manually add those Twitter contacts in Friendica from whom you would like to see posts here.')],
+					'$auto_follow' => ['twitter-auto_follow', DI::l10n()->t('Follow in fediverse'), $auto_followenabled, DI::l10n()->t('Automatically subscribe to the contact in the fediverse, when a fediverse account is mentioned in name or description and we are following the Twitter contact.')],
+				]);
+
+				// Enable the default submit button
+				$submit = null;
+			} catch (TwitterOAuthException $e) {
+				$html = '<p>' . DI::l10n()->t('An error occured: ') . $e->getMessage() . '</p>';
+			}
+		}
+	}
 
 	$data = [
 		'connector' => 'twitter',
-		'title'     => DI::l10n()->t('Twitter Export'),
+		'title'     => DI::l10n()->t('Twitter Import/Export/Mirror'),
 		'enabled'   => $enabled,
 		'image'     => 'images/twitter.png',
 		'html'      => $html,
+		'submit'    => $submit ?? null,
 	];
 }
 
@@ -169,31 +425,64 @@ function twitter_hook_fork(array &$b)
 
 	$post = $b['data'];
 
-	if (
-		$post['deleted'] || ($post['private'] == Item::PRIVATE) || ($post['created'] !== $post['edited']) ||
-		!strstr($post['postopts'], 'twitter') || ($post['gravity'] != Item::GRAVITY_PARENT)
-	) {
+	// Deletion checks are done in twitter_delete_item()
+	if ($post['deleted']) {
+		return;
+	}
+
+	// Editing is not supported by the addon
+	if ($post['created'] !== $post['edited']) {
+		DI::logger()->info('Editing is not supported by the addon');
 		$b['execute'] = false;
 		return;
 	}
+
+	// if post comes from twitter don't send it back
+	if (($post['extid'] == Protocol::TWITTER) || twitter_get_id($post['extid'])) {
+		DI::logger()->info('If post comes from twitter don\'t send it back');
+		$b['execute'] = false;
+		return;
+	}
+
+	if (substr($post['app'] ?? '', 0, 7) == 'Twitter') {
+		DI::logger()->info('No Twitter app');
+		$b['execute'] = false;
+		return;
+	}
+
+	if (DI::pConfig()->get($post['uid'], 'twitter', 'import')) {
+		// Don't fork if it isn't a reply to a twitter post
+		if (($post['parent'] != $post['id']) && !Post::exists(['id' => $post['parent'], 'network' => Protocol::TWITTER])) {
+			Logger::notice('No twitter parent found', ['item' => $post['id']]);
+			$b['execute'] = false;
+			return;
+		}
+	} else {
+		// Comments are never exported when we don't import the twitter timeline
+		if (!strstr($post['postopts'] ?? '', 'twitter') || ($post['parent'] != $post['id']) || $post['private']) {
+			DI::logger()->info('Comments are never exported when we don\'t import the twitter timeline');
+			$b['execute'] = false;
+			return;
+		}
+    }
 }
 
 function twitter_post_local(array &$b)
 {
+	if ($b['edit']) {
+		return;
+	}
+
 	if (!DI::userSession()->getLocalUserId() || (DI::userSession()->getLocalUserId() != $b['uid'])) {
 		return;
 	}
 
-	if ($b['edit'] || ($b['private'] == Item::PRIVATE) || ($b['gravity'] != Item::GRAVITY_PARENT)) {
-		return;
-	}
-
-	$twitter_post   = (bool)DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'post');
-	$twitter_enable = (($twitter_post && !empty($_REQUEST['twitter_enable'])) ? (bool)$_REQUEST['twitter_enable'] : false);
+	$twitter_post = intval(DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'post'));
+	$twitter_enable = (($twitter_post && !empty($_REQUEST['twitter_enable'])) ? intval($_REQUEST['twitter_enable']) : 0);
 
 	// if API is used, default to the chosen settings
 	if ($b['api_source'] && intval(DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'twitter', 'post_by_default'))) {
-		$twitter_enable = true;
+		$twitter_enable = 1;
 	}
 
 	if (!$twitter_enable) {
@@ -207,98 +496,398 @@ function twitter_post_local(array &$b)
 	$b['postopts'] .= 'twitter';
 }
 
+function twitter_probe_detect(array &$hookData)
+{
+	// Don't overwrite an existing result
+	if (isset($hookData['result'])) {
+		return;
+	}
+
+	// Avoid a lookup for the wrong network
+	if (!in_array($hookData['network'], ['', Protocol::TWITTER])) {
+		return;
+	}
+
+	if (preg_match('=([^@]+)@(?:mobile\.)?twitter\.com$=i', $hookData['uri'], $matches)) {
+		$nick = $matches[1];
+	} elseif (preg_match('=^https?://(?:mobile\.)?twitter\.com/(.+)=i', $hookData['uri'], $matches)) {
+		if (strpos($matches[1], '/') !== false) {
+			// Status case: https://twitter.com/<nick>/status/<status id>
+			// Not a contact
+			$hookData['result'] = false;
+			return;
+		}
+
+		$nick = $matches[1];
+	} else {
+		return;
+	}
+
+	$user = twitter_fetchuser($nick);
+
+	if ($user) {
+		$hookData['result'] = twitter_user_to_contact($user) ?: null;
+	}
+
+	// Authoritative probe should set the result even if the probe was unsuccessful
+	if ($hookData['network'] == Protocol::TWITTER && empty($hookData['result'])) {
+		$hookData['result'] = [];
+	}
+}
+
+function twitter_item_by_link(array &$hookData)
+{
+	// Don't overwrite an existing result
+	if (isset($hookData['item_id'])) {
+		return;
+	}
+
+	// Relevancy check
+	if (!preg_match('#^https?://(?:mobile\.|www\.)?twitter.com/[^/]+/status/(\d+).*#', $hookData['uri'], $matches)) {
+		return;
+	}
+
+	// From now on, any early return should abort the whole chain since we've established it was a Twitter URL
+	$hookData['item_id'] = false;
+
+	// Node-level configuration check
+	if (empty(DI::config()->get('twitter', 'consumerkey')) || empty(DI::config()->get('twitter', 'consumersecret'))) {
+		return;
+	}
+
+	// No anonymous import
+	if (!$hookData['uid']) {
+		return;
+	}
+
+	if (
+		empty(DI::pConfig()->get($hookData['uid'], 'twitter', 'oauthtoken'))
+		|| empty(DI::pConfig()->get($hookData['uid'], 'twitter', 'oauthsecret'))
+	) {
+		DI::sysmsg()->addNotice(DI::l10n()->t('Please connect a Twitter account in your Social Network settings to import Twitter posts.'));
+		return;
+	}
+
+	$status = twitter_statuses_show($matches[1]);
+
+	if (empty($status->id_str)) {
+		DI::sysmsg()->addNotice(DI::l10n()->t('Twitter post not found.'));
+		return;
+	}
+
+	$item = twitter_createpost($hookData['uid'], $status, [], true, false, false);
+	if (!empty($item)) {
+		$hookData['item_id'] = Item::insert($item);
+	}
+}
+
+function twitter_api_post(string $apiPath, string $pid, int $uid): ?object
+{
+	if (empty($pid)) {
+		return null;
+	}
+
+	return twitter_api_call($uid, $apiPath, ['id' => $pid]);
+}
+
+function twitter_api_call(int $uid, string $apiPath, array $parameters = []): ?object
+{
+	$ckey = DI::config()->get('twitter', 'consumerkey');
+	$csecret = DI::config()->get('twitter', 'consumersecret');
+	$otoken = DI::pConfig()->get($uid, 'twitter', 'oauthtoken');
+	$osecret = DI::pConfig()->get($uid, 'twitter', 'oauthsecret');
+
+	// If the addon is not configured (general or for this user) quit here
+	if (empty($ckey) || empty($csecret) || empty($otoken) || empty($osecret)) {
+		return null;
+	}
+
+	try {
+		$connection = new TwitterOAuth($ckey, $csecret, $otoken, $osecret);
+		$result = $connection->post($apiPath, $parameters);
+
+		if ($connection->getLastHttpCode() != 200) {
+			throw new Exception($result->errors[0]->message ?? json_encode($result), $connection->getLastHttpCode());
+		}
+
+		if (!empty($result->errors)) {
+			throw new Exception($result->errors[0]->message, $result->errors[0]->code);
+		}
+
+		Logger::info('[twitter] API call successful', ['apiPath' => $apiPath, 'parameters' => $parameters]);
+		Logger::debug('[twitter] API call result', ['apiPath' => $apiPath, 'parameters' => $parameters, 'result' => $result]);
+
+		return $result;
+	} catch (TwitterOAuthException $twitterOAuthException) {
+		Logger::notice('Unable to communicate with twitter', ['apiPath' => $apiPath, 'parameters' => $parameters, 'code' => $twitterOAuthException->getCode(), 'exception' => $twitterOAuthException]);
+		return null;
+	} catch (Exception $e) {
+		Logger::notice('[twitter] API call failed', ['apiPath' => $apiPath, 'parameters' => $parameters, 'code' => $e->getCode(), 'message' => $e->getMessage()]);
+		return null;
+	}
+}
+
+function twitter_get_id(string $uri)
+{
+	if ((substr($uri, 0, 9) != 'twitter::') || (strlen($uri) <= 9)) {
+		return 0;
+	}
+
+	$id = substr($uri, 9);
+	if (!is_numeric($id)) {
+		return 0;
+	}
+
+	return (int)$id;
+}
+
 function twitter_post_hook(array &$b)
 {
 	DI::logger()->debug('Invoke post hook', $b);
 
-	if (($b['gravity'] != Item::GRAVITY_PARENT) || !strstr($b['postopts'], 'twitter') || ($b['private'] == Item::PRIVATE) || $b['deleted'] || ($b['created'] !== $b['edited'])) {
+	if ($b['deleted']) {
+		twitter_delete_item($b);
+		return;
+	}
+
+	// Post to Twitter
+	if (!DI::pConfig()->get($b['uid'], 'twitter', 'import')
+		&& ($b['private'] || ($b['created'] !== $b['edited']))) {
 		return;
 	}
 
 	$b['body'] = Post\Media::addAttachmentsToBody($b['uri-id'], DI::contentItem()->addSharedPost($b));
 
+	$thr_parent = null;
+
+	if ($b['parent'] != $b['id']) {
+		Logger::debug('Got comment', ['item' => $b]);
+
+		// Looking if its a reply to a twitter post
+		if (!twitter_get_id($b['parent-uri']) &&
+			!twitter_get_id($b['extid']) &&
+			!twitter_get_id($b['thr-parent'])) {
+			Logger::info('No twitter post', ['parent' => $b['parent']]);
+			return;
+		}
+
+		$condition = ['uri' => $b['thr-parent'], 'uid' => $b['uid']];
+		$thr_parent = Post::selectFirst(['uri', 'extid', 'author-link', 'author-nick', 'author-network'], $condition);
+		if (!DBA::isResult($thr_parent)) {
+			Logger::notice('No parent found', ['thr-parent' => $b['thr-parent']]);
+			return;
+		}
+
+		if ($thr_parent['author-network'] == Protocol::TWITTER) {
+			$nickname = '@[url=' . $thr_parent['author-link'] . ']' . $thr_parent['author-nick'] . '[/url]';
+			$nicknameplain = '@' . $thr_parent['author-nick'];
+
+			Logger::info('Comparing', ['nickname' => $nickname, 'nicknameplain' => $nicknameplain, 'body' => $b['body']]);
+			if ((strpos($b['body'], $nickname) === false) && (strpos($b['body'], $nicknameplain) === false)) {
+				$b['body'] = $nickname . ' ' . $b['body'];
+			}
+		}
+
+		Logger::debug('Parent found', ['parent' => $thr_parent]);
+	} else {
+		if ($b['private'] || !strstr($b['postopts'], 'twitter')) {
+			return;
+		}
+
+		// Dont't post if the post doesn't belong to us.
+		// This is a check for forum postings
+		$self = DBA::selectFirst('contact', ['id'], ['uid' => $b['uid'], 'self' => true]);
+		if ($b['contact-id'] != $self['id']) {
+			return;
+		}
+	}
+
+	if ($b['verb'] == Activity::LIKE) {
+		Logger::info('Like', ['uid' => $b['uid'], 'id' => twitter_get_id($b['thr-parent'])]);
+
+		twitter_api_post('favorites/create', twitter_get_id($b['thr-parent']), $b['uid']);
+
+		return;
+	}
+
+	if ($b['verb'] == Activity::ANNOUNCE) {
+		Logger::info('Retweet', ['uid' => $b['uid'], 'id' => twitter_get_id($b['thr-parent'])]);
+		twitter_retweet($b['uid'], twitter_get_id($b['thr-parent']));
+		return;
+	}
+
+	if ($b['created'] !== $b['edited']) {
+		return;
+	}
+
+	// if post comes from twitter don't send it back
+	if (($b['extid'] == Protocol::TWITTER) || twitter_get_id($b['extid'])) {
+		return;
+	}
+
+	if ($b['app'] == 'Twitter') {
+		return;
+	}
+
 	Logger::notice('twitter post invoked', ['id' => $b['id'], 'guid' => $b['guid']]);
 
 	DI::pConfig()->load($b['uid'], 'twitter');
 
-	$api_key       = DI::pConfig()->get($b['uid'], 'twitter', 'api_key');
-	$api_secret    = DI::pConfig()->get($b['uid'], 'twitter', 'api_secret');
-	$access_token  = DI::pConfig()->get($b['uid'], 'twitter', 'access_token');
-	$access_secret = DI::pConfig()->get($b['uid'], 'twitter', 'access_secret');
+	$ckey    = DI::config()->get('twitter', 'consumerkey');
+	$csecret = DI::config()->get('twitter', 'consumersecret');
+	$otoken  = DI::pConfig()->get($b['uid'], 'twitter', 'oauthtoken');
+	$osecret = DI::pConfig()->get($b['uid'], 'twitter', 'oauthsecret');
 
-	if (empty($api_key) || empty($api_secret) || empty($access_token) || empty($access_secret)) {
-		Logger::info('Missing keys, secrets or tokens.');
-		return;
-	}
+	if ($ckey && $csecret && $otoken && $osecret) {
+		Logger::info('We have customer key and oauth stuff, going to send.');
 
-	$msgarr = Plaintext::getPost($b, 280, true, BBCode::TWITTER);
-	Logger::debug('Got plaintext', ['id' => $b['id'], 'message' => $msgarr]);
-
-	$media_ids = [];
-
-	if (!empty($msgarr['images']) || !empty($msgarr['remote_images'])) {
-		Logger::info('Got images', ['id' => $b['id'], 'images' => $msgarr['images'] ?? []]);
-
-		$retrial = Worker::getRetrial();
-		if ($retrial > 4) {
+		// If it's a repeated message from twitter then do a native retweet and exit
+		if (twitter_is_retweet($b['uid'], $b['body'])) {
 			return;
 		}
-		foreach ($msgarr['images'] ?? [] as $image) {
-			if (count($media_ids) == 4) {
-				continue;
+
+		Codebird::setConsumerKey($ckey, $csecret);
+		$cb = Codebird::getInstance();
+		$cb->setToken($otoken, $osecret);
+
+		$connection = new TwitterOAuth($ckey, $csecret, $otoken, $osecret);
+
+		// Set the timeout for upload to 30 seconds
+		$connection->setTimeouts(10, 30);
+
+		$max_char = 280;
+
+		// Handling non-native reshares
+		$b['body'] = Friendica\Content\Text\BBCode::convertShare(
+			$b['body'],
+			function (array $attributes, array $author_contact, $content, $is_quote_share) {
+				return twitter_convert_share($attributes, $author_contact, $content, $is_quote_share);
 			}
+		);
+
+		$b['body'] = twitter_update_mentions($b['body']);
+
+		$msgarr = Plaintext::getPost($b, $max_char, true, BBCode::TWITTER);
+		Logger::info('Got plaintext', ['id' => $b['id'], 'message' => $msgarr]);
+		$msg = $msgarr['text'];
+
+		if (($msg == '') && isset($msgarr['title'])) {
+			$msg = Plaintext::shorten($msgarr['title'], $max_char - 50, $b['uid']);
+		}
+
+		// Add the link to the body if the type isn't a photo or there are more than 4 images in the post
+		if (!empty($msgarr['url']) && (strpos($msg, $msgarr['url']) === false) && (($msgarr['type'] != 'photo') || empty($msgarr['images']) || (count($msgarr['images']) > 4))) {
+			$msg .= "\n" . $msgarr['url'];
+		}
+
+		if (empty($msg)) {
+			Logger::notice('Empty message', ['id' => $b['id']]);
+			return;
+		}
+
+		// and now tweet it :-)
+		$post = [];
+
+		if (!empty($msgarr['images']) || !empty($msgarr['remote_images'])) {
+			Logger::info('Got images', ['id' => $b['id'], 'images' => $msgarr['images'] ?? [], 'remote_images' => $msgarr['remote_images'] ?? []]);
 			try {
-				$media_ids[] = twitter_upload_image($b['uid'], $image, $retrial);
-			} catch (RequestException $exception) {
-				Logger::warning('Error while uploading image', ['image' => $image, 'code' => $exception->getCode(), 'message' => $exception->getMessage()]);
+				$media_ids = [];
+				foreach ($msgarr['images'] ?? [] as $image) {
+					if (count($media_ids) == 4) {
+						continue;
+					}
+					try {
+						$media_ids[] = twitter_upload_image($connection, $cb, $image, $b);
+					} catch (\Throwable $th) {
+						Logger::warning('Error while uploading image', ['code' => $th->getCode(), 'message' => $th->getMessage()]);
+					}
+				}
+
+				foreach ($msgarr['remote_images'] ?? [] as $image) {
+					if (count($media_ids) == 4) {
+						continue;
+					}
+					try {
+						$media_ids[] = twitter_upload_image($connection, $cb, $image, $b);
+					} catch (\Throwable $th) {
+						Logger::warning('Error while uploading image', ['code' => $th->getCode(), 'message' => $th->getMessage()]);
+					}
+				}
+				$post['media_ids'] = implode(',', $media_ids);
+				if (empty($post['media_ids'])) {
+					unset($post['media_ids']);
+				}
+			} catch (Exception $e) {
+				Logger::warning('Exception when trying to send to Twitter', ['id' => $b['id'], 'message' => $e->getMessage()]);
+			}
+		}
+
+		if (!DI::pConfig()->get($b['uid'], 'twitter', 'thread') || empty($msgarr['parts']) || (count($msgarr['parts']) == 1)) {
+			Logger::debug('Post single message', ['id' => $b['id']]);
+
+			$post['status'] = $msg;
+
+			if ($thr_parent) {
+				$post['in_reply_to_status_id'] = twitter_get_id($thr_parent['uri']);
+			}
+
+			$result = $connection->post('statuses/update', $post);
+			Logger::info('twitter_post send', ['id' => $b['id'], 'result' => $result]);
+
+			if (!empty($result->source)) {
+				DI::keyValue()->set('twitter_application_name', strip_tags($result->source));
+			}
+
+			if (!empty($result->errors)) {
+				Logger::error('Send to Twitter failed', ['id' => $b['id'], 'error' => $result->errors]);
 				Worker::defer();
-				return;
+			} elseif ($thr_parent) {
+				Logger::notice('Post send, updating extid', ['id' => $b['id'], 'extid' => $result->id_str]);
+				Item::update(['extid' => 'twitter::' . $result->id_str], ['id' => $b['id']]);
+			}
+		} else {
+			if ($thr_parent) {
+				$in_reply_to_status_id = twitter_get_id($thr_parent['uri']);
+			} else {
+				$in_reply_to_status_id = 0;
+			}
+
+			Logger::debug('Post message thread', ['id' => $b['id'], 'parts' => count($msgarr['parts'])]);
+			foreach ($msgarr['parts'] as $key => $part) {
+				$post['status'] = $part;
+
+				if ($in_reply_to_status_id) {
+					$post['in_reply_to_status_id'] = $in_reply_to_status_id;
+				}
+
+				$result = $connection->post('statuses/update', $post);
+				Logger::debug('twitter_post send', ['part' => $key, 'id' => $b['id'], 'result' => $result]);
+
+				if (!empty($result->errors)) {
+					Logger::warning('Send to Twitter failed', ['part' => $key, 'id' => $b['id'], 'error' => $result->errors]);
+					Worker::defer();
+					break;
+				} elseif ($key == 0) {
+					Logger::debug('Updating extid', ['part' => $key, 'id' => $b['id'], 'extid' => $result->id_str]);
+					Item::update(['extid' => 'twitter::' . $result->id_str], ['id' => $b['id']]);
+				}
+
+				if (!empty($result->source)) {
+					$application_name = strip_tags($result->source);
+				}
+
+				$in_reply_to_status_id = $result->id_str;
+				unset($post['media_ids']);
+			}
+
+			if (!empty($application_name)) {
+				DI::keyValue()->set('twitter_application_name', strip_tags($application_name));
 			}
 		}
 	}
-
-	$in_reply_to_tweet_id = 0;
-
-	Logger::debug('Post message', ['id' => $b['id'], 'parts' => count($msgarr['parts'])]);
-	foreach ($msgarr['parts'] as $key => $part) {
-		try {
-			$id = twitter_post_status($b['uid'], $part, $media_ids, $in_reply_to_tweet_id);
-			Logger::info('twitter_post send', ['part' => $key, 'id' => $b['id'], 'result' => $id]);
-		} catch (RequestException $exception) {
-			Logger::warning('Error while posting message', ['part' => $key, 'id' => $b['id'], 'code' => $exception->getCode(), 'message' => $exception->getMessage()]);
-			$status = [
-				'code'    => $exception->getCode(),
-				'reason'  => $exception->getResponse()->getReasonPhrase(),
-				'content' => $exception->getMessage()
-			];
-			DI::pConfig()->set($b['uid'], 'twitter', 'last_status', $status);
-			if ($key == 0) {
-				Worker::defer();
-			}
-			break;
-		}
-
-		$in_reply_to_tweet_id = $id;
-		$media_ids = [];
-	}
 }
 
-function twitter_post_status(int $uid, string $status, array $media_ids = [], string $in_reply_to_tweet_id = ''): string
-{
-	$parameters = ['text' => $status];
-	if (!empty($media_ids)) {
-		$parameters['media'] = ['media_ids' => $media_ids];
-	}
-	if (!empty($in_reply_to_tweet_id)) {
-		$parameters['reply'] = ['in_reply_to_tweet_id' => $in_reply_to_tweet_id];
-	}
-
-	$response = twitter_post($uid, 'https://api.twitter.com/2/tweets', 'json', $parameters);
-
-	return $response->data->id;
-}
-
-function twitter_upload_image(int $uid, array $image, int $retrial)
+function twitter_upload_image($connection, $cb, array $image, array $item)
 {
 	if (!empty($image['id'])) {
 		$photo = Photo::selectFirst([], ['id' => $image['id']]);
@@ -306,110 +895,1606 @@ function twitter_upload_image(int $uid, array $image, int $retrial)
 		$photo = Photo::createPhotoForExternalResource($image['url']);
 	}
 
-	$picturedata = Photo::getImageForPhoto($photo);
+	$tempfile = tempnam(System::getTempPath(), 'cache');
+	file_put_contents($tempfile, Photo::getImageForPhoto($photo));
 
-	$picture = new Image($picturedata, $photo['type'], $photo['filename']);
-	$height  = $picture->getHeight();
-	$width   = $picture->getWidth();
-	$size    = strlen($picturedata);
+	Logger::info('Uploading', ['id' => $item['id'], 'image' => $image]);
+	$media = $connection->upload('media/upload', ['media' => $tempfile]);
 
-	$picture     = Photo::resizeToFileSize($picture, TWITTER_IMAGE_SIZE[$retrial]);
-	$new_height  = $picture->getHeight();
-	$new_width   = $picture->getWidth();
-	$picturedata = $picture->asString();
-	$new_size    = strlen($picturedata);
-
-	Logger::info('Uploading', ['uid' => $uid, 'retrial' => $retrial, 'height' => $new_height, 'width' => $new_width, 'size' => $new_size, 'orig-height' => $height, 'orig-width' => $width, 'orig-size' => $size, 'image' => $image]);
-	$media = twitter_post($uid, 'https://upload.twitter.com/1.1/media/upload.json', 'form_params', ['media' => base64_encode($picturedata)]);
-	Logger::info('Uploading done', ['uid' => $uid, 'retrial' => $retrial, 'height' => $new_height, 'width' => $new_width, 'size' => $new_size, 'orig-height' => $height, 'orig-width' => $width, 'orig-size' => $size, 'image' => $image]);
+	unlink($tempfile);
 
 	if (isset($media->media_id_string)) {
 		$media_id = $media->media_id_string;
 
 		if (!empty($image['description'])) {
-			$data = [
-				'media_id' => $media->media_id_string,
-				'alt_text' => [
-					'text' => substr($image['description'], 0, 1000)
-				]
-			];
-			$ret = twitter_post($uid, 'https://upload.twitter.com/1.1/media/metadata/create.json', 'json', $data);
-			Logger::info('Metadata create', ['uid' => $uid, 'data' => $data, 'return' => $ret]);
+			$data = ['media_id' => $media->media_id_string,
+				'alt_text' => ['text' => substr($image['description'], 0, 420)]];
+			$ret = $cb->media_metadata_create($data);
+			Logger::info('Metadata create', ['id' => $item['id'], 'data' => $data, 'return' => $ret]);
 		}
 	} else {
-		Logger::error('Failed upload', ['uid' => $uid, 'size' => strlen($picturedata), 'image' => $image['url'], 'return' => $media]);
+		Logger::error('Failed upload', ['id' => $item['id'], 'image' => $image['url'], 'return' => $media]);
 		throw new Exception('Failed upload of ' . $image['url']);
 	}
 
 	return $media_id;
 }
 
-function twitter_post(int $uid, string $url, string $type, array $data): stdClass
+function twitter_delete_item(array $item)
 {
-	$stack = HandlerStack::create();
+	if (!$item['deleted']) {
+		return;
+	}
 
-	$middleware = new Oauth1([
-		'consumer_key'    => DI::pConfig()->get($uid, 'twitter', 'api_key'),
-		'consumer_secret' => DI::pConfig()->get($uid, 'twitter', 'api_secret'),
-		'token'           => DI::pConfig()->get($uid, 'twitter', 'access_token'),
-		'token_secret'    => DI::pConfig()->get($uid, 'twitter', 'access_secret'),
-	]);
+	if ($item['parent'] != $item['id']) {
+		Logger::debug('Deleting comment/announce', ['item' => $item]);
 
-	$stack->push($middleware);
+		// Looking if it's a reply to a twitter post
+		if (!twitter_get_id($item['parent-uri']) &&
+			!twitter_get_id($item['extid']) &&
+			!twitter_get_id($item['thr-parent'])) {
+			Logger::info('No twitter post', ['parent' => $item['parent']]);
+			return;
+		}
 
-	$client = new Client([
-		'handler' => $stack
-	]);
+		$condition = ['uri' => $item['thr-parent'], 'uid' => $item['uid']];
+		$thr_parent = Post::selectFirst(['uri', 'extid', 'author-link', 'author-nick', 'author-network'], $condition);
+		if (!DBA::isResult($thr_parent)) {
+			Logger::notice('No parent found', ['thr-parent' => $item['thr-parent']]);
+			return;
+		}
 
-	$response = $client->post($url, ['auth' => 'oauth', $type => $data]);
-	$body     = $response->getBody()->getContents();
+		Logger::debug('Parent found', ['parent' => $thr_parent]);
+	} else {
+		if (!strstr($item['extid'], 'twitter')) {
+			DI::logger()->info('Not a Twitter post', ['extid' => $item['extid']]);
+			return;
+		}
 
-	$status = [
-		'code'    => $response->getStatusCode(),
-		'reason'  => $response->getReasonPhrase(),
-		'content' => $body
-	];
+		// Don't delete if the post doesn't belong to us.
+		// This is a check for forum postings
+		$self = DBA::selectFirst('contact', ['id'], ['uid' => $item['uid'], 'self' => true]);
+		if ($item['contact-id'] != $self['id']) {
+			DI::logger()->info('Don\'t delete if the post doesn\'t belong to the user', ['contact-id' => $item['contact-id'], 'self' => $self['id']]);
+			return;
+		}
+	}
 
-	DI::pConfig()->set($uid, 'twitter', 'last_status', $status);
+	/**
+	 * @TODO Remaining caveat: Comments posted on Twitter and imported in Friendica do not trigger any Notifier task,
+	 *       possibly because they are private to the user and don't require any remote deletion notifications sent.
+	 *       Comments posted on Friendica and mirrored on Twitter trigger the Notifier task and the Twitter counter-part
+	 *       will be deleted accordingly.
+	 */
+	if ($item['verb'] == Activity::POST) {
+		Logger::info('Delete post/comment', ['uid' => $item['uid'], 'id' => twitter_get_id($item['extid'])]);
+		twitter_api_post('statuses/destroy', twitter_get_id($item['extid']), $item['uid']);
+		return;
+	}
 
-	$content = json_decode($body) ?? new stdClass;
-	Logger::debug('Success', ['content' => $content]);
-	return $content;
-}
+	if ($item['verb'] == Activity::LIKE) {
+		Logger::info('Unlike', ['uid' => $item['uid'], 'id' => twitter_get_id($item['thr-parent'])]);
+		twitter_api_post('favorites/destroy', twitter_get_id($item['thr-parent']), $item['uid']);
+		return;
+	}
 
-function twitter_test_connection(int $uid)
-{
-	$stack = HandlerStack::create();
-
-	$middleware = new Oauth1([
-		'consumer_key'    => DI::pConfig()->get($uid, 'twitter', 'api_key'),
-		'consumer_secret' => DI::pConfig()->get($uid, 'twitter', 'api_secret'),
-		'token'           => DI::pConfig()->get($uid, 'twitter', 'access_token'),
-		'token_secret'    => DI::pConfig()->get($uid, 'twitter', 'access_secret'),
-	]);
-
-	$stack->push($middleware);
-
-	$client = new Client([
-		'handler' => $stack
-	]);
-
-	try {
-		$response = $client->get('https://api.twitter.com/2/users/me', ['auth' => 'oauth']);
-		$status = [
-			'code'   => $response->getStatusCode(),
-			'reason'  => $response->getReasonPhrase(),
-			'content' => $response->getBody()->getContents()
-		];
-		DI::pConfig()->set(1, 'twitter', 'last_status',  $status);
-		Logger::info('Test successful', ['uid' => $uid]);
-	} catch (RequestException $exception) {
-		$status = [
-			'code'    => $exception->getCode(),
-			'reason'  => $exception->getResponse()->getReasonPhrase(),
-			'content' => $exception->getMessage()
-		];
-		DI::pConfig()->set(1, 'twitter', 'last_status',  $status);
-		Logger::info('Test failed', ['uid' => $uid]);
+	if ($item['verb'] == Activity::ANNOUNCE && !empty($thr_parent['uri'])) {
+		Logger::info('Unretweet', ['uid' => $item['uid'], 'extid' => $thr_parent['uri'], 'id' => twitter_get_id($thr_parent['uri'])]);
+		twitter_api_post('statuses/unretweet', twitter_get_id($thr_parent['uri']), $item['uid']);
+		return;
 	}
 }
+
+function twitter_addon_admin_post()
+{
+	DI::config()->set('twitter', 'consumerkey', trim($_POST['consumerkey'] ?? ''));
+	DI::config()->set('twitter', 'consumersecret', trim($_POST['consumersecret'] ?? ''));
+}
+
+function twitter_addon_admin(string &$o)
+{
+	$t = Renderer::getMarkupTemplate('admin.tpl', 'addon/twitter/');
+
+	$o = Renderer::replaceMacros($t, [
+		'$submit' => DI::l10n()->t('Save Settings'),
+		// name, label, value, help, [extra values]
+		'$consumerkey' => ['consumerkey', DI::l10n()->t('Consumer key'), DI::config()->get('twitter', 'consumerkey'), ''],
+		'$consumersecret' => ['consumersecret', DI::l10n()->t('Consumer secret'), DI::config()->get('twitter', 'consumersecret'), ''],
+	]);
+}
+
+function twitter_cron()
+{
+	$last = DI::keyValue()->get('twitter_last_poll');
+
+	$poll_interval = intval(DI::config()->get('twitter', 'poll_interval'));
+	if (!$poll_interval) {
+		$poll_interval = TWITTER_DEFAULT_POLL_INTERVAL;
+	}
+
+	if ($last) {
+		$next = $last + ($poll_interval * 60);
+		if ($next > time()) {
+			Logger::notice('twitter: poll intervall not reached');
+			return;
+		}
+	}
+	Logger::notice('twitter: cron_start');
+
+	$pconfigs = DBA::selectToArray('pconfig', [], ['cat' => 'twitter', 'k' => 'mirror_posts', 'v' => true]);
+	foreach ($pconfigs as $rr) {
+		Logger::notice('Fetching', ['user' => $rr['uid']]);
+		Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/twitter/twitter_sync.php', 1, (int) $rr['uid']);
+	}
+
+	$abandon_days = intval(DI::config()->get('system', 'account_abandon_days'));
+	if ($abandon_days < 1) {
+		$abandon_days = 0;
+	}
+
+	$abandon_limit = date(DateTimeFormat::MYSQL, time() - $abandon_days * 86400);
+
+	$pconfigs = DBA::selectToArray('pconfig', [], ['cat' => 'twitter', 'k' => 'import', 'v' => true]);
+	foreach ($pconfigs as $rr) {
+		if ($abandon_days != 0) {
+			if (!DBA::exists('user', ["`uid` = ? AND `login_date` >= ?", $rr['uid'], $abandon_limit])) {
+				Logger::notice('abandoned account: timeline from user will not be imported', ['user' => $rr['uid']]);
+				continue;
+			}
+		}
+
+		Logger::notice('importing timeline', ['user' => $rr['uid']]);
+		Worker::add(['priority' => Worker::PRIORITY_MEDIUM, 'force_priority' => true], 'addon/twitter/twitter_sync.php', 2, (int) $rr['uid']);
+		/*
+			// To-Do
+			// check for new contacts once a day
+			$last_contact_check = DI::pConfig()->get($rr['uid'],'pumpio','contact_check');
+			if($last_contact_check)
+			$next_contact_check = $last_contact_check + 86400;
+			else
+			$next_contact_check = 0;
+
+			if($next_contact_check <= time()) {
+			pumpio_getallusers($rr["uid"]);
+			DI::pConfig()->set($rr['uid'],'pumpio','contact_check',time());
+			}
+			*/
+	}
+
+	Logger::notice('twitter: cron_end');
+
+	DI::keyValue()->set('twitter_last_poll', time());
+}
+
+function twitter_expire()
+{
+	$days = DI::config()->get('twitter', 'expire');
+
+	if ($days == 0) {
+		return;
+	}
+
+	Logger::notice('Start deleting expired posts');
+
+	$r = Post::select(['id', 'guid'], ['deleted' => true, 'network' => Protocol::TWITTER]);
+	while ($row = Post::fetch($r)) {
+		Logger::info('[twitter] Delete expired item', ['id' => $row['id'], 'guid' => $row['guid'], 'callstack' => \Friendica\Core\System::callstack()]);
+		Item::markForDeletionById($row['id']);
+	}
+	DBA::close($r);
+
+	Logger::notice('End deleting expired posts');
+
+	Logger::notice('Start expiry');
+
+	$pconfigs = DBA::selectToArray('pconfig', [], ['cat' => 'twitter', 'k' => 'import', 'v' => true]);
+	foreach ($pconfigs as $rr) {
+		Logger::notice('twitter_expire', ['user' => $rr['uid']]);
+		Item::expire($rr['uid'], $days, Protocol::TWITTER, true);
+	}
+
+	Logger::notice('End expiry');
+}
+
+function twitter_prepare_body(array &$b)
+{
+	if ($b['item']['network'] != Protocol::TWITTER) {
+		return;
+	}
+
+	if ($b['preview']) {
+		$max_char = 280;
+		$item = $b['item'];
+		$item['plink'] = DI::baseUrl() . '/display/' . $item['guid'];
+
+		$condition = ['uri' => $item['thr-parent'], 'uid' => DI::userSession()->getLocalUserId()];
+		$orig_post = Post::selectFirst(['author-link'], $condition);
+		if (DBA::isResult($orig_post)) {
+			$nicknameplain = preg_replace("=https?://twitter.com/(.*)=ism", "$1", $orig_post['author-link']);
+			$nickname = '@[url=' . $orig_post['author-link'] . ']' . $nicknameplain . '[/url]';
+			$nicknameplain = '@' . $nicknameplain;
+
+			if ((strpos($item['body'], $nickname) === false) && (strpos($item['body'], $nicknameplain) === false)) {
+				$item['body'] = $nickname . ' ' . $item['body'];
+			}
+		}
+
+		$msgarr = Plaintext::getPost($item, $max_char, true, BBCode::TWITTER);
+		$msg = $msgarr['text'];
+
+		if (isset($msgarr['url']) && ($msgarr['type'] != 'photo')) {
+			$msg .= ' ' . $msgarr['url'];
+		}
+
+		if (isset($msgarr['image'])) {
+			$msg .= ' ' . $msgarr['image'];
+		}
+
+		$b['html'] = nl2br(htmlspecialchars($msg));
+	}
+}
+
+function twitter_statuses_show(string $id, TwitterOAuth $twitterOAuth = null)
+{
+	if ($twitterOAuth === null) {
+		$ckey = DI::config()->get('twitter', 'consumerkey');
+		$csecret = DI::config()->get('twitter', 'consumersecret');
+
+		if (empty($ckey) || empty($csecret)) {
+			return new stdClass();
+		}
+
+		$twitterOAuth = new TwitterOAuth($ckey, $csecret);
+	}
+
+	$parameters = ['trim_user' => false, 'tweet_mode' => 'extended', 'id' => $id, 'include_ext_alt_text' => true];
+
+	return $twitterOAuth->get('statuses/show', $parameters);
+}
+
+/**
+ * Parse Twitter status URLs since Twitter removed OEmbed
+ *
+ * @param array $b Expected format:
+ *                 [
+ *                      'url' => [URL to parse],
+ *                      'format' => 'json'|'',
+ *                      'text' => Output parameter
+ *                 ]
+ * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+ */
+function twitter_parse_link(array &$b)
+{
+	// Only handle Twitter status URLs
+	if (!preg_match('#^https?://(?:mobile\.|www\.)?twitter.com/[^/]+/status/(\d+).*#', $b['url'], $matches)) {
+		return;
+	}
+
+	$status = twitter_statuses_show($matches[1]);
+
+	if (empty($status->id)) {
+		return;
+	}
+
+	$item = twitter_createpost(0, $status, [], true, false, true);
+	if (empty($item)) {
+		return;
+	}
+
+	if ($b['format'] == 'json') {
+		$images = [];
+		foreach ($status->extended_entities->media ?? [] as $media) {
+			if (!empty($media->media_url_https)) {
+				$images[] = [
+					'src'    => $media->media_url_https,
+					'width'  => $media->sizes->thumb->w,
+					'height' => $media->sizes->thumb->h,
+				];
+			}
+		}
+
+		$b['text'] = [
+			'data' => [
+				'type' => 'link',
+				'url' => $item['plink'],
+				'title' => DI::l10n()->t('%s on Twitter', $status->user->name),
+				'text' => BBCode::toPlaintext($item['body'], false),
+				'images' => $images,
+			],
+			'contentType' => 'attachment',
+			'success' => true,
+		];
+	} else {
+		$b['text'] = BBCode::getShareOpeningTag(
+			$item['author-name'],
+			$item['author-link'],
+			$item['author-avatar'],
+			$item['plink'],
+			$item['created']
+		);
+		$b['text'] .= $item['body'] . '[/share]';
+	}
+}
+
+
+/*********************
+ *
+ * General functions
+ *
+ *********************/
+
+
+/**
+ * @brief Build the item array for the mirrored post
+ *
+ * @param integer $uid User id
+ * @param object $post Twitter object with the post
+ *
+ * @return array item data to be posted
+ */
+function twitter_do_mirrorpost(int $uid, $post)
+{
+	$datarray['uid'] = $uid;
+	$datarray['extid'] = 'twitter::' . $post->id;
+	$datarray['title'] = '';
+
+	if (!empty($post->retweeted_status)) {
+		// We don't support nested shares, so we mustn't show quotes as shares on retweets
+		$item = twitter_createpost($uid, $post->retweeted_status, ['id' => 0], false, false, true, -1);
+
+		if (empty($item)) {
+			return [];
+		}
+
+		$datarray['body'] = "\n" . BBCode::getShareOpeningTag(
+			$item['author-name'],
+			$item['author-link'],
+			$item['author-avatar'],
+			$item['plink'],
+			$item['created']
+		);
+
+		$datarray['body'] .= $item['body'] . '[/share]';
+	} else {
+		$item = twitter_createpost($uid, $post, ['id' => 0], false, false, false, -1);
+
+		if (empty($item)) {
+			return [];
+		}
+
+		$datarray['body'] = $item['body'];
+	}
+
+	$datarray['app'] = $item['app'];
+	$datarray['verb'] = $item['verb'];
+
+	if (isset($item['location'])) {
+		$datarray['location'] = $item['location'];
+	}
+
+	if (isset($item['coord'])) {
+		$datarray['coord'] = $item['coord'];
+	}
+
+	return $datarray;
+}
+
+/**
+ * Fetches the Twitter user's own posts
+ *
+ * @param int $uid
+ * @return void
+ * @throws Exception
+ */
+function twitter_fetchtimeline(int $uid): void
+{
+	$ckey    = DI::config()->get('twitter', 'consumerkey');
+	$csecret = DI::config()->get('twitter', 'consumersecret');
+	$otoken  = DI::pConfig()->get($uid, 'twitter', 'oauthtoken');
+	$osecret = DI::pConfig()->get($uid, 'twitter', 'oauthsecret');
+	$lastid  = DI::pConfig()->get($uid, 'twitter', 'lastid');
+
+	$application_name = DI::keyValue()->get('twitter_application_name') ?? '';
+
+	if ($application_name == '') {
+		$application_name = DI::baseUrl()->getHost();
+	}
+
+	$connection = new TwitterOAuth($ckey, $csecret, $otoken, $osecret);
+
+	// Ensure to have the own contact
+	try {
+		twitter_fetch_own_contact($uid);
+	} catch (TwitterOAuthException $e) {
+		Logger::notice('Error fetching own contact', ['uid' => $uid, 'message' => $e->getMessage()]);
+		return;
+	}
+
+	$parameters = [
+		'exclude_replies' => true,
+		'trim_user' => false,
+		'contributor_details' => true,
+		'include_rts' => true,
+		'tweet_mode' => 'extended',
+		'include_ext_alt_text' => true,
+	];
+
+	$first_time = ($lastid == '');
+
+	if ($lastid != '') {
+		$parameters['since_id'] = $lastid;
+	}
+
+	try {
+		$items = $connection->get('statuses/user_timeline', $parameters);
+	} catch (TwitterOAuthException $e) {
+		Logger::notice('Error fetching timeline', ['uid' => $uid, 'message' => $e->getMessage()]);
+		return;
+	}
+
+	if (!is_array($items)) {
+		Logger::notice('No items', ['user' => $uid]);
+		return;
+	}
+
+	$posts = array_reverse($items);
+
+	Logger::notice('Start processing posts', ['from' => $lastid, 'user' => $uid, 'count' => count($posts)]);
+
+	if (count($posts)) {
+		foreach ($posts as $post) {
+			if ($post->id_str > $lastid) {
+				$lastid = $post->id_str;
+				DI::pConfig()->set($uid, 'twitter', 'lastid', $lastid);
+			}
+
+			if ($first_time) {
+				Logger::notice('First time, continue');
+				continue;
+			}
+
+			if (stristr($post->source, $application_name)) {
+				Logger::notice('Source is application name', ['source' => $post->source, 'application_name' => $application_name]);
+				continue;
+			}
+			Logger::info('Preparing mirror post', ['twitter-id' => $post->id_str, 'uid' => $uid]);
+
+			$mirrorpost = twitter_do_mirrorpost($uid, $post);
+
+			if (empty($mirrorpost['body'])) {
+				Logger::notice('Body is empty', ['post' => $post, 'mirrorpost' => $mirrorpost]);
+				continue;
+			}
+
+			Logger::info('Posting mirror post', ['twitter-id' => $post->id_str, 'uid' => $uid]);
+
+			Post\Delayed::add($mirrorpost['extid'], $mirrorpost, Worker::PRIORITY_MEDIUM, Post\Delayed::UNPREPARED);
+		}
+	}
+	DI::pConfig()->set($uid, 'twitter', 'lastid', $lastid);
+	Logger::info('Last ID for user ' . $uid . ' is now ' . $lastid);
+}
+
+function twitter_fix_avatar($avatar)
+{
+	$new_avatar = str_replace('_normal.', '_400x400.', $avatar);
+
+	$info = Images::getInfoFromURLCached($new_avatar);
+	if (!$info) {
+		$new_avatar = $avatar;
+	}
+
+	return $new_avatar;
+}
+
+function twitter_get_relation($uid, $target, $contact = [])
+{
+	if (isset($contact['rel'])) {
+		$relation = $contact['rel'];
+	} else {
+		$relation = 0;
+	}
+
+	$ckey = DI::config()->get('twitter', 'consumerkey');
+	$csecret = DI::config()->get('twitter', 'consumersecret');
+	$otoken = DI::pConfig()->get($uid, 'twitter', 'oauthtoken');
+	$osecret = DI::pConfig()->get($uid, 'twitter', 'oauthsecret');
+	$own_id = DI::pConfig()->get($uid, 'twitter', 'own_id');
+
+	$connection = new TwitterOAuth($ckey, $csecret, $otoken, $osecret);
+	$parameters = ['source_id' => $own_id, 'target_screen_name' => $target];
+
+	try {
+		$status = $connection->get('friendships/show', $parameters);
+		if ($connection->getLastHttpCode() !== 200) {
+			throw new Exception($status->errors[0]->message ?? 'HTTP response code ' . $connection->getLastHttpCode(), $status->errors[0]->code ?? $connection->getLastHttpCode());
+		}
+
+		$following = $status->relationship->source->following;
+		$followed = $status->relationship->source->followed_by;
+
+		if ($following && !$followed) {
+			$relation = Contact::SHARING;
+		} elseif (!$following && $followed) {
+			$relation = Contact::FOLLOWER;
+		} elseif ($following && $followed) {
+			$relation = Contact::FRIEND;
+		} elseif (!$following && !$followed) {
+			$relation = 0;
+		}
+
+		Logger::info('Fetched friendship relation', ['user' => $uid, 'target' => $target, 'relation' => $relation]);
+	} catch (Throwable $e) {
+		Logger::notice('Error fetching friendship status', ['uid' => $uid, 'target' => $target, 'message' => $e->getMessage()]);
+	}
+
+	return $relation;
+}
+
+/**
+ * @param $data
+ * @return array
+ */
+function twitter_user_to_contact($data)
+{
+	if (empty($data->id_str)) {
+		return [];
+	}
+
+	$baseurl = 'https://twitter.com';
+	$url = $baseurl . '/' . $data->screen_name;
+	$addr = $data->screen_name . '@twitter.com';
+
+	$fields = [
+		'url'      => $url,
+		'nurl'     => Strings::normaliseLink($url),
+		'uri-id'   => ItemURI::getIdByURI($url),
+		'network'  => Protocol::TWITTER,
+		'alias'    => 'twitter::' . $data->id_str,
+		'baseurl'  => $baseurl,
+		'name'     => $data->name,
+		'nick'     => $data->screen_name,
+		'addr'     => $addr,
+		'location' => $data->location,
+		'about'    => $data->description,
+		'photo'    => twitter_fix_avatar($data->profile_image_url_https),
+		'header'   => $data->profile_banner_url ?? $data->profile_background_image_url_https,
+	];
+
+	return $fields;
+}
+
+function twitter_get_contact($data, int $uid = 0)
+{
+	$contact = DBA::selectFirst('contact', ['id'], ['uid' => $uid, 'alias' => 'twitter::' . $data->id_str]);
+	if (DBA::isResult($contact)) {
+		return $contact['id'];
+	} else {
+		return twitter_fetch_contact($uid, $data, false);
+	}
+}
+
+function twitter_fetch_contact($uid, $data, $create_user)
+{
+	$fields = twitter_user_to_contact($data);
+
+	if (empty($fields)) {
+		return -1;
+	}
+
+	// photo comes from twitter_user_to_contact but shouldn't be saved directly in the contact row
+	$avatar = $fields['photo'];
+	unset($fields['photo']);
+
+	// Update the public contact
+	$pcontact = DBA::selectFirst('contact', ['id'], ['uid' => 0, 'alias' => 'twitter::' . $data->id_str]);
+	if (DBA::isResult($pcontact)) {
+		$cid = $pcontact['id'];
+	} else {
+		$cid = Contact::getIdForURL($fields['url'], 0, false, $fields);
+	}
+
+	if (!empty($cid)) {
+		Contact::update($fields, ['id' => $cid]);
+		Contact::updateAvatar($cid, $avatar);
+	} else {
+		Logger::notice('No contact found', ['fields' => $fields]);
+	}
+
+	$contact = DBA::selectFirst('contact', [], ['uid' => $uid, 'alias' => 'twitter::' . $data->id_str]);
+	if (!DBA::isResult($contact) && empty($cid)) {
+		Logger::notice('User contact not found', ['uid' => $uid, 'twitter-id' => $data->id_str]);
+		return 0;
+	} elseif (!$create_user) {
+		return $cid;
+	}
+
+	if (!DBA::isResult($contact)) {
+		$relation = twitter_get_relation($uid, $data->screen_name);
+
+		// create contact record
+		$fields['uid'] = $uid;
+		$fields['created'] = DateTimeFormat::utcNow();
+		$fields['poll'] = 'twitter::' . $data->id_str;
+		$fields['rel'] = $relation;
+		$fields['priority'] = 1;
+		$fields['writable'] = true;
+		$fields['blocked'] = false;
+		$fields['readonly'] = false;
+		$fields['pending'] = false;
+
+		if (!Contact::insert($fields)) {
+			return false;
+		}
+
+		$contact_id = DBA::lastInsertId();
+
+		Group::addMember(User::getDefaultGroup($uid), $contact_id);
+	} else {
+		if ($contact['readonly'] || $contact['blocked']) {
+			Logger::notice('Contact is blocked or readonly.', ['nickname' => $contact['nick']]);
+			return -1;
+		}
+
+		$contact_id = $contact['id'];
+		$update = false;
+
+		// Update the contact relation once per day
+		if ($contact['updated'] < DateTimeFormat::utc('now -24 hours')) {
+			$fields['rel'] = twitter_get_relation($uid, $data->screen_name, $contact);
+			$update = true;
+		}
+
+		if ($contact['name'] != $data->name) {
+			$fields['name-date'] = $fields['uri-date'] = DateTimeFormat::utcNow();
+			$update = true;
+		}
+
+		if ($contact['nick'] != $data->screen_name) {
+			$fields['uri-date'] = DateTimeFormat::utcNow();
+			$update = true;
+		}
+
+		if (($contact['location'] != $data->location) || ($contact['about'] != $data->description)) {
+			$update = true;
+		}
+
+		if ($update) {
+			$fields['updated'] = DateTimeFormat::utcNow();
+			Contact::update($fields, ['id' => $contact['id']]);
+			Logger::info('Updated contact', ['id' => $contact['id'], 'nick' => $data->screen_name]);
+		}
+	}
+
+	Contact::updateAvatar($contact_id, $avatar);
+
+	if (Contact::isSharing($contact_id, $uid, true) && DI::pConfig()->get($uid, 'twitter', 'auto_follow')) {
+		twitter_auto_follow($uid, $data);
+	}
+
+	return $contact_id;
+}
+
+/**
+ * Follow a fediverse account that is proived in the name or the profile
+ *
+ * @param integer $uid
+ * @param object $data
+ */
+function twitter_auto_follow(int $uid, object $data)
+{
+	$addrpattern = '([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,6})';
+
+	// Search for user@domain.tld in the name
+	if (preg_match('#' . $addrpattern . '#', $data->name, $match)) {
+		if (twitter_add_contact($match[1], true, $uid)) {
+			return;
+		}
+	}
+
+	// Search for @user@domain.tld in the description
+	if (preg_match('#@' . $addrpattern . '#', $data->description, $match)) {
+		if (twitter_add_contact($match[1], true, $uid)) {
+			return;
+		}
+	}
+
+	// Search for user@domain.tld in the description
+	// We don't probe here, since this could be a mail address
+	if (preg_match('#' . $addrpattern . '#', $data->description, $match)) {
+		if (twitter_add_contact($match[1], false, $uid)) {
+			return;
+		}
+	}
+
+	// Search for profile links in the description
+	foreach ($data->entities->description->urls as $url) {
+		if (!empty($url->expanded_url)) {
+			// We only probe on Mastodon style URL to reduce the number of unsuccessful probes
+			twitter_add_contact($url->expanded_url, strpos($url->expanded_url, '@'), $uid);
+		}
+	}
+}
+
+/**
+ * Check if the provided address is a fediverse account and adds it
+ *
+ * @param string $addr
+ * @param boolean $probe
+ * @param integer $uid
+ * @return boolean
+ */
+function twitter_add_contact(string $addr, bool $probe, int $uid): bool
+{
+	$contact = Contact::getByURL($addr, $probe ? null : false, ['id', 'url', 'network']);
+	if (empty($contact)) {
+		Logger::debug('Not a contact address', ['uid' => $uid, 'probe' => $probe, 'addr' => $addr]);
+		return false;
+	}
+
+	if (!in_array($contact['network'], Protocol::FEDERATED)) {
+		Logger::debug('Not a federated network', ['uid' => $uid, 'addr' => $addr, 'contact' => $contact]);
+		return false;
+	}
+
+	if (Contact::isSharing($contact['id'], $uid)) {
+		Logger::debug('Contact has already been added', ['uid' => $uid, 'addr' => $addr, 'contact' => $contact]);
+		return true;
+	}
+
+	Logger::info('Add contact', ['uid' => $uid, 'addr' => $addr, 'contact' => $contact]);
+	Worker::add(Worker::PRIORITY_LOW, 'AddContact', $uid, $contact['url']);
+
+	return true;
+}
+
+/**
+ * @param string $screen_name
+ * @return stdClass|null
+ * @throws Exception
+ */
+function twitter_fetchuser($screen_name)
+{
+	$ckey = DI::config()->get('twitter', 'consumerkey');
+	$csecret = DI::config()->get('twitter', 'consumersecret');
+
+	try {
+		// Fetching user data
+		$connection = new TwitterOAuth($ckey, $csecret);
+		$parameters = ['screen_name' => $screen_name];
+		$user = $connection->get('users/show', $parameters);
+	} catch (TwitterOAuthException $e) {
+		Logger::notice('Error fetching user', ['user' => $screen_name, 'message' => $e->getMessage()]);
+		return null;
+	}
+
+	if (!is_object($user)) {
+		return null;
+	}
+
+	return $user;
+}
+
+/**
+ * Replaces Twitter entities with Friendica-friendly links.
+ *
+ * The Twitter API gives indices for each entity, which allows for fine-grained replacement.
+ *
+ * First, we need to collect everything that needs to be replaced, what we will replace it with, and the start index.
+ * Then we sort the indices decreasingly, and we replace from the end of the body to the start in order for the next
+ * index to be correct even after the last replacement.
+ *
+ * @param string   $body
+ * @param stdClass $status
+ * @return array
+ * @throws \Friendica\Network\HTTPException\InternalServerErrorException
+ */
+function twitter_expand_entities($body, stdClass $status)
+{
+	$plain = $body;
+	$contains_urls = false;
+
+	$taglist = [];
+
+	$replacementList = [];
+
+	foreach ($status->entities->hashtags AS $hashtag) {
+		$replace = '#[url=' . DI::baseUrl() . '/search?tag=' . $hashtag->text . ']' . $hashtag->text . '[/url]';
+		$taglist['#' . $hashtag->text] = ['#', $hashtag->text, ''];
+
+		$replacementList[$hashtag->indices[0]] = [
+			'replace' => $replace,
+			'length' => $hashtag->indices[1] - $hashtag->indices[0],
+		];
+	}
+
+	foreach ($status->entities->user_mentions AS $mention) {
+		$replace = '@[url=https://twitter.com/' . rawurlencode($mention->screen_name) . ']' . $mention->screen_name . '[/url]';
+		$taglist['@' . $mention->screen_name] = ['@', $mention->screen_name, 'https://twitter.com/' . rawurlencode($mention->screen_name)];
+
+		$replacementList[$mention->indices[0]] = [
+			'replace' => $replace,
+			'length' => $mention->indices[1] - $mention->indices[0],
+		];
+	}
+
+	foreach ($status->entities->urls ?? [] as $url) {
+		$plain = str_replace($url->url, '', $plain);
+
+		if ($url->url && $url->expanded_url && $url->display_url) {
+			// Quote tweet, we just remove the quoted tweet URL from the body, the share block will be added later.
+			if (!empty($status->quoted_status) && isset($status->quoted_status_id_str)
+				&& substr($url->expanded_url, -strlen($status->quoted_status_id_str)) == $status->quoted_status_id_str
+			) {
+				$replacementList[$url->indices[0]] = [
+					'replace' => '',
+					'length' => $url->indices[1] - $url->indices[0],
+				];
+				continue;
+			}
+
+			$contains_urls = true;
+
+			$expanded_url = $url->expanded_url;
+
+			// Quickfix: Workaround for URL with '[' and ']' in it
+			if (strpos($expanded_url, '[') || strpos($expanded_url, ']')) {
+				$expanded_url = $url->url;
+			}
+
+			$replacementList[$url->indices[0]] = [
+				'replace' => '[url=' . $expanded_url . ']' . $url->display_url . '[/url]',
+				'length' => $url->indices[1] - $url->indices[0],
+			];
+		}
+	}
+
+	krsort($replacementList);
+
+	foreach ($replacementList as $startIndex => $parameters) {
+		$body = Strings::substringReplace($body, $parameters['replace'], $startIndex, $parameters['length']);
+	}
+
+	$body = trim($body);
+
+	return ['body' => trim($body), 'plain' => trim($plain), 'taglist' => $taglist, 'urls' => $contains_urls];
+}
+
+/**
+ * Store entity attachments
+ *
+ * @param integer $uriId
+ * @param object $post Twitter object with the post
+ */
+function twitter_store_attachments(int $uriId, $post)
+{
+	if (!empty($post->extended_entities->media)) {
+		foreach ($post->extended_entities->media AS $medium) {
+			switch ($medium->type) {
+				case 'photo':
+					$attachment = ['uri-id' => $uriId, 'type' => Post\Media::IMAGE];
+
+					$attachment['url'] = $medium->media_url_https . '?name=large';
+					$attachment['width'] = $medium->sizes->large->w;
+					$attachment['height'] = $medium->sizes->large->h;
+
+					if ($medium->sizes->small->w != $attachment['width']) {
+						$attachment['preview'] = $medium->media_url_https . '?name=small';
+						$attachment['preview-width'] = $medium->sizes->small->w;
+						$attachment['preview-height'] = $medium->sizes->small->h;
+					}
+
+					$attachment['name'] = $medium->display_url ?? null;
+					$attachment['description'] = $medium->ext_alt_text ?? null;
+					Logger::debug('Photo attachment', ['attachment' => $attachment]);
+					Post\Media::insert($attachment);
+					break;
+				case 'video':
+				case 'animated_gif':
+					$attachment = ['uri-id' => $uriId, 'type' => Post\Media::VIDEO];
+					if (is_array($medium->video_info->variants)) {
+						$bitrate = 0;
+						// We take the video with the highest bitrate
+						foreach ($medium->video_info->variants AS $variant) {
+							if (($variant->content_type == 'video/mp4') && ($variant->bitrate >= $bitrate)) {
+								$attachment['url'] = $variant->url;
+								$bitrate = $variant->bitrate;
+							}
+						}
+					}
+
+					$attachment['name'] = $medium->display_url ?? null;
+					$attachment['preview'] = $medium->media_url_https . ':small';
+					$attachment['preview-width'] = $medium->sizes->small->w;
+					$attachment['preview-height'] = $medium->sizes->small->h;
+					$attachment['description'] = $medium->ext_alt_text ?? null;
+					Logger::debug('Video attachment', ['attachment' => $attachment]);
+					Post\Media::insert($attachment);
+					break;
+				default:
+					Logger::notice('Unknown media type', ['medium' => $medium]);
+			}
+		}
+	}
+
+	if (!empty($post->entities->urls)) {
+		foreach ($post->entities->urls as $url) {
+			$attachment = ['uri-id' => $uriId, 'type' => Post\Media::UNKNOWN, 'url' => $url->expanded_url, 'name' => $url->display_url];
+			Logger::debug('Attached link', ['attachment' => $attachment]);
+			Post\Media::insert($attachment);
+		}
+	}
+}
+
+/**
+ * @brief Fetch media entities and add media links to the body
+ *
+ * @param object  $post      Twitter object with the post
+ * @param array   $postarray Array of the item that is about to be posted
+ * @param integer $uriId URI Id used to store tags. -1 = don't store tags for this post.
+ */
+function twitter_media_entities($post, array &$postarray, int $uriId = -1)
+{
+	// There are no media entities? So we quit.
+	if (empty($post->extended_entities->media)) {
+		return;
+	}
+
+	// This is a pure media post, first search for all media urls
+	$media = [];
+	foreach ($post->extended_entities->media AS $medium) {
+		if (!isset($media[$medium->url])) {
+			$media[$medium->url] = '';
+		}
+		switch ($medium->type) {
+			case 'photo':
+				if (!empty($medium->ext_alt_text)) {
+					Logger::info('Got text description', ['alt_text' => $medium->ext_alt_text]);
+					$media[$medium->url] .= "\n[img=" . $medium->media_url_https .']' . $medium->ext_alt_text . '[/img]';
+				} else {
+					$media[$medium->url] .= "\n[img]" . $medium->media_url_https . '[/img]';
+				}
+
+				$postarray['object-type'] = Activity\ObjectType::IMAGE;
+				$postarray['post-type'] = Item::PT_IMAGE;
+				break;
+			case 'video':
+				// Currently deactivated, since this causes the video to be display before the content
+				// We have to figure out a better way for declaring the post type and the display style.
+				//$postarray['post-type'] = Item::PT_VIDEO;
+			case 'animated_gif':
+				if (!empty($medium->ext_alt_text)) {
+					Logger::info('Got text description', ['alt_text' => $medium->ext_alt_text]);
+					$media[$medium->url] .= "\n[img=" . $medium->media_url_https .']' . $medium->ext_alt_text . '[/img]';
+				} else {
+					$media[$medium->url] .= "\n[img]" . $medium->media_url_https . '[/img]';
+				}
+
+				$postarray['object-type'] = Activity\ObjectType::VIDEO;
+				if (is_array($medium->video_info->variants)) {
+					$bitrate = 0;
+					// We take the video with the highest bitrate
+					foreach ($medium->video_info->variants AS $variant) {
+						if (($variant->content_type == 'video/mp4') && ($variant->bitrate >= $bitrate)) {
+							$media[$medium->url] = "\n[video]" . $variant->url . '[/video]';
+							$bitrate = $variant->bitrate;
+						}
+					}
+				}
+				break;
+		}
+	}
+
+	if ($uriId != -1) {
+		foreach ($media AS $key => $value) {
+			$postarray['body'] = str_replace($key, '', $postarray['body']);
+		}
+		return;
+	}
+
+	// Now we replace the media urls.
+	foreach ($media AS $key => $value) {
+		$postarray['body'] = str_replace($key, "\n" . $value . "\n", $postarray['body']);
+	}
+}
+
+/**
+ * Undocumented function
+ *
+ * @param integer $uid User ID
+ * @param object $post Incoming Twitter post
+ * @param array $self
+ * @param bool $create_user Should users be created?
+ * @param bool $only_existing_contact Only import existing contacts if set to "true"
+ * @param bool $noquote
+ * @param integer $uriId URI Id used to store tags. 0 = create a new one; -1 = don't store tags for this post.
+ * @return array item array
+ */
+function twitter_createpost(int $uid, $post, array $self, $create_user, bool $only_existing_contact, bool $noquote, int $uriId = 0): array
+{
+	$postarray = [];
+	$postarray['network'] = Protocol::TWITTER;
+	$postarray['uid'] = $uid;
+	$postarray['wall'] = 0;
+	$postarray['uri'] = 'twitter::' . $post->id_str;
+	$postarray['protocol'] = Conversation::PARCEL_TWITTER;
+	$postarray['source'] = json_encode($post);
+	$postarray['direction'] = Conversation::PULL;
+
+	if (empty($uriId)) {
+		$uriId = $postarray['uri-id'] = ItemURI::insert(['uri' => $postarray['uri']]);
+	}
+
+	// Don't import our own comments
+	if (Post::exists(['extid' => $postarray['uri'], 'uid' => $uid])) {
+		Logger::info('Item found', ['extid' => $postarray['uri']]);
+		return [];
+	}
+
+	$contactid = 0;
+
+	if ($post->in_reply_to_status_id_str != '') {
+		$thr_parent = 'twitter::' . $post->in_reply_to_status_id_str;
+
+		$item = Post::selectFirst(['uri'], ['uri' => $thr_parent, 'uid' => $uid]);
+		if (!DBA::isResult($item)) {
+			$item = Post::selectFirst(['uri'], ['extid' => $thr_parent, 'uid' => $uid, 'gravity' => Item::GRAVITY_COMMENT]);
+		}
+
+		if (DBA::isResult($item)) {
+			$postarray['thr-parent'] = $item['uri'];
+			$postarray['object-type'] = Activity\ObjectType::COMMENT;
+		} else {
+			$postarray['object-type'] = Activity\ObjectType::NOTE;
+		}
+
+		// Is it me?
+		$own_id = DI::pConfig()->get($uid, 'twitter', 'own_id');
+
+		if ($post->user->id_str == $own_id) {
+			$self = Contact::selectFirst(['id', 'name', 'url', 'photo'], ['self' => true, 'uid' => $uid]);
+			if (DBA::isResult($self)) {
+				$contactid = $self['id'];
+
+				$postarray['owner-id']     = Contact::getIdForURL($self['url']);
+				$postarray['owner-name']   = $self['name'];
+				$postarray['owner-link']   = $self['url'];
+				$postarray['owner-avatar'] = $self['photo'];
+			} else {
+				Logger::error('No self contact found', ['uid' => $uid]);
+				return [];
+			}
+		}
+		// Don't create accounts of people who just comment something
+		$create_user = false;
+	} else {
+		$postarray['object-type'] = Activity\ObjectType::NOTE;
+	}
+
+	if ($contactid == 0) {
+		$contactid = twitter_fetch_contact($uid, $post->user, $create_user);
+
+		$postarray['owner-id']     = twitter_get_contact($post->user);
+		$postarray['owner-name']   = $post->user->name;
+		$postarray['owner-link']   = 'https://twitter.com/' . $post->user->screen_name;
+		$postarray['owner-avatar'] = twitter_fix_avatar($post->user->profile_image_url_https);
+	}
+
+	if (($contactid == 0) && !$only_existing_contact) {
+		$contactid = $self['id'];
+	} elseif ($contactid <= 0) {
+		Logger::info('Contact ID is zero or less than zero.');
+		return [];
+	}
+
+	$postarray['contact-id']    = $contactid;
+	$postarray['verb']          = Activity::POST;
+	$postarray['author-id']     = $postarray['owner-id'];
+	$postarray['author-name']   = $postarray['owner-name'];
+	$postarray['author-link']   = $postarray['owner-link'];
+	$postarray['author-avatar'] = $postarray['owner-avatar'];
+	$postarray['plink']         = 'https://twitter.com/' . $post->user->screen_name . '/status/' . $post->id_str;
+	$postarray['app']           = strip_tags($post->source);
+
+	if ($post->user->protected) {
+		$postarray['private']   = Item::PRIVATE;
+		$postarray['allow_cid'] = '<' . $self['id'] . '>';
+	} else {
+		$postarray['private']   = Item::UNLISTED;
+		$postarray['allow_cid'] = '';
+	}
+
+	if (!empty($post->full_text)) {
+		$postarray['body'] = $post->full_text;
+	} else {
+		$postarray['body'] = $post->text;
+	}
+
+	// When the post contains links then use the correct object type
+	if (count($post->entities->urls) > 0) {
+		$postarray['object-type'] = Activity\ObjectType::BOOKMARK;
+	}
+
+	// Search for media links
+	twitter_media_entities($post, $postarray, $uriId);
+
+	$converted = twitter_expand_entities($postarray['body'], $post);
+
+	// When the post contains external links then images or videos are just "decorations".
+	if (!empty($converted['urls'])) {
+		$postarray['post-type'] = Item::PT_NOTE;
+	}
+
+	$postarray['body'] = $converted['body'];
+	$postarray['created'] = DateTimeFormat::utc($post->created_at);
+	$postarray['edited'] = DateTimeFormat::utc($post->created_at);
+
+	if ($uriId > 0) {
+		twitter_store_tags($uriId, $converted['taglist']);
+		twitter_store_attachments($uriId, $post);
+	}
+
+	if (!empty($post->place->name)) {
+		$postarray['location'] = $post->place->name;
+	}
+	if (!empty($post->place->full_name)) {
+		$postarray['location'] = $post->place->full_name;
+	}
+	if (!empty($post->geo->coordinates)) {
+		$postarray['coord'] = $post->geo->coordinates[0] . ' ' . $post->geo->coordinates[1];
+	}
+	if (!empty($post->coordinates->coordinates)) {
+		$postarray['coord'] = $post->coordinates->coordinates[1] . ' ' . $post->coordinates->coordinates[0];
+	}
+	if (!empty($post->retweeted_status)) {
+		$retweet = twitter_createpost($uid, $post->retweeted_status, $self, false, false, $noquote);
+
+		if (empty($retweet)) {
+			return [];
+		}
+
+		if (!$noquote) {
+			// Store the original tweet
+			Item::insert($retweet);
+
+			// CHange the other post into a reshare activity
+			$postarray['verb'] = Activity::ANNOUNCE;
+			$postarray['gravity'] = Item::GRAVITY_ACTIVITY;
+			$postarray['object-type'] = Activity\ObjectType::NOTE;
+
+			$postarray['thr-parent'] = $retweet['uri'];
+		} else {
+			$retweet['source']       = $postarray['source'];
+			$retweet['direction']    = $postarray['direction'];
+			$retweet['private']      = $postarray['private'];
+			$retweet['allow_cid']    = $postarray['allow_cid'];
+			$retweet['contact-id']   = $postarray['contact-id'];
+			$retweet['owner-id']     = $postarray['owner-id'];
+			$retweet['owner-name']   = $postarray['owner-name'];
+			$retweet['owner-link']   = $postarray['owner-link'];
+			$retweet['owner-avatar'] = $postarray['owner-avatar'];
+
+			$postarray = $retweet;
+		}
+	}
+
+	if (!empty($post->quoted_status)) {
+		if ($noquote) {
+			// To avoid recursive share blocks we just provide the link to avoid removing quote context.
+			$postarray['body'] .= "\n\nhttps://twitter.com/" . $post->quoted_status->user->screen_name . "/status/" . $post->quoted_status->id_str;
+		} else {
+			$quoted = twitter_createpost(0, $post->quoted_status, $self, false, false, true);
+			if (!empty($quoted)) {
+				Item::insert($quoted);
+				$post = Post::selectFirst(['guid', 'uri-id'], ['uri' => $quoted['uri'], 'uid' => 0]);
+				Logger::info('Stored quoted post', ['uid' => $uid, 'uri-id' => $uriId, 'post' => $post]);
+
+				$postarray['body'] .= "\n" . BBCode::getShareOpeningTag(
+						$quoted['author-name'],
+						$quoted['author-link'],
+						$quoted['author-avatar'],
+						$quoted['plink'],
+						$quoted['created'],
+						$post['guid'] ?? ''
+					);
+
+				$postarray['body'] .= $quoted['body'] . '[/share]';
+			} else {
+				// Quoted post author is blocked/ignored, so we just provide the link to avoid removing quote context.
+				$postarray['body'] .= "\n\nhttps://twitter.com/" . $post->quoted_status->user->screen_name . '/status/' . $post->quoted_status->id_str;
+			}
+		}
+	}
+
+	return $postarray;
+}
+
+/**
+ * Store tags and mentions
+ *
+ * @param integer $uriId
+ * @param array $taglist
+ * @return void
+ */
+function twitter_store_tags(int $uriId, array $taglist)
+{
+	foreach ($taglist as $tag) {
+		Tag::storeByHash($uriId, $tag[0], $tag[1], $tag[2]);
+	}
+}
+
+function twitter_fetchparentposts(int $uid, $post, TwitterOAuth $connection, array $self)
+{
+	Logger::info('Fetching parent posts', ['user' => $uid, 'post' => $post->id_str]);
+
+	$posts = [];
+
+	while (!empty($post->in_reply_to_status_id_str)) {
+		try {
+			$post = twitter_statuses_show($post->in_reply_to_status_id_str, $connection);
+		} catch (TwitterOAuthException $e) {
+			Logger::notice('Error fetching parent post', ['uid' => $uid, 'post' => $post->id_str, 'message' => $e->getMessage()]);
+			break;
+		}
+
+		if (empty($post)) {
+			Logger::info("twitter_fetchparentposts: Can't fetch post");
+			break;
+		}
+
+		if (empty($post->id_str)) {
+			Logger::info('twitter_fetchparentposts: This is not a post', ['post' => $post]);
+			break;
+		}
+
+		if (Post::exists(['uri' => 'twitter::' . $post->id_str, 'uid' => $uid])) {
+			break;
+		}
+
+		$posts[] = $post;
+	}
+
+	Logger::info('twitter_fetchparentposts: Fetching ' . count($posts) . ' parents');
+
+	$posts = array_reverse($posts);
+
+	if (!empty($posts)) {
+		foreach ($posts as $post) {
+			$postarray = twitter_createpost($uid, $post, $self, false, !DI::pConfig()->get($uid, 'twitter', 'create_user'), false);
+
+			if (empty($postarray)) {
+				continue;
+			}
+
+			$item = Item::insert($postarray);
+
+			$postarray['id'] = $item;
+
+			Logger::notice('twitter_fetchparentpost: User ' . $self['nick'] . ' posted parent timeline item ' . $item);
+		}
+	}
+}
+
+/**
+ * Fetches the posts received by the Twitter user
+ *
+ * @param int $uid
+ * @return void
+ * @throws Exception
+ */
+function twitter_fetchhometimeline(int $uid): void
+{
+	$ckey    = DI::config()->get('twitter', 'consumerkey');
+	$csecret = DI::config()->get('twitter', 'consumersecret');
+	$otoken  = DI::pConfig()->get($uid, 'twitter', 'oauthtoken');
+	$osecret = DI::pConfig()->get($uid, 'twitter', 'oauthsecret');
+	$create_user = DI::pConfig()->get($uid, 'twitter', 'create_user');
+	$mirror_posts = DI::pConfig()->get($uid, 'twitter', 'mirror_posts');
+
+	Logger::info('Fetching timeline', ['uid' => $uid]);
+
+	$application_name = DI::keyValue()->get('twitter_application_name') ?? '';
+
+	if ($application_name == '') {
+		$application_name = DI::baseUrl()->getHost();
+	}
+
+	$connection = new TwitterOAuth($ckey, $csecret, $otoken, $osecret);
+
+	try {
+		$own_contact = twitter_fetch_own_contact($uid);
+	} catch (TwitterOAuthException $e) {
+		Logger::notice('Error fetching own contact', ['uid' => $uid, 'message' => $e->getMessage()]);
+		return;
+	}
+
+	$contact = Contact::selectFirst(['nick'], ['id' => $own_contact, 'uid' => $uid]);
+	if (DBA::isResult($contact)) {
+		$own_id = $contact['nick'];
+	} else {
+		Logger::notice('Own twitter contact not found', ['uid' => $uid]);
+		return;
+	}
+
+	$self = User::getOwnerDataById($uid);
+	if ($self === false) {
+		Logger::warning('Own contact not found', ['uid' => $uid]);
+		return;
+	}
+
+	$parameters = [
+		'exclude_replies' => false,
+		'trim_user' => false,
+		'contributor_details' => true,
+		'include_rts' => true,
+		'tweet_mode' => 'extended',
+		'include_ext_alt_text' => true,
+		//'count' => 200,
+	];
+
+	// Fetching timeline
+	$lastid = DI::pConfig()->get($uid, 'twitter', 'lasthometimelineid');
+
+	$first_time = ($lastid == '');
+
+	if ($lastid != '') {
+		$parameters['since_id'] = $lastid;
+	}
+
+	try {
+		$items = $connection->get('statuses/home_timeline', $parameters);
+	} catch (TwitterOAuthException $e) {
+		Logger::notice('Error fetching home timeline', ['uid' => $uid, 'message' => $e->getMessage()]);
+		return;
+	}
+
+	if (!is_array($items)) {
+		Logger::notice('home timeline is no array', ['items' => $items]);
+		return;
+	}
+
+	if (empty($items)) {
+		Logger::info('No new timeline content', ['uid' => $uid]);
+		return;
+	}
+
+	$posts = array_reverse($items);
+
+	Logger::notice('Processing timeline', ['lastid' => $lastid, 'uid' => $uid, 'count' => count($posts)]);
+
+	if (count($posts)) {
+		foreach ($posts as $post) {
+			if ($post->id_str > $lastid) {
+				$lastid = $post->id_str;
+				DI::pConfig()->set($uid, 'twitter', 'lasthometimelineid', $lastid);
+			}
+
+			if ($first_time) {
+				continue;
+			}
+
+			if (stristr($post->source, $application_name) && $post->user->screen_name == $own_id) {
+				Logger::info('Skip previously sent post');
+				continue;
+			}
+
+			if ($mirror_posts && $post->user->screen_name == $own_id && $post->in_reply_to_status_id_str == '') {
+				Logger::info('Skip post that will be mirrored');
+				continue;
+			}
+
+			if ($post->in_reply_to_status_id_str != '') {
+				twitter_fetchparentposts($uid, $post, $connection, $self);
+			}
+
+			Logger::info('Preparing post ' . $post->id_str . ' for user ' . $uid);
+
+			$postarray = twitter_createpost($uid, $post, $self, $create_user, true, false);
+
+			if (empty($postarray)) {
+				Logger::info('Empty post ' . $post->id_str . ' and user ' . $uid);
+				continue;
+			}
+
+			$notify = false;
+
+			if (empty($postarray['thr-parent'])) {
+				$contact = DBA::selectFirst('contact', [], ['id' => $postarray['contact-id'], 'self' => false]);
+				if (DBA::isResult($contact) && Item::isRemoteSelf($contact, $postarray)) {
+					$notify = Worker::PRIORITY_MEDIUM;
+				}
+			}
+
+			$postarray['wall'] = (bool)$notify;
+
+			$item = Item::insert($postarray, $notify);
+			$postarray['id'] = $item;
+
+			Logger::notice('User ' . $uid . ' posted home timeline item ' . $item);
+		}
+	}
+	DI::pConfig()->set($uid, 'twitter', 'lasthometimelineid', $lastid);
+
+	Logger::info('Last timeline ID for user ' . $uid . ' is now ' . $lastid);
+
+	// Fetching mentions
+	$lastid = DI::pConfig()->get($uid, 'twitter', 'lastmentionid');
+
+	$first_time = ($lastid == '');
+
+	if ($lastid != '') {
+		$parameters['since_id'] = $lastid;
+	}
+
+	try {
+		$items = $connection->get('statuses/mentions_timeline', $parameters);
+	} catch (TwitterOAuthException $e) {
+		Logger::notice('Error fetching mentions', ['uid' => $uid, 'message' => $e->getMessage()]);
+		return;
+	}
+
+	if (!is_array($items)) {
+		Logger::notice('mentions are no arrays', ['items' => $items]);
+		return;
+	}
+
+	$posts = array_reverse($items);
+
+	Logger::info('Fetching mentions for user ' . $uid . ' ' . sizeof($posts) . ' items');
+
+	if (count($posts)) {
+		foreach ($posts as $post) {
+			if ($post->id_str > $lastid) {
+				$lastid = $post->id_str;
+			}
+
+			if ($first_time) {
+				continue;
+			}
+
+			if ($post->in_reply_to_status_id_str != '') {
+				twitter_fetchparentposts($uid, $post, $connection, $self);
+			}
+
+			$postarray = twitter_createpost($uid, $post, $self, false, !$create_user, false);
+
+			if (empty($postarray)) {
+				continue;
+			}
+
+			$item = Item::insert($postarray);
+
+			Logger::notice('User ' . $uid . ' posted mention timeline item ' . $item);
+		}
+	}
+
+	DI::pConfig()->set($uid, 'twitter', 'lastmentionid', $lastid);
+
+	Logger::info('Last mentions ID for user ' . $uid . ' is now ' . $lastid);
+}
+
+function twitter_fetch_own_contact(int $uid)
+{
+	$ckey    = DI::config()->get('twitter', 'consumerkey');
+	$csecret = DI::config()->get('twitter', 'consumersecret');
+	$otoken  = DI::pConfig()->get($uid, 'twitter', 'oauthtoken');
+	$osecret = DI::pConfig()->get($uid, 'twitter', 'oauthsecret');
+
+	$own_id = DI::pConfig()->get($uid, 'twitter', 'own_id');
+
+	$contact_id = 0;
+
+	if ($own_id == '') {
+		$connection = new TwitterOAuth($ckey, $csecret, $otoken, $osecret);
+
+		// Fetching user data
+		// get() may throw TwitterOAuthException, but we will catch it later
+		$user = $connection->get('account/verify_credentials');
+		if (empty($user->id_str)) {
+			return false;
+		}
+
+		DI::pConfig()->set($uid, 'twitter', 'own_id', $user->id_str);
+
+		$contact_id = twitter_fetch_contact($uid, $user, true);
+	} else {
+		$contact = Contact::selectFirst(['id'], ['uid' => $uid, 'alias' => 'twitter::' . $own_id]);
+		if (DBA::isResult($contact)) {
+			$contact_id = $contact['id'];
+		} else {
+			DI::pConfig()->delete($uid, 'twitter', 'own_id');
+		}
+	}
+
+	return $contact_id;
+}
+
+function twitter_is_retweet(int $uid, string $body): bool
+{
+	$body = trim($body);
+
+	// Skip if it isn't a pure repeated messages
+	// Does it start with a share?
+	if (strpos($body, '[share') > 0) {
+		return false;
+	}
+
+	// Does it end with a share?
+	if (strlen($body) > (strrpos($body, '[/share]') + 8)) {
+		return false;
+	}
+
+	$attributes = preg_replace("/\[share(.*?)\]\s?(.*?)\s?\[\/share\]\s?/ism", "$1", $body);
+	// Skip if there is no shared message in there
+	if ($body == $attributes) {
+		return false;
+	}
+
+	$link = '';
+	preg_match("/link='(.*?)'/ism", $attributes, $matches);
+	if (!empty($matches[1])) {
+		$link = $matches[1];
+	}
+
+	preg_match('/link="(.*?)"/ism', $attributes, $matches);
+	if (!empty($matches[1])) {
+		$link = $matches[1];
+	}
+
+	$id = preg_replace("=https?://twitter.com/(.*)/status/(.*)=ism", "$2", $link);
+	if ($id == $link) {
+		return false;
+	}
+	return twitter_retweet($uid, $id);
+}
+
+function twitter_retweet(int $uid, int $id, int $item_id = 0): bool
+{
+	Logger::info('Retweeting', ['user' => $uid, 'id' => $id]);
+
+	$result = twitter_api_post('statuses/retweet', $id, $uid);
+
+	Logger::info('Retweeted', ['user' => $uid, 'id' => $id, 'result' => $result]);
+
+	if (!empty($item_id) && !empty($result->id_str)) {
+		Logger::notice('Update extid', ['id' => $item_id, 'extid' => $result->id_str]);
+		Item::update(['extid' => 'twitter::' . $result->id_str], ['id' => $item_id]);
+	}
+
+	return !isset($result->errors);
+}
+
+function twitter_update_mentions(string $body): string
+{
+	$URLSearchString = '^\[\]';
+	$return = preg_replace_callback(
+		"/@\[url\=([$URLSearchString]*)\](.*?)\[\/url\]/ism",
+		function ($matches) {
+			if (strpos($matches[1], 'twitter.com')) {
+				$return = '@' . substr($matches[1], strrpos($matches[1], '/') + 1);
+			} else {
+				$return = $matches[2] . ' (' . $matches[1] . ')';
+			}
+
+			return $return;
+		},
+		$body
+	);
+
+	return $return;
+}
+
+function twitter_convert_share(array $attributes, array $author_contact, string $content, bool $is_quote_share): string
+{
+	if (empty($author_contact)) {
+		return $content . "\n\n" . $attributes['link'];
+	}
+
+	if (!empty($author_contact['network']) && ($author_contact['network'] == Protocol::TWITTER)) {
+		$mention = '@' . $author_contact['nick'];
+	} else {
+		$mention = $author_contact['addr'];
+	}
+
+	return ($is_quote_share ? "\n\n" : '' ) . 'RT ' . $mention . ': ' . $content . "\n\n" . $attributes['link'];
+}