log uid but ignore results

Reviewed-on: friendica/friendica-addons#1387

bluesky/README.md

@@ -0,0 +1,10 @@
+Bluesky Addon
+==============
+
+This addon currently supports posting to Bluesky.
+The import is under development and will hopefully come soon.
+
+No setup is needed for the admins to make it work for their users.
+
+Bluesky itself is under development as well. It is planned to make it decentral.
+The addon is prepared to support different servers. But it isn't enabled yet.

bluesky/bluesky.php

@@ -0,0 +1,387 @@
+<?php
+/**
+ * Name: Bluesky Connector
+ * Description: Post to Bluesky
+ * Version: 1.0
+ * Author: Michael Vogel <https://pirati.ca/profile/heluecht>
+ */
+
+use Friendica\Content\Text\BBCode;
+use Friendica\Content\Text\Plaintext;
+use Friendica\Core\Config\Util\ConfigFileManager;
+use Friendica\Core\Hook;
+use Friendica\Core\Logger;
+use Friendica\Core\Renderer;
+use Friendica\DI;
+use Friendica\Model\Item;
+use Friendica\Model\Photo;
+use Friendica\Network\HTTPClient\Client\HttpClientAccept;
+use Friendica\Network\HTTPClient\Client\HttpClientOptions;
+use Friendica\Util\DateTimeFormat;
+
+function bluesky_install()
+{
+	Hook::register('load_config',             __FILE__, 'bluesky_load_config');
+	Hook::register('hook_fork',               __FILE__, 'bluesky_hook_fork');
+	Hook::register('post_local',              __FILE__, 'bluesky_post_local');
+	Hook::register('notifier_normal',         __FILE__, 'bluesky_send');
+	Hook::register('jot_networks',            __FILE__, 'bluesky_jot_nets');
+	Hook::register('connector_settings',      __FILE__, 'bluesky_settings');
+	Hook::register('connector_settings_post', __FILE__, 'bluesky_settings_post');
+}
+
+function bluesky_load_config(ConfigFileManager $loader)
+{
+	DI::app()->getConfigCache()->load($loader->loadAddonConfig('bluesky'), \Friendica\Core\Config\ValueObject\Cache::SOURCE_STATIC);
+}
+
+function bluesky_settings(array &$data)
+{
+	if (!DI::userSession()->getLocalUserId()) {
+		return;
+	}
+
+	$enabled     = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'post') ?? false;
+	$def_enabled = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'post_by_default') ?? false;
+	$host        = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'host') ?: 'https://bsky.social';
+	$handle      = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'handle');
+	$did         = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'did');
+	$token       = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'access_token');
+
+	$status = $token ? DI::l10n()->t("You are authenticated to Bluesky. For security reasons the password isn't stored.") : DI::l10n()->t('You are not authenticated. Please enter the app password.');
+
+	$t    = Renderer::getMarkupTemplate('connector_settings.tpl', 'addon/bluesky/');
+	$html = Renderer::replaceMacros($t, [
+		'$enable'    => ['bluesky', DI::l10n()->t('Enable Bluesky Post Addon'), $enabled],
+		'$bydefault' => ['bluesky_bydefault', DI::l10n()->t('Post to Bluesky by default'), $def_enabled],
+		'$host'      => ['bluesky_host', DI::l10n()->t('Bluesky host'), $host, '', '', 'readonly'],
+		'$handle'    => ['bluesky_handle', DI::l10n()->t('Bluesky handle'), $handle],
+		'$did'       => ['bluesky_did', DI::l10n()->t('Bluesky DID'), $did, DI::l10n()->t('This is the unique identifier. It will be fetched automatically, when the handle is entered.'), '', 'readonly'],
+		'$password'  => ['bluesky_password', DI::l10n()->t('Bluesky app password'), '', DI::l10n()->t("Please don't add your real password here, but instead create a specific app password in the Bluesky settings.")],
+		'$status'    => $status
+	]);
+
+	$data = [
+		'connector' => 'bluesky',
+		'title'     => DI::l10n()->t('Bluesky Export'),
+		'image'     => 'images/bluesky.jpg',
+		'enabled'   => $enabled,
+		'html'      => $html,
+	];
+}
+
+function bluesky_settings_post(array &$b)
+{
+	if (empty($_POST['bluesky-submit'])) {
+		return;
+	}
+
+	$old_host   = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'host');
+	$old_handle = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'handle');
+	$old_did    = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'did');
+
+	$host   = $_POST['bluesky_host'];
+	$handle = $_POST['bluesky_handle'];
+
+	DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'post',            intval($_POST['bluesky']));
+	DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'post_by_default', intval($_POST['bluesky_bydefault']));
+	DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'host',            $host);
+	DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'handle',          $handle);
+
+	if (!empty($host) && !empty($handle)) {
+		if (empty($old_did) || $old_host != $host || $old_handle != $handle) {
+			DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'bluesky', 'did', bluesky_get_did(DI::userSession()->getLocalUserId()));
+		}
+	} else {
+		DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'bluesky', 'did');
+	}
+
+	if (!empty($_POST['bluesky_password'])) {
+		bluesky_create_token(DI::userSession()->getLocalUserId(), $_POST['bluesky_password']);
+	}
+
+}
+
+function bluesky_jot_nets(array &$jotnets_fields)
+{
+	if (!DI::userSession()->getLocalUserId()) {
+		return;
+	}
+
+	if (DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'post')) {
+		$jotnets_fields[] = [
+			'type'  => 'checkbox',
+			'field' => [
+				'bluesky_enable',
+				DI::l10n()->t('Post to Bluesky'),
+				DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'post_by_default')
+			]
+		];
+	}
+}
+
+function bluesky_hook_fork(array &$b)
+{
+	if ($b['name'] != 'notifier_normal') {
+		return;
+	}
+
+	$post = $b['data'];
+
+	if (($post['created'] !== $post['edited']) && !$post['deleted']) {
+		DI::logger()->info('Editing is not supported by the addon');
+		$b['execute'] = false;
+		return;
+	}
+
+	if (!strstr($post['postopts'] ?? '', 'bluesky') || ($post['parent'] != $post['id']) || $post['private']) {
+		$b['execute'] = false;
+		return;
+	}
+}
+
+function bluesky_post_local(array &$b)
+{
+	if ($b['edit']) {
+		return;
+	}
+
+	if (!DI::userSession()->getLocalUserId() || (DI::userSession()->getLocalUserId() != $b['uid'])) {
+		return;
+	}
+
+	if ($b['private'] || $b['parent']) {
+		return;
+	}
+
+	$bluesky_post   = intval(DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'post'));
+	$bluesky_enable = (($bluesky_post && !empty($_REQUEST['bluesky_enable'])) ? intval($_REQUEST['bluesky_enable']) : 0);
+
+	// if API is used, default to the chosen settings
+	if ($b['api_source'] && intval(DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'bluesky', 'post_by_default'))) {
+		$bluesky_enable = 1;
+	}
+
+	if (!$bluesky_enable) {
+		return;
+	}
+
+	if (strlen($b['postopts'])) {
+		$b['postopts'] .= ',';
+	}
+
+	$b['postopts'] .= 'bluesky';
+}
+
+function bluesky_send(array &$b)
+{
+	if (($b['created'] !== $b['edited']) && !$b['deleted']) {
+		return;
+	}
+
+	if ($b['gravity'] != Item::GRAVITY_PARENT) {
+		return;
+	} elseif ($b['private'] || !strstr($b['postopts'], 'bluesky')) {
+		return;
+	}
+
+	bluesky_create_post($b);
+}
+
+function bluesky_create_post(array $item)
+{
+	$uid = $item['uid'];
+	$token = bluesky_get_token($uid);
+	if (empty($token)) {
+		return;
+	}
+
+	$did  = DI::pConfig()->get($uid, 'bluesky', 'did');
+
+	$msg = Plaintext::getPost($item, 300, false, BBCode::CONNECTORS);
+	$parent = $root = [];
+	foreach ($msg['parts'] as $key => $part) {
+		$record = [
+			'text'      => $part,
+			'createdAt' => DateTimeFormat::utcNow(DateTimeFormat::ATOM),
+			'$type'     => 'app.bsky.feed.post'
+		];
+
+		if (!empty($root)) {
+			$record['reply'] = ['root' => $root, 'parent' => $parent];
+		}
+
+		if ($key == count($msg['parts']) - 1) {
+			$record = bluesky_add_embed($uid, $msg, $record);
+		}
+
+		$post = [
+			'collection' => 'app.bsky.feed.post',
+			'repo'       => $did,
+			'record'     => $record
+		];
+
+		$parent = bluesky_post($uid, '/xrpc/com.atproto.repo.createRecord', json_encode($post), ['Content-type' => 'application/json', 'Authorization' => ['Bearer ' . $token]]);
+		if (empty($parent)) {
+			return;
+		}
+		Logger::debug('Posting done', ['return' => $parent]);
+		if (empty($root)) {
+			$root = $parent;
+		}
+	}
+}
+
+function bluesky_add_embed(int $uid, array $msg, array $record): array
+{
+	if (($msg['type'] != 'link') && !empty($msg['images'])) {
+		$images = [];
+		foreach ($msg['images'] as $image) {
+			$photo = Photo::selectFirst(['resource-id'], ['id' => $image['id']]);
+			$photo = Photo::selectFirst([], ["`resource-id` = ? AND `scale` > ?", $photo['resource-id'], 0], ['order' => ['scale']]);
+			$blob = bluesky_upload_blob($uid, $photo);
+			if (!empty($blob) && count($images) < 4) {
+				$images[] = ['alt' => $image['description'], 'image' => $blob];
+			}
+		}
+		if (!empty($images)) {
+			$record['embed'] = ['$type' => 'app.bsky.embed.images', 'images' => $images];
+		}
+	} elseif ($msg['type'] == 'link') {
+		$record['embed'] = [
+			'$type'    => 'app.bsky.embed.external',
+			'external' => [
+				'uri'         => $msg['url'],
+				'title'       => $msg['title'],
+				'description' => $msg['description'],
+			]
+		];
+		if (!empty($msg['image'])) {
+			$photo = Photo::createPhotoForExternalResource($msg['image']);
+			$blob = bluesky_upload_blob($uid, $photo);
+			if (!empty($blob)) {
+				$record['embed']['external']['thumb'] = $blob;
+			}
+		}
+	}
+	return $record;
+}
+
+function bluesky_upload_blob(int $uid, array $photo): ?stdClass
+{
+	$content = Photo::getImageForPhoto($photo);
+	$data = bluesky_post($uid, '/xrpc/com.atproto.repo.uploadBlob', $content, ['Content-type' => $photo['type'], 'Authorization' => ['Bearer ' . bluesky_get_token($uid)]]);
+	if (empty($data)) {
+		return null;
+	}
+
+	Logger::debug('Uploaded blob', ['return' => $data]);
+	return $data->blob;
+}
+
+function bluesky_get_timeline(int $uid)
+{
+	$data = bluesky_get($uid, '/xrpc/app.bsky.feed.getTimeline', HttpClientAccept::JSON, [HttpClientOptions::HEADERS => ['Authorization' => ['Bearer ' . bluesky_get_token($uid)]]]);
+	if (empty($data)) {
+		return;
+	}
+
+	if (empty($data->feed)) {
+		return;
+	}
+
+	foreach ($data->feed as $entry) {
+		// TODO Add Functionality to read the timeline
+		print_r($entry);
+	}
+}
+
+function bluesky_get_did(int $uid): string
+{
+	$data = bluesky_get($uid, '/xrpc/com.atproto.identity.resolveHandle?handle=' . DI::pConfig()->get($uid, 'bluesky', 'handle'));
+	if (empty($data)) {
+		return '';
+	}
+	Logger::debug('Got DID', ['return' => $data]);
+	return $data->did;
+}
+
+function bluesky_get_token(int $uid): string
+{
+	$token   = DI::pConfig()->get($uid, 'bluesky', 'access_token');
+	$created = DI::pConfig()->get($uid, 'bluesky', 'token_created');
+	if (empty($token)) {
+		return '';
+	}
+
+	if ($created + 300 < time()) {
+		return bluesky_refresh_token($uid);
+	}
+	return $token;
+}
+
+function bluesky_refresh_token(int $uid): string
+{
+	$token = DI::pConfig()->get($uid, 'bluesky', 'refresh_token');
+
+	$data = bluesky_post($uid, '/xrpc/com.atproto.server.refreshSession', '', ['Authorization' => ['Bearer ' . $token]]);
+	if (empty($data)) {
+		return '';
+	}
+
+	Logger::debug('Refreshed token', ['return' => $data]);
+	DI::pConfig()->set($uid, 'bluesky', 'access_token', $data->accessJwt);
+	DI::pConfig()->set($uid, 'bluesky', 'refresh_token', $data->refreshJwt);
+	DI::pConfig()->set($uid, 'bluesky', 'token_created', time());
+	return $data->accessJwt;
+}
+
+function bluesky_create_token(int $uid, string $password): string
+{
+	$did = DI::pConfig()->get($uid, 'bluesky', 'did');
+
+	$data = bluesky_post($uid, '/xrpc/com.atproto.server.createSession', json_encode(['identifier' => $did, 'password' => $password]), ['Content-type' => 'application/json']);
+	if (empty($data)) {
+		return '';
+	}
+
+	Logger::debug('Created token', ['return' => $data]);
+	DI::pConfig()->set($uid, 'bluesky', 'access_token', $data->accessJwt);
+	DI::pConfig()->set($uid, 'bluesky', 'refresh_token', $data->refreshJwt);
+	DI::pConfig()->set($uid, 'bluesky', 'token_created', time());
+	return $data->accessJwt;
+}
+
+function bluesky_post(int $uid, string $url, string $params, array $headers): ?stdClass
+{
+	try {
+		$curlResult = DI::httpClient()->post(DI::pConfig()->get($uid, 'bluesky', 'host') . $url, $params, $headers);
+	} catch (\Exception $e) {
+		Logger::notice('Exception on post', ['exception' => $e]);
+		return null;
+	}
+
+	if (!$curlResult->isSuccess()) {
+		Logger::notice('API Error', ['error' => json_decode($curlResult->getBody()) ?: $curlResult->getBody()]);
+		return null;
+	}
+
+	return json_decode($curlResult->getBody());
+}
+
+function bluesky_get(int $uid, string $url, string $accept_content = HttpClientAccept::DEFAULT, array $opts = []): ?stdClass
+{
+	try {
+		$curlResult = DI::httpClient()->get(DI::pConfig()->get($uid, 'bluesky', 'host') . $url, $accept_content, $opts);
+	} catch (\Exception $e) {
+		Logger::notice('Exception on get', ['exception' => $e]);
+		return null;
+	}
+
+	if (!$curlResult->isSuccess()) {
+		Logger::notice('API Error', ['error' => json_decode($curlResult->getBody()) ?: $curlResult->getBody()]);
+		return null;
+	}
+
+	return json_decode($curlResult->getBody());
+}

bluesky/lang/C/messages.po

@@ -0,0 +1,72 @@
+# ADDON bluesky
+# Copyright (C) 
+# This file is distributed under the same license as the Friendica bluesky addon package.
+# 
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: \n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2023-05-21 19:24+0000\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"Language: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+#: bluesky.php:51
+msgid ""
+"You are authenticated to Bluesky. For security reasons the password isn't "
+"stored."
+msgstr ""
+
+#: bluesky.php:51
+msgid "You are not authenticated. Please enter the app password."
+msgstr ""
+
+#: bluesky.php:55
+msgid "Enable Bluesky Post Addon"
+msgstr ""
+
+#: bluesky.php:56
+msgid "Post to Bluesky by default"
+msgstr ""
+
+#: bluesky.php:57
+msgid "Bluesky host"
+msgstr ""
+
+#: bluesky.php:58
+msgid "Bluesky handle"
+msgstr ""
+
+#: bluesky.php:59
+msgid "Bluesky DID"
+msgstr ""
+
+#: bluesky.php:59
+msgid ""
+"This is the unique identifier. It will be fetched automatically, when the "
+"handle is entered."
+msgstr ""
+
+#: bluesky.php:60
+msgid "Bluesky app password"
+msgstr ""
+
+#: bluesky.php:60
+msgid ""
+"Please don't add your real password here, but instead create a specific app "
+"password in the Bluesky settings."
+msgstr ""
+
+#: bluesky.php:66
+msgid "Bluesky Export"
+msgstr ""
+
+#: bluesky.php:116
+msgid "Post to Bluesky"
+msgstr ""

bluesky/templates/connector_settings.tpl

@@ -0,0 +1,7 @@
+<p>{{$status}}</p>
+{{include file="field_checkbox.tpl" field=$enable}}
+{{include file="field_checkbox.tpl" field=$bydefault}}
+{{include file="field_input.tpl" field=$host}}
+{{include file="field_input.tpl" field=$handle}}
+{{include file="field_input.tpl" field=$did}}
+{{include file="field_input.tpl" field=$password}}

ifttt/ifttt.php

@@ -180,5 +180,5 @@ function ifttt_message($uid, $item)
 		$link = hash('ripemd128', $item['msg']);
 	}
 
-	Post\Delayed::add($link, $post, Worker::PRIORITY_MEDIUM, Post\Delayed::PREPARED);
+	Post\Delayed::add($link, $post, Worker::PRIORITY_MEDIUM, Post\Delayed::UNPREPARED);
 }

mailstream/lang/hu/messages.po

@@ -4,15 +4,15 @@
 # 
 # 
 # Translators:
-# Balázs Úr, 2020-2021
+# Balázs Úr, 2020-2021,2023
 msgid ""
 msgstr ""
 "Project-Id-Version: friendica\n"
 "Report-Msgid-Bugs-To: \n"
 "POT-Creation-Date: 2021-11-21 19:15-0500\n"
 "PO-Revision-Date: 2014-06-23 09:54+0000\n"
-"Last-Translator: Balázs Úr, 2020-2021\n"
-"Language-Team: Hungarian (http://www.transifex.com/Friendica/friendica/language/hu/)\n"
+"Last-Translator: Balázs Úr, 2020-2021,2023\n"
+"Language-Team: Hungarian (http://app.transifex.com/Friendica/friendica/language/hu/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
@@ -31,68 +31,72 @@ msgstr "E-mail-cím, ahonnan úgy tűnik, hogy a folyam elemei származnak."
 msgid "Save Settings"
 msgstr "Beállítások mentése"
 
-#: mailstream.php:301
+#: mailstream.php:311
 msgid "Re:"
 msgstr "Vá:"
 
-#: mailstream.php:314 mailstream.php:317
+#: mailstream.php:324 mailstream.php:327
 msgid "Friendica post"
 msgstr "Friendica-bejegyzés"
 
-#: mailstream.php:320
+#: mailstream.php:330
 msgid "Diaspora post"
 msgstr "Diaspora-bejegyzés"
 
-#: mailstream.php:330
+#: mailstream.php:340
 msgid "Feed item"
 msgstr "Hírforráselem"
 
-#: mailstream.php:333
+#: mailstream.php:343
 msgid "Email"
 msgstr "E-mail"
 
-#: mailstream.php:335
+#: mailstream.php:345
 msgid "Friendica Item"
 msgstr "Friendica-elem"
 
-#: mailstream.php:404
+#: mailstream.php:419
 msgid "Upstream"
 msgstr "Távoli"
 
-#: mailstream.php:405
+#: mailstream.php:420
+msgid "URI"
+msgstr "URI"
+
+#: mailstream.php:421
 msgid "Local"
 msgstr "Helyi"
 
-#: mailstream.php:481
+#: mailstream.php:499
 msgid "Enabled"
 msgstr "Engedélyezve"
 
-#: mailstream.php:486
+#: mailstream.php:504
 msgid "Email Address"
 msgstr "E-mail-cím"
 
-#: mailstream.php:488
+#: mailstream.php:506
 msgid "Leave blank to use your account email address"
 msgstr "Hagyja üresen a fiókja e-mail-címének használatához"
 
-#: mailstream.php:492
+#: mailstream.php:510
 msgid "Exclude Likes"
 msgstr "Kedvelések kizárása"
 
-#: mailstream.php:494
+#: mailstream.php:512
 msgid "Check this to omit mailing \"Like\" notifications"
 msgstr "Jelölje be ezt a „Tetszik” értesítések elküldésének kihagyásához"
 
-#: mailstream.php:498
+#: mailstream.php:516
 msgid "Attach Images"
 msgstr "Képek csatolása"
 
-#: mailstream.php:500
+#: mailstream.php:518
 msgid ""
 "Download images in posts and attach them to the email.  Useful for reading "
 "email while offline."
 msgstr "Képek letöltése a bejegyzésekből és csatolás az e-mailhez. Hasznos az e-mailek kapcsolat nélküli olvasásakor."
 
-#: mailstream.php:507
+#: mailstream.php:525
 msgid "Mail Stream Settings"
 msgstr "Levelezőfolyam beállításai"

mailstream/lang/hu/strings.php

@@ -15,6 +15,7 @@ $a->strings['Feed item'] = 'Hírforráselem';
 $a->strings['Email'] = 'E-mail';
 $a->strings['Friendica Item'] = 'Friendica-elem';
 $a->strings['Upstream'] = 'Távoli';
+$a->strings['URI'] = 'URI';
 $a->strings['Local'] = 'Helyi';
 $a->strings['Enabled'] = 'Engedélyezve';
 $a->strings['Email Address'] = 'E-mail-cím';

mailstream/mailstream.php

@@ -66,10 +66,9 @@ function mailstream_module() {}
 /**
  * Adds an item in "addon features" in the admin menu of the site
  *
- * @param App $a App object (unused)
  * @param string        $o HTML form data
  */
-function mailstream_addon_admin(App $a, string &$o)
+function mailstream_addon_admin(string &$o)
 {
 	$frommail = DI::config()->get('mailstream', 'frommail');
 	$template = Renderer::getMarkupTemplate('admin.tpl', 'addon/mailstream/');
@@ -103,14 +102,14 @@ function mailstream_addon_admin_post()
  */
 function mailstream_generate_id(string $uri): string
 {
-	$host = DI::baseUrl()->getHostname();
+	$host = DI::baseUrl()->getHost();
 	$resource = hash('md5', $uri);
 	$message_id = "<" . $resource . "@" . $host . ">";
 	Logger::debug('mailstream: Generated message ID ' . $message_id . ' for URI ' . $uri);
 	return $message_id;
 }
 
-function mailstream_send_hook(App $a, array $data)
+function mailstream_send_hook(array $data)
 {
 	$criteria = array('uid' => $data['uid'], 'contact-id' => $data['contact-id'], 'uri' => $data['uri']);
 	$item = Post::selectFirst([], $criteria);
@@ -138,17 +137,17 @@ function mailstream_send_hook(App $a, array $data)
  * mailstream is enabled and the necessary data is available, forks a
  * workerqueue item to send the email.
  *
- * @param App $a    App object (unused)
  * @param array     $item content of the item (may or may not already be stored in the item table)
  * @return void
  */
-function mailstream_post_hook(App $a, array &$item)
+function mailstream_post_hook(array &$item)
 {
 	mailstream_check_version();
+	Logger::debug('@@@ mailstream_post_hook', ['item-uid' => $item['uid']]);
 
 	if (!DI::pConfig()->get($item['uid'], 'mailstream', 'enabled')) {
-		Logger::debug('mailstream: not enabled.', ['item' => $item['id'], ' uid ' => $item['uid']]);
-		return;
+		Logger::debug('mailstream: not enabled for item ' . $item['id'] . ' uid ' . $item['uid']);
+		// return;
 	}
 	if (!$item['uid']) {
 		Logger::debug('mailstream: no uid for item ' . $item['id']);
@@ -414,7 +413,7 @@ function mailstream_send(string $message_id, array $item, array $user): bool
 		$template = Renderer::getMarkupTemplate('mail.tpl', 'addon/mailstream/');
 		$mail->AltBody = BBCode::toPlaintext($item['body']);
 		$item['body'] = BBCode::convertForUriId($item['uri-id'], $item['body'], BBCode::CONNECTORS);
-		$item['url'] = DI::baseUrl()->get() . '/display/' . $item['guid'];
+		$item['url'] = DI::baseUrl() . '/display/' . $item['guid'];
 		$mail->Body = Renderer::replaceMacros($template, [
 						 '$upstream' => DI::l10n()->t('Upstream'),
 						 '$uri' => DI::l10n()->t('URI'),
@@ -468,7 +467,7 @@ function mailstream_convert_table_entries()
 					'message_id' => $ms_item_id['message-id'],
 					'tries' => 0);
 		if (!$ms_item_id['message-id'] || !strlen($ms_item_id['message-id'])) {
-			Logger::info('mailstream_convert_table_entries: item has no message-id.', 'item' => $ms_item_id['id'], 'uri' => $ms_item_id['uri']]);
+			Logger::info('mailstream_convert_table_entries: item has no message-id.', ['item' => $ms_item_id['id'], 'uri' => $ms_item_id['uri']]);
 							continue;
 		}
 		Logger::info('mailstream_convert_table_entries: convert item to workerqueue', $send_hook_data);
@@ -480,11 +479,10 @@ function mailstream_convert_table_entries()
 /**
  * Form for configuring mailstream features for a user
  *
- * @param App   $a    App object
  * @param array $data Hook data array
  * @throws \Friendica\Network\HTTPException\ServiceUnavailableException
  */
-function mailstream_addon_settings(App &$a, array &$data)
+function mailstream_addon_settings(array &$data)
 {
 	$enabled   = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'mailstream', 'enabled');
 	$address   = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'mailstream', 'address');
@@ -528,11 +526,10 @@ function mailstream_addon_settings(App &$a, array &$data)
 
 /**
  * Process data submitted to user's mailstream features form
- * @param App $a
  * @param array          $post POST data
  * @return void
  */
-function mailstream_addon_settings_post(App $a, array $post)
+function mailstream_addon_settings_post(array $post)
 {
 	if (!DI::userSession()->getLocalUserId() || empty($post['mailstream-submit'])) {
 		return;

phototrack/database.sql

@@ -0,0 +1,23 @@
+CREATE TABLE IF NOT EXISTS `phototrack_photo_use` (
+       `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+       `resource-id` char(64) NOT NULL,
+       `table` char(64) NOT NULL,
+       `field` char(64) NOT NULL,
+       `row-id` int(11) NOT NULL,
+       `checked` timestamp NOT NULL DEFAULT now(),
+       PRIMARY KEY (`id`),
+       INDEX `resource-id` (`resource-id`),
+       INDEX `row` (`table`,`field`,`row-id`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+CREATE TABLE IF NOT EXISTS `phototrack_row_check` (
+       `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+       `table` char(64) NOT NULL,
+       `row-id` int(11) NOT NULL,
+       `checked` timestamp NOT NULL DEFAULT now(),
+       PRIMARY KEY (`id`),
+       INDEX `row` (`table`,`row-id`),
+       INDEX `checked` (`checked`)
+) ENGINE=MyISAM DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+SELECT TRUE

phototrack/phototrack.php

@@ -0,0 +1,274 @@
+<?php
+/**
+ * Name: Photo Track
+ * Description: Track which photos are actually being used and delete any others
+ * Version: 1.0
+ * Author: Matthew Exon <http://mat.exon.name>
+ */
+
+/*
+ * List of tables and the fields that are checked:
+ * 
+ * contact: photo thumb micro about
+ * fcontact: photo
+ * fsuggest: photo
+ * gcontact: photo about
+ * item: body
+ * item-content: body
+ * mail: from-photo
+ * notify: photo
+ * profile: photo thumb about
+ */
+
+use Friendica\Core\Addon;
+use Friendica\Core\Logger;
+use Friendica\Object\Image;
+use Friendica\Database\DBA;
+use Friendica\Util\Images;
+use Friendica\Util\DateTimeFormat;
+use Friendica\DI;
+
+if (!defined('PHOTOTRACK_DEFAULT_BATCH_SIZE')) {
+    define('PHOTOTRACK_DEFAULT_BATCH_SIZE', 1000);
+}
+// Time in *minutes* between searching for photo uses
+if (!defined('PHOTOTRACK_DEFAULT_SEARCH_INTERVAL')) {
+    define('PHOTOTRACK_DEFAULT_SEARCH_INTERVAL', 10);
+}
+
+function phototrack_install() {
+    global $db;
+
+    Addon::registerHook('post_local_end', 'addon/phototrack/phototrack.php', 'phototrack_post_local_end');
+    Addon::registerHook('post_remote_end', 'addon/phototrack/phototrack.php', 'phototrack_post_remote_end');
+    Addon::registerHook('notifier_end', 'addon/phototrack/phototrack.php', 'phototrack_notifier_end');
+    Addon::registerHook('cron', 'addon/phototrack/phototrack.php', 'phototrack_cron');
+
+    if (DI::config()->get('phototrack', 'dbversion') != '0.1') {
+        $schema = file_get_contents(dirname(__file__).'/database.sql');
+        $arr = explode(';', $schema);
+        foreach ($arr as $a) {
+            if (!DBA::e($a)) {
+                Logger::warning('Unable to create database table: ' . DBA::errorMessage());
+                return;
+            }
+        }
+        DI::config()->set('phototrack', 'dbversion', '0.1');
+    }
+}
+
+function phototrack_uninstall() {
+    Addon::unregisterHook('post_local_end', 'addon/phototrack/phototrack.php', 'phototrack_post_local_end');
+    Addon::unregisterHook('post_remote_end', 'addon/phototrack/phototrack.php', 'phototrack_post_remote_end');
+    Addon::unregisterHook('notifier_end', 'addon/phototrack/phototrack.php', 'phototrack_notifier_end');
+    Addon::unregisterHook('cron', 'addon/phototrack/phototrack.php', 'phototrack_cron');
+}
+
+function phototrack_module() {}
+
+function phototrack_finished_row($table, $id) {
+    $existing = DBA::selectFirst('phototrack_row_check', ['id'], ['table' => $table, 'row-id' => $id]);
+    if (!is_bool($existing)) {
+        DBA::update('phototrack_row_check', ['checked' => DateTimeFormat::utcNow()], ['table' => $table, 'row-id' => $id]);
+    }
+    else {
+        DBA::insert('phototrack_row_check', ['table' => $table, 'row-id' => $id, 'checked' => DateTimeFormat::utcNow()]);
+    }
+}
+
+function phototrack_photo_use($photo, $table, $field, $id) {
+    Logger::debug('@@@ phototrack_photo_use ' . $photo);
+    foreach (Images::supportedTypes() as $m => $e) {
+        $photo = str_replace(".$e", '', $photo);
+    }
+    if (substr($photo, -2, 1) == '-') {
+        $resolution = intval(substr($photo,-1,1));
+        $photo = substr($photo,0,-2);
+    }
+    if (strlen($photo) != 32) {
+        return;
+    }
+    $r = DBA::selectFirst('photo', ['resource-id'], ['resource-id' => $photo]);
+    if (!DBA::isResult($r)) {
+        return;
+    }
+    $rid = $r['resource-id'];
+    $existing = DBA::selectFirst('phototrack_photo_use', ['id'], ['resource-id' => $rid, 'table' => $table, 'field' => $field, 'row-id' => $id]);
+    if (DBA::isResult($existing)) {
+        DBA::update('phototrack_photo_use', ['checked' => DateTimeFormat::utcNow()], ['resource-id' => $rid, 'table' => $table, 'field' => $field, 'row-id' => $id]);
+    }
+    else {
+        DBA::insert('phototrack_photo_use', ['resource-id' => $rid, 'table' => $table, 'field' => $field, 'row-id' => $id, 'checked' => DateTimeFormat::utcNow()]);
+    }
+}
+
+function phototrack_check_field_url($a, $table, $field, $id, $url) {
+    Logger::info('@@@ phototrack_check_field_url table ' . $table . ' field ' . $field . ' id ' . $id . ' url ' . $url);
+    $baseurl = DI::baseUrl()->get(true);
+    if (strpos($url, $baseurl) === FALSE) {
+        return;
+    }
+    else {
+        $url = substr($url, strlen($baseurl));
+        Logger::info('@@@ phototrack_check_field_url funny url stuff ' . $url . ' base ' . $baseurl);
+    }
+    if (strpos($url, '/photo/') === FALSE) {
+        return;
+    }
+    else {
+        $url = substr($url, strlen('/photo/'));
+        Logger::info('@@@ phototrack_check_field_url more url stuff ' . $url);
+    }
+    if (preg_match('/([0-9a-z]{32})/', $url, $matches)) {
+        $rid = $matches[0];
+        Logger::info('@@@ phototrack_check_field_url rid ' . $rid);
+        phototrack_photo_use($rid, $table, $field, $id);
+    }
+}
+
+function phototrack_check_field_bbcode($a, $table, $field, $id, $value) {
+    $baseurl = DI::baseUrl()->get(true);
+    $matches = array();
+    preg_match_all("/\[img(\=([0-9]*)x([0-9]*))?\](.*?)\[\/img\]/ism", $value, $matches);
+    foreach ($matches[4] as $url) {
+        phototrack_check_field_url($a, $table, $field, $id, $url);
+    }
+}
+
+function phototrack_post_local_end(&$a, &$item) {
+    phototrack_check_row($a, 'item', $item);
+    phototrack_check_row($a, 'item-content', $item);
+}
+
+function phototrack_post_remote_end(&$a, &$item) {
+    phototrack_check_row($a, 'item', $item);
+    phototrack_check_row($a, 'item-content', $item);
+}
+
+function phototrack_notifier_end($item) {
+}
+
+function phototrack_check_row($a, $table, $row) {
+    switch ($table) {
+        case 'item':
+            $fields = array(
+                'body' => 'bbcode');
+            break;
+        case 'item-content':
+            $fields = array(
+                'body' => 'bbcode');
+            break;
+        case 'contact':
+            $fields = array(
+                'photo' => 'url',
+                'thumb' => 'url',
+                'micro' => 'url',
+                'about' => 'bbcode');
+            break;
+        case 'fcontact':
+            $fields = array(
+                'photo' => 'url');
+            break;
+        case 'fsuggest':
+            $fields = array(
+                'photo' => 'url');
+            break;
+        case 'gcontact':
+            $fields = array(
+                'photo' => 'url',
+                'about' => 'bbcode');
+            break;
+        default: $fields = array(); break;
+    }
+    foreach ($fields as $field => $type) {
+        switch ($type) {
+            case 'bbcode': phototrack_check_field_bbcode($a, $table, $field, $row['id'], $row[$field]); break;
+            case 'url': phototrack_check_field_url($a, $table, $field, $row['id'], $row[$field]); break;
+        }
+    }
+    phototrack_finished_row($table, $row['id']);
+}
+
+function phototrack_batch_size() {
+    $batch_size = DI::config()->get('phototrack', 'batch_size');
+    if ($batch_size > 0) {
+        return $batch_size;
+    }
+    return PHOTOTRACK_DEFAULT_BATCH_SIZE;
+}
+
+function phototrack_search_table($a, $table) {
+    $batch_size = phototrack_batch_size();
+    $rows = DBA::p("SELECT `$table`.* FROM `$table` LEFT OUTER JOIN phototrack_row_check ON ( phototrack_row_check.`table` = '$table' AND phototrack_row_check.`row-id` = `$table`.id ) WHERE ( ( phototrack_row_check.checked IS NULL ) OR ( phototrack_row_check.checked < DATE_SUB(NOW(), INTERVAL 1 MONTH) ) ) ORDER BY phototrack_row_check.checked LIMIT $batch_size");
+    if (DBA::isResult($rows)) {
+        while ($row = DBA::fetch($rows)) {
+            phototrack_check_row($a, $table, $row);
+        }
+    }
+    $r = DBA::p("SELECT COUNT(*) FROM `$table` LEFT OUTER JOIN phototrack_row_check ON ( phototrack_row_check.`table` = '$table' AND phototrack_row_check.`row-id` = `$table`.id ) WHERE ( ( phototrack_row_check.checked IS NULL ) OR ( phototrack_row_check.checked < DATE_SUB(NOW(), INTERVAL 1 MONTH) ) )");
+    Logger::info("@@@ phototrack_search_table " . print_r(DBA::fetch($r)));
+    $remaining = DBA::fetch($r)['count'];
+    Logger::info('phototrack: searched ' . DBA::numRows($rows) . ' rows in table ' . $table . ', ' . $remaining . ' still remaining to search');
+    return $remaining;
+}
+
+function phototrack_cron_time() {
+    $prev_remaining = DI::config()->get('phototrack', 'remaining_items');
+    if ($prev_remaining > 10 * phototrack_batch_size()) {
+        Logger::debug('phototrack: more than ' . (10 * phototrack_batch_size()) . ' items remaining');
+        return true;
+    }
+    $last = DI::config()->get('phototrack', 'last_search');
+    $search_interval = intval(DI::config()->get('phototrack', 'search_interval'));
+    if (!$search_interval) {
+        $search_interval = PHOTOTRACK_DEFAULT_SEARCH_INTERVAL;
+    }
+    if ($last) {
+        $next = $last + ($search_interval * 60);
+        if ($next > time()) {
+            Logger::debug('phototrack: search interval not reached');
+            return false;
+        }
+    }
+    return true;
+}
+
+function phototrack_cron($a, $b) {
+    if (!phototrack_cron_time()) {
+        return;
+    }
+    DI::config()->set('phototrack', 'last_search', time());
+
+    $remaining = 0;
+    $remaining += phototrack_search_table($a, 'item');
+    $remaining += phototrack_search_table($a, 'item-content');
+    $remaining += phototrack_search_table($a, 'contact');
+    $remaining += phototrack_search_table($a, 'fcontact');
+    $remaining += phototrack_search_table($a, 'fsuggest');
+    $remaining += phototrack_search_table($a, 'gcontact');
+
+    DI::config()->set('phototrack', 'remaining_items', $remaining);
+    if ($remaining === 0) {
+        phototrack_tidy();
+    }
+}
+
+function phototrack_tidy() {
+    $batch_size = phototrack_batch_size();
+    DBA::e('CREATE TABLE IF NOT EXISTS `phototrack-temp` (`resource-id` char(255) not null)');
+    DBA::e('INSERT INTO `phototrack-temp` SELECT DISTINCT(`resource-id`) FROM photo WHERE photo.`created` < DATE_SUB(NOW(), INTERVAL 2 MONTH)');
+    $rows = DBA::p('SELECT `phototrack-temp`.`resource-id` FROM `phototrack-temp` LEFT OUTER JOIN phototrack_photo_use ON (`phototrack-temp`.`resource-id` = phototrack_photo_use.`resource-id`) WHERE phototrack_photo_use.id IS NULL limit ' . /*$batch_size*/1000);
+    if (DBA::isResult($rows)) {
+        foreach ($rows as $row) {
+            Logger::debug('phototrack: remove photo ' . $row['resource-id']);
+            DBA::e('DELETE FROM photo WHERE `resource-id` = "' . $row['resource-id'] . '"');
+        }
+        Logger::info('phototrack_tidy: deleted ' . DBA::numRows($rows) . ' photos');
+    }
+    DBA::e('DROP TABLE `phototrack-temp`');
+    $rows = DBA::p('SELECT id FROM phototrack_photo_use WHERE checked < DATE_SUB(NOW(), INTERVAL 14 DAY)');
+    foreach ($rows as $row) {
+        DBA::e( 'DELETE FROM phototrack_photo_use WHERE id = ' . $row['id']);
+    }
+    Logger::info('phototrack_tidy: deleted ' . DBA::numRows($rows) . ' phototrack_photo_use rows');
+}

piwik/README.md

@@ -31,7 +31,7 @@ Open the `config/node.config.php` file and add "piwik" to the list of activated
 		],
 	]
 
-You can change 4 more configuration variables for the addon in the `config/piwik.config.php` file:
+You can change 5 more configuration variables for the addon in the `config/piwik.config.php` file:
 
 	return [
 		'piwik' => [
@@ -39,6 +39,7 @@ You can change 4 more configuration variables for the addon in the `config/piwik
 			'sideid' => 1,
 			'optout' => true,
 			'async' => false,
+			'shortendpoint' => false,
 		],
 	];
 
@@ -50,7 +51,7 @@ Configuration fields
 * The *optout* parameter (true|false) defines whether or not a short notice about the utilization of Piwik will be displayed on every page of your Friendica site (at the bottom of the page with some spacing to the
 other content). Part of the note is a link that allows the visitor to set an _opt-out_ cookie which will prevent visits from that user be tracked by piwik.
 * The *async* parameter (true|false) defines whether or not to use asynchronous tracking so pages load (or appear to load) faster.
-
+* The *shortendpoint* parameter (true|false) defines whether or not to use a short path to the tracking script: "/js/" instead of "/piwik.js".
 Currently the optional notice states the following:
 
 >    This website is tracked using the Piwik analytics tool. If you do not want that your visits are logged this way you can set a cookie to prevent Piwik from tracking further visits of the site (opt-out).

piwik/config/piwik.config.php

@@ -25,5 +25,9 @@ return [
 		// async (Boolean)
 		// This defines whether or not to use asynchronous tracking so pages load (or appear to load) faster.
 		'async' => false,
+
+		// shortendpoint (Boolean)
+		// This defines whether or not to use a short path to the tracking script: "/js/" instead of "/piwik.js".
+		'shortendpoint' => false,
 	],
 ];

piwik/lang/C/messages.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: \n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2021-02-01 18:15+0100\n"
+"POT-Creation-Date: 2023-05-01 07:39+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -17,13 +17,13 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 
-#: piwik.php:87
+#: piwik.php:96
 msgid ""
 "This website is tracked using the <a href='http://www.matomo.org'>Matomo</a> "
 "analytics tool."
 msgstr ""
 
-#: piwik.php:90
+#: piwik.php:99
 #, php-format
 msgid ""
 "If you do not want that your visits are logged in this way you <a "
@@ -31,28 +31,32 @@ msgid ""
 "visits of the site</a> (opt-out)."
 msgstr ""
 
-#: piwik.php:97
+#: piwik.php:108
 msgid "Save Settings"
 msgstr ""
 
-#: piwik.php:98
+#: piwik.php:109
 msgid "Matomo (Piwik) Base URL"
 msgstr ""
 
-#: piwik.php:98
+#: piwik.php:109
 msgid ""
 "Absolute path to your Matomo (Piwik) installation. (without protocol (http/"
 "s), with trailing slash)"
 msgstr ""
 
-#: piwik.php:99
+#: piwik.php:110
 msgid "Site ID"
 msgstr ""
 
-#: piwik.php:100
+#: piwik.php:111
 msgid "Show opt-out cookie link?"
 msgstr ""
 
-#: piwik.php:101
+#: piwik.php:112
 msgid "Asynchronous tracking"
 msgstr ""
+
+#: piwik.php:113
+msgid "Shortcut path to the script ('/js/' instead of '/piwik.js')"
+msgstr ""

piwik/lang/de/messages.po

@@ -7,15 +7,15 @@
 # Andreas H., 2014-2015
 # Till Mohr <tmtrfx@till-mohr.de>, 2021
 # Tobias Diekershoff <tobias.diekershoff@gmx.net>, 2014
-# Tobias Diekershoff <tobias.diekershoff@gmx.net>, 2019
+# Tobias Diekershoff <tobias.diekershoff@gmx.net>, 2019,2023
 # Ulf Rompe <transifex.com@rompe.org>, 2019
 msgid ""
 msgstr ""
 "Project-Id-Version: friendica\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2021-02-01 18:15+0100\n"
+"POT-Creation-Date: 2023-05-01 07:39+0200\n"
 "PO-Revision-Date: 2014-06-23 11:18+0000\n"
-"Last-Translator: Till Mohr <tmtrfx@till-mohr.de>, 2021\n"
+"Last-Translator: Tobias Diekershoff <tobias.diekershoff@gmx.net>, 2019,2023\n"
 "Language-Team: German (http://app.transifex.com/Friendica/friendica/language/de/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
@@ -23,13 +23,13 @@ msgstr ""
 "Language: de\n"
 "Plural-Forms: nplurals=2; plural=(n != 1);\n"
 
-#: piwik.php:87
+#: piwik.php:96
 msgid ""
 "This website is tracked using the <a href='http://www.matomo.org'>Matomo</a>"
 " analytics tool."
 msgstr "Diese Website benutzt <a href='http://www.matomo.org'>Matomo</a>, eine Open-Source-Software zur statistischen Auswertung der Besucherzugriffe."
 
-#: piwik.php:90
+#: piwik.php:99
 #, php-format
 msgid ""
 "If you do not want that your visits are logged in this way you <a "
@@ -37,28 +37,32 @@ msgid ""
 "visits of the site</a> (opt-out)."
 msgstr "Wenn du nicht willst, dass Deine Besuche auf diese Weise gespeichert werden, kannst du <a href='%s'>ein Cookie setzen</a>. Dann wird Matomo / Piwik dich auf dieser Website nicht mehr verfolgen (opt-out)."
 
-#: piwik.php:97
+#: piwik.php:108
 msgid "Save Settings"
 msgstr "Einstellungen speichern"
 
-#: piwik.php:98
+#: piwik.php:109
 msgid "Matomo (Piwik) Base URL"
 msgstr "Matomo-Basis-URL (Piwik-Basis-URL)"
 
-#: piwik.php:98
+#: piwik.php:109
 msgid ""
 "Absolute path to your Matomo (Piwik) installation. (without protocol "
 "(http/s), with trailing slash)"
 msgstr "Absoluter Pfad zu deiner Matomo-/Piwik-Installation (ohne \"http://\" oder \"https://\"), mit abschließendem Schrägstrich"
 
-#: piwik.php:99
+#: piwik.php:110
 msgid "Site ID"
 msgstr "Seiten-ID"
 
-#: piwik.php:100
+#: piwik.php:111
 msgid "Show opt-out cookie link?"
 msgstr "Link zum Setzen des Opt-Out-Cookies anzeigen?"
 
-#: piwik.php:101
+#: piwik.php:112
 msgid "Asynchronous tracking"
 msgstr "Asynchrones Tracking"
+
+#: piwik.php:113
+msgid "Shortcut path to the script ('/js/' instead of '/piwik.js')"
+msgstr "Shortcut Pfad zum Script ('/js/' anstelle von '/piwik.js')"

piwik/lang/de/strings.php

@@ -13,3 +13,4 @@ $a->strings['Absolute path to your Matomo (Piwik) installation. (without protoco
 $a->strings['Site ID'] = 'Seiten-ID';
 $a->strings['Show opt-out cookie link?'] = 'Link zum Setzen des Opt-Out-Cookies anzeigen?';
 $a->strings['Asynchronous tracking'] = 'Asynchrones Tracking';
+$a->strings['Shortcut path to the script (\'/js/\' instead of \'/piwik.js\')'] = 'Shortcut Pfad zum Script (\'/js/\' anstelle von \'/piwik.js\')';

piwik/lang/en-gb/messages.po

@@ -20,13 +20,13 @@ msgstr ""
 "Language: en_GB\n"
 "Plural-Forms: nplurals=2; plural=(n != 1);\n"
 
-#: piwik.php:94
+#: piwik.php:95
 msgid ""
 "This website is tracked using the <a href='http://www.matomo.org'>Matomo</a>"
 " analytics tool."
 msgstr "This website is tracking, using the <a href='http://www.matomo.org'>Matomo</a> analytics tool."
 
-#: piwik.php:97
+#: piwik.php:98
 #, php-format
 msgid ""
 "If you do not want that your visits are logged in this way you <a "
@@ -34,32 +34,36 @@ msgid ""
 "visits of the site</a> (opt-out)."
 msgstr "If you do not want that your visits logged in this way you <a href='%s'>can set a cookie to prevent Matomo / Piwik from tracking further visits of the site</a> (opt-out)."
 
-#: piwik.php:104
+#: piwik.php:107
 msgid "Save Settings"
 msgstr "Save settings"
 
-#: piwik.php:105
+#: piwik.php:108
 msgid "Matomo (Piwik) Base URL"
 msgstr "Matomo (Piwik) Base URL"
 
-#: piwik.php:105
+#: piwik.php:108
 msgid ""
 "Absolute path to your Matomo (Piwik) installation. (without protocol "
 "(http/s), with trailing slash)"
 msgstr "Absolute path to your Matomo (Piwik) installation. (without protocol (http/s), with trailing slash)"
 
-#: piwik.php:106
+#: piwik.php:109
 msgid "Site ID"
 msgstr "Site ID"
 
-#: piwik.php:107
+#: piwik.php:110
 msgid "Show opt-out cookie link?"
 msgstr "Show opt-out cookie link?"
 
-#: piwik.php:108
+#: piwik.php:111
 msgid "Asynchronous tracking"
 msgstr "Asynchronous tracking"
 
+#: piwik.php:112
+msgid "Shortcut path to the script ('/js/' instead of '/piwik.js')"
+msgstr "Shortcut path to the script ('/js/' instead of '/piwik.js')"
+
 #: piwik.php:120
 msgid "Settings updated."
-msgstr "Settings updated."
+msgstr "Settings updated."

piwik/lang/en-gb/strings.php

@@ -14,3 +14,4 @@ $a->strings['Site ID'] = 'Site ID';
 $a->strings['Show opt-out cookie link?'] = 'Show opt-out cookie link?';
 $a->strings['Asynchronous tracking'] = 'Asynchronous tracking';
 $a->strings['Settings updated.'] = 'Settings updated.';
+$a->strings["Shortcut path to the script ('/js/' instead of '/piwik.js')"] = "Shortcut path to the script ('/js/' instead of '/piwik.js')";

piwik/lang/hu/messages.po

@@ -4,28 +4,28 @@
 # 
 # 
 # Translators:
-# Balázs Úr, 2020-2021
+# Balázs Úr, 2020-2021,2023
 msgid ""
 msgstr ""
 "Project-Id-Version: friendica\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2021-02-01 18:15+0100\n"
+"POT-Creation-Date: 2023-05-01 07:39+0200\n"
 "PO-Revision-Date: 2014-06-23 11:18+0000\n"
-"Last-Translator: Balázs Úr, 2020-2021\n"
-"Language-Team: Hungarian (http://www.transifex.com/Friendica/friendica/language/hu/)\n"
+"Last-Translator: Balázs Úr, 2020-2021,2023\n"
+"Language-Team: Hungarian (http://app.transifex.com/Friendica/friendica/language/hu/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Language: hu\n"
 "Plural-Forms: nplurals=2; plural=(n != 1);\n"
 
-#: piwik.php:87
+#: piwik.php:96
 msgid ""
 "This website is tracked using the <a href='http://www.matomo.org'>Matomo</a>"
 " analytics tool."
 msgstr "Ez a weboldal a <a href='http://www.matomo.org'>Matomo</a> analitikai eszköz használatával van követve."
 
-#: piwik.php:90
+#: piwik.php:99
 #, php-format
 msgid ""
 "If you do not want that your visits are logged in this way you <a "
@@ -33,28 +33,32 @@ msgid ""
 "visits of the site</a> (opt-out)."
 msgstr "Ha nem szeretné, hogy a látogatásai ilyen módon naplózva legyenek, akkor <a href='%s'>beállíthat egy sütit annak megakadályozásához, hogy a Matomo vagy a Piwik kövesse az oldal további meglátogatásait</a> (lemondás)."
 
-#: piwik.php:97
+#: piwik.php:108
 msgid "Save Settings"
 msgstr "Beállítások mentése"
 
-#: piwik.php:98
+#: piwik.php:109
 msgid "Matomo (Piwik) Base URL"
 msgstr "Matomo (Piwik) alap URL"
 
-#: piwik.php:98
+#: piwik.php:109
 msgid ""
 "Absolute path to your Matomo (Piwik) installation. (without protocol "
 "(http/s), with trailing slash)"
 msgstr "Abszolút útvonal a Matomo (Piwik) telepítéséhez (http vagy https protokoll nélkül, de lezáró perjellel)."
 
-#: piwik.php:99
+#: piwik.php:110
 msgid "Site ID"
 msgstr "Oldalazonosító"
 
-#: piwik.php:100
+#: piwik.php:111
 msgid "Show opt-out cookie link?"
 msgstr "Megjeleníti a lemondó süti hivatkozását?"
 
-#: piwik.php:101
+#: piwik.php:112
 msgid "Asynchronous tracking"
 msgstr "Aszinkron követés"
+
+#: piwik.php:113
+msgid "Shortcut path to the script ('/js/' instead of '/piwik.js')"
+msgstr "A parancsfájl rövidített útvonala („/js/” a „/piwik.js” helyett)"

piwik/lang/hu/strings.php

@@ -13,3 +13,4 @@ $a->strings['Absolute path to your Matomo (Piwik) installation. (without protoco
 $a->strings['Site ID'] = 'Oldalazonosító';
 $a->strings['Show opt-out cookie link?'] = 'Megjeleníti a lemondó süti hivatkozását?';
 $a->strings['Asynchronous tracking'] = 'Aszinkron követés';
+$a->strings['Shortcut path to the script (\'/js/\' instead of \'/piwik.js\')'] = 'A parancsfájl rövidített útvonala („/js/” a „/piwik.js” helyett)';

piwik/lang/ru/strings.php

@@ -9,3 +9,4 @@ $a->strings["Site ID"] = "ID сайта";
 $a->strings["Show opt-out cookie link?"] = "Показать ссылку opt-out cookie?";
 $a->strings["Asynchronous tracking"] = "Асинхронное отслеживание";
 $a->strings["Settings updated."] = "Настройки обновлены.";
+$a->strings["Shortcut path to the script ('/js/' instead of '/piwik.js')"] = "Сокращенный путь к скрипту ('/js/' вместо '/piwik.js')";

piwik/piwik.php

@@ -25,6 +25,7 @@
  *              'sideid' => '',
  *              'optout' => true,
  *              'async' => false,
+ *              'shortendpoint' => false,
  *          ],
  *      ];
  *
@@ -60,7 +61,7 @@ function piwik_analytics(string &$b)
 	 *   associated CSS file. We just have to tell Friendica to get it
 	 *   into the page header.
 	 */
-	DI::page()['htmlhead'] .= '<link rel="stylesheet"  type="text/css" href="' . DI::baseUrl() . '/addon/piwik/piwik.css' . '" media="all" />';
+	DI::page()->registerStylesheet('addon/piwik/piwik.css', 'all');
 
 	/*
 	 *   Get the configuration values.
@@ -69,16 +70,21 @@ function piwik_analytics(string &$b)
 	$siteid  = DI::config()->get('piwik', 'siteid');
 	$optout  = DI::config()->get('piwik', 'optout');
 	$async   = DI::config()->get('piwik', 'async');
+	$shortendpoint = DI::config()->get('piwik', 'shortendpoint');
 
 	/*
 	 *   Add the Piwik tracking code for the site.
 	 *   If async is set to true use asynchronous tracking
 	 */
+	
+	$scriptAsyncValue = $async ? 'true' : 'false';
+	$scriptPhpEndpoint = $shortendpoint ? 'js/' : 'piwik.php';
+	$scriptJsEndpoint = $shortendpoint ? 'js/' : 'piwik.js';
+
+	$b .= "<!-- Piwik --> <script type=\"text/javascript\"> var _paq = _paq || []; _paq.push(['trackPageView']); _paq.push(['enableLinkTracking']); (function() { var u=((\"https:\" == document.location.protocol) ? \"https\" : \"http\") + \"://$baseurl\"; _paq.push(['setTrackerUrl', u+'$scriptPhpEndpoint']); _paq.push(['setSiteId', $siteid]); var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.type='text/javascript'; g.defer=true; g.async=$scriptAsyncValue; g.src=u+'$scriptJsEndpoint'; s.parentNode.insertBefore(g,s); })(); </script> <!-- End Piwik Code -->\r\n";
+
 	if ($async) {
-	  $b .= "<!-- Piwik --> <script type=\"text/javascript\"> var _paq = _paq || []; _paq.push(['trackPageView']); _paq.push(['enableLinkTracking']); (function() { var u=((\"https:\" == document.location.protocol) ? \"https\" : \"http\") + \"://".$baseurl."\"; _paq.push(['setTrackerUrl', u+'piwik.php']); _paq.push(['setSiteId', ".$siteid."]); var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.type='text/javascript'; g.defer=true; g.async=true; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s); })(); </script> <!-- End Piwik Code -->\r\n";
-	  $b .= "<div id='piwik-code-block'> <!-- Piwik -->\r\n<noscript><p><img src=\"//".$baseurl."piwik.php?idsite=".$siteid."\" style=\"border:0\" alt=\"\" /></p></noscript>\r\n <!-- End Piwik Tracking Tag --> </div>";
-	} else {
-	  $b .= "<!-- Piwik --> <script type=\"text/javascript\"> var _paq = _paq || []; _paq.push(['trackPageView']); _paq.push(['enableLinkTracking']); (function() { var u=((\"https:\" == document.location.protocol) ? \"https\" : \"http\") + \"://".$baseurl."\"; _paq.push(['setTrackerUrl', u+'piwik.php']); _paq.push(['setSiteId', ".$siteid."]); var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.type='text/javascript'; g.defer=true; g.async=false; g.src=u+'piwik.js'; s.parentNode.insertBefore(g,s); })(); </script> <!-- End Piwik Code -->\r\n";
+		$b .= "<div id='piwik-code-block'> <!-- Piwik -->\r\n<noscript><p><img src=\"//$baseurl$scriptPhpEndpoint?idsite=$siteid\" style=\"border:0\" alt=\"\" /></p></noscript>\r\n <!-- End Piwik Tracking Tag --> </div>";
 	}
 
 	/*
@@ -89,7 +95,7 @@ function piwik_analytics(string &$b)
 		$b .= "<div id='piwik-optout-link'>";
 		$b .= DI::l10n()->t("This website is tracked using the <a href='http://www.matomo.org'>Matomo</a> analytics tool.");
 		$b .= " ";
-		$the_url =  "http://".$baseurl ."index.php?module=CoreAdminHome&action=optOut";
+		$the_url =  "http://{$baseurl}index.php?module=CoreAdminHome&action=optOut";
 		$b .= DI::l10n()->t("If you do not want that your visits are logged in this way you <a href='%s'>can set a cookie to prevent Matomo / Piwik from tracking further visits of the site</a> (opt-out).", $the_url);
 		$b .= "</div>";
 	}
@@ -104,6 +110,7 @@ function piwik_addon_admin (string &$o)
 		'$siteid' => ['siteid', DI::l10n()->t('Site ID'), DI::config()->get('piwik','siteid' ), ''],
 		'$optout' => ['optout', DI::l10n()->t('Show opt-out cookie link?'), DI::config()->get('piwik','optout' ), ''],
 		'$async' => ['async', DI::l10n()->t('Asynchronous tracking'), DI::config()->get('piwik','async' ), ''],
+		'$shortendpoint' => ['shortendpoint', DI::l10n()->t("Shortcut path to the script ('/js/' instead of '/piwik.js')"), DI::config()->get('piwik','shortendpoint' ), ''],
 	]);
 }
 
@@ -113,4 +120,5 @@ function piwik_addon_admin_post()
 	DI::config()->set('piwik', 'siteid', trim($_POST['siteid'] ?? ''));
 	DI::config()->set('piwik', 'optout', trim($_POST['optout'] ?? ''));
 	DI::config()->set('piwik', 'async', trim($_POST['async'] ?? ''));
+	DI::config()->set('piwik', 'shortendpoint', trim($_POST['shortendpoint'] ?? ''));
 }

piwik/templates/admin.tpl

@@ -2,4 +2,5 @@
 {{include file="field_input.tpl" field=$siteid}}
 {{include file="field_checkbox.tpl" field=$optout}}
 {{include file="field_checkbox.tpl" field=$async}}
+{{include file="field_checkbox.tpl" field=$shortendpoint}}
 <div class="submit"><input type="submit" name="page_site" value="{{$submit}}" /></div>

publicise/publicise.php

@@ -0,0 +1,11 @@
+"SELECT `uid` FROM `contact` WHERE `id` = %d AND `reason` = 'publicise'", intval($item['contact-id']));
+    if (!$r1) {
+        return;
+    }
+
+    Logger::debug('Publicise: moving to wall: ' . $item['uid'] . ' ' . $item['contact-id'] . ' ' . $item['uri']);
+    $item['type'] = 'wall';
+    $item['wall'] = 1;
+    $item['private'] = 0;
+}
+

publicise/templates/admin.tpl

@@ -0,0 +1,39 @@
+{{*
+ *	AUTOMATICALLY GENERATED TEMPLATE
+ *	DO NOT EDIT THIS FILE, CHANGES WILL BE OVERWRITTEN
+ *
+ *}}
+<form method="post">
+  <table>
+    <thead>
+      <tr>
+        <th>{{$feed_t}}</th>
+        <th>{{$publicised_t}}</th>
+        <th>{{$comments_t}}</th>
+        <th>{{$expire_t}}</th>
+      </tr>
+    </thead>
+    <tbody>
+{{foreach $feeds as $f}}
+      <tr>
+        <td>
+          <a href="{{$f.url}}">
+            <img style="vertical-align:middle" src='{{$f.micro}}'>
+            <span style="margin-left:1em">{{$f.name}}</span>
+          </a>
+        </td>
+        <td>
+{{include file="field_yesno.tpl" field=$f.enabled}}
+        </td>
+        <td>
+{{include file="field_yesno.tpl" field=$f.comments}}
+        </td>
+        <td>
+          <input name="publicise-expire-{{$f.id}}" value="{{$f.expire}}">
+        </td>
+      </tr>
+{{/foreach}}
+    </tbody>
+  </table>
+  <input type="submit" size="70" value="{{$submit_t}}">
+</form>

retriever/database.sql

@@ -0,0 +1,42 @@
+CREATE TABLE IF NOT EXISTS `retriever_rule` (
+       `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+       `uid` int(11) NOT NULL,
+       `contact-id` int(11) NOT NULL,
+       `data` mediumtext NULL DEFAULT NULL,
+       PRIMARY KEY (`id`),
+       KEY `uid` (`uid`),
+       KEY `contact-id` (`contact-id`)
+) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
+CREATE TABLE IF NOT EXISTS `retriever_item` (
+       `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+       `item-uri` varbinary(255) NOT NULL,
+       `item-uid` int(10) unsigned NOT NULL DEFAULT '0',
+       `contact-id` int(10) unsigned NOT NULL DEFAULT '0',
+       `resource` int(11) NOT NULL,
+       `finished` tinyint(1) unsigned NOT NULL DEFAULT '0',
+       KEY `resource` (`resource`),
+       KEY `finished` (`finished`),
+       KEY `item-uid` (`item-uid`),
+       KEY `all` (`item-uri`, `item-uid`, `contact-id`),
+       PRIMARY KEY (`id`)
+) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
+
+CREATE TABLE IF NOT EXISTS `retriever_resource` (
+       `id` int(11) unsigned NOT NULL AUTO_INCREMENT,
+       `item-uid` int(10) unsigned NOT NULL DEFAULT '0',
+       `contact-id` int(10) unsigned NOT NULL DEFAULT '0',
+       `type` char(255) NULL DEFAULT NULL,
+       `binary` int(1) NOT NULL DEFAULT 0,
+       `url` varbinary(700) NOT NULL,
+       `created` timestamp NOT NULL DEFAULT now(),
+       `completed` timestamp NULL DEFAULT NULL,
+       `last-try` timestamp NULL DEFAULT NULL,
+       `num-tries` int(11) NOT NULL DEFAULT 0,
+       `data` mediumblob NULL DEFAULT NULL,
+       `http-code` smallint(1) unsigned NULL DEFAULT NULL,
+       `redirect-url` varbinary(700) NOT NULL,
+       KEY `url` (`url`),
+       KEY `completed` (`completed`),
+       PRIMARY KEY (`id`)
+) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

retriever/templates/admin.tpl

@@ -0,0 +1,9 @@
+{{*
+ *	AUTOMATICALLY GENERATED TEMPLATE
+ *	DO NOT EDIT THIS FILE, CHANGES WILL BE OVERWRITTEN
+ *
+ *}}
+{{include file="field_input.tpl" field=$downloads_per_cron}}
+{{include file="field_checkbox.tpl" field=$allow_images}}
+<div class="submit"><input type="submit" name="page_site" value="{{$submit}}"></div>
+

retriever/templates/extract.tpl

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+  <xsl:output method="html" indent="yes" version="4.0"/>
+
+  <xsl:template match="text()"/>
+{{function clause_xpath}}{{if !$clause.attribute}}{{$clause.element}}{{elseif $clause.attribute == 'class'}}{{$clause.element}}[contains(concat(' ', normalize-space(@class), ' '), '{{$clause.value}}')]{{else}}{{$clause.element}}[@{{$clause.attribute}}='{{$clause.value}}']{{/if}}{{/function}}
+{{foreach $spec.include as $clause}}
+
+  <xsl:template match="{{clause_xpath clause=$clause}}">
+    <xsl:copy>
+      <xsl:apply-templates select="node()|@*" mode="remove"/>
+    </xsl:copy>
+  </xsl:template>{{/foreach}}
+{{foreach $spec.exclude as $clause}}
+
+  <xsl:template match="{{clause_xpath clause=$clause}}" mode="remove"/>{{/foreach}}
+
+  <xsl:template match="node()|@*" mode="remove">
+    <xsl:copy>
+      <xsl:apply-templates select="node()|@*" mode="remove"/>
+    </xsl:copy>
+  </xsl:template>
+
+</xsl:stylesheet>

retriever/templates/fix-urls.tpl

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<!-- attempt to replace relative URLs with absolute URLs -->
+<!-- http://stackoverflow.com/questions/3824631/replace-href-value-in-anchor-tags-of-html-using-xslt -->
+
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+  <xsl:output method="html" indent="yes" version="4.0"/>
+
+  <xsl:template match="node()|@*">
+    <xsl:copy>
+      <xsl:apply-templates select="node()|@*"/>
+    </xsl:copy>
+  </xsl:template>
+
+  <xsl:template match="*/@src[starts-with(.,'.')]">
+    <xsl:attribute name="src">
+      <xsl:value-of select="concat('{{$dirurl}}',.)"/>
+    </xsl:attribute>
+  </xsl:template>
+  <xsl:template match="*/@src[starts-with(.,'/')]">
+    <xsl:attribute name="src">
+      <xsl:value-of select="concat('{{$rooturl}}',.)"/>
+    </xsl:attribute>
+  </xsl:template>
+
+</xsl:stylesheet>

retriever/templates/help.tpl

@@ -0,0 +1,163 @@
+<h2>Retriever Plugin Help</h2>
+<p>
+This plugin replaces the short excerpts you normally get in RSS feeds
+with the full content of the article from the source website.  You
+specify which part of the page you're interested in with a set of
+rules.  When each item arrives, the plugin downloads the full page
+from the website, extracts content using the rules, and replaces the
+original article.
+</p>
+<p>
+There's a few reasons you may want to do this.  The source website
+might be slow or overloaded.  The source website might be
+untrustworthy, in which case using Friendica to scrub the HTML is a
+good idea.  You might be on a LAN that blacklists certain websites.
+It also works neatly with the mailstream plugin, allowing you to read
+a news stream comfortably without needing continuous Internet
+connectivity.
+</p>
+<p>
+However, setting up retriever can be quite tricky since it depends on
+the internal design of the website.  That was designed to make life
+easy for the website's developers, not for you.  You'll need to have
+some familiarity with HTML, and be willing to adapt when the website
+suddenly changes everything without notice.
+</p>
+<h3>Configuring Retriever for a feed</h3>
+<p>
+To set up retriever for an RSS feed, go to the "Contacts" page and
+find your feed.  Then click on the drop-down menu on the contact.
+Select "Retriever" to get to the retriever configuration.
+</p>
+<p>
+The "Include" configuration section specifies parts of the page to
+include in the article.  Each row has three components:
+</p>
+<ul>
+<li>An HTML tag (e.g. "div", "span", "p")</li>
+<li>An attribute (usually "class" or "id")</li>
+<li>A value for the attribute</li>
+</ul>
+<p>
+A simple case is when the article is wrapped in a "div" element:
+</p>
+<pre>
+    ...
+    &lt;div class="ArticleWrapper"&gt;
+      &lt;h2&gt;Man Bites Dog&lt;/h2&gt;
+      &lt;img src="mbd.jpg"&gt;
+      &lt;p&gt;
+        Residents of the sleepy community of Nowheresville were
+        shocked yesterday by the sight of creepy local weirdo Jim
+        McOddman assaulting innocent local dog Snufflekins with his
+        false teeth.
+      &lt;/p&gt;
+      ...
+    &lt;/div&gt;
+    ...
+</pre>
+<p>
+You then specify the tag "div", attribute "class", and value
+"ArticleWrapper".  Everything else in the page, such as navigation
+panels and menus and footers and so on, will be discarded.  If there
+is more than one section of the page you want to include, specify each
+one on a separate row.  If the matching section contains some sections
+you want to remove, specify those in the "Exclude" section in the same
+way.
+</p>
+<p>
+Once you've got a configuration that you think will work, you can try
+it out on some existing articles.  Type a number into the
+"Retrospectively Apply" box and click "Submit".  After a while
+(exactly how long depends on your system's cron configuration) the new
+articles should be available.
+</p>
+<h3>Techniques</h3>
+<p>
+You can leave the attribute and value blank to include all the
+corresponding elements with the specified tag name.  You can also use
+a tag name of just an asterisk ("*"), which will match any element type with the
+specified attribute regardless of the tag.
+</p>
+<p>
+Note that the "class" attribute is a special case.  Many web page
+templates will put multiple different classes in the same element,
+separated by spaces.  If you specify an attribute of "class" it will
+match an element if any of its classes matches the specified value.
+For example:
+</p>
+<pre>
+    &lt;div class="article breaking-news"&gt;
+</pre>
+<p>
+In this case you can specify a value of "article", or "breaking-news".
+You can also specify "article breaking-news", but that won't match if
+the website suddenly changes to "breaking-news article", so that's not
+recommended.
+</p>
+<p>
+One useful trick you can try is using the website's "print" pages.
+Many news sites have print versions of all their articles.  These are
+usually drastically simplified compared to the live website page.
+Sometimes this is a good way to get the whole article when it's
+normally split across multiple pages.
+</p>
+<p>
+Hopefully the URL for the print page is a predictable variant of the
+normal article URL.  For example, an article URL like:
+</p>
+<pre>
+    http://www.newssite.com/article-8636.html
+</pre>
+<p>
+...might have a print version at:
+</p>
+<pre>
+    http://www.newssite.com/print/article-8636.html
+</pre>
+<p>
+To change the URL used to retrieve the page, use the "URL Pattern" and
+"URL Replace" fields.  The pattern is a regular expression matching
+part of the URL to replace.  In this case, you might use a pattern of
+"/article" and a replace string of "/print/article".  A common pattern
+is simply a dollar sign ("$"), used to add the replace string to the end of the URL.
+</p>
+<h3>Background Processing</h3>
+<p>
+Note that retrieving and processing the articles can take some time,
+so it's done in the background.  Incoming articles will be marked as
+invisible while they're in the process of being downloaded.  If a URL
+fails, the plugin will keep trying at progressively longer intervals
+for up to a month, in case the website is temporarily overloaded or
+the network is down.
+</p>
+{{if $allow_images}}
+<h3>Retrieving Images</h3>
+<p>
+Retriever can also optionally download images and store them in the
+local Friendica instance.  Just check the "Download Images" box.  You
+can also download images in every item from your network, whether it's
+an RSS feed or not.  Go to the "Settings" page and
+click <a href="$config">"Plugin settings"</a>.  Then check the "All
+Photos" box in the "Retriever Settings" section and click "Submit".
+</p>
+{{/if}}
+<h2>Configure Feeds:</h2>
+<div>
+{{foreach $feeds as $feed}}
+<div class="contact-entry-wrapper" id="contact-entry-wrapper-{{$feed.id}}">
+	<a href="{{$feed.url}} title="{{$feed.img_hover}}">
+		<div class="contact-entry-photo-wrapper">
+			<div class="contact-entry-photo mframe" id="contact-entry-photo-{{$feed.id}}">
+				<img src="{{$feed.thumb}}" {{$feed.sparkle}} alt="{{$feed.name}}"/>
+			</div>
+		</div>
+		<div class="contact-entry-desc">
+			<div class="contact-entry-name" id="contact-entry-name-{{$feed.id}}">
+				{{$feed.name}}
+			</div>
+		</div>
+	</a>
+</div>
+{{/foreach}}
+</div>

retriever/templates/rule-config.tpl

@@ -0,0 +1,154 @@
+<div class="settings-block">
+  <script language="javascript">
+function retriever_add_row(id)
+{
+    var tbody = document.getElementById(id);
+    var last = tbody.rows[tbody.childElementCount - 1];
+    var count = +last.id.replace(id + '-', '');
+    count++;
+    var row = document.createElement('tr');
+    row.id = id + '-' + count;
+    var cell1 = document.createElement('td');
+    var inptag = document.createElement('input');
+    inptag.name = row.id + '-element';
+    cell1.appendChild(inptag);
+    row.appendChild(cell1);
+    var cell2 = document.createElement('td');
+    var inpatt = document.createElement('input');
+    inpatt.name = row.id + '-attribute';
+    cell2.appendChild(inpatt);
+    row.appendChild(cell2);
+    var cell3 = document.createElement('td');
+    var inpval = document.createElement('input');
+    inpval.name = row.id + '-value';
+    cell3.appendChild(inpval);
+    row.appendChild(cell3);
+    var cell4 = document.createElement('td');
+    var butrem = document.createElement('input');
+    butrem.id = row.id + '-rem';
+    butrem.type = 'button';
+    butrem.onclick = function(){retriever_remove_row(id, count)};
+    butrem.value = '{{$remove_t}}';
+    cell4.appendChild(butrem);
+    row.appendChild(cell4);
+    tbody.appendChild(row);
+}
+
+function retriever_remove_row(id, number)
+{
+    var tbody = document.getElementById(id);
+    var row = document.getElementById(id + '-' + number);
+    tbody.removeChild(row);
+}
+
+function retriever_toggle_url_block()
+{
+    var pattern = document.querySelector("#id_retriever_pattern").parentNode;
+    if (document.querySelector("#id_retriever_modurl").checked) {
+        pattern.style.display = "block";
+    }
+    else {
+        pattern.style.display = "none";
+    }
+
+    var replace = document.querySelector("#id_retriever_replace").parentNode;
+    if (document.querySelector("#id_retriever_modurl").checked) {
+        replace.style.display = "block";
+    }
+    else {
+        replace.style.display = "none";
+    }
+}
+
+function retriever_toggle_cookiedata_block()
+{
+    var div = document.querySelector("#id_retriever_cookiedata").parentNode;
+    if (document.querySelector("#id_retriever_storecookies").checked) {
+        div.style.display = "block";
+    }
+    else {
+        div.style.display = "none";
+    }
+}
+
+document.addEventListener('DOMContentLoaded', function() {
+    retriever_toggle_url_block();
+    document.querySelector("#id_retriever_modurl").addEventListener('change', retriever_toggle_url_block, false);
+    retriever_toggle_cookiedata_block();
+    document.querySelector("#id_retriever_storecookies").addEventListener('change', retriever_toggle_cookiedata_block, false);
+}, false);
+  </script>
+  <h2>{{$title}}</h2>
+  <p><a href="{{$help}}">{{$help_t}}</a></p>
+  <form method="post">
+    <input type="hidden" name="id" value="{{$id}}">
+{{include file="field_checkbox.tpl" field=$enable}}
+    <h3>{{$include_t}}:</h3>
+    <div>
+      <table>
+        <thead>
+          <tr><th>{{$tag_t}}</th><th>{{$attribute_t}}</th><th>{{$value_t}}</th></tr>
+        </thead>
+        <tbody id="retriever-include">
+{{if $include}}
+  {{foreach $include as $k=>$m}}
+          <tr id="retriever-include-{{$k}}">
+            <td><input name="retriever-include-{{$k}}-element" value="{{$m.element}}"></td>
+            <td><input name="retriever-include-{{$k}}-attribute" value="{{$m.attribute}}"></td>
+            <td><input name="retriever-include-{{$k}}-value" value="{{$m.value}}"></td>
+            <td><input id="retrieve-include-{{$k}}-rem" type="button" onclick="retriever_remove_row('retriever-include', {{$k}})" value="{{$remove_t}}"></td>
+          </tr>
+  {{/foreach}}
+{{else}}
+          <tr id="retriever-include-0">
+            <td><input name="retriever-include-0-element"></td>
+            <td><input name="retriever-include-0-attribute"></td>
+            <td><input name="retriever-include-0-value"></td>
+            <td><input id="retrieve-include-0-rem" type="button" onclick="retriever_remove_row('retriever-include', 0)" value="{{$remove_t}}"></td>
+          </tr>
+{{/if}}
+        </tbody>
+      </table>
+      <input type="button" onclick="retriever_add_row('retriever-include')" value="{{$add_t}}">
+    </div>
+    <h3>{{$exclude_t}}:</h3>
+    <div>
+      <table>
+        <thead>
+          <tr><th>{{$tag_t}}</th><th>{{$attribute_t}}</th><th>{{$value_t}}</th></tr>
+        </thead>
+        <tbody id="retriever-exclude">
+{{if $exclude}}
+  {{foreach $exclude as $k=>$r}}
+          <tr id="retriever-exclude-{{$k}}">
+            <td><input name="retriever-exclude-{{$k}}-element" value="{{$r.element}}"></td>
+            <td><input name="retriever-exclude-{{$k}}-attribute" value="{{$r.attribute}}"></td>
+            <td><input name="retriever-exclude-{{$k}}-value" value="{{$r.value}}"></td>
+            <td><input id="retrieve-exclude-{{$k}}-rem" type="button" onclick="retriever_remove_row('retriever-exclude', {{$k}})" value="{{$remove_t}}"></td>
+          </tr>
+  {{/foreach}}
+{{else}}
+          <tr id="retriever-exclude-0">
+            <td><input name="retriever-exclude-0-element"></td>
+            <td><input name="retriever-exclude-0-attribute"></td>
+            <td><input name="retriever-exclude-0-value"></td>
+            <td><input id="retrieve-exclude-0-rem" type="button" onclick="retriever_remove_row('retriever-exclude', 0)" value="{{$remove_t}}"></td>
+          </tr>
+{{/if}}
+        </tbody>
+      </table>
+      <input type="button" onclick="retriever_add_row('retriever-exclude')" value="{{$add_t}}">
+    </div>
+{{include file="field_checkbox.tpl" field=$modurl}}
+{{include file="field_input.tpl" field=$pattern}}
+{{include file="field_input.tpl" field=$replace}}
+{{if $allow_images}}
+{{include file="field_checkbox.tpl" field=$images}}
+{{/if}}
+{{include file="field_textarea.tpl" field=$customxslt}}
+{{include file="field_checkbox.tpl" field=$storecookies}}
+{{include file="field_textarea.tpl" field=$cookiedata}}
+{{include file="field_input.tpl" field=$retrospective}}
+    <input type="submit" size="70" value="{{$submit_t}}">
+  </form>
+</div>

retriever/templates/settings.tpl

@@ -0,0 +1,16 @@
+<span id="settings_retriever_inflated" class="settings-block fakelink" style="display: block;" onclick="openClose('settings_retriever_expanded'); openClose('settings_retriever_inflated');">
+	<h3>{{$title}}</h3>
+</span>
+<div id="settings_retriever_expanded" class="settings-block" style="display: none;">
+	<span class="fakelink" onclick="openClose('settings_retriever_expanded'); openClose('settings_retriever_inflated');">
+		<h3>{{$title}}</h3>
+	</span>
+	<p>
+		<a href="{{$help}}">Get Help</a>
+	</p>
+{{if $allow_images}}
+{{include file="field_checkbox.tpl" field=$allphotos}}
+{{/if}}
+{{include file="field_checkbox.tpl" field=$oembed}}
+	<input type="submit" value="{{$submit}}">
+</div>

tumblr/lang/C/messages.po

@@ -8,7 +8,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: \n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2023-04-22 10:00+0000\n"
+"POT-Creation-Date: 2023-04-29 06:56+0000\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -17,58 +17,71 @@ msgstr ""
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 
-#: tumblr.php:60
+#: tumblr.php:243
 msgid "Permission denied."
 msgstr ""
 
-#: tumblr.php:111
-msgid "Could not connect to Tumblr. Refresh the page or try again later."
-msgstr ""
-
-#: tumblr.php:159
-msgid "Unable to authenticate"
-msgstr ""
-
-#: tumblr.php:174
+#: tumblr.php:296
 msgid "Save Settings"
 msgstr ""
 
-#: tumblr.php:176
+#: tumblr.php:297
 msgid "Consumer Key"
 msgstr ""
 
-#: tumblr.php:177
+#: tumblr.php:298
 msgid "Consumer Secret"
 msgstr ""
 
-#: tumblr.php:212
+#: tumblr.php:299
+msgid "Maximum tags"
+msgstr ""
+
+#: tumblr.php:299
+msgid ""
+"Maximum number of tags that a user can follow. Enter 0 to deactivate the "
+"feature."
+msgstr ""
+
+#: tumblr.php:336
 msgid "Post to page:"
 msgstr ""
 
-#: tumblr.php:218
+#: tumblr.php:342
 msgid "(Re-)Authenticate your tumblr page"
 msgstr ""
 
-#: tumblr.php:219
+#: tumblr.php:343
 msgid "You are not authenticated to tumblr"
 msgstr ""
 
-#: tumblr.php:224
+#: tumblr.php:348
 msgid "Enable Tumblr Post Addon"
 msgstr ""
 
-#: tumblr.php:225
+#: tumblr.php:349
 msgid "Post to Tumblr by default"
 msgstr ""
 
-#: tumblr.php:226
+#: tumblr.php:350
 msgid "Import the remote timeline"
 msgstr ""
 
-#: tumblr.php:232
+#: tumblr.php:351
+msgid "Subscribed tags"
+msgstr ""
+
+#: tumblr.php:351
+#, php-format
+msgid ""
+"Comma separated list of up to %d tags that will be imported additionally to "
+"the timeline"
+msgstr ""
+
+#: tumblr.php:357
 msgid "Tumblr Import/Export"
 msgstr ""
 
-#: tumblr.php:250
+#: tumblr.php:375
 msgid "Post to Tumblr"
 msgstr ""

tumblr/lang/hu/messages.po

@@ -4,69 +4,86 @@
 # 
 # 
 # Translators:
-# Balázs Úr, 2020-2021
+# Balázs Úr, 2020-2021,2023
 msgid ""
 msgstr ""
 "Project-Id-Version: friendica\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2021-11-21 19:17-0500\n"
+"POT-Creation-Date: 2023-04-29 06:56+0000\n"
 "PO-Revision-Date: 2014-06-23 12:58+0000\n"
-"Last-Translator: Balázs Úr, 2020-2021\n"
-"Language-Team: Hungarian (http://www.transifex.com/Friendica/friendica/language/hu/)\n"
+"Last-Translator: Balázs Úr, 2020-2021,2023\n"
+"Language-Team: Hungarian (http://app.transifex.com/Friendica/friendica/language/hu/)\n"
 "MIME-Version: 1.0\n"
 "Content-Type: text/plain; charset=UTF-8\n"
 "Content-Transfer-Encoding: 8bit\n"
 "Language: hu\n"
 "Plural-Forms: nplurals=2; plural=(n != 1);\n"
 
-#: tumblr.php:39
+#: tumblr.php:243
 msgid "Permission denied."
 msgstr "Engedély megtagadva."
 
-#: tumblr.php:69
+#: tumblr.php:296
 msgid "Save Settings"
 msgstr "Beállítások mentése"
 
-#: tumblr.php:71
+#: tumblr.php:297
 msgid "Consumer Key"
 msgstr "Felhasználói kulcs"
 
-#: tumblr.php:72
+#: tumblr.php:298
 msgid "Consumer Secret"
 msgstr "Felhasználói titok"
 
-#: tumblr.php:177
-msgid "You are now authenticated to tumblr."
-msgstr "Most már hitelesítve van a Tumblr-hez."
+#: tumblr.php:299
+msgid "Maximum tags"
+msgstr "Legtöbb címke"
 
-#: tumblr.php:178
-msgid "return to the connector page"
-msgstr "visszatérés a csatlakozó oldalára"
+#: tumblr.php:299
+msgid ""
+"Maximum number of tags that a user can follow. Enter 0 to deactivate the "
+"feature."
+msgstr "A címkék legnagyobb száma, amit egy felhasználó követhet. Adjon meg 0 értéket a funkció kikapcsolásához."
 
-#: tumblr.php:194
-msgid "Post to Tumblr"
-msgstr "Beküldése a Tumblr-re"
-
-#: tumblr.php:225
+#: tumblr.php:336
 msgid "Post to page:"
 msgstr "Beküldés az oldalra:"
 
-#: tumblr.php:231
+#: tumblr.php:342
 msgid "(Re-)Authenticate your tumblr page"
 msgstr "A Tumblr-oldal (újra)hitelesítése"
 
-#: tumblr.php:232
+#: tumblr.php:343
 msgid "You are not authenticated to tumblr"
 msgstr "Nincs hitelesítve van a Tumblr-hez"
 
-#: tumblr.php:237
+#: tumblr.php:348
 msgid "Enable Tumblr Post Addon"
 msgstr "A Tumblr-beküldő bővítmény engedélyezése"
 
-#: tumblr.php:238
+#: tumblr.php:349
 msgid "Post to Tumblr by default"
 msgstr "Beküldés a Tumblr-re alapértelmezetten"
 
-#: tumblr.php:244
-msgid "Tumblr Export"
-msgstr "Tumblr exportálás"
+#: tumblr.php:350
+msgid "Import the remote timeline"
+msgstr "A távoli idővonal importálása"
+
+#: tumblr.php:351
+msgid "Subscribed tags"
+msgstr "Feliratkozott címkék"
+
+#: tumblr.php:351
+#, php-format
+msgid ""
+"Comma separated list of up to %d tags that will be imported additionally to "
+"the timeline"
+msgstr "Legfeljebb %d címke vesszővel elválasztott listája, amelyek szintén importálásra kerülnek az idővonalon felül"
+
+#: tumblr.php:357
+msgid "Tumblr Import/Export"
+msgstr "Tumblr importálás és exportálás"
+
+#: tumblr.php:375
+msgid "Post to Tumblr"
+msgstr "Beküldése a Tumblr-re"

tumblr/lang/hu/strings.php

@@ -9,12 +9,15 @@ $a->strings['Permission denied.'] = 'Engedély megtagadva.';
 $a->strings['Save Settings'] = 'Beállítások mentése';
 $a->strings['Consumer Key'] = 'Felhasználói kulcs';
 $a->strings['Consumer Secret'] = 'Felhasználói titok';
-$a->strings['You are now authenticated to tumblr.'] = 'Most már hitelesítve van a Tumblr-hez.';
-$a->strings['return to the connector page'] = 'visszatérés a csatlakozó oldalára';
-$a->strings['Post to Tumblr'] = 'Beküldése a Tumblr-re';
+$a->strings['Maximum tags'] = 'Legtöbb címke';
+$a->strings['Maximum number of tags that a user can follow. Enter 0 to deactivate the feature.'] = 'A címkék legnagyobb száma, amit egy felhasználó követhet. Adjon meg 0 értéket a funkció kikapcsolásához.';
 $a->strings['Post to page:'] = 'Beküldés az oldalra:';
 $a->strings['(Re-)Authenticate your tumblr page'] = 'A Tumblr-oldal (újra)hitelesítése';
 $a->strings['You are not authenticated to tumblr'] = 'Nincs hitelesítve van a Tumblr-hez';
 $a->strings['Enable Tumblr Post Addon'] = 'A Tumblr-beküldő bővítmény engedélyezése';
 $a->strings['Post to Tumblr by default'] = 'Beküldés a Tumblr-re alapértelmezetten';
-$a->strings['Tumblr Export'] = 'Tumblr exportálás';
+$a->strings['Import the remote timeline'] = 'A távoli idővonal importálása';
+$a->strings['Subscribed tags'] = 'Feliratkozott címkék';
+$a->strings['Comma separated list of up to %d tags that will be imported additionally to the timeline'] = 'Legfeljebb %d címke vesszővel elválasztott listája, amelyek szintén importálásra kerülnek az idővonalon felül';
+$a->strings['Tumblr Import/Export'] = 'Tumblr importálás és exportálás';
+$a->strings['Post to Tumblr'] = 'Beküldése a Tumblr-re';

tumblr/templates/admin.tpl

@@ -1,3 +1,4 @@
 {{include file="field_input.tpl" field=$consumer_key}}
 {{include file="field_input.tpl" field=$consumer_secret}}
+{{include file="field_input.tpl" field=$max_tags}}
 <div class="submit"><input type="submit" name="page_site" value="{{$submit}}" /></div>

tumblr/templates/connector_settings.tpl

@@ -3,6 +3,7 @@
 {{include file="field_checkbox.tpl" field=$enable}}
 {{include file="field_checkbox.tpl" field=$bydefault}}
 {{include file="field_checkbox.tpl" field=$import}}
+{{include file="field_input.tpl" field=$tags}}
 
 {{if $page_select}}
 	{{include file="field_select.tpl" field=$page_select}}

tumblr/tumblr.php

@@ -39,6 +39,7 @@ use GuzzleHttp\HandlerStack;
 use GuzzleHttp\Subscriber\Oauth\Oauth1;
 
 define('TUMBLR_DEFAULT_POLL_INTERVAL', 10); // given in minutes
+define('TUMBLR_DEFAULT_MAXIMUM_TAGS', 10);
 
 function tumblr_install()
 {
@@ -69,7 +70,7 @@ function tumblr_load_config(ConfigFileManager $loader)
 
 function tumblr_check_item_notification(array &$notification_data)
 {
-	if (!tumblr_enabled_for_user($notification_data['uid'])) { 
+	if (!tumblr_enabled_for_user($notification_data['uid'])) {
 		return;
 	}
 
@@ -78,7 +79,7 @@ function tumblr_check_item_notification(array &$notification_data)
 		return;
 	}
 
-	$own_user = Contact::selectFirst(['url', 'alias'], ['uid' => $notification_data['uid'], 'poll' => 'tumblr::'.$page]);
+	$own_user = Contact::selectFirst(['url', 'alias'], ['uid' => $notification_data['uid'], 'poll' => 'tumblr::' . $page]);
 	if ($own_user) {
 		$notification_data['profiles'][] = $own_user['url'];
 		$notification_data['profiles'][] = $own_user['alias'];
@@ -97,9 +98,12 @@ function tumblr_probe_detect(array &$hookData)
 		return;
 	}
 
-	Logger::debug('Search for tumblr blog', ['url' => $hookData['uri']]);
-
 	$hookData['result'] = tumblr_get_contact_by_url($hookData['uri']);
+
+	// Authoritative probe should set the result even if the probe was unsuccessful
+	if ($hookData['network'] == Protocol::TUMBLR && empty($hookData['result'])) {
+		$hookData['result'] = [];
+	}
 }
 
 function tumblr_item_by_link(array &$hookData)
@@ -116,7 +120,7 @@ function tumblr_item_by_link(array &$hookData)
 	if (!preg_match('#^https?://www\.tumblr.com/blog/view/(.+)/(\d+).*#', $hookData['uri'], $matches) && !preg_match('#^https?://www\.tumblr.com/(.+)/(\d+).*#', $hookData['uri'], $matches)) {
 		return;
 	}
-	
+
 	Logger::debug('Found tumblr post', ['url' => $hookData['uri'], 'blog' => $matches[1], 'id' => $matches[2]]);
 
 	$parameters = ['id' => $matches[2], 'reblog_info' => false, 'notes_info' => false, 'npf' => false];
@@ -128,7 +132,7 @@ function tumblr_item_by_link(array &$hookData)
 
 	Logger::debug('Got post', ['blog' => $matches[1], 'id' => $matches[2], 'result' => $result->response->posts]);
 	if (!empty($result->response->posts)) {
-		$hookData['item_id'] = tumblr_process_post($result->response->posts[0], $hookData['uid']);
+		$hookData['item_id'] = tumblr_process_post($result->response->posts[0], $hookData['uid'], Item::PR_FETCHED);
 	}
 }
 
@@ -297,6 +301,7 @@ function tumblr_addon_admin(string &$o)
 		'$submit' => DI::l10n()->t('Save Settings'),
 		'$consumer_key'    => ['consumer_key', DI::l10n()->t('Consumer Key'), DI::config()->get('tumblr', 'consumer_key'), ''],
 		'$consumer_secret' => ['consumer_secret', DI::l10n()->t('Consumer Secret'), DI::config()->get('tumblr', 'consumer_secret'), ''],
+		'$max_tags'        => ['max_tags', DI::l10n()->t('Maximum tags'), DI::config()->get('tumblr', 'max_tags') ?? TUMBLR_DEFAULT_MAXIMUM_TAGS, DI::l10n()->t('Maximum number of tags that a user can follow. Enter 0 to deactivate the feature.')],
 	]);
 }
 
@@ -304,6 +309,7 @@ function tumblr_addon_admin_post()
 {
 	DI::config()->set('tumblr', 'consumer_key', trim($_POST['consumer_key'] ?? ''));
 	DI::config()->set('tumblr', 'consumer_secret', trim($_POST['consumer_secret'] ?? ''));
+	DI::config()->set('tumblr', 'max_tags', max(0, intval($_POST['max_tags'] ?? '')));
 }
 
 function tumblr_settings(array &$data)
@@ -312,10 +318,14 @@ function tumblr_settings(array &$data)
 		return;
 	}
 
-	$enabled     = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'tumblr', 'post', false);
-	$def_enabled = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'tumblr', 'post_by_default', false);
-	$import      = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'tumblr', 'import', false);
+	$enabled     = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'tumblr', 'post') ?? false;
+	$def_enabled = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'tumblr', 'post_by_default') ?? false;
+	$import      = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'tumblr', 'import') ?? false;
+	$tags        = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'tumblr', 'tags') ?? [];
 
+	$max_tags = DI::config()->get('tumblr', 'max_tags') ?? TUMBLR_DEFAULT_MAXIMUM_TAGS;
+
+	$tags_str = implode(', ', $tags);
 	$cachekey = 'tumblr-blogs-' . DI::userSession()->getLocalUserId();
 	$blogs = DI::cache()->get($cachekey);
 	if (empty($blogs)) {
@@ -343,6 +353,7 @@ function tumblr_settings(array &$data)
 		'$enable'      => ['tumblr', DI::l10n()->t('Enable Tumblr Post Addon'), $enabled],
 		'$bydefault'   => ['tumblr_bydefault', DI::l10n()->t('Post to Tumblr by default'), $def_enabled],
 		'$import'      => ['tumblr_import', DI::l10n()->t('Import the remote timeline'), $import],
+		'$tags'        => ['tags', DI::l10n()->t('Subscribed tags'), $tags_str, DI::l10n()->t('Comma separated list of up to %d tags that will be imported additionally to the timeline', $max_tags)],
 		'$page_select' => $page_select ?? '',
 	]);
 
@@ -380,6 +391,16 @@ function tumblr_settings_post(array &$b)
 		DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'tumblr', 'page',            $_POST['tumblr_page']);
 		DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'tumblr', 'post_by_default', intval($_POST['tumblr_bydefault']));
 		DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'tumblr', 'import',          intval($_POST['tumblr_import']));
+
+		$max_tags = DI::config()->get('tumblr', 'max_tags') ?? TUMBLR_DEFAULT_MAXIMUM_TAGS;
+		$tags     = [];
+		foreach (explode(',', $_POST['tags']) as $tag) {
+			if (count($tags) < $max_tags) {
+				$tags[] = trim($tag, ' #');
+			}
+		}
+
+		DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'tumblr', 'tags', $tags);
 	}
 }
 
@@ -395,7 +416,7 @@ function tumblr_cron()
 	if ($last) {
 		$next = $last + ($poll_interval * 60);
 		if ($next > time()) {
-			Logger::notice('poll intervall not reached');
+			Logger::notice('poll interval not reached');
 			return;
 		}
 	}
@@ -419,6 +440,7 @@ function tumblr_cron()
 
 		Logger::notice('importing timeline - start', ['user' => $pconfig['uid']]);
 		tumblr_fetch_dashboard($pconfig['uid']);
+		tumblr_fetch_tags($pconfig['uid']);
 		Logger::notice('importing timeline - done', ['user' => $pconfig['uid']]);
 	}
 
@@ -682,6 +704,29 @@ function tumblr_get_post_from_uri(string $uri): array
 	return $post;
 }
 
+/**
+ * Fetch posts for user defined hashtags for the given user
+ *
+ * @param integer $uid
+ * @return void
+ */
+function tumblr_fetch_tags(int $uid)
+{
+	if (!DI::config()->get('tumblr', 'max_tags') ?? TUMBLR_DEFAULT_MAXIMUM_TAGS) {
+		return;
+	}
+
+	foreach (DI::pConfig()->get($uid, 'tumblr', 'tags') ?? [] as $tag) {
+		$data = tumblr_get($uid, 'tagged', ['tag' => $tag]);
+		foreach (array_reverse($data->response) as $post) {
+			$id = tumblr_process_post($post, $uid, Item::PR_TAG);
+			if (!empty($id)) {
+				Logger::debug('Tag post imported', ['tag' => $tag, 'id' => $id]);
+			}
+		}
+	}
+}
+
 /**
  * Fetch the dashboard (timeline) for the given user
  *
@@ -690,8 +735,6 @@ function tumblr_get_post_from_uri(string $uri): array
  */
 function tumblr_fetch_dashboard(int $uid)
 {
-	$page = tumblr_get_page($uid);
-
 	$parameters = ['reblog_info' => false, 'notes_info' => false, 'npf' => false];
 
 	$last = DI::pConfig()->get($uid, 'tumblr', 'last_id');
@@ -710,36 +753,36 @@ function tumblr_fetch_dashboard(int $uid)
 	}
 
 	foreach (array_reverse($dashboard->response->posts) as $post) {
-		$uri = 'tumblr::' . $post->id_string . ':' . $post->reblog_key;
-
 		if ($post->id > $last) {
 			$last = $post->id;
 		}
 
-		Logger::debug('Importing post', ['uid' => $uid, 'created' => date(DateTimeFormat::MYSQL, $post->timestamp), 'uri' => $uri]);
-
-		if (Post::exists(['uri' => $uri, 'uid' => $uid]) || ($post->blog->uuid == $page)) {
-			DI::pConfig()->set($uid, 'tumblr', 'last_id', $last);
-			continue;
-		}
-
-		tumblr_process_post($post, $uid, $uri);
+		Logger::debug('Importing post', ['uid' => $uid, 'created' => date(DateTimeFormat::MYSQL, $post->timestamp), 'id' => $post->id_string]);
 
+		tumblr_process_post($post, $uid, Item::PR_NONE);
 
 		DI::pConfig()->set($uid, 'tumblr', 'last_id', $last);
 	}
 }
 
-function tumblr_process_post(stdClass $post, int $uid, string $uri = ''): int
+function tumblr_process_post(stdClass $post, int $uid, int $post_reason): int
 {
-	if (empty($uri)) {
-		$uri = 'tumblr::' . $post->id_string . ':' . $post->reblog_key;
+	$uri = 'tumblr::' . $post->id_string . ':' . $post->reblog_key;
+
+	if (Post::exists(['uri' => $uri, 'uid' => $uid]) || ($post->blog->uuid == tumblr_get_page($uid))) {
+		return 0;
 	}
 
 	$item = tumblr_get_header($post, $uri, $uid);
 
 	$item = tumblr_get_content($item, $post);
 
+	$item['post-reason'] = $post_reason;
+
+	if (!empty($post->followed)) {
+		$item['post-reason'] = Item::PR_FOLLOWER;
+	}
+
 	$id = item::insert($item);
 
 	if ($id) {
@@ -1049,7 +1092,7 @@ function tumblr_insert_contact(stdClass $blog, int $uid)
 		'url'      => $url,
 		'nurl'     => Strings::normaliseLink($url),
 		'alias'    => $blog->url,
-		'name'     => $blog->title,
+		'name'     => $blog->title ?: $blog->name,
 		'nick'     => $blog->name,
 		'addr'     => $blog->name . '@tumblr.com',
 		'about'    => HTML::toBBCode($blog->description),
@@ -1099,7 +1142,7 @@ function tumblr_update_contact(stdClass $blog, int $uid, int $cid, int $pcid)
 		'nurl'    => Strings::normaliseLink($url),
 		'uri-id'  => $uri_id,
 		'alias'   => $info->response->blog->url,
-		'name'    => $info->response->blog->title,
+		'name'    => $info->response->blog->title ?: $info->response->blog->name,
 		'nick'    => $info->response->blog->name,
 		'addr'    => $info->response->blog->name . '@tumblr.com',
 		'about'   => HTML::toBBCode($info->response->blog->description),
@@ -1163,7 +1206,7 @@ function tumblr_get_blogs(int $uid): array
 	return $blogs;
 }
 
-function tumblr_enabled_for_user(int $uid) 
+function tumblr_enabled_for_user(int $uid)
 {
 	return !empty($uid) && !empty(DI::pConfig()->get($uid, 'tumblr', 'access_token')) &&
 		!empty(DI::pConfig()->get($uid, 'tumblr', 'refresh_token')) &&
@@ -1175,24 +1218,25 @@ function tumblr_enabled_for_user(int $uid)
  * Get a contact array from a Tumblr url
  *
  * @param string $url
- * @return array
+ * @return array|null
+ * @throws \Friendica\Network\HTTPException\InternalServerErrorException
  */
-function tumblr_get_contact_by_url(string $url): array
+function tumblr_get_contact_by_url(string $url): ?array
 {
 	$consumer_key = DI::config()->get('tumblr', 'consumer_key');
 	if (empty($consumer_key)) {
-		return [];
+		return null;
 	}
 
 	if (!preg_match('#^https?://tumblr.com/(.+)#', $url, $matches) && !preg_match('#^https?://www\.tumblr.com/(.+)#', $url, $matches) && !preg_match('#^https?://(.+)\.tumblr.com#', $url, $matches)) {
 		try {
 			$curlResult = DI::httpClient()->get($url);
 		} catch (\Exception $e) {
-			return [];
+			return null;
 		}
 		$html = $curlResult->getBody();
 		if (empty($html)) {
-			return [];
+			return null;
 		}
 		$doc = new DOMDocument();
 		@$doc->loadHTML($html);
@@ -1206,14 +1250,21 @@ function tumblr_get_contact_by_url(string $url): array
 	}
 
 	if (empty($blog)) {
-		return [];
+		return null;
 	}
 
+	Logger::debug('Update Tumblr blog data', ['url' => $url]);
+
 	$curlResult = DI::httpClient()->get('https://api.tumblr.com/v2/blog/' . $blog . '/info?api_key=' . $consumer_key);
 	$body = $curlResult->getBody();
 	$data = json_decode($body);
 	if (empty($data)) {
-		return [];
+		return null;
+	}
+
+	if (is_array($data->response->blog)) {
+		Logger::warning('Unexpected blog format', ['blog' => $blog, 'data' => $data]);
+		return null;
 	}
 
 	$baseurl = 'https://tumblr.com';
@@ -1228,7 +1279,7 @@ function tumblr_get_contact_by_url(string $url): array
 		'notify'   => '',
 		'poll'     => 'tumblr::' . $data->response->blog->uuid,
 		'poco'     => '',
-		'name'     => $data->response->blog->title,
+		'name'     => $data->response->blog->title ?: $data->response->blog->name,
 		'nick'     => $data->response->blog->name,
 		'network'  => Protocol::TUMBLR,
 		'baseurl'  => $baseurl,
@@ -1236,8 +1287,8 @@ function tumblr_get_contact_by_url(string $url): array
 		'priority' => 0,
 		'guid'     => $data->response->blog->uuid,
 		'about'    => HTML::toBBCode($data->response->blog->description),
-    	'photo'    => $data->response->blog->avatar[0]->url,
-    	'header'   => $data->response->blog->theme->header_image_focused,
+		'photo'    => $data->response->blog->avatar[0]->url,
+		'header'   => $data->response->blog->theme->header_image_focused,
 	];
 }
 
@@ -1363,13 +1414,13 @@ function tumblr_get_token(int $uid, string $code = ''): string
 			Logger::info('Error fetching token', ['uid' => $uid, 'code' => $code, 'result' => $curlResult->getBody(), 'parameters' => $parameters]);
 			return '';
 		}
-	
+
 		$result = json_decode($curlResult->getBody());
 		if (empty($result)) {
 			Logger::info('Invalid result when updating token', ['uid' => $uid]);
 			return '';
 		}
-	
+
 		$expires_at = time() + $result->expires_in;
 		Logger::debug('Renewed token', ['uid' => $uid, 'expires_at' => date('c', $expires_at)]);
 	}

twitter/twitter.php

@@ -526,7 +526,12 @@ function twitter_probe_detect(array &$hookData)
 	$user = twitter_fetchuser($nick);
 
 	if ($user) {
-		$hookData['result'] = twitter_user_to_contact($user);
+		$hookData['result'] = twitter_user_to_contact($user) ?: null;
+	}
+
+	// Authoritative probe should set the result even if the probe was unsuccessful
+	if ($hookData['network'] == Protocol::TWITTER && empty($hookData['result'])) {
+		$hookData['result'] = [];
 	}
 }
 
@@ -1363,7 +1368,7 @@ function twitter_fetchtimeline(int $uid): void
 
 			Logger::info('Posting mirror post', ['twitter-id' => $post->id_str, 'uid' => $uid]);
 
-			Post\Delayed::add($mirrorpost['extid'], $mirrorpost, Worker::PRIORITY_MEDIUM, Post\Delayed::PREPARED);
+			Post\Delayed::add($mirrorpost['extid'], $mirrorpost, Worker::PRIORITY_MEDIUM, Post\Delayed::UNPREPARED);
 		}
 	}
 	DI::pConfig()->set($uid, 'twitter', 'lastid', $lastid);