From 57ad3c7f46ab6ef1a06b19c188b64fcc41b7ff19 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Thu, 20 Jun 2024 20:32:05 +0100 Subject: [PATCH 01/10] fix whitespace --- retriever/retriever.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 275e84e7..4f87fe1b 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -661,9 +661,9 @@ function retriever_extract(DOMDocument $doc, array $retriever) { */ function retriever_globalise_urls(DOMDocument $doc, array $resource) { $components = parse_url($resource['redirect-url']); - if (!array_key_exists('scheme', $components) || !array_key_exists('host', $components) || !array_key_exists('path', $components)) { + if (!array_key_exists('scheme', $components) || !array_key_exists('host', $components) || !array_key_exists('path', $components)) { return $doc; - } + } $rooturl = $components['scheme'] . "://" . $components['host']; $dirurl = $rooturl . dirname($components['path']) . "/"; $params = array('$dirurl' => $dirurl, '$rooturl' => $rooturl); From e5ada18f8ee5868e01c0e30a2050977a2a86edd7 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Thu, 20 Jun 2024 20:32:52 +0100 Subject: [PATCH 02/10] globalise_urls works better when retrospectively applying --- retriever/retriever.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 4f87fe1b..639d96c6 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -660,7 +660,11 @@ function retriever_extract(DOMDocument $doc, array $retriever) { * @return DOMDocument New DOM document with global URLs */ function retriever_globalise_urls(DOMDocument $doc, array $resource) { - $components = parse_url($resource['redirect-url']); + $url = $resource['redirect-url']; + if ($url == "") { + $url = $resource['url']; + } + $components = parse_url($url); if (!array_key_exists('scheme', $components) || !array_key_exists('host', $components) || !array_key_exists('path', $components)) { return $doc; } From be09d37331473884cf07f7a931f0846c60074487 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Thu, 20 Jun 2024 20:33:32 +0100 Subject: [PATCH 03/10] globalise urls now handles relative urls --- retriever/templates/fix-urls.tpl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/retriever/templates/fix-urls.tpl b/retriever/templates/fix-urls.tpl index 248d4770..1d59938c 100644 --- a/retriever/templates/fix-urls.tpl +++ b/retriever/templates/fix-urls.tpl @@ -22,5 +22,10 @@ + + + + + From 59de855d0f185b6764ac4a3675b48de37f31eda9 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Sun, 30 Jun 2024 10:38:00 +0100 Subject: [PATCH 04/10] a bit more defensiveness about add_retriever_item --- retriever/retriever.php | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 639d96c6..c0f6f935 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -448,7 +448,9 @@ function retriever_on_item_insert(array $retriever, array &$item) { } $resource = add_retriever_resource($url, $item['uid'], $item['contact-id']); - $retriever_item_id = add_retriever_item($item, $resource); + if (is_array($resource)) { + $retriever_item_id = add_retriever_item($item, $resource); + } } /** @@ -744,12 +746,15 @@ function retrieve_images(array &$item) { } if (strpos($url, (string)(DI::baseUrl())) === FALSE) { $resource = add_retriever_resource($url, $item['uid'], $item['contact-id'], true); + if (!is_array($resource)) { + Logger::error('retrieve_images: could not add resource', ['url' => $url, 'uid' => $item['uid'], 'contact-id' => $item['contact-id']]); + continue; + } if (!$resource['completed']) { add_retriever_item($item, $resource); + continue; } - else { - retriever_transform_images($item, $resource); - } + retriever_transform_images($item, $resource); } } } From 7417ce0ad148fd0d2c3aa3a9a3a4bc2ebececbea Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Sun, 30 Jun 2024 10:44:04 +0100 Subject: [PATCH 05/10] Another attempt to resolve local urls --- retriever/retriever.php | 2 +- retriever/templates/fix-urls.tpl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index c0f6f935..31db037e 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -671,7 +671,7 @@ function retriever_globalise_urls(DOMDocument $doc, array $resource) { return $doc; } $rooturl = $components['scheme'] . "://" . $components['host']; - $dirurl = $rooturl . dirname($components['path']) . "/"; + $dirurl = $rooturl . dirname($components['path']); $params = array('$dirurl' => $dirurl, '$rooturl' => $rooturl); $fix_urls_template = Renderer::getMarkupTemplate('fix-urls.tpl', 'addon/retriever/'); $fix_urls_xslt = Renderer::replaceMacros($fix_urls_template, $params); diff --git a/retriever/templates/fix-urls.tpl b/retriever/templates/fix-urls.tpl index 1d59938c..ae2452fc 100644 --- a/retriever/templates/fix-urls.tpl +++ b/retriever/templates/fix-urls.tpl @@ -14,7 +14,7 @@ - + @@ -22,7 +22,7 @@ - + From 1311fe1e76d764b656caebc58d19cc2cb7ae75ff Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Tue, 9 Jul 2024 20:14:59 +0100 Subject: [PATCH 06/10] Revert "log uid but ignore results" This reverts commit 0f5ba218f69f9b7bb32dc061108370d5bf95c366. --- mailstream/mailstream.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mailstream/mailstream.php b/mailstream/mailstream.php index 218289c3..60be0b62 100644 --- a/mailstream/mailstream.php +++ b/mailstream/mailstream.php @@ -133,8 +133,8 @@ function mailstream_post_hook(array &$item) return; } if (!DI::pConfig()->get($item['uid'], 'mailstream', 'enabled')) { - Logger::debug('mailstream: not enabled for item ' . $item['id'] . ' uid ' . $item['uid']); - // return; + Logger::debug('mailstream: not enabled.', ['item' => $item['id'], ' uid ' => $item['uid']]); + return; } if (!$item['contact-id']) { Logger::debug('no contact-id', ['item' => $item['id']]); From bc9ca2eae845e1dd862779f706c6aa58e905472b Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Tue, 9 Jul 2024 20:12:09 +0100 Subject: [PATCH 07/10] Mailstream: respect blocked/ignored/collapsed contact settings --- mailstream/mailstream.php | 48 ++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/mailstream/mailstream.php b/mailstream/mailstream.php index 60be0b62..52474749 100644 --- a/mailstream/mailstream.php +++ b/mailstream/mailstream.php @@ -106,12 +106,40 @@ function mailstream_send_hook(array $data) $user = User::getById($item['uid']); if (empty($user)) { - Logger::error('could not find user', ['uid' => $item['uid']]); + Logger::error('mailstream_send_hook could not find user', ['uid' => $item['uid']]); return; } - if (!mailstream_send($data['message_id'], $item, $user)) { - Logger::debug('send failed, will retry', $data); + $author = DBA::selectFirst('contact', ['nick', 'blocked', 'uri-id'], ['id' => $data['author-id'], 'self' => false]); + if (!DBA::isResult($author)) { + Logger::error('mailstream_send_hook could not find author', ['guid' => $item['guid'], 'author-id' => $data['author-id']]); + return; + } + if ($author['blocked']) { + Logger::info('mailstream_send_hook author is blocked', ['guid' => $item['guid'], 'author-id' => $data['author-id']]); + return; + } + $collapsed = false; + $user_contact = DBA::selectFirst('user-contact', ['cid', 'blocked', 'ignored', 'collapsed'], ['uid' => $item['uid'], 'uri-id' => $item['author-uri-id']]); + if (!DBA::isResult($user_contact)) { + $user_contact = DBA::selectFirst('user-contact', ['cid', 'blocked', 'ignored', 'collapsed'], ['uid' => $item['uid'], 'cid' => $item['author-id']]); + } + if (DBA::isResult($user_contact)) { + if ($user_contact['blocked']) { + Logger::info('mailstream_send_hook author is blocked', ['guid' => $item['guid'], 'cid' => $user_contact['cid']]); + return; + } + if ($user_contact['ignored']) { + Logger::info('mailstream_send_hook author is ignored', ['guid' => $item['guid'], 'cid' => $user_contact['cid']]); + return; + } + if ($user_contact['collapsed']) { + $collapsed = true; + } + } + + if (!mailstream_send($data['message_id'], $item, $user, $collapsed)) { + Logger::debug('mailstream_send_hook send failed, will retry', $data); if (!Worker::defer()) { Logger::error('failed and could not defer', $data); } @@ -164,6 +192,7 @@ function mailstream_post_hook(array &$item) $send_hook_data = [ 'uid' => $item['uid'], 'contact-id' => $item['contact-id'], + 'author-id' => $item['author-id'], 'uri' => $item['uri'], 'message_id' => $message_id, 'tries' => 0, @@ -350,10 +379,11 @@ function mailstream_subject(array $item): string * @param string $message_id ID of the message (RFC 1036) * @param array $item content of the item * @param array $user results from the user table + * @param bool $collapsed true if the content should be hidden * * @return bool True if this message has been completed. False if it should be retried. */ -function mailstream_send(string $message_id, array $item, array $user): bool +function mailstream_send(string $message_id, array $item, array $user, bool $collapsed): bool { if (!is_array($item)) { Logger::error('item is empty', ['message_id' => $message_id]); @@ -371,10 +401,16 @@ function mailstream_send(string $message_id, array $item, array $user): bool require_once(dirname(__file__) . '/phpmailer/class.phpmailer.php'); - $item['body'] = Post\Media::addAttachmentsToBody($item['uri-id'], $item['body']); + if ($collapsed) { + $item['body'] = DI::l10n()->t('Content from %s is collapsed', $item['author-name']); + } else { + $item['body'] = Post\Media::addAttachmentsToBody($item['uri-id'], $item['body']); + } $attachments = []; - mailstream_do_images($item, $attachments); + if (!$collapsed) { + mailstream_do_images($item, $attachments); + } $frommail = DI::config()->get('mailstream', 'frommail'); if ($frommail == '') { $frommail = 'friendica@localhost.local'; From daacd19f68124013bdb08a5af21f4dae278c73d9 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Wed, 1 Jan 2025 16:03:56 +0100 Subject: [PATCH 08/10] improved logging --- mailstream/mailstream.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mailstream/mailstream.php b/mailstream/mailstream.php index 52474749..08b95dd5 100644 --- a/mailstream/mailstream.php +++ b/mailstream/mailstream.php @@ -465,9 +465,9 @@ function mailstream_send(string $message_id, array $item, array $user, bool $col 'address' => $address ]); } catch (phpmailerException $e) { - Logger::debug('PHPMailer exception sending message', ['id' => $message_id, 'error' => $e->errorMessage()]); + Logger::debug('mailstream_send PHPMailer exception sending message', ['item uri' => $item['uri'], 'message_id' => $message_id, 'error' => $e->errorMessage()]); } catch (Exception $e) { - Logger::debug('exception sending message', ['id' => $message_id, 'error' => $e->getMessage()]); + Logger::debug('mailstream_send exception sending message', ['item uri' => $item['uri'], 'message_id' => $message_id, 'error' => $e->errorMessage()]); } return true; From 380dab7e95267a2dcda1d2e80cd4c7a0c2b365a7 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Mon, 6 Jan 2025 17:04:48 +0100 Subject: [PATCH 09/10] Retriever: use new HTTP client API --- retriever/retriever.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 31db037e..bf06815c 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -16,6 +16,8 @@ use Friendica\Core\System; use Friendica\Content\Text\HTML; use Friendica\Content\Text\BBCode; use Friendica\Model\Photo; +use Friendica\Network\HTTPClient\Client\HttpClientAccept; +use Friendica\Network\HTTPClient\Client\HttpClientOptions; use Friendica\Object\Image; use Friendica\Util\Network; use Friendica\Database\DBA; @@ -291,13 +293,12 @@ function retrieve_resource(array $resource) { try { Logger::debug('retrieve_resource: ' . ($resource['num-tries'] + 1) . ' attempt at resource ' . $resource['id'] . ' ' . $resource['url']); - $redirects = 0; $cookiejar = ''; if (array_key_exists('storecookies', $rule_data) && $rule_data['storecookies']) { $cookiejar = tempnam(System::getTempPath(), 'cookiejar-retriever-'); file_put_contents($cookiejar, $rule_data['cookiedata']); } - $fetch_result = DI::httpClient()->fetchFull($resource['url'], $redirects, 0, $cookiejar); + $fetch_result = DI::httpClient()->get($resource['url'], HttpClientAccept::DEFAULT, [HttpClientOptions::COOKIEJAR => $cookiejar]); if (array_key_exists('storecookies', $rule_data) && $rule_data['storecookies']) { $retriever_rule['data']['cookiedata'] = file_get_contents($cookiejar); DBA::update('retriever_rule', ['data' => json_encode($retriever_rule['data'])], ['id' => intval($retriever_rule["id"])], $retriever_rule); From c6bd06d3d796f5412d7d69fc3221a789ff328084 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Sun, 19 Jan 2025 14:42:20 +0100 Subject: [PATCH 10/10] restore retriever configuration --- retriever/retriever.php | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index bf06815c..da7148ce 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -37,6 +37,7 @@ function retriever_install() { Hook::register('addon_settings_post', 'addon/retriever/retriever.php', 'retriever_addon_settings_post'); Hook::register('post_remote', 'addon/retriever/retriever.php', 'retriever_post_remote_hook'); Hook::register('contact_photo_menu', 'addon/retriever/retriever.php', 'retriever_contact_photo_menu'); + Hook::register('retriever_mod_post', 'addon/retriever/retriever.php', 'retriever_content'); Hook::register('cron', 'addon/retriever/retriever.php', 'retriever_cron'); if (DI::config()->get('retriever', 'dbversion') == '0.14') { @@ -77,6 +78,7 @@ function retriever_uninstall() { Hook::unregister('addon_settings', 'addon/retriever/retriever.php', 'retriever_addon_settings'); Hook::unregister('addon_settings_post', 'addon/retriever/retriever.php', 'retriever_addon_settings_post'); Hook::unregister('contact_photo_menu', 'addon/retriever/retriever.php', 'retriever_contact_photo_menu'); + Hook::unregister('retriever_mod_post', 'addon/retriever/retriever.php', 'retriever_content'); Hook::unregister('cron', 'addon/retriever/retriever.php', 'retriever_cron'); } @@ -856,11 +858,12 @@ function retriever_transform_images(array &$item, array $resource) { * @brief Displays the retriever configuration page for a contact. Alternatively, if the user clicked the "help" button, display the help content. */ function retriever_content() { + $e = new \Exception; if (!DI::userSession()->getLocalUserId()) { DI::page()['content'] .= "

Please log in

"; return; } - if (isset(DI::args()->getArgv()[1]) and DI::args()->getArgv()[1] === 'help') { + if (DI::args()->get(1) === 'help') { $feeds = DBA::selectToArray('contact', ['id', 'name', 'thumb'], ['uid' => DI::userSession()->getLocalUserId(), 'network' => 'feed']); for ($i = 0; $i < count($feeds); ++$i) { $feeds[$i]['url'] = DI::baseUrl() . '/retriever/' . $feeds[$i]['id']; @@ -872,8 +875,8 @@ function retriever_content() { '$feeds' => $feeds)); return; } - if (isset(DI::args()->getArgv()[1])) { - $arg1 = DI::args()->getArgv()[1]; + if (DI::args()->get(1)) { + $arg1 = DI::args()->get(1); $retriever_rule = get_retriever_rule($arg1, DI::userSession()->getLocalUserId(), false); if (!$retriever_rule) { $retriever_rule = ['id' => 0, 'data' => ['enable' => 0, 'modurl' => '', 'pattern' => '', 'replace' => '', 'images' => 0, 'storecookies' => 0, 'cookiedata' => '', 'customxslt' => '', 'include' => '', 'exclude' => '']];