From 8e96300d1b43fbd4d109cc6e52e466038d1fcbc9 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Sun, 26 May 2024 13:23:00 +0100 Subject: [PATCH 01/10] adaptation for 2024.03 --- retriever/retriever.php | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 6769a60c..66a44fdb 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -143,7 +143,6 @@ $retriever_item_count = 0; * @param int $max_items Maximum number of items to retrieve in this call */ function retriever_retrieve_items(int $max_items) { - Logger::debug('@@@ retriever_retrieve_items started'); global $retriever_item_count; $retriever_schedule = array(array(1,'minute'), @@ -166,7 +165,7 @@ function retriever_retrieve_items(int $max_items) { $retrieve_items = $max_items - $retriever_item_count; do { Logger::debug('retriever_retrieve_items: asked for maximum ' . $max_items . ', already retrieved ' . intval($retriever_item_count) . ', retrieve ' . $retrieve_items); - $retriever_resources = DBA::selectToArray('retriever_resource', [], ['`completed` IS NULL AND (`last-try` IS NULL OR ' . implode($schedule_clauses, ' OR ') . ')'], ['order' => ['last-try' => 0], 'limit' => $retrieve_items]); + $retriever_resources = DBA::selectToArray('retriever_resource', [], ['`completed` IS NULL AND (`last-try` IS NULL OR ' . implode(' OR ', $schedule_clauses) . ')'], ['order' => ['last-try' => 0], 'limit' => $retrieve_items]); if (!is_array($retriever_resources)) { break; } @@ -182,7 +181,6 @@ function retriever_retrieve_items(int $max_items) { } while ($retrieve_items > 0); Logger::debug('retriever_retrieve_items: finished retrieving items'); - Logger::debug('@@@ retriever_retrieve_items finished'); } /** @@ -191,11 +189,9 @@ function retriever_retrieve_items(int $max_items) { * @param int $max_items Maximum number of items to retrieve in this call */ function retriever_clean_up_completed_resources(int $max_items) { - Logger::debug('@@@ retriever_clean_up_completed_resources started'); // TODO: figure out how to do this with DBA module $r = DBA::p("SELECT retriever_resource.`id` as resource, retriever_item.`id` as item FROM retriever_resource, retriever_item, retriever_rule WHERE retriever_item.`finished` = 0 AND retriever_item.`resource` = retriever_resource.`id` AND retriever_resource.`completed` IS NOT NULL AND retriever_item.`contact-id` = retriever_rule.`contact-id` AND retriever_item.`item-uid` = retriever_rule.`uid` LIMIT $max_items"); if (!DBA::isResult($r)) { - Logger::debug('@@@ retriever_clean_up_completed_resources nothing to do'); return; } Logger::debug('retriever_clean_up_completed_resources: items waiting even though resource has completed: ' . DBA::numRows($r)); @@ -221,7 +217,6 @@ function retriever_clean_up_completed_resources(int $max_items) { DBA::update('retriever_item', ['finished' => 1], ['id' => intval($retriever_item['id'])], ['finished' => 0]); retriever_check_item_completed($item); } - Logger::debug('@@@ retriever_clean_up_completed_resources finished'); } /** @@ -699,9 +694,6 @@ function retriever_get_body(array $item) { Logger::warning('retriever_get_body: item-content uri-id ' . $item['uri-id'] . ' has no body'); return $item['body']; } - if ($content['body'] != $item['body']) { - Logger::warning('@@@ this is probably bad @@@ content: ' . $content['body'] . ' @@@ item: ' . $item['body']); - } return $content['body']; } @@ -783,7 +775,7 @@ function retriever_check_item_completed(array &$item) * @param array $resource The resource that has just been completed */ function retriever_apply_completed_resource_to_item(array $retriever, array &$item, array $resource) { - Logger::debug('retriever_apply_completed_resource_to_item: retriever ' . ($retriever ? $retriever['id'] : 'none') . ' resource ' . $resource['url'] . ' plink ' . $item['plink']); + Logger::debug('retriever_apply_completed_resource_to_item', ['retriever' => $retriever ? $retriever['id'] : 'none', 'resource' => $resource['url'], 'plink' => $item['plink']]); if (strpos($resource['type'], 'image') !== false) { retriever_transform_images($item, $resource); } @@ -832,7 +824,7 @@ function retriever_transform_images(array &$item, array $resource) { return; } try { - $photo = Photo::store($image, $uid, $cid, $rid, $filename, $album, 0, 0, "", "", "", "", $desc); + $photo = Photo::store($image, $uid, $cid, $rid, $filename, $album, $scale, Photo::DEFAULT, '', '', '', '', $desc); } catch (Exception $e) { Logger::error('retriever_transform_images: unable to store photo ' . $resource['url'] . ' error: ' . $e->getMessage()); return; From fb944ccc06270688d67afe0f11881199925ece16 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Thu, 20 Jun 2024 20:31:07 +0100 Subject: [PATCH 02/10] Fix broken images that have been broken for ages --- retriever/retriever.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 66a44fdb..275e84e7 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -829,7 +829,7 @@ function retriever_transform_images(array &$item, array $resource) { Logger::error('retriever_transform_images: unable to store photo ' . $resource['url'] . ' error: ' . $e->getMessage()); return; } - $new_url = DI::baseUrl() . '/photo/' . $rid . '-0.' . $image->getExt(); + $new_url = DI::baseUrl() . '/photo/' . $rid . '-0' . $image->getExt(); if (!strlen($new_url)) { Logger::warning('retriever_transform_images: no replacement URL for image ' . $resource['url']); return; From ed49acf61cacaf7df7fbf6045001b90e24150cde Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Thu, 20 Jun 2024 20:32:05 +0100 Subject: [PATCH 03/10] fix whitespace --- retriever/retriever.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 275e84e7..4f87fe1b 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -661,9 +661,9 @@ function retriever_extract(DOMDocument $doc, array $retriever) { */ function retriever_globalise_urls(DOMDocument $doc, array $resource) { $components = parse_url($resource['redirect-url']); - if (!array_key_exists('scheme', $components) || !array_key_exists('host', $components) || !array_key_exists('path', $components)) { + if (!array_key_exists('scheme', $components) || !array_key_exists('host', $components) || !array_key_exists('path', $components)) { return $doc; - } + } $rooturl = $components['scheme'] . "://" . $components['host']; $dirurl = $rooturl . dirname($components['path']) . "/"; $params = array('$dirurl' => $dirurl, '$rooturl' => $rooturl); From c7cb79c49532217e77b29d21c106aaa29b887ae4 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Thu, 20 Jun 2024 20:32:52 +0100 Subject: [PATCH 04/10] globalise_urls works better when retrospectively applying --- retriever/retriever.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 4f87fe1b..639d96c6 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -660,7 +660,11 @@ function retriever_extract(DOMDocument $doc, array $retriever) { * @return DOMDocument New DOM document with global URLs */ function retriever_globalise_urls(DOMDocument $doc, array $resource) { - $components = parse_url($resource['redirect-url']); + $url = $resource['redirect-url']; + if ($url == "") { + $url = $resource['url']; + } + $components = parse_url($url); if (!array_key_exists('scheme', $components) || !array_key_exists('host', $components) || !array_key_exists('path', $components)) { return $doc; } From 13f3b44522d7ced4074151621c4f10fc1aaaf9ff Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Thu, 20 Jun 2024 20:33:32 +0100 Subject: [PATCH 05/10] globalise urls now handles relative urls --- retriever/templates/fix-urls.tpl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/retriever/templates/fix-urls.tpl b/retriever/templates/fix-urls.tpl index 248d4770..1d59938c 100644 --- a/retriever/templates/fix-urls.tpl +++ b/retriever/templates/fix-urls.tpl @@ -22,5 +22,10 @@ + + + + + From fafe6a2ea42e69bf077ccf1107be62ae99d70eb4 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Sun, 30 Jun 2024 10:38:00 +0100 Subject: [PATCH 06/10] a bit more defensiveness about add_retriever_item --- retriever/retriever.php | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 639d96c6..c0f6f935 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -448,7 +448,9 @@ function retriever_on_item_insert(array $retriever, array &$item) { } $resource = add_retriever_resource($url, $item['uid'], $item['contact-id']); - $retriever_item_id = add_retriever_item($item, $resource); + if (is_array($resource)) { + $retriever_item_id = add_retriever_item($item, $resource); + } } /** @@ -744,12 +746,15 @@ function retrieve_images(array &$item) { } if (strpos($url, (string)(DI::baseUrl())) === FALSE) { $resource = add_retriever_resource($url, $item['uid'], $item['contact-id'], true); + if (!is_array($resource)) { + Logger::error('retrieve_images: could not add resource', ['url' => $url, 'uid' => $item['uid'], 'contact-id' => $item['contact-id']]); + continue; + } if (!$resource['completed']) { add_retriever_item($item, $resource); + continue; } - else { - retriever_transform_images($item, $resource); - } + retriever_transform_images($item, $resource); } } } From dec73dca940cc60244d3296b70a9b00dae102bb9 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Sun, 30 Jun 2024 10:44:04 +0100 Subject: [PATCH 07/10] Another attempt to resolve local urls --- retriever/retriever.php | 2 +- retriever/templates/fix-urls.tpl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index c0f6f935..31db037e 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -671,7 +671,7 @@ function retriever_globalise_urls(DOMDocument $doc, array $resource) { return $doc; } $rooturl = $components['scheme'] . "://" . $components['host']; - $dirurl = $rooturl . dirname($components['path']) . "/"; + $dirurl = $rooturl . dirname($components['path']); $params = array('$dirurl' => $dirurl, '$rooturl' => $rooturl); $fix_urls_template = Renderer::getMarkupTemplate('fix-urls.tpl', 'addon/retriever/'); $fix_urls_xslt = Renderer::replaceMacros($fix_urls_template, $params); diff --git a/retriever/templates/fix-urls.tpl b/retriever/templates/fix-urls.tpl index 1d59938c..ae2452fc 100644 --- a/retriever/templates/fix-urls.tpl +++ b/retriever/templates/fix-urls.tpl @@ -14,7 +14,7 @@ - + @@ -22,7 +22,7 @@ - + From bc5604137daedc0d2b5a464f1906d2122393037e Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Tue, 9 Jul 2024 20:14:59 +0100 Subject: [PATCH 08/10] Revert "log uid but ignore results" This reverts commit 0f5ba218f69f9b7bb32dc061108370d5bf95c366. --- mailstream/mailstream.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mailstream/mailstream.php b/mailstream/mailstream.php index d814e28c..cb79ef0c 100644 --- a/mailstream/mailstream.php +++ b/mailstream/mailstream.php @@ -164,8 +164,8 @@ function mailstream_post_hook(array &$item) mailstream_check_version(); if (!DI::pConfig()->get($item['uid'], 'mailstream', 'enabled')) { - Logger::debug('mailstream: not enabled for item ' . $item['id'] . ' uid ' . $item['uid']); - // return; + Logger::debug('mailstream: not enabled.', ['item' => $item['id'], ' uid ' => $item['uid']]); + return; } if (!$item['uid']) { Logger::debug('mailstream: no uid for item ' . $item['id']); From 7f4a2ff5d059b5687b07e7465baa460d7ed8a209 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Tue, 9 Jul 2024 20:13:00 +0100 Subject: [PATCH 09/10] More comprehensible check for root user contact --- mailstream/mailstream.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mailstream/mailstream.php b/mailstream/mailstream.php index cb79ef0c..d9685af3 100644 --- a/mailstream/mailstream.php +++ b/mailstream/mailstream.php @@ -163,12 +163,12 @@ function mailstream_post_hook(array &$item) { mailstream_check_version(); - if (!DI::pConfig()->get($item['uid'], 'mailstream', 'enabled')) { - Logger::debug('mailstream: not enabled.', ['item' => $item['id'], ' uid ' => $item['uid']]); + if ($item['uid'] === 0) { + Logger::debug('mailstream: root user, skipping item ' . $item['id']); return; } - if (!$item['uid']) { - Logger::debug('mailstream: no uid for item ' . $item['id']); + if (!DI::pConfig()->get($item['uid'], 'mailstream', 'enabled')) { + Logger::debug('mailstream: not enabled.', ['item' => $item['id'], ' uid ' => $item['uid']]); return; } if (!$item['contact-id']) { From 942e5135da06e6fecdc8d45d6a67ec872dcbab1f Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Tue, 9 Jul 2024 20:12:09 +0100 Subject: [PATCH 10/10] Mailstream: respect blocked/ignored/collapsed contact settings --- mailstream/mailstream.php | 46 +++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/mailstream/mailstream.php b/mailstream/mailstream.php index d9685af3..92f29f93 100644 --- a/mailstream/mailstream.php +++ b/mailstream/mailstream.php @@ -129,21 +129,35 @@ function mailstream_send_hook(array $data) return; } - $contact = DBA::selectFirst('contact', [], ['id' => $item['contact-id'], 'self' => false]); - if (!DBA::isResult($contact)) { - Logger::error('mailstream_send_hook could not find contact', ['guid' => $item['guid'], 'contact-id' => $item['contact-id']]); + $author = DBA::selectFirst('contact', ['nick', 'blocked', 'uri-id'], ['id' => $data['author-id'], 'self' => false]); + if (!DBA::isResult($author)) { + Logger::error('mailstream_send_hook could not find author', ['guid' => $item['guid'], 'author-id' => $data['author-id']]); return; } - if ($contact['blocked']) { - Logger::error('mailstream_send_hook contact is blocked', ['guid' => $item['guid'], 'contact-id' => $item['contact-id']]); + if ($author['blocked']) { + Logger::info('mailstream_send_hook author is blocked', ['guid' => $item['guid'], 'author-id' => $data['author-id']]); return; } - if (array_key_exists('ignored', $contact) && $contact['ignored']) { - Logger::error('mailstream_send_hook contact is ignored', ['guid' => $item['guid'], 'contact-id' => $item['contact-id']]); - return; + $collapsed = false; + $user_contact = DBA::selectFirst('user-contact', ['cid', 'blocked', 'ignored', 'collapsed'], ['uid' => $item['uid'], 'uri-id' => $item['author-uri-id']]); + if (!DBA::isResult($user_contact)) { + $user_contact = DBA::selectFirst('user-contact', ['cid', 'blocked', 'ignored', 'collapsed'], ['uid' => $item['uid'], 'cid' => $item['author-id']]); + } + if (DBA::isResult($user_contact)) { + if ($user_contact['blocked']) { + Logger::info('mailstream_send_hook author is blocked', ['guid' => $item['guid'], 'cid' => $user_contact['cid']]); + return; + } + if ($user_contact['ignored']) { + Logger::info('mailstream_send_hook author is ignored', ['guid' => $item['guid'], 'cid' => $user_contact['cid']]); + return; + } + if ($user_contact['collapsed']) { + $collapsed = true; + } } - if (!mailstream_send($data['message_id'], $item, $user)) { + if (!mailstream_send($data['message_id'], $item, $user, $collapsed)) { Logger::debug('mailstream_send_hook send failed, will retry', $data); if (!Worker::defer()) { Logger::error('mailstream_send_hook failed and could not defer', $data); @@ -199,6 +213,7 @@ function mailstream_post_hook(array &$item) $send_hook_data = [ 'uid' => $item['uid'], 'contact-id' => $item['contact-id'], + 'author-id' => $item['author-id'], 'uri' => $item['uri'], 'message_id' => $message_id, 'tries' => 0, @@ -383,10 +398,11 @@ function mailstream_subject(array $item): string * @param string $message_id ID of the message (RFC 1036) * @param array $item content of the item * @param array $user results from the user table + * @param bool $collapsed true if the content should be hidden * * @return bool True if this message has been completed. False if it should be retried. */ -function mailstream_send(string $message_id, array $item, array $user): bool +function mailstream_send(string $message_id, array $item, array $user, bool $collapsed): bool { if (!is_array($item)) { Logger::error('mailstream_send item is empty', ['message_id' => $message_id]); @@ -405,10 +421,16 @@ function mailstream_send(string $message_id, array $item, array $user): bool require_once (dirname(__file__) . '/phpmailer/class.phpmailer.php'); - $item['body'] = Post\Media::addAttachmentsToBody($item['uri-id'], $item['body']); + if ($collapsed) { + $item['body'] = DI::l10n()->t('Content from %s is collapsed', $item['author-name']); + } else { + $item['body'] = Post\Media::addAttachmentsToBody($item['uri-id'], $item['body']); + } $attachments = []; - mailstream_do_images($item, $attachments); + if (!$collapsed) { + mailstream_do_images($item, $attachments); + } $frommail = DI::config()->get('mailstream', 'frommail'); if ($frommail == '') { $frommail = 'friendica@localhost.local';