From 860583eaa9c0e1772b0d20c3344b11c90a3a7e13 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Sun, 26 May 2024 13:23:00 +0100 Subject: [PATCH 01/10] adaptation for 2024.03 --- retriever/retriever.php | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 6769a60c..66a44fdb 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -143,7 +143,6 @@ $retriever_item_count = 0; * @param int $max_items Maximum number of items to retrieve in this call */ function retriever_retrieve_items(int $max_items) { - Logger::debug('@@@ retriever_retrieve_items started'); global $retriever_item_count; $retriever_schedule = array(array(1,'minute'), @@ -166,7 +165,7 @@ function retriever_retrieve_items(int $max_items) { $retrieve_items = $max_items - $retriever_item_count; do { Logger::debug('retriever_retrieve_items: asked for maximum ' . $max_items . ', already retrieved ' . intval($retriever_item_count) . ', retrieve ' . $retrieve_items); - $retriever_resources = DBA::selectToArray('retriever_resource', [], ['`completed` IS NULL AND (`last-try` IS NULL OR ' . implode($schedule_clauses, ' OR ') . ')'], ['order' => ['last-try' => 0], 'limit' => $retrieve_items]); + $retriever_resources = DBA::selectToArray('retriever_resource', [], ['`completed` IS NULL AND (`last-try` IS NULL OR ' . implode(' OR ', $schedule_clauses) . ')'], ['order' => ['last-try' => 0], 'limit' => $retrieve_items]); if (!is_array($retriever_resources)) { break; } @@ -182,7 +181,6 @@ function retriever_retrieve_items(int $max_items) { } while ($retrieve_items > 0); Logger::debug('retriever_retrieve_items: finished retrieving items'); - Logger::debug('@@@ retriever_retrieve_items finished'); } /** @@ -191,11 +189,9 @@ function retriever_retrieve_items(int $max_items) { * @param int $max_items Maximum number of items to retrieve in this call */ function retriever_clean_up_completed_resources(int $max_items) { - Logger::debug('@@@ retriever_clean_up_completed_resources started'); // TODO: figure out how to do this with DBA module $r = DBA::p("SELECT retriever_resource.`id` as resource, retriever_item.`id` as item FROM retriever_resource, retriever_item, retriever_rule WHERE retriever_item.`finished` = 0 AND retriever_item.`resource` = retriever_resource.`id` AND retriever_resource.`completed` IS NOT NULL AND retriever_item.`contact-id` = retriever_rule.`contact-id` AND retriever_item.`item-uid` = retriever_rule.`uid` LIMIT $max_items"); if (!DBA::isResult($r)) { - Logger::debug('@@@ retriever_clean_up_completed_resources nothing to do'); return; } Logger::debug('retriever_clean_up_completed_resources: items waiting even though resource has completed: ' . DBA::numRows($r)); @@ -221,7 +217,6 @@ function retriever_clean_up_completed_resources(int $max_items) { DBA::update('retriever_item', ['finished' => 1], ['id' => intval($retriever_item['id'])], ['finished' => 0]); retriever_check_item_completed($item); } - Logger::debug('@@@ retriever_clean_up_completed_resources finished'); } /** @@ -699,9 +694,6 @@ function retriever_get_body(array $item) { Logger::warning('retriever_get_body: item-content uri-id ' . $item['uri-id'] . ' has no body'); return $item['body']; } - if ($content['body'] != $item['body']) { - Logger::warning('@@@ this is probably bad @@@ content: ' . $content['body'] . ' @@@ item: ' . $item['body']); - } return $content['body']; } @@ -783,7 +775,7 @@ function retriever_check_item_completed(array &$item) * @param array $resource The resource that has just been completed */ function retriever_apply_completed_resource_to_item(array $retriever, array &$item, array $resource) { - Logger::debug('retriever_apply_completed_resource_to_item: retriever ' . ($retriever ? $retriever['id'] : 'none') . ' resource ' . $resource['url'] . ' plink ' . $item['plink']); + Logger::debug('retriever_apply_completed_resource_to_item', ['retriever' => $retriever ? $retriever['id'] : 'none', 'resource' => $resource['url'], 'plink' => $item['plink']]); if (strpos($resource['type'], 'image') !== false) { retriever_transform_images($item, $resource); } @@ -832,7 +824,7 @@ function retriever_transform_images(array &$item, array $resource) { return; } try { - $photo = Photo::store($image, $uid, $cid, $rid, $filename, $album, 0, 0, "", "", "", "", $desc); + $photo = Photo::store($image, $uid, $cid, $rid, $filename, $album, $scale, Photo::DEFAULT, '', '', '', '', $desc); } catch (Exception $e) { Logger::error('retriever_transform_images: unable to store photo ' . $resource['url'] . ' error: ' . $e->getMessage()); return; From d529f6294bfa37d0b47644f04b384df5a93121d7 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Thu, 20 Jun 2024 20:31:07 +0100 Subject: [PATCH 02/10] Fix broken images that have been broken for ages --- retriever/retriever.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 66a44fdb..275e84e7 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -829,7 +829,7 @@ function retriever_transform_images(array &$item, array $resource) { Logger::error('retriever_transform_images: unable to store photo ' . $resource['url'] . ' error: ' . $e->getMessage()); return; } - $new_url = DI::baseUrl() . '/photo/' . $rid . '-0.' . $image->getExt(); + $new_url = DI::baseUrl() . '/photo/' . $rid . '-0' . $image->getExt(); if (!strlen($new_url)) { Logger::warning('retriever_transform_images: no replacement URL for image ' . $resource['url']); return; From 0665e2c36c0a92acecb4416e71ecb0bcf4c20b5e Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Thu, 20 Jun 2024 20:32:05 +0100 Subject: [PATCH 03/10] fix whitespace --- retriever/retriever.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 275e84e7..4f87fe1b 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -661,9 +661,9 @@ function retriever_extract(DOMDocument $doc, array $retriever) { */ function retriever_globalise_urls(DOMDocument $doc, array $resource) { $components = parse_url($resource['redirect-url']); - if (!array_key_exists('scheme', $components) || !array_key_exists('host', $components) || !array_key_exists('path', $components)) { + if (!array_key_exists('scheme', $components) || !array_key_exists('host', $components) || !array_key_exists('path', $components)) { return $doc; - } + } $rooturl = $components['scheme'] . "://" . $components['host']; $dirurl = $rooturl . dirname($components['path']) . "/"; $params = array('$dirurl' => $dirurl, '$rooturl' => $rooturl); From 478df813ed2c526d691e30865e12cd992e1ed4ed Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Thu, 20 Jun 2024 20:32:52 +0100 Subject: [PATCH 04/10] globalise_urls works better when retrospectively applying --- retriever/retriever.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 4f87fe1b..639d96c6 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -660,7 +660,11 @@ function retriever_extract(DOMDocument $doc, array $retriever) { * @return DOMDocument New DOM document with global URLs */ function retriever_globalise_urls(DOMDocument $doc, array $resource) { - $components = parse_url($resource['redirect-url']); + $url = $resource['redirect-url']; + if ($url == "") { + $url = $resource['url']; + } + $components = parse_url($url); if (!array_key_exists('scheme', $components) || !array_key_exists('host', $components) || !array_key_exists('path', $components)) { return $doc; } From 2952a154c7c02debfdd0e4c499cea2dd9a103271 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Thu, 20 Jun 2024 20:33:32 +0100 Subject: [PATCH 05/10] globalise urls now handles relative urls --- retriever/templates/fix-urls.tpl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/retriever/templates/fix-urls.tpl b/retriever/templates/fix-urls.tpl index 248d4770..1d59938c 100644 --- a/retriever/templates/fix-urls.tpl +++ b/retriever/templates/fix-urls.tpl @@ -22,5 +22,10 @@ + + + + + From 649a9b280ac4bb4165df58640b6026fcb1f1575c Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Sun, 30 Jun 2024 10:38:00 +0100 Subject: [PATCH 06/10] a bit more defensiveness about add_retriever_item --- retriever/retriever.php | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 639d96c6..c0f6f935 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -448,7 +448,9 @@ function retriever_on_item_insert(array $retriever, array &$item) { } $resource = add_retriever_resource($url, $item['uid'], $item['contact-id']); - $retriever_item_id = add_retriever_item($item, $resource); + if (is_array($resource)) { + $retriever_item_id = add_retriever_item($item, $resource); + } } /** @@ -744,12 +746,15 @@ function retrieve_images(array &$item) { } if (strpos($url, (string)(DI::baseUrl())) === FALSE) { $resource = add_retriever_resource($url, $item['uid'], $item['contact-id'], true); + if (!is_array($resource)) { + Logger::error('retrieve_images: could not add resource', ['url' => $url, 'uid' => $item['uid'], 'contact-id' => $item['contact-id']]); + continue; + } if (!$resource['completed']) { add_retriever_item($item, $resource); + continue; } - else { - retriever_transform_images($item, $resource); - } + retriever_transform_images($item, $resource); } } } From 41a718cd8253752ec040bbc6300b3db2ec85fdd3 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Sun, 30 Jun 2024 10:44:04 +0100 Subject: [PATCH 07/10] Another attempt to resolve local urls --- retriever/retriever.php | 2 +- retriever/templates/fix-urls.tpl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index c0f6f935..31db037e 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -671,7 +671,7 @@ function retriever_globalise_urls(DOMDocument $doc, array $resource) { return $doc; } $rooturl = $components['scheme'] . "://" . $components['host']; - $dirurl = $rooturl . dirname($components['path']) . "/"; + $dirurl = $rooturl . dirname($components['path']); $params = array('$dirurl' => $dirurl, '$rooturl' => $rooturl); $fix_urls_template = Renderer::getMarkupTemplate('fix-urls.tpl', 'addon/retriever/'); $fix_urls_xslt = Renderer::replaceMacros($fix_urls_template, $params); diff --git a/retriever/templates/fix-urls.tpl b/retriever/templates/fix-urls.tpl index 1d59938c..ae2452fc 100644 --- a/retriever/templates/fix-urls.tpl +++ b/retriever/templates/fix-urls.tpl @@ -14,7 +14,7 @@ - + @@ -22,7 +22,7 @@ - + From aa197997722d0e4a5205c7bde6e67b233861cb95 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Tue, 9 Jul 2024 20:14:59 +0100 Subject: [PATCH 08/10] Revert "log uid but ignore results" This reverts commit 0f5ba218f69f9b7bb32dc061108370d5bf95c366. --- mailstream/mailstream.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mailstream/mailstream.php b/mailstream/mailstream.php index 425d5ec8..ff77d126 100644 --- a/mailstream/mailstream.php +++ b/mailstream/mailstream.php @@ -133,8 +133,8 @@ function mailstream_post_hook(array &$item) return; } if (!DI::pConfig()->get($item['uid'], 'mailstream', 'enabled')) { - Logger::debug('mailstream: not enabled for item ' . $item['id'] . ' uid ' . $item['uid']); - // return; + Logger::debug('mailstream: not enabled.', ['item' => $item['id'], ' uid ' => $item['uid']]); + return; } if (!$item['contact-id']) { Logger::debug('no contact-id', ['item' => $item['id']]); From 5a1169bb0155bf802f0bc6f50c37440e3a8dcaee Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Tue, 9 Jul 2024 20:12:09 +0100 Subject: [PATCH 09/10] Mailstream: respect blocked/ignored/collapsed contact settings --- mailstream/mailstream.php | 48 ++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/mailstream/mailstream.php b/mailstream/mailstream.php index ff77d126..a59c3b68 100644 --- a/mailstream/mailstream.php +++ b/mailstream/mailstream.php @@ -106,12 +106,40 @@ function mailstream_send_hook(array $data) $user = User::getById($item['uid']); if (empty($user)) { - Logger::error('could not find user', ['uid' => $item['uid']]); + Logger::error('mailstream_send_hook could not find user', ['uid' => $item['uid']]); return; } - if (!mailstream_send($data['message_id'], $item, $user)) { - Logger::debug('send failed, will retry', $data); + $author = DBA::selectFirst('contact', ['nick', 'blocked', 'uri-id'], ['id' => $data['author-id'], 'self' => false]); + if (!DBA::isResult($author)) { + Logger::error('mailstream_send_hook could not find author', ['guid' => $item['guid'], 'author-id' => $data['author-id']]); + return; + } + if ($author['blocked']) { + Logger::info('mailstream_send_hook author is blocked', ['guid' => $item['guid'], 'author-id' => $data['author-id']]); + return; + } + $collapsed = false; + $user_contact = DBA::selectFirst('user-contact', ['cid', 'blocked', 'ignored', 'collapsed'], ['uid' => $item['uid'], 'uri-id' => $item['author-uri-id']]); + if (!DBA::isResult($user_contact)) { + $user_contact = DBA::selectFirst('user-contact', ['cid', 'blocked', 'ignored', 'collapsed'], ['uid' => $item['uid'], 'cid' => $item['author-id']]); + } + if (DBA::isResult($user_contact)) { + if ($user_contact['blocked']) { + Logger::info('mailstream_send_hook author is blocked', ['guid' => $item['guid'], 'cid' => $user_contact['cid']]); + return; + } + if ($user_contact['ignored']) { + Logger::info('mailstream_send_hook author is ignored', ['guid' => $item['guid'], 'cid' => $user_contact['cid']]); + return; + } + if ($user_contact['collapsed']) { + $collapsed = true; + } + } + + if (!mailstream_send($data['message_id'], $item, $user, $collapsed)) { + Logger::debug('mailstream_send_hook send failed, will retry', $data); if (!Worker::defer()) { Logger::error('failed and could not defer', $data); } @@ -164,6 +192,7 @@ function mailstream_post_hook(array &$item) $send_hook_data = [ 'uid' => $item['uid'], 'contact-id' => $item['contact-id'], + 'author-id' => $item['author-id'], 'uri' => $item['uri'], 'message_id' => $message_id, 'tries' => 0, @@ -350,10 +379,11 @@ function mailstream_subject(array $item): string * @param string $message_id ID of the message (RFC 1036) * @param array $item content of the item * @param array $user results from the user table + * @param bool $collapsed true if the content should be hidden * * @return bool True if this message has been completed. False if it should be retried. */ -function mailstream_send(string $message_id, array $item, array $user): bool +function mailstream_send(string $message_id, array $item, array $user, bool $collapsed): bool { if (!is_array($item)) { Logger::error('item is empty', ['message_id' => $message_id]); @@ -371,10 +401,16 @@ function mailstream_send(string $message_id, array $item, array $user): bool require_once(dirname(__file__) . '/phpmailer/class.phpmailer.php'); - $item['body'] = Post\Media::addAttachmentsToBody($item['uri-id'], $item['body']); + if ($collapsed) { + $item['body'] = DI::l10n()->t('Content from %s is collapsed', $item['author-name']); + } else { + $item['body'] = Post\Media::addAttachmentsToBody($item['uri-id'], $item['body']); + } $attachments = []; - mailstream_do_images($item, $attachments); + if (!$collapsed) { + mailstream_do_images($item, $attachments); + } $frommail = DI::config()->get('mailstream', 'frommail'); if ($frommail == '') { $frommail = 'friendica@localhost.local'; From ba4d2b7ad0e75b86b6edd42e4312e4dfaeeb4971 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Wed, 1 Jan 2025 16:03:56 +0100 Subject: [PATCH 10/10] improved logging --- mailstream/mailstream.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mailstream/mailstream.php b/mailstream/mailstream.php index a59c3b68..9984d9f7 100644 --- a/mailstream/mailstream.php +++ b/mailstream/mailstream.php @@ -465,9 +465,9 @@ function mailstream_send(string $message_id, array $item, array $user, bool $col 'address' => $address ]); } catch (phpmailerException $e) { - Logger::debug('PHPMailer exception sending message', ['id' => $message_id, 'error' => $e->errorMessage()]); + Logger::debug('mailstream_send PHPMailer exception sending message', ['item uri' => $item['uri'], 'message_id' => $message_id, 'error' => $e->errorMessage()]); } catch (Exception $e) { - Logger::debug('exception sending message', ['id' => $message_id, 'error' => $e->getMessage()]); + Logger::debug('mailstream_send exception sending message', ['item uri' => $item['uri'], 'message_id' => $message_id, 'error' => $e->errorMessage()]); } return true;