From 0f5ba218f69f9b7bb32dc061108370d5bf95c366 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Sun, 7 May 2023 13:37:00 +0200 Subject: [PATCH 01/10] log uid but ignore results --- mailstream/mailstream.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mailstream/mailstream.php b/mailstream/mailstream.php index b513cf36..f0dcf341 100644 --- a/mailstream/mailstream.php +++ b/mailstream/mailstream.php @@ -143,10 +143,11 @@ function mailstream_send_hook(array $data) function mailstream_post_hook(array &$item) { mailstream_check_version(); + Logger::debug('@@@ mailstream_post_hook', ['item-uid' => $item['uid']]); if (!DI::pConfig()->get($item['uid'], 'mailstream', 'enabled')) { - Logger::debug('mailstream: not enabled.', ['item' => $item['id'], ' uid ' => $item['uid']]); - return; + Logger::debug('mailstream: not enabled for item ' . $item['id'] . ' uid ' . $item['uid']); + // return; } if (!$item['uid']) { Logger::debug('mailstream: no uid for item ' . $item['id']); From 11df57f3ea2bd854e3a614325619fd54f28e2a89 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Tue, 26 Dec 2023 13:25:19 +0100 Subject: [PATCH 02/10] adapt to latest release --- retriever/retriever.php | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 9370271c..6e499d94 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -740,7 +740,7 @@ function retrieve_images(array &$item) { if (!$url) { continue; } - if (strpos($url, DI::baseUrl()->get(true)) === FALSE) { + if (strpos($url, DI::baseUrl()) === FALSE) { $resource = add_retriever_resource($url, $item['uid'], $item['contact-id'], true); if (!$resource['completed']) { add_retriever_item($item, $resource); @@ -826,7 +826,7 @@ function retriever_transform_images(array &$item, array $resource) { return; } $photo = Photo::store($image, $uid, $cid, $rid, $filename, $album, 0, 0, "", "", "", "", $desc); - $new_url = DI::baseUrl()->get(true) . '/photo/' . $rid . '-0.' . $image->getExt(); + $new_url = DI::baseUrl() . '/photo/' . $rid . '-0.' . $image->getExt(); if (!strlen($new_url)) { Logger::warning('retriever_transform_images: no replacement URL for image ' . $resource['url']); return; @@ -845,31 +845,31 @@ function retriever_transform_images(array &$item, array $resource) { * @param App $a The App object */ function retriever_content(App $a) { - if (!Session::getLocalUser()) { + if (!DI::userSession()->getLocalUserId()) { $a->page['content'] .= "

Please log in

"; return; } if (isset(DI::args()->getArgv()[1]) and DI::args()->getArgv()[1] === 'help') { - $feeds = DBA::selectToArray('contact', ['id', 'name', 'thumb'], ['uid' => Session::getLocalUser(), 'network' => 'feed']); + $feeds = DBA::selectToArray('contact', ['id', 'name', 'thumb'], ['uid' => DI::userSession()->getLocalUserId(), 'network' => 'feed']); for ($i = 0; $i < count($feeds); ++$i) { - $feeds[$i]['url'] = DI::baseUrl()->get(true) . '/retriever/' . $feeds[$i]['id']; + $feeds[$i]['url'] = DI::baseUrl() . '/retriever/' . $feeds[$i]['id']; } $template = Renderer::getMarkupTemplate('/help.tpl', 'addon/retriever/'); $a->page['content'] .= Renderer::replaceMacros($template, array( - '$config' => DI::baseUrl()->get(true) . '/settings/addon', + '$config' => DI::baseUrl() . '/settings/addon', '$allow_images' => DI::config()->get('retriever', 'allow_images'), '$feeds' => $feeds)); return; } if (isset(DI::args()->getArgv()[1])) { $arg1 = DI::args()->getArgv()[1]; - $retriever_rule = get_retriever_rule($arg1, Session::getLocalUser(), false); + $retriever_rule = get_retriever_rule($arg1, DI::userSession()->getLocalUserId(), false); if (!$retriever_rule) { $retriever_rule = ['id' => 0, 'data' => ['enable' => 0, 'modurl' => '', 'pattern' => '', 'replace' => '', 'images' => 0, 'storecookies' => 0, 'cookiedata' => '', 'customxslt' => '', 'include' => '', 'exclude' => '']]; } if (!empty($_POST["id"])) { - $retriever_rule = get_retriever_rule($arg1, Session::getLocalUser(), true); + $retriever_rule = get_retriever_rule($arg1, DI::userSession()->getLocalUserId(), true); $retriever_rule['data'] = array(); foreach (array('modurl', 'pattern', 'replace', 'enable', 'images', 'customxslt', 'storecookies', 'cookiedata') as $setting) { if (empty($_POST['retriever_' . $setting])) { @@ -951,7 +951,7 @@ function retriever_content(App $a) { $retriever_rule['data']['customxslt'], DI::l10n()->t("When standard rules aren't enough, apply custom XSLT to the article")), '$title' => DI::l10n()->t('Retrieve Feed Content'), - '$help' => DI::baseUrl()->get(true) . '/retriever/help', + '$help' => DI::baseUrl() . '/retriever/help', '$help_t' => DI::l10n()->t('Get Help'), '$submit_t' => DI::l10n()->t('Submit'), '$submit' => DI::l10n()->t('Save Settings'), @@ -972,15 +972,14 @@ function retriever_content(App $a) { /** * @brief Hook that adds the retriever option to the contact menu * - * @param App $a The App object (by ref) * @param array $args Contact menu details to be filled in (by ref) */ -function retriever_contact_photo_menu(App &$a, array &$args) { +function retriever_contact_photo_menu(array &$args) { if (!$args) { return; } if ($args["contact"]["network"] == "feed") { - $args["menu"]['retriever'] = array(DI::l10n()->t('Retriever'), DI::baseUrl()->get(true) . '/retriever/' . $args["contact"]['id']); + $args["menu"]['retriever'] = array(DI::l10n()->t('Retriever'), DI::baseUrl() . '/retriever/' . $args["contact"]['id']); } } @@ -1018,12 +1017,12 @@ function retriever_post_remote_hook(App &$a, array &$item) { * @param string $s HTML string to which to append settings content (by ref) */ function retriever_addon_settings(App &$a, string &$s) { - $all_photos = DI::config()->get(Session::getLocalUser(), 'retriever', 'all_photos'); - $oembed = DI::config()->get(Session::getLocalUser(), 'retriever', 'oembed'); + $all_photos = DI::config()->get(DI::userSession()->getLocalUserId(), 'retriever', 'all_photos'); + $oembed = DI::config()->get(DI::userSession()->getLocalUserId(), 'retriever', 'oembed'); $template = Renderer::getMarkupTemplate('/settings.tpl', 'addon/retriever/'); $config = array('$submit' => DI::l10n()->t('Save Settings'), '$title' => DI::l10n()->t('Retriever Settings'), - '$help' => DI::baseUrl()->get(true) . '/retriever/help', + '$help' => DI::baseUrl() . '/retriever/help', '$allow_images' => DI::config()->get('retriever', 'allow_images')); $config['$allphotos'] = array('retriever_all_photos', DI::l10n()->t('All Photos'), @@ -1044,15 +1043,15 @@ function retriever_addon_settings(App &$a, string &$s) { */ function retriever_addon_settings_post(App $a, array $post) { if ($post['retriever_all_photos']) { - DI::config()->set(Session::getLocalUser(), 'retriever', 'all_photos', $post['retriever_all_photos']); + DI::config()->set(DI::userSession()->getLocalUserId(), 'retriever', 'all_photos', $post['retriever_all_photos']); } else { - DI::config()->delete(Session::getLocalUser(), 'retriever', 'all_photos'); + DI::config()->delete(DI::userSession()->getLocalUserId(), 'retriever', 'all_photos'); } if ($post['retriever_oembed']) { - DI::config()->set(Session::getLocalUser(), 'retriever', 'oembed', $post['retriever_oembed']); + DI::config()->set(DI::userSession()->getLocalUserId(), 'retriever', 'oembed', $post['retriever_oembed']); } else { - DI::config()->delete(Session::getLocalUser(), 'retriever', 'oembed'); + DI::config()->delete(DI::userSession()->getLocalUserId(), 'retriever', 'oembed'); } } From 4d15daed7040b3dd543c53669e66a635d0cad024 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Tue, 26 Dec 2023 13:44:36 +0100 Subject: [PATCH 03/10] more adaption to latest release --- retriever/retriever.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 6e499d94..8e721eaf 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -88,10 +88,9 @@ function retriever_module() {} /** * @brief Admin page hook for retriever plugin * - * @param App $a App object (unused) * @param string $o HTML to append content to (by ref) */ -function retriever_addon_admin(App $a, string &$o) { +function retriever_addon_admin(string &$o) { $template = Renderer::getMarkupTemplate('admin.tpl', 'addon/retriever/'); $downloads_per_cron = DI::config()->get('retriever', 'downloads_per_cron'); @@ -986,10 +985,9 @@ function retriever_contact_photo_menu(array &$args) { /** * @brief Hook for processing new incoming items * - * @param App $a The App object (by ref) * @param array $item New item, which has not yet been inserted into database (by ref) */ -function retriever_post_remote_hook(App &$a, array &$item) { +function retriever_post_remote_hook(array &$item) { Logger::info('retriever_post_remote_hook: ' . $item['uri'] . ' ' . $item['uid'] . ' ' . $item['contact-id']); $retriever_rule = get_retriever_rule($item['contact-id'], $item["uid"], false); From 05b6b05d682337efc6e3e06619091dd1876fbcc8 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Tue, 26 Dec 2023 15:14:35 +0100 Subject: [PATCH 04/10] some changes that were long overdue --- retriever/retriever.php | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 8e721eaf..bcc0963a 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -125,12 +125,12 @@ function retriever_addon_admin_post () { * @brief Cron jobs for retriever plugin */ function retriever_cron() { - $downloads_per_cron = DI::config()->get('retriever', 'downloads_per_cron'); + $downloads_per_cron = DI::config()->get('retriever', 'downloads_per_cron', '100'); // Do this first, otherwise it can interfere with retriever_retrieve_items - retriever_clean_up_completed_resources($downloads_per_cron); + retriever_clean_up_completed_resources((int)$downloads_per_cron); - retriever_retrieve_items($downloads_per_cron); + retriever_retrieve_items((int)$downloads_per_cron); retriever_tidy(); } @@ -476,7 +476,7 @@ function add_retriever_resource(string $url, string $uid, string $cid, bool $bin return $resource; } - DBA::insert('retriever_resource', ['item-uid' => intval($uid), 'contact-id' => intval($cid), 'type' => $type, 'binary' => ($binary ? 1 : 0), 'url' => $url, 'completed' => DateTimeFormat::utcNow(), 'data' => $data]); + DBA::insert('retriever_resource', ['item-uid' => intval($uid), 'contact-id' => intval($cid), 'type' => $type, 'binary' => ($binary ? 1 : 0), 'url' => $url, 'completed' => DateTimeFormat::utcNow(), 'data' => $data, 'redirect-url' => '']); $resource = DBA::selectFirst('retriever_resource', [], ['url' => $url, 'item-uid' => intval($uid), 'contact-id' => intval($cid)]); if ($resource) { retriever_resource_completed($resource); @@ -495,7 +495,7 @@ function add_retriever_resource(string $url, string $uid, string $cid, bool $bin return $resource; } - DBA::insert('retriever_resource', ['item-uid' => intval($uid), 'contact-id' => intval($cid), 'binary' => ($binary ? 1 : 0), 'url' => $url]); + DBA::insert('retriever_resource', ['item-uid' => intval($uid), 'contact-id' => intval($cid), 'binary' => ($binary ? 1 : 0), 'url' => $url, 'redirect-url' => '']); return DBA::selectFirst('retriever_resource', [], ['url' => $url, 'item-uid' => intval($uid), 'contact-id' => intval($cid)]); } From 87f7f8f8cbbaed0454f0c4747e27f85c338ed7f7 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Tue, 26 Dec 2023 15:52:57 +0100 Subject: [PATCH 05/10] more overdue adaptations --- retriever/retriever.php | 42 +++++++++++++++++--------------- retriever/templates/settings.tpl | 13 +--------- 2 files changed, 23 insertions(+), 32 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index bcc0963a..37715126 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -840,12 +840,10 @@ function retriever_transform_images(array &$item, array $resource) { /** * @brief Displays the retriever configuration page for a contact. Alternatively, if the user clicked the "help" button, display the help content. - * - * @param App $a The App object */ -function retriever_content(App $a) { +function retriever_content() { if (!DI::userSession()->getLocalUserId()) { - $a->page['content'] .= "

Please log in

"; + DI::page()['content'] .= "

Please log in

"; return; } if (isset(DI::args()->getArgv()[1]) and DI::args()->getArgv()[1] === 'help') { @@ -854,7 +852,7 @@ function retriever_content(App $a) { $feeds[$i]['url'] = DI::baseUrl() . '/retriever/' . $feeds[$i]['id']; } $template = Renderer::getMarkupTemplate('/help.tpl', 'addon/retriever/'); - $a->page['content'] .= Renderer::replaceMacros($template, array( + DI::page()['content'] .= Renderer::replaceMacros($template, array( '$config' => DI::baseUrl() . '/settings/addon', '$allow_images' => DI::config()->get('retriever', 'allow_images'), '$feeds' => $feeds)); @@ -895,12 +893,12 @@ function retriever_content(App $a) { } } DBA::update('retriever_rule', ['data' => json_encode($retriever_rule['data'])], ['id' => intval($retriever_rule["id"])], ['data' => '']); - $a->page['content'] .= "

Settings Updated"; + DI::page()['content'] .= "

Settings Updated"; if (!empty($_POST["retriever_retrospective"])) { apply_retrospective($retriever_rule, $_POST["retriever_retrospective"]); - $a->page['content'] .= " and retrospectively applied to " . $_POST["retriever_retrospective"] . " posts"; + DI::page()['content'] .= " and retrospectively applied to " . $_POST["retriever_retrospective"] . " posts"; } - $a->page['content'] .= ".

"; + DI::page()['content'] .= ".

"; } $template = Renderer::getMarkupTemplate('/rule-config.tpl', 'addon/retriever/'); @@ -1011,12 +1009,11 @@ function retriever_post_remote_hook(array &$item) { /** * @brief Hook for adding per-user retriever settings to the user's settings page * - * @param App $a The App object (by ref) - * @param string $s HTML string to which to append settings content (by ref) + * @param array $data Hook data array */ -function retriever_addon_settings(App &$a, string &$s) { - $all_photos = DI::config()->get(DI::userSession()->getLocalUserId(), 'retriever', 'all_photos'); - $oembed = DI::config()->get(DI::userSession()->getLocalUserId(), 'retriever', 'oembed'); +function retriever_addon_settings(array &$data) { + $all_photos = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'retriever', 'all_photos'); + $oembed = DI::pConfig()->get(DI::userSession()->getLocalUserId(), 'retriever', 'oembed'); $template = Renderer::getMarkupTemplate('/settings.tpl', 'addon/retriever/'); $config = array('$submit' => DI::l10n()->t('Save Settings'), '$title' => DI::l10n()->t('Retriever Settings'), @@ -1030,26 +1027,31 @@ function retriever_addon_settings(App &$a, string &$s) { DI::l10n()->t('Resolve OEmbed'), $oembed, DI::l10n()->t('Check this to attempt to retrieve embedded content for all posts')); - $s .= Renderer::replaceMacros($template, $config); + $html = Renderer::replaceMacros($template, $config); + $data = [ + 'addon' => 'retriever', + 'title' => DI::l10n()->t('Retriever Settings'), + 'html' => $html, + ]; } /** * @brief Hook for processing post results from user's settings page * - * @param App $a The App object * @param array $post Posted content + * @return void */ -function retriever_addon_settings_post(App $a, array $post) { +function retriever_addon_settings_post(array $post) { if ($post['retriever_all_photos']) { - DI::config()->set(DI::userSession()->getLocalUserId(), 'retriever', 'all_photos', $post['retriever_all_photos']); + DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'retriever', 'all_photos', $post['retriever_all_photos']); } else { - DI::config()->delete(DI::userSession()->getLocalUserId(), 'retriever', 'all_photos'); + DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'retriever', 'all_photos'); } if ($post['retriever_oembed']) { - DI::config()->set(DI::userSession()->getLocalUserId(), 'retriever', 'oembed', $post['retriever_oembed']); + DI::pConfig()->set(DI::userSession()->getLocalUserId(), 'retriever', 'oembed', $post['retriever_oembed']); } else { - DI::config()->delete(DI::userSession()->getLocalUserId(), 'retriever', 'oembed'); + DI::pConfig()->delete(DI::userSession()->getLocalUserId(), 'retriever', 'oembed'); } } diff --git a/retriever/templates/settings.tpl b/retriever/templates/settings.tpl index 3151fd72..f6437be9 100644 --- a/retriever/templates/settings.tpl +++ b/retriever/templates/settings.tpl @@ -1,16 +1,5 @@ - -

{{$title}}

- - From b5a2e9662fc9414c5a6b6797d3d6983558f89574 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Tue, 26 Dec 2023 16:33:37 +0000 Subject: [PATCH 06/10] debugging some issues --- retriever/retriever.php | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 37715126..4d9ae127 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -143,6 +143,7 @@ $retriever_item_count = 0; * @param int $max_items Maximum number of items to retrieve in this call */ function retriever_retrieve_items(int $max_items) { + Logger::debug('@@@ retriever_retrieve_items started'); global $retriever_item_count; $retriever_schedule = array(array(1,'minute'), @@ -181,6 +182,7 @@ function retriever_retrieve_items(int $max_items) { } while ($retrieve_items > 0); Logger::debug('retriever_retrieve_items: finished retrieving items'); + Logger::debug('@@@ retriever_retrieve_items finished'); } /** @@ -189,9 +191,11 @@ function retriever_retrieve_items(int $max_items) { * @param int $max_items Maximum number of items to retrieve in this call */ function retriever_clean_up_completed_resources(int $max_items) { + Logger::debug('@@@ retriever_clean_up_completed_resources started'); // TODO: figure out how to do this with DBA module $r = DBA::p("SELECT retriever_resource.`id` as resource, retriever_item.`id` as item FROM retriever_resource, retriever_item, retriever_rule WHERE retriever_item.`finished` = 0 AND retriever_item.`resource` = retriever_resource.`id` AND retriever_resource.`completed` IS NOT NULL AND retriever_item.`contact-id` = retriever_rule.`contact-id` AND retriever_item.`item-uid` = retriever_rule.`uid` LIMIT $max_items"); if (!DBA::isResult($r)) { + Logger::debug('@@@ retriever_clean_up_completed_resources nothing to do'); return; } Logger::debug('retriever_clean_up_completed_resources: items waiting even though resource has completed: ' . DBA::numRows($r)); @@ -217,6 +221,7 @@ function retriever_clean_up_completed_resources(int $max_items) { DBA::update('retriever_item', ['finished' => 1], ['id' => intval($retriever_item['id'])], ['finished' => 0]); retriever_check_item_completed($item); } + Logger::debug('@@@ retriever_clean_up_completed_resources finished'); } /** @@ -376,7 +381,9 @@ function retriever_item_completed(string $retriever_item_id, array $resource) { // Note: the retriever might be null. Doesn't matter. $retriever_rule = get_retriever_rule($retriever_item['contact-id'], $retriever_item['item-uid'], false); - retriever_apply_completed_resource_to_item($retriever_rule, $item, $resource); + if ($retriever_rule) { + retriever_apply_completed_resource_to_item($retriever_rule, $item, $resource); + } DBA::update('retriever_item', ['finished' => 1], ['id' => intval($retriever_item['id'])], ['finished' => 0]); retriever_check_item_completed($item); @@ -739,7 +746,7 @@ function retrieve_images(array &$item) { if (!$url) { continue; } - if (strpos($url, DI::baseUrl()) === FALSE) { + if (strpos($url, (string)(DI::baseUrl())) === FALSE) { $resource = add_retriever_resource($url, $item['uid'], $item['contact-id'], true); if (!$resource['completed']) { add_retriever_item($item, $resource); @@ -824,7 +831,12 @@ function retriever_transform_images(array &$item, array $resource) { Logger::warning('retriever_transform_images: invalid image found at URL ' . $resource['url'] . ' for item ' . $item['id']); return; } - $photo = Photo::store($image, $uid, $cid, $rid, $filename, $album, 0, 0, "", "", "", "", $desc); + try { + $photo = Photo::store($image, $uid, $cid, $rid, $filename, $album, 0, 0, "", "", "", "", $desc); + } catch (Exception $e) { + Logger::error('retriever_transform_images: unable to store photo ' . $resource['url'] . ' error: ' . $e->getMessage()); + return; + } $new_url = DI::baseUrl() . '/photo/' . $rid . '-0.' . $image->getExt(); if (!strlen($new_url)) { Logger::warning('retriever_transform_images: no replacement URL for image ' . $resource['url']); From cd2809575c07d9abbdf8846f8c3b76a30fa7f81b Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Tue, 26 Dec 2023 16:34:57 +0000 Subject: [PATCH 07/10] some more robust mailstream stuff --- mailstream/mailstream.php | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/mailstream/mailstream.php b/mailstream/mailstream.php index f0dcf341..cfc665f0 100644 --- a/mailstream/mailstream.php +++ b/mailstream/mailstream.php @@ -118,9 +118,28 @@ function mailstream_send_hook(array $data) return; } + if ($item['deleted']) { + Logger::debug('mailstream_send_hook skipping deleted item', ['guid' => $item['guid']]); + return; + } + $user = User::getById($item['uid']); if (empty($user)) { - Logger::error('mailstream_send_hook could not fund user', ['uid' => $item['uid']]); + Logger::error('mailstream_send_hook could not find user', ['uid' => $item['uid']]); + return; + } + + $contact = DBA::selectFirst('contact', [], ['id' => $item['contact-id'], 'self' => false]); + if (!DBA::isResult($contact)) { + Logger::error('mailstream_send_hook could not find contact', ['guid' => $item['guid'], 'contact-id' => $item['contact-id']]); + return; + } + if ($contact['blocked']) { + Logger::error('mailstream_send_hook contact is blocked', ['guid' => $item['guid'], 'contact-id' => $item['contact-id']]); + return; + } + if (array_key_exists('ignored', $contact) && $contact['ignored']) { + Logger::error('mailstream_send_hook contact is ignored', ['guid' => $item['guid'], 'contact-id' => $item['contact-id']]); return; } @@ -143,7 +162,6 @@ function mailstream_send_hook(array $data) function mailstream_post_hook(array &$item) { mailstream_check_version(); - Logger::debug('@@@ mailstream_post_hook', ['item-uid' => $item['uid']]); if (!DI::pConfig()->get($item['uid'], 'mailstream', 'enabled')) { Logger::debug('mailstream: not enabled for item ' . $item['id'] . ' uid ' . $item['uid']); From e0226981a1dedf79c5859c4bc77cf776a7d7db8d Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Tue, 26 Dec 2023 16:35:27 +0000 Subject: [PATCH 08/10] trying to get phototrack to work --- phototrack/phototrack.php | 52 +++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/phototrack/phototrack.php b/phototrack/phototrack.php index 0ede2a1c..97fde076 100644 --- a/phototrack/phototrack.php +++ b/phototrack/phototrack.php @@ -102,8 +102,8 @@ function phototrack_photo_use($photo, $table, $field, $id) { } } -function phototrack_check_field_url($a, $table, $field, $id, $url) { - Logger::info('@@@ phototrack_check_field_url table ' . $table . ' field ' . $field . ' id ' . $id . ' url ' . $url); +function phototrack_check_field_url($a, $table, $id_field, $field, $id, $url) { + Logger::info('@@@ phototrack_check_field_url table ' . $table . ' id_field ' . $id_field . ' field ' . $field . ' id ' . $id . ' url ' . $url); $baseurl = DI::baseUrl()->get(true); if (strpos($url, $baseurl) === FALSE) { return; @@ -126,35 +126,32 @@ function phototrack_check_field_url($a, $table, $field, $id, $url) { } } -function phototrack_check_field_bbcode($a, $table, $field, $id, $value) { +function phototrack_check_field_bbcode($a, $table, $id_field, $field, $id, $value) { + Logger::info('@@@ phototrack_check_field_url table ' . $table . ' id_field ' . $id_field . ' field ' . $field . ' id ' . $id . ' value ' . $value); $baseurl = DI::baseUrl()->get(true); $matches = array(); preg_match_all("/\[img(\=([0-9]*)x([0-9]*))?\](.*?)\[\/img\]/ism", $value, $matches); foreach ($matches[4] as $url) { - phototrack_check_field_url($a, $table, $field, $id, $url); + phototrack_check_field_url($a, $table, $id_field, $field, $id, $url); } } function phototrack_post_local_end(&$a, &$item) { - phototrack_check_row($a, 'item', $item); - phototrack_check_row($a, 'item-content', $item); + phototrack_check_row($a, 'item', 'id', $item); + phototrack_check_row($a, 'item-content', 'id', $item); } function phototrack_post_remote_end(&$a, &$item) { - phototrack_check_row($a, 'item', $item); - phototrack_check_row($a, 'item-content', $item); + phototrack_check_row($a, 'item', 'id', $item); + phototrack_check_row($a, 'item-content', 'id', $item); } function phototrack_notifier_end($item) { } -function phototrack_check_row($a, $table, $row) { +function phototrack_check_row($a, $table, $id_field, $row) { switch ($table) { - case 'item': - $fields = array( - 'body' => 'bbcode'); - break; - case 'item-content': + case 'post-content': $fields = array( 'body' => 'bbcode'); break; @@ -182,8 +179,8 @@ function phototrack_check_row($a, $table, $row) { } foreach ($fields as $field => $type) { switch ($type) { - case 'bbcode': phototrack_check_field_bbcode($a, $table, $field, $row['id'], $row[$field]); break; - case 'url': phototrack_check_field_url($a, $table, $field, $row['id'], $row[$field]); break; + case 'bbcode': phototrack_check_field_bbcode($a, $table, $id_field, $field, $row['id'], $row[$field]); break; + case 'url': phototrack_check_field_url($a, $table, $id_field, $field, $row['id'], $row[$field]); break; } } phototrack_finished_row($table, $row['id']); @@ -197,15 +194,15 @@ function phototrack_batch_size() { return PHOTOTRACK_DEFAULT_BATCH_SIZE; } -function phototrack_search_table($a, $table) { +function phototrack_search_table($a, $table, $id_field) { $batch_size = phototrack_batch_size(); - $rows = DBA::p("SELECT `$table`.* FROM `$table` LEFT OUTER JOIN phototrack_row_check ON ( phototrack_row_check.`table` = '$table' AND phototrack_row_check.`row-id` = `$table`.id ) WHERE ( ( phototrack_row_check.checked IS NULL ) OR ( phototrack_row_check.checked < DATE_SUB(NOW(), INTERVAL 1 MONTH) ) ) ORDER BY phototrack_row_check.checked LIMIT $batch_size"); + $rows = DBA::p("SELECT `$table`.* FROM `$table` LEFT OUTER JOIN phototrack_row_check ON ( phototrack_row_check.`table` = '$table' AND phototrack_row_check.`row-id` = `$table`.$id_field ) WHERE ( ( phototrack_row_check.checked IS NULL ) OR ( phototrack_row_check.checked < DATE_SUB(NOW(), INTERVAL 1 MONTH) ) ) ORDER BY phototrack_row_check.checked LIMIT $batch_size"); if (DBA::isResult($rows)) { while ($row = DBA::fetch($rows)) { - phototrack_check_row($a, $table, $row); + phototrack_check_row($a, $table, $id_field, $row); } } - $r = DBA::p("SELECT COUNT(*) FROM `$table` LEFT OUTER JOIN phototrack_row_check ON ( phototrack_row_check.`table` = '$table' AND phototrack_row_check.`row-id` = `$table`.id ) WHERE ( ( phototrack_row_check.checked IS NULL ) OR ( phototrack_row_check.checked < DATE_SUB(NOW(), INTERVAL 1 MONTH) ) )"); + $r = DBA::p("SELECT COUNT(*) FROM `$table` LEFT OUTER JOIN phototrack_row_check ON ( phototrack_row_check.`table` = '$table' AND phototrack_row_check.`row-id` = `$table`.$id_field ) WHERE ( ( phototrack_row_check.checked IS NULL ) OR ( phototrack_row_check.checked < DATE_SUB(NOW(), INTERVAL 1 MONTH) ) )"); Logger::info("@@@ phototrack_search_table " . print_r(DBA::fetch($r))); $remaining = DBA::fetch($r)['count']; Logger::info('phototrack: searched ' . DBA::numRows($rows) . ' rows in table ' . $table . ', ' . $remaining . ' still remaining to search'); @@ -230,22 +227,23 @@ function phototrack_cron_time() { return false; } } + Logger::debug('@@@ phototrack: search interval reached last ' . $last . ' search interval ' . $search_interval); return true; } function phototrack_cron($a, $b) { + return; // @@@ something is broken if (!phototrack_cron_time()) { return; } DI::config()->set('phototrack', 'last_search', time()); $remaining = 0; - $remaining += phototrack_search_table($a, 'item'); - $remaining += phototrack_search_table($a, 'item-content'); - $remaining += phototrack_search_table($a, 'contact'); - $remaining += phototrack_search_table($a, 'fcontact'); - $remaining += phototrack_search_table($a, 'fsuggest'); - $remaining += phototrack_search_table($a, 'gcontact'); + $remaining += phototrack_search_table($a, 'post-content', 'uri-id'); + $remaining += phototrack_search_table($a, 'contact', 'id'); + $remaining += phototrack_search_table($a, 'fcontact', 'id'); + $remaining += phototrack_search_table($a, 'fsuggest', 'id'); + $remaining += phototrack_search_table($a, 'gcontact', 'id'); DI::config()->set('phototrack', 'remaining_items', $remaining); if ($remaining === 0) { @@ -266,7 +264,7 @@ function phototrack_tidy() { Logger::info('phototrack_tidy: deleted ' . DBA::numRows($rows) . ' photos'); } DBA::e('DROP TABLE `phototrack-temp`'); - $rows = DBA::p('SELECT id FROM phototrack_photo_use WHERE checked < DATE_SUB(NOW(), INTERVAL 14 DAY)'); + $rows = DBA::p('SELECT id FROM phototrack_photo_use WHERE checked < DATE_SUB(NOW(), INTERVAL 2 MONTH)'); foreach ($rows as $row) { DBA::e( 'DELETE FROM phototrack_photo_use WHERE id = ' . $row['id']); } From 26e95978b2fdca68206f9d7e10828c7266b0620a Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Wed, 3 Apr 2024 18:32:58 +0800 Subject: [PATCH 09/10] adaptation for 2024.03 --- retriever/retriever.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 4d9ae127..6769a60c 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -308,7 +308,7 @@ function retrieve_resource(array $resource) { DBA::update('retriever_rule', ['data' => json_encode($retriever_rule['data'])], ['id' => intval($retriever_rule["id"])], $retriever_rule); unlink($cookiejar); } - $resource['data'] = $fetch_result->getBody(); + $resource['data'] = $fetch_result->getBodyString(); $resource['http-code'] = $fetch_result->getReturnCode(); $resource['type'] = $fetch_result->getContentType(); $resource['redirect-url'] = $fetch_result->getRedirectUrl(); From 5b88c3a87991aef6068a6fb5831a6cc4cacf0147 Mon Sep 17 00:00:00 2001 From: Matthew Exon Date: Sun, 26 May 2024 13:23:00 +0100 Subject: [PATCH 10/10] adaptation for 2024.03 --- retriever/retriever.php | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/retriever/retriever.php b/retriever/retriever.php index 6769a60c..66a44fdb 100644 --- a/retriever/retriever.php +++ b/retriever/retriever.php @@ -143,7 +143,6 @@ $retriever_item_count = 0; * @param int $max_items Maximum number of items to retrieve in this call */ function retriever_retrieve_items(int $max_items) { - Logger::debug('@@@ retriever_retrieve_items started'); global $retriever_item_count; $retriever_schedule = array(array(1,'minute'), @@ -166,7 +165,7 @@ function retriever_retrieve_items(int $max_items) { $retrieve_items = $max_items - $retriever_item_count; do { Logger::debug('retriever_retrieve_items: asked for maximum ' . $max_items . ', already retrieved ' . intval($retriever_item_count) . ', retrieve ' . $retrieve_items); - $retriever_resources = DBA::selectToArray('retriever_resource', [], ['`completed` IS NULL AND (`last-try` IS NULL OR ' . implode($schedule_clauses, ' OR ') . ')'], ['order' => ['last-try' => 0], 'limit' => $retrieve_items]); + $retriever_resources = DBA::selectToArray('retriever_resource', [], ['`completed` IS NULL AND (`last-try` IS NULL OR ' . implode(' OR ', $schedule_clauses) . ')'], ['order' => ['last-try' => 0], 'limit' => $retrieve_items]); if (!is_array($retriever_resources)) { break; } @@ -182,7 +181,6 @@ function retriever_retrieve_items(int $max_items) { } while ($retrieve_items > 0); Logger::debug('retriever_retrieve_items: finished retrieving items'); - Logger::debug('@@@ retriever_retrieve_items finished'); } /** @@ -191,11 +189,9 @@ function retriever_retrieve_items(int $max_items) { * @param int $max_items Maximum number of items to retrieve in this call */ function retriever_clean_up_completed_resources(int $max_items) { - Logger::debug('@@@ retriever_clean_up_completed_resources started'); // TODO: figure out how to do this with DBA module $r = DBA::p("SELECT retriever_resource.`id` as resource, retriever_item.`id` as item FROM retriever_resource, retriever_item, retriever_rule WHERE retriever_item.`finished` = 0 AND retriever_item.`resource` = retriever_resource.`id` AND retriever_resource.`completed` IS NOT NULL AND retriever_item.`contact-id` = retriever_rule.`contact-id` AND retriever_item.`item-uid` = retriever_rule.`uid` LIMIT $max_items"); if (!DBA::isResult($r)) { - Logger::debug('@@@ retriever_clean_up_completed_resources nothing to do'); return; } Logger::debug('retriever_clean_up_completed_resources: items waiting even though resource has completed: ' . DBA::numRows($r)); @@ -221,7 +217,6 @@ function retriever_clean_up_completed_resources(int $max_items) { DBA::update('retriever_item', ['finished' => 1], ['id' => intval($retriever_item['id'])], ['finished' => 0]); retriever_check_item_completed($item); } - Logger::debug('@@@ retriever_clean_up_completed_resources finished'); } /** @@ -699,9 +694,6 @@ function retriever_get_body(array $item) { Logger::warning('retriever_get_body: item-content uri-id ' . $item['uri-id'] . ' has no body'); return $item['body']; } - if ($content['body'] != $item['body']) { - Logger::warning('@@@ this is probably bad @@@ content: ' . $content['body'] . ' @@@ item: ' . $item['body']); - } return $content['body']; } @@ -783,7 +775,7 @@ function retriever_check_item_completed(array &$item) * @param array $resource The resource that has just been completed */ function retriever_apply_completed_resource_to_item(array $retriever, array &$item, array $resource) { - Logger::debug('retriever_apply_completed_resource_to_item: retriever ' . ($retriever ? $retriever['id'] : 'none') . ' resource ' . $resource['url'] . ' plink ' . $item['plink']); + Logger::debug('retriever_apply_completed_resource_to_item', ['retriever' => $retriever ? $retriever['id'] : 'none', 'resource' => $resource['url'], 'plink' => $item['plink']]); if (strpos($resource['type'], 'image') !== false) { retriever_transform_images($item, $resource); } @@ -832,7 +824,7 @@ function retriever_transform_images(array &$item, array $resource) { return; } try { - $photo = Photo::store($image, $uid, $cid, $rid, $filename, $album, 0, 0, "", "", "", "", $desc); + $photo = Photo::store($image, $uid, $cid, $rid, $filename, $album, $scale, Photo::DEFAULT, '', '', '', '', $desc); } catch (Exception $e) { Logger::error('retriever_transform_images: unable to store photo ' . $resource['url'] . ' error: ' . $e->getMessage()); return;