restore retriever configuration

This reverts commit 0f5ba218f6.

mailstream/mailstream.php

@@ -106,12 +106,40 @@ function mailstream_send_hook(array $data)
 
 	$user = User::getById($item['uid']);
 	if (empty($user)) {
-		Logger::error('could not find user', ['uid' => $item['uid']]);
+		Logger::error('mailstream_send_hook could not find user', ['uid' => $item['uid']]);
 		return;
 	}
 
-	if (!mailstream_send($data['message_id'], $item, $user)) {
-		Logger::debug('send failed, will retry', $data);
+	$author = DBA::selectFirst('contact', ['nick', 'blocked', 'uri-id'], ['id' => $data['author-id'], 'self' => false]);
+	if (!DBA::isResult($author)) {
+		Logger::error('mailstream_send_hook could not find author', ['guid' => $item['guid'], 'author-id' => $data['author-id']]);
+		return;
+	}
+	if ($author['blocked']) {
+		Logger::info('mailstream_send_hook author is blocked', ['guid' => $item['guid'], 'author-id' => $data['author-id']]);
+		return;
+	}
+	$collapsed = false;
+	$user_contact = DBA::selectFirst('user-contact', ['cid', 'blocked', 'ignored', 'collapsed'], ['uid' => $item['uid'], 'uri-id' => $item['author-uri-id']]);
+	if (!DBA::isResult($user_contact)) {
+		$user_contact = DBA::selectFirst('user-contact', ['cid', 'blocked', 'ignored', 'collapsed'], ['uid' => $item['uid'], 'cid' => $item['author-id']]);
+	}
+	if (DBA::isResult($user_contact)) {
+		if ($user_contact['blocked']) {
+			Logger::info('mailstream_send_hook author is blocked', ['guid' => $item['guid'], 'cid' => $user_contact['cid']]);
+			return;
+		}
+		if ($user_contact['ignored']) {
+			Logger::info('mailstream_send_hook author is ignored', ['guid' => $item['guid'], 'cid' => $user_contact['cid']]);
+			return;
+		}
+		if ($user_contact['collapsed']) {
+			$collapsed = true;
+		}
+	}
+
+	if (!mailstream_send($data['message_id'], $item, $user, $collapsed)) {
+		Logger::debug('mailstream_send_hook send failed, will retry', $data);
 		if (!Worker::defer()) {
 			Logger::error('failed and could not defer', $data);
 		}
@@ -133,8 +161,8 @@ function mailstream_post_hook(array &$item)
 		return;
 	}
 	if (!DI::pConfig()->get($item['uid'], 'mailstream', 'enabled')) {
-		Logger::debug('mailstream: not enabled for item ' . $item['id'] . ' uid ' . $item['uid']);
-		// return;
+		Logger::debug('mailstream: not enabled.', ['item' => $item['id'], ' uid ' => $item['uid']]);
+		return;
 	}
 	if (!$item['contact-id']) {
 		Logger::debug('no contact-id', ['item' => $item['id']]);
@@ -164,6 +192,7 @@ function mailstream_post_hook(array &$item)
 	$send_hook_data = [
 		'uid' => $item['uid'],
 		'contact-id' => $item['contact-id'],
+		'author-id' => $item['author-id'],
 		'uri' => $item['uri'],
 		'message_id' => $message_id,
 		'tries' => 0,
@@ -350,10 +379,11 @@ function mailstream_subject(array $item): string
  * @param string $message_id ID of the message (RFC 1036)
  * @param array  $item       content of the item
  * @param array  $user       results from the user table
+ * @param bool   $collapsed  true if the content should be hidden
  *
  * @return bool True if this message has been completed.  False if it should be retried.
  */
-function mailstream_send(string $message_id, array $item, array $user): bool
+function mailstream_send(string $message_id, array $item, array $user, bool $collapsed): bool
 {
 	if (!is_array($item)) {
 		Logger::error('item is empty', ['message_id' => $message_id]);
@@ -371,10 +401,16 @@ function mailstream_send(string $message_id, array $item, array $user): bool
 
 	require_once(dirname(__file__) . '/phpmailer/class.phpmailer.php');
 
+	if ($collapsed) {
+		$item['body'] = DI::l10n()->t('Content from %s is collapsed', $item['author-name']);
+	} else {
 		$item['body'] = Post\Media::addAttachmentsToBody($item['uri-id'], $item['body']);
+	}
 
 	$attachments = [];
+	if (!$collapsed) {
 		mailstream_do_images($item, $attachments);
+	}
 	$frommail = DI::config()->get('mailstream', 'frommail');
 	if ($frommail == '') {
 		$frommail = 'friendica@localhost.local';
@@ -429,9 +465,9 @@ function mailstream_send(string $message_id, array $item, array $user): bool
 			'address' => $address
 		]);
 	} catch (phpmailerException $e) {
-		Logger::debug('PHPMailer exception sending message', ['id' => $message_id, 'error' => $e->errorMessage()]);
+		Logger::debug('mailstream_send PHPMailer exception sending message', ['item uri' => $item['uri'], 'message_id' => $message_id, 'error' => $e->errorMessage()]);
 	} catch (Exception $e) {
-		Logger::debug('exception sending message', ['id' => $message_id, 'error' => $e->getMessage()]);
+		Logger::debug('mailstream_send exception sending message', ['item uri' => $item['uri'], 'message_id' => $message_id, 'error' => $e->errorMessage()]);
 	}
 
 	return true;

retriever/retriever.php

@@ -16,6 +16,8 @@ use Friendica\Core\System;
 use Friendica\Content\Text\HTML;
 use Friendica\Content\Text\BBCode;
 use Friendica\Model\Photo;
+use Friendica\Network\HTTPClient\Client\HttpClientAccept;
+use Friendica\Network\HTTPClient\Client\HttpClientOptions;
 use Friendica\Object\Image;
 use Friendica\Util\Network;
 use Friendica\Database\DBA;
@@ -35,6 +37,7 @@ function retriever_install() {
 	Hook::register('addon_settings_post', 'addon/retriever/retriever.php', 'retriever_addon_settings_post');
 	Hook::register('post_remote', 'addon/retriever/retriever.php', 'retriever_post_remote_hook');
 	Hook::register('contact_photo_menu', 'addon/retriever/retriever.php', 'retriever_contact_photo_menu');
+	Hook::register('retriever_mod_post', 'addon/retriever/retriever.php', 'retriever_content');
 	Hook::register('cron', 'addon/retriever/retriever.php', 'retriever_cron');
 
 	if (DI::config()->get('retriever', 'dbversion') == '0.14') {
@@ -75,6 +78,7 @@ function retriever_uninstall() {
 	Hook::unregister('addon_settings', 'addon/retriever/retriever.php', 'retriever_addon_settings');
 	Hook::unregister('addon_settings_post', 'addon/retriever/retriever.php', 'retriever_addon_settings_post');
 	Hook::unregister('contact_photo_menu', 'addon/retriever/retriever.php', 'retriever_contact_photo_menu');
+	Hook::unregister('retriever_mod_post', 'addon/retriever/retriever.php', 'retriever_content');
 	Hook::unregister('cron', 'addon/retriever/retriever.php', 'retriever_cron');
 }
 
@@ -291,13 +295,12 @@ function retrieve_resource(array $resource) {
 
 	try {
 		Logger::debug('retrieve_resource: ' . ($resource['num-tries'] + 1) . ' attempt at resource ' . $resource['id'] . ' ' . $resource['url']);
-		$redirects = 0;
 		$cookiejar = '';
 		if (array_key_exists('storecookies', $rule_data) && $rule_data['storecookies']) {
 			$cookiejar = tempnam(System::getTempPath(), 'cookiejar-retriever-');
 			file_put_contents($cookiejar, $rule_data['cookiedata']);
 		}
-		$fetch_result = DI::httpClient()->fetchFull($resource['url'], $redirects, 0, $cookiejar);
+		$fetch_result = DI::httpClient()->get($resource['url'], HttpClientAccept::DEFAULT, [HttpClientOptions::COOKIEJAR => $cookiejar]);
 		if (array_key_exists('storecookies', $rule_data) && $rule_data['storecookies']) {
 			$retriever_rule['data']['cookiedata'] = file_get_contents($cookiejar);
 			DBA::update('retriever_rule', ['data' => json_encode($retriever_rule['data'])], ['id' => intval($retriever_rule["id"])], $retriever_rule);
@@ -448,8 +451,10 @@ function retriever_on_item_insert(array $retriever, array &$item) {
 	}
 
 	$resource = add_retriever_resource($url, $item['uid'], $item['contact-id']);
+	if (is_array($resource)) {
 		$retriever_item_id = add_retriever_item($item, $resource);
 	}
+}
 
 /**
  * @brief Creates a new resource to be downloaded from the supplied URL.  Unique resources are created for each URL, UID and contact ID, because different contact IDs may have different rules for how to retrieve them.  If the URL is actually a data URL, the resource is completed immediately.
@@ -660,12 +665,16 @@ function retriever_extract(DOMDocument $doc, array $retriever) {
  * @return DOMDocument New DOM document with global URLs
  */
 function retriever_globalise_urls(DOMDocument $doc, array $resource) {
-	$components = parse_url($resource['redirect-url']);
+	$url = $resource['redirect-url'];
+	if ($url == "") {
+		$url = $resource['url'];
+	}
+	$components = parse_url($url);
 	if (!array_key_exists('scheme', $components) || !array_key_exists('host', $components) || !array_key_exists('path', $components)) {
 		return $doc;
 	}
 	$rooturl = $components['scheme'] . "://" . $components['host'];
-	$dirurl = $rooturl . dirname($components['path']) . "/";
+	$dirurl = $rooturl . dirname($components['path']);
 	$params = array('$dirurl' => $dirurl, '$rooturl' => $rooturl);
 	$fix_urls_template = Renderer::getMarkupTemplate('fix-urls.tpl', 'addon/retriever/');
 	$fix_urls_xslt = Renderer::replaceMacros($fix_urls_template, $params);
@@ -740,15 +749,18 @@ function retrieve_images(array &$item) {
 		}
 		if (strpos($url, (string)(DI::baseUrl())) === FALSE) {
 			$resource = add_retriever_resource($url, $item['uid'], $item['contact-id'], true);
+			if (!is_array($resource)) {
+				Logger::error('retrieve_images: could not add resource', ['url' => $url, 'uid' => $item['uid'], 'contact-id' => $item['contact-id']]);
+				continue;
+			}
 			if (!$resource['completed']) {
 				add_retriever_item($item, $resource);
+				continue;
 			}
-			else {
 			retriever_transform_images($item, $resource);
 		}
 	}
 }
-}
 
 /**
  * @brief Checks if an item has been completed, i.e. all its associated retriever_item rows have been retrieved.  If so, update the item to be visible again.
@@ -846,11 +858,12 @@ function retriever_transform_images(array &$item, array $resource) {
  * @brief Displays the retriever configuration page for a contact.  Alternatively, if the user clicked the "help" button, display the help content.
  */
 function retriever_content() {
+    $e = new \Exception;
 	if (!DI::userSession()->getLocalUserId()) {
 		DI::page()['content'] .= "<p>Please log in</p>";
 		return;
 	}
-	if (isset(DI::args()->getArgv()[1]) and DI::args()->getArgv()[1] === 'help') {
+	if (DI::args()->get(1) === 'help') {
 		$feeds = DBA::selectToArray('contact', ['id', 'name', 'thumb'], ['uid' => DI::userSession()->getLocalUserId(), 'network' => 'feed']);
 		for ($i = 0; $i < count($feeds); ++$i) {
 			$feeds[$i]['url'] = DI::baseUrl() . '/retriever/' . $feeds[$i]['id'];
@@ -862,8 +875,8 @@ function retriever_content() {
 								       '$feeds' => $feeds));
 		return;
 	}
-	if (isset(DI::args()->getArgv()[1])) {
-		$arg1 = DI::args()->getArgv()[1];
+	if (DI::args()->get(1)) {
+		$arg1 = DI::args()->get(1);
 		$retriever_rule = get_retriever_rule($arg1, DI::userSession()->getLocalUserId(), false);
 		if (!$retriever_rule) {
 			$retriever_rule = ['id' => 0, 'data' => ['enable' => 0, 'modurl' => '', 'pattern' => '', 'replace' => '', 'images' => 0, 'storecookies' => 0, 'cookiedata' => '', 'customxslt' => '', 'include' => '', 'exclude' => '']];

retriever/templates/fix-urls.tpl

@@ -14,7 +14,7 @@
 
   <xsl:template match="*/@src[starts-with(.,'.')]">
     <xsl:attribute name="src">
-      <xsl:value-of select="concat('{{$dirurl}}',.)"/>
+      <xsl:value-of select="concat('{{$dirurl}}/',.)"/>
     </xsl:attribute>
   </xsl:template>
   <xsl:template match="*/@src[starts-with(.,'/')]">
@@ -22,5 +22,10 @@
       <xsl:value-of select="concat('{{$rooturl}}',.)"/>
     </xsl:attribute>
   </xsl:template>
+  <xsl:template match="*/@src[not(starts-with(.,'/')) and not(contains(.,':'))]">
+    <xsl:attribute name="src">
+      <xsl:value-of select="concat('{{$dirurl}}',.)"/>
+    </xsl:attribute>
+  </xsl:template>
 
 </xsl:stylesheet>