From da663db1cb1d9ab0c0eb06ec8e5408033265c942 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 23 Nov 2019 23:43:59 +0000 Subject: [PATCH 1/9] Discourse addon created --- discourse/README | 97 +++++++++++++++++ discourse/discourse.php | 224 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 321 insertions(+) create mode 100644 discourse/README create mode 100644 discourse/discourse.php diff --git a/discourse/README b/discourse/README new file mode 100644 index 00000000..e4a8b9d3 --- /dev/null +++ b/discourse/README @@ -0,0 +1,97 @@ +NSFW + +"Not safe for work" + +Scans the message content for the string 'nsfw' +(case insensitive) and if found replaces the content +with a "click to open/close" link, default is closed. + +If you click on the 'Not safe for work' addon under +/settings/addon a text field appears, where you can +extend the list of search terms. The terms must be +seperated by commas. + +It is also possible to enter profile URLs as values. +This is quite useful for the case, that you perhaps +don't want to see postings by person_A, but person_B +is one of your contacts and person_B used to reshare +postings by person_A. + +You can also make use of regular expressions. +They also have to be seperated by commas and the +regex itself has to be enclosed with slashes: + + ... nsfw, //, politics,... + +--------------- +A few examples: +--------------- + +1) +Let's say you don't want to see postings which contain +the term 'fake news' + +The term could appear in several ways: + +fakenews, fake news, fake_news, fake-news, f@ke news, +f4ke news, f4k3 n3ws, and so on and so on and so on. + +You could write every possible version of it as single +item into your NSFW-filter list, but this can also be +done with a single regex, which matches all of them: + + /f[@4a]k[3e][-_ ]n[3e]w[sz]/ + + +2) +Another use case could be, that you are simply not +interested in postings about christmas. + + /christmas(?:[-_ ]?(?:tree|time|eve|pudding))?/ + + +ATTENTION: + +It is absolutely important, that you use grouping +parentheses instead of capturing parentheses!! + +Grouping parentheses are: + + (?: ) + +If you use capturing parentheses, which are + + ( ) + +it will produce errors and the regex won't work and +at least your targets will not get collapsed. + + + +3) +Another possibility is the usage of a so called +'lookbehind' construct. I'll give an example followed +by a descripton: + + /(? + * + */ +//use DOMDocument; +//use DOMXPath; +use Friendica\App; +use Friendica\Core\Hook; +use Friendica\Core\L10n; +use Friendica\Core\Logger; +use Friendica\Core\PConfig; +use Friendica\Util\XML; +use Friendica\Content\Text\Markdown; +use Friendica\Util\Network; +Use Friendica\Util\DateTimeFormat; + +function discourse_install() +{ + Hook::register('email_getmessage', __FILE__, 'discourse_email_getmessage'); + Hook::register('email_getmessage_end', __FILE__, 'discourse_email_getmessage_end'); + Hook::register('addon_settings', __FILE__, 'discourse_addon_settings'); + Hook::register('addon_settings_post', __FILE__, 'discourse_addon_settings_post'); +} + +function discourse_uninstall() +{ + Hook::unregister('email_getmessage', __FILE__, 'discourse_email_getmessage'); + Hook::unregister('email_getmessage_end', __FILE__, 'discourse_email_getmessage_end'); + Hook::unregister('addon_settings', __FILE__, 'discourse_addon_settings'); + Hook::unregister('addon_settings_post', __FILE__, 'discourse_addon_settings_post'); +} + +function discourse_addon_settings(App $a, &$s) +{ +} + +function discourse_addon_settings_post(App $a) +{ +} + +function discourse_email_getmessage(App $a, &$message) +{ +// Logger::info('Got raw message', $message); + // Remove the title on comments, they don't serve any purpose there + if ($message['item']['parent-uri'] != $message['item']['uri']) { + unset($message['item']['title']); + } + + if (preg_match('=topic/(.*)/(.*)@(.*)=', $message['item']['uri'], $matches)) { + Logger::info('Got post data', ['topic' => $matches[1], 'post' => $matches[2], 'host' => $matches[3]]); + if (discourse_fetch_post_from_api($message, $matches[2], $matches[3])) { + return; + } + } + + // Search in the text part for the link to the discourse entry and the text body + // The text body is used as alternative, if the fetched HTML isn't working + if (!empty($message['text'])) { + discourse_get_text($message); + } + + if (!empty($message['item']['plink'])) { + if (preg_match('=(http.*)/t/.*/(.*\d)/(.*\d)=', $message['item']['plink'], $matches)) { + if (discourse_fetch_topic_from_api($message, $matches[1], $matches[1], $matches[1])) { + return; + } + } + } + + // Search in the HTML part for the discourse entry and the author profile + if (!empty($message['html'])) { + discourse_get_html($message); + } +} + +function discourse_fetch_topic_from_api(&$message, $host, $topic, $pid) +{ + $url = $host . '/t/' . $topic . '/posts.json?posts_ids[]=' . $pid; + $curlResult = Network::curl($url); + if (!$curlResult->isSuccess()) { + return false; + } + $raw = $curlResult->getBody(); + $data = json_decode($raw, true); + $posts = $data['post_stream']['posts']; + foreach($posts as $post) { + if ($post['post_number'] != $pid) { + continue; + } + Logger::info('Got post data from topic', $post); + discourse_process_post($message, $post); + return true; + } + return false; +} + +function discourse_fetch_post_from_api(&$message, $post, $host) +{ + $url = "https://" . $host . '/posts/' . $post . '.json'; + $curlResult = Network::curl($url); + if (!$curlResult->isSuccess()) { + return false; + } + + $raw = $curlResult->getBody(); + $data = json_decode($raw, true); + if (empty($data)) { + return false; + } + + discourse_process_post($message, $data); + + Logger::info('Got API data', $message); + return true; +} + +function discourse_process_post(&$message, $post) +{ + if ($post['post_number'] == 1) { + // Thread information + } + + $nick = $post['username']; + $name = $post['name']; + // User information + + $message['html'] = $post['cooked']; + $message['text'] = $post['raw']; + $message['item']['created'] = DateTimeFormat::utc($post['created_at']); +} + +function discourse_get_html(&$message) +{ + $doc = new DOMDocument(); + $doc2 = new DOMDocument(); + $doc->preserveWhiteSpace = false; + + $html = mb_convert_encoding($message['html'], 'HTML-ENTITIES', "UTF-8"); + @$doc->loadHTML($html, LIBXML_HTML_NODEFDTD); + + $xpath = new DomXPath($doc); + + // Fetch the first 'div' before the 'hr' -hopefully this fits for all systems + $result = $xpath->query("//hr//preceding::div[1]"); + $div = $doc2->importNode($result->item(0), true); + $doc2->appendChild($div); + $message['html'] = $doc2->saveHTML(); + Logger::info('Found html body', ['html' => $message['html']]); + + $profile = discourse_get_profile($xpath); + if (!empty($profile)) { + Logger::info('Found profile', $profile); +/* + $message['item']['author-avatar'] = $contact['avatar']; + $message['item']['author-link'] = $profile['link']; + $message['item']['author-name'] = $profile['name']; +*/ + } +} + +function discourse_get_text(&$message) +{ + $text = $message['text']; + $text = str_replace("\r", '', $text); + $pos = strpos($text, "\n---\n"); + if ($pos > 0) { + $message['text'] = trim(substr($text, 0, $pos)); + Logger::info('Found text body', ['text' => $message['text']]); + + $message['text'] = Markdown::toBBCode($message['text']); + + $text = substr($text, $pos); + if (preg_match('=\((http.*?)\)=', $text, $link)) { + $message['item']['plink'] = $link[1]; + Logger::info('Found plink', ['plink' => $message['item']['plink']]); + } + } else { + Logger::info('No separator found', ['text' => $text]); + } +} + +function discourse_get_profile($xpath) +{ + $profile = []; + $list = $xpath->query("//td//following::img"); + foreach ($list as $node) { + $attr = []; + foreach ($node->attributes as $attribute) { + $attr[$attribute->name] = $attribute->value; + } + + if (!empty($attr['src']) && !empty($attr['title']) + && !empty($attr['width']) && !empty($attr['height']) + && ($attr['width'] == $attr['height'])) { + $profile = ['avatar' => $attr['src'], 'name' => $attr['title']]; + break; + } + } + + $list = $xpath->query("//td//following::a"); + foreach ($list as $node) { + if (!empty(trim($node->textContent)) && $node->attributes->length) { + $attr = []; + foreach ($node->attributes as $attribute) { + $attr[$attribute->name] = $attribute->value; + } + if (!empty($attr['href']) && (strpos($attr['href'], '/' . $profile['name']))) { + $profile['link'] = $attr['href']; + break; + } + } + } + return $profile; +} + +function discourse_email_getmessage_end(App $a, &$message) +{ +// Logger::info('Got converted message', $message); +} From 0465de62dc695b7d6ed153a590f6b0960b1cf599 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 25 Nov 2019 11:14:22 +0000 Subject: [PATCH 2/9] Discourse contacts are generated, URI and Parent-URI are regenerated --- discourse/discourse.php | 174 +++++++++++++++++++++++++++++----------- 1 file changed, 129 insertions(+), 45 deletions(-) diff --git a/discourse/discourse.php b/discourse/discourse.php index 1df248a5..6a203950 100644 --- a/discourse/discourse.php +++ b/discourse/discourse.php @@ -14,9 +14,13 @@ use Friendica\Core\Hook; use Friendica\Core\L10n; use Friendica\Core\Logger; use Friendica\Core\PConfig; +use Friendica\Core\Protocol; +use Friendica\Database\DBA; +use Friendica\Model\Contact; use Friendica\Util\XML; use Friendica\Content\Text\Markdown; use Friendica\Util\Network; +use Friendica\Util\Strings; Use Friendica\Util\DateTimeFormat; function discourse_install() @@ -46,62 +50,81 @@ function discourse_addon_settings_post(App $a) function discourse_email_getmessage(App $a, &$message) { // Logger::info('Got raw message', $message); - // Remove the title on comments, they don't serve any purpose there - if ($message['item']['parent-uri'] != $message['item']['uri']) { - unset($message['item']['title']); - } - if (preg_match('=topic/(.*)/(.*)@(.*)=', $message['item']['uri'], $matches)) { +/* if (preg_match('=topic/(.*)/(.*)@(.*)=', $message['item']['uri'], $matches)) { Logger::info('Got post data', ['topic' => $matches[1], 'post' => $matches[2], 'host' => $matches[3]]); if (discourse_fetch_post_from_api($message, $matches[2], $matches[3])) { return; } } - +*/ // Search in the text part for the link to the discourse entry and the text body // The text body is used as alternative, if the fetched HTML isn't working if (!empty($message['text'])) { - discourse_get_text($message); + $message = discourse_get_text($message); } if (!empty($message['item']['plink'])) { if (preg_match('=(http.*)/t/.*/(.*\d)/(.*\d)=', $message['item']['plink'], $matches)) { - if (discourse_fetch_topic_from_api($message, $matches[1], $matches[1], $matches[1])) { + if (discourse_fetch_topic_from_api($message, $matches[1], $matches[2], $matches[3])) { return; } } } - + Logger::info('Stop'); +die('Test'); // Search in the HTML part for the discourse entry and the author profile if (!empty($message['html'])) { - discourse_get_html($message); + $message = discourse_get_html($message); + } + + // Remove the title on comments, they don't serve any purpose there + if ($message['item']['parent-uri'] != $message['item']['uri']) { + unset($message['item']['title']); } } -function discourse_fetch_topic_from_api(&$message, $host, $topic, $pid) +function discourse_fetch_post($host, $topic, $pid) { - $url = $host . '/t/' . $topic . '/posts.json?posts_ids[]=' . $pid; + $url = $host . '/t/' . $topic . '/' . $pid . '.json'; $curlResult = Network::curl($url); if (!$curlResult->isSuccess()) { + Logger::info('No success', ['url' => $url]); return false; } + $raw = $curlResult->getBody(); $data = json_decode($raw, true); $posts = $data['post_stream']['posts']; foreach($posts as $post) { if ($post['post_number'] != $pid) { + // Test + discourse_get_user($post, $host); continue; } Logger::info('Got post data from topic', $post); - discourse_process_post($message, $post); - return true; + return $post; } + + Logger::info('Post not found', ['host' => $host, 'topic' => $topic, 'pid' => $pid]); return false; } +function discourse_fetch_topic_from_api(&$message, $host, $topic, $pid) +{ + $post = discourse_fetch_post($host, $topic, $pid); + if (empty($post)) { + return false; + } + + $message = discourse_process_post($message, $post, $host); + return true; +} + function discourse_fetch_post_from_api(&$message, $post, $host) { - $url = "https://" . $host . '/posts/' . $post . '.json'; + $hostaddr = 'https://' . $host; + $url = $hostaddr . '/posts/' . $post . '.json'; $curlResult = Network::curl($url); if (!$curlResult->isSuccess()) { return false; @@ -113,28 +136,84 @@ function discourse_fetch_post_from_api(&$message, $post, $host) return false; } - discourse_process_post($message, $data); + $message = discourse_process_post($message, $data, $hostaddr); Logger::info('Got API data', $message); return true; } -function discourse_process_post(&$message, $post) +function discourse_get_user($post, $hostaddr) { - if ($post['post_number'] == 1) { - // Thread information + $host = parse_url($hostaddr, PHP_URL_HOST); + + $contact = []; + // display_username + // user_id + $contact['uid'] = 0; + $contact['network'] = Protocol::DISCOURSE; + $contact['name'] = $contact['nick'] = $post['username']; + if (!empty($post['name'])) { + $contact['name'] = $post['name']; } - $nick = $post['username']; - $name = $post['name']; - // User information + $contact['about'] = $post['user_title']; - $message['html'] = $post['cooked']; - $message['text'] = $post['raw']; - $message['item']['created'] = DateTimeFormat::utc($post['created_at']); + if (parse_url($post['avatar_template'], PHP_URL_SCHEME)) { + $contact['photo'] = str_replace('{size}', '300', $post['avatar_template']); + } else { + $contact['photo'] = $hostaddr . str_replace('{size}', '300', $post['avatar_template']); + } + + $contact['addr'] = $contact['nick'] . '@' . $host; + $contact['contact-type'] = Contact::TYPE_PERSON; + $contact['url'] = $hostaddr . '/u/' . $contact['nick']; + $contact['nurl'] = Strings::normaliseLink($contact['url']); + $contact['baseurl'] = $hostaddr; + Logger::info('Contact', $contact); + $contact['id'] = Contact::getIdForURL($contact['url'], 0, true, $contact); + if (!empty($contact['id'])) { + $avatar = $contact['photo']; + unset($contact['photo']); + DBA::update('contact', $contact, ['id' => $contact['id']]); + Contact::updateAvatar($avatar, 0, $contact['id']); + $contact['photo'] = $avatar; + } + + return $contact; } -function discourse_get_html(&$message) +function discourse_process_post($message, $post, $hostaddr) +{ + $host = parse_url($hostaddr, PHP_URL_HOST); + + $message['html'] = $post['cooked']; + + $contact = discourse_get_user($post, $hostaddr); + $message['item']['author-id'] = $contact['id']; + $message['item']['author-link'] = $contact['url']; + $message['item']['author-name'] = $contact['name']; + $message['item']['author-avatar'] = $contact['photo']; + $message['item']['created'] = DateTimeFormat::utc($post['created_at']); + $message['item']['plink'] = $hostaddr . '/t/' . $post['topic_slug'] . '/' . $post['topic_id'] . '/' . $post['post_number']; + + if ($post['post_number'] == 1) { + $message['item']['parent-uri'] = $message['item']['uri'] = 'topic/' . $post['topic_id'] . '@' . $host; + // To-Do: Thread information + } else { + $message['item']['uri'] = 'topic/' . $post['topic_id'] . '/' . $post['id'] . '@' . $host; + unset($message['item']['title']); + if (empty($post['reply_to_post_number']) || $post['reply_to_post_number'] == 1) { + $message['item']['parent-uri'] = 'topic/' . $post['topic_id'] . '@' . $host; + } else { + $reply = discourse_fetch_post($hostaddr, $post['topic_id'], $post['reply_to_post_number']); + $message['item']['parent-uri'] = 'topic/' . $post['topic_id'] . '/' . $reply['id'] . '@' . $host; + } + } + + return $message; +} + +function discourse_get_html($message) { $doc = new DOMDocument(); $doc2 = new DOMDocument(); @@ -155,33 +234,38 @@ function discourse_get_html(&$message) $profile = discourse_get_profile($xpath); if (!empty($profile)) { Logger::info('Found profile', $profile); -/* - $message['item']['author-avatar'] = $contact['avatar']; - $message['item']['author-link'] = $profile['link']; + $message['item']['author-id'] = Contact::getIdForURL($profile['url'], 0, true, $profile); + $message['item']['author-link'] = $profile['url']; $message['item']['author-name'] = $profile['name']; -*/ + $message['item']['author-avatar'] = $profile['photo']; } + + return $message; } -function discourse_get_text(&$message) +function discourse_get_text($message) { $text = $message['text']; $text = str_replace("\r", '', $text); $pos = strpos($text, "\n---\n"); - if ($pos > 0) { - $message['text'] = trim(substr($text, 0, $pos)); - Logger::info('Found text body', ['text' => $message['text']]); - - $message['text'] = Markdown::toBBCode($message['text']); - - $text = substr($text, $pos); - if (preg_match('=\((http.*?)\)=', $text, $link)) { - $message['item']['plink'] = $link[1]; - Logger::info('Found plink', ['plink' => $message['item']['plink']]); - } - } else { + if ($pos == 0) { Logger::info('No separator found', ['text' => $text]); + return $message; } + + $message['text'] = trim(substr($text, 0, $pos)); + + Logger::info('Found text body', ['text' => $message['text']]); + + $message['text'] = Markdown::toBBCode($message['text']); + + $text = substr($text, $pos); + Logger::info('Found footer', ['text' => $text]); + if (preg_match('=\((http.*/t/.*/.*\d/.*\d)\)=', $text, $link)) { + $message['item']['plink'] = $link[1]; + Logger::info('Found plink', ['plink' => $message['item']['plink']]); + } + return $message; } function discourse_get_profile($xpath) @@ -197,7 +281,7 @@ function discourse_get_profile($xpath) if (!empty($attr['src']) && !empty($attr['title']) && !empty($attr['width']) && !empty($attr['height']) && ($attr['width'] == $attr['height'])) { - $profile = ['avatar' => $attr['src'], 'name' => $attr['title']]; + $profile = ['photo' => $attr['src'], 'name' => $attr['title']]; break; } } @@ -210,7 +294,7 @@ function discourse_get_profile($xpath) $attr[$attribute->name] = $attribute->value; } if (!empty($attr['href']) && (strpos($attr['href'], '/' . $profile['name']))) { - $profile['link'] = $attr['href']; + $profile['url'] = $attr['href']; break; } } From 9bc1f9cee66cb575cde390dd64ccb52d3520bc04 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 25 Nov 2019 22:23:19 +0000 Subject: [PATCH 3/9] README updated, code restructured --- discourse/README | 117 +++++++++------------------------------- discourse/discourse.php | 71 ++++++++++++------------ 2 files changed, 60 insertions(+), 128 deletions(-) diff --git a/discourse/README b/discourse/README index e4a8b9d3..65c54d20 100644 --- a/discourse/README +++ b/discourse/README @@ -1,97 +1,28 @@ -NSFW +Discourse connector +=================== -"Not safe for work" +The Discourse connectors detects incoming mails from Discourse and +improves them by fetching the content and user data via API. -Scans the message content for the string 'nsfw' -(case insensitive) and if found replaces the content -with a "click to open/close" link, default is closed. +Prerequisites +------------- +The user has to configure the mail interface so that the user's mails +can be fetched via Friendica. Then the user has to activate the +mailing list mode in Discourse. -If you click on the 'Not safe for work' addon under -/settings/addon a text field appears, where you can -extend the list of search terms. The terms must be -seperated by commas. +The mailing list mode in Discourse knows two different options: +1. Get all posts - including your own. This will create duplicates + if you post via Friendica. +2. Don't get your own posts. Then you will missing all your posts + that you made directly on Discourse. Since you cannot create + a new post via this connector (only comments are possible) + this is not a goog choice either. -It is also possible to enter profile URLs as values. -This is quite useful for the case, that you perhaps -don't want to see postings by person_A, but person_B -is one of your contacts and person_B used to reshare -postings by person_A. - -You can also make use of regular expressions. -They also have to be seperated by commas and the -regex itself has to be enclosed with slashes: - - ... nsfw, //, politics,... - ---------------- -A few examples: ---------------- - -1) -Let's say you don't want to see postings which contain -the term 'fake news' - -The term could appear in several ways: - -fakenews, fake news, fake_news, fake-news, f@ke news, -f4ke news, f4k3 n3ws, and so on and so on and so on. - -You could write every possible version of it as single -item into your NSFW-filter list, but this can also be -done with a single regex, which matches all of them: - - /f[@4a]k[3e][-_ ]n[3e]w[sz]/ - - -2) -Another use case could be, that you are simply not -interested in postings about christmas. - - /christmas(?:[-_ ]?(?:tree|time|eve|pudding))?/ - - -ATTENTION: - -It is absolutely important, that you use grouping -parentheses instead of capturing parentheses!! - -Grouping parentheses are: - - (?: ) - -If you use capturing parentheses, which are - - ( ) - -it will produce errors and the regex won't work and -at least your targets will not get collapsed. - - - -3) -Another possibility is the usage of a so called -'lookbehind' construct. I'll give an example followed -by a descripton: - - /(? $matches[1], 'post' => $matches[2], 'host' => $matches[3]]); - if (discourse_fetch_post_from_api($message, $matches[2], $matches[3])) { - return; - } + // We do assume that all Discourse servers are running with SSL + if (preg_match('=topic/(.*\d)/(.*\d)@(.*)=', $message['item']['uri'], $matches) && + discourse_fetch_post_from_api($message, $matches[2], $matches[3])) { + Logger::info('Fetched comment via API', ['host' => $matches[3], 'topic' => $matches[1], 'post' => $matches[2]]); + return; } -*/ + + if (preg_match('=topic/(.*\d)@(.*)=', $message['item']['uri'], $matches) && + discourse_fetch_topic_from_api($message, 'https://' . $matches[2], $matches[1], 1)) { + discourse_fetch_post_from_api($message, $matches[2], $matches[3]); + Logger::info('Fetched starting post via API', ['host' => $matches[2], 'topic' => $matches[1]]); + return; + } + // Search in the text part for the link to the discourse entry and the text body - // The text body is used as alternative, if the fetched HTML isn't working if (!empty($message['text'])) { $message = discourse_get_text($message); } - if (!empty($message['item']['plink'])) { - if (preg_match('=(http.*)/t/.*/(.*\d)/(.*\d)=', $message['item']['plink'], $matches)) { - if (discourse_fetch_topic_from_api($message, $matches[1], $matches[2], $matches[3])) { - return; - } - } + if (empty($message['item']['plink']) || !preg_match('=(http.*)/t/.*/(.*\d)/(.*\d)=', $message['item']['plink'], $matches)) { + Logger::info('This is no Discourse post'); } - Logger::info('Stop'); -die('Test'); + + if (discourse_fetch_topic_from_api($message, $matches[1], $matches[2], $matches[3])) { + Logger::info('Fetched post from via API', ['host' => $matches[1], 'topic' => $matches[2], 'id' => $matches[3]]); + return; + } + + Logger::info('Fallback mode'); // Search in the HTML part for the discourse entry and the author profile if (!empty($message['html'])) { $message = discourse_get_html($message); @@ -98,8 +103,7 @@ function discourse_fetch_post($host, $topic, $pid) $posts = $data['post_stream']['posts']; foreach($posts as $post) { if ($post['post_number'] != $pid) { - // Test - discourse_get_user($post, $host); + /// @todo Possibly fetch missing posts here continue; } Logger::info('Got post data from topic', $post); @@ -146,9 +150,11 @@ function discourse_get_user($post, $hostaddr) { $host = parse_url($hostaddr, PHP_URL_HOST); + // Currently unused contact fields: + // - display_username + // - user_id + $contact = []; - // display_username - // user_id $contact['uid'] = 0; $contact['network'] = Protocol::DISCOURSE; $contact['name'] = $contact['nick'] = $post['username']; @@ -198,7 +204,7 @@ function discourse_process_post($message, $post, $hostaddr) if ($post['post_number'] == 1) { $message['item']['parent-uri'] = $message['item']['uri'] = 'topic/' . $post['topic_id'] . '@' . $host; - // To-Do: Thread information + /// @ToDo Fetch thread information } else { $message['item']['uri'] = 'topic/' . $post['topic_id'] . '/' . $post['id'] . '@' . $host; unset($message['item']['title']); @@ -224,7 +230,7 @@ function discourse_get_html($message) $xpath = new DomXPath($doc); - // Fetch the first 'div' before the 'hr' -hopefully this fits for all systems + // Fetch the first 'div' before the 'hr' - hopefully this fits for all systems $result = $xpath->query("//hr//preceding::div[1]"); $div = $doc2->importNode($result->item(0), true); $doc2->appendChild($div); @@ -232,7 +238,7 @@ function discourse_get_html($message) Logger::info('Found html body', ['html' => $message['html']]); $profile = discourse_get_profile($xpath); - if (!empty($profile)) { + if (!empty($profile['url'])) { Logger::info('Found profile', $profile); $message['item']['author-id'] = Contact::getIdForURL($profile['url'], 0, true, $profile); $message['item']['author-link'] = $profile['url']; @@ -301,8 +307,3 @@ function discourse_get_profile($xpath) } return $profile; } - -function discourse_email_getmessage_end(App $a, &$message) -{ -// Logger::info('Got converted message', $message); -} From 0235126589e0ad23c3e72a34203402b73bf3dea0 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 25 Nov 2019 22:58:01 +0000 Subject: [PATCH 4/9] Settings page added --- discourse/discourse.php | 55 ++++++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/discourse/discourse.php b/discourse/discourse.php index 3b0f7cf4..d8bb95f2 100644 --- a/discourse/discourse.php +++ b/discourse/discourse.php @@ -14,6 +14,7 @@ use Friendica\Core\Hook; use Friendica\Core\L10n; use Friendica\Core\Logger; use Friendica\Core\PConfig; +use Friendica\Core\Renderer; use Friendica\Core\Protocol; use Friendica\Database\DBA; use Friendica\Model\Contact; @@ -25,9 +26,9 @@ Use Friendica\Util\DateTimeFormat; function discourse_install() { - Hook::register('email_getmessage', __FILE__, 'discourse_email_getmessage'); - Hook::register('addon_settings', __FILE__, 'discourse_settings'); - Hook::register('addon_settings_post', __FILE__, 'discourse_settings_post'); + Hook::register('email_getmessage', __FILE__, 'discourse_email_getmessage'); + Hook::register('connector_settings', __FILE__, 'discourse_settings'); + Hook::register('connector_settings_post', __FILE__, 'discourse_settings_post'); } function discourse_uninstall() @@ -37,29 +38,64 @@ function discourse_uninstall() Hook::unregister('connector_settings_post', __FILE__, 'discourse_settings_post'); } -function discourse_addon_settings(App $a, &$s) +function discourse_settings(App $a, &$s) { + if (!local_user()) { + return; + } + + $enabled = intval(PConfig::get(local_user(), 'discourse', 'enabled')); + $css = ($enabled ? '' : '-disabled'); + + $s .= ''; + $s .= '

' . L10n::t('Discourse') . '

'; + $s .= ''; + $s .= ''; } -function discourse_addon_settings_post(App $a) +function discourse_settings_post(App $a) { + if (!local_user() || empty($_POST['discourse-submit'])) { + return; + } + + PConfig::set(local_user(), 'discourse', 'enabled', intval($_POST['enabled'])); } function discourse_email_getmessage(App $a, &$message) { -// Logger::info('Got raw message', $message); + if (empty($message['item']['uid'])) { + return; + } + + if (!PConfig::get($message['item']['uid'], 'discourse', 'enabled')) { + return; + } // We do assume that all Discourse servers are running with SSL if (preg_match('=topic/(.*\d)/(.*\d)@(.*)=', $message['item']['uri'], $matches) && discourse_fetch_post_from_api($message, $matches[2], $matches[3])) { - Logger::info('Fetched comment via API', ['host' => $matches[3], 'topic' => $matches[1], 'post' => $matches[2]]); + Logger::info('Fetched comment via API (message-id mode)', ['host' => $matches[3], 'topic' => $matches[1], 'post' => $matches[2]]); return; } if (preg_match('=topic/(.*\d)@(.*)=', $message['item']['uri'], $matches) && discourse_fetch_topic_from_api($message, 'https://' . $matches[2], $matches[1], 1)) { discourse_fetch_post_from_api($message, $matches[2], $matches[3]); - Logger::info('Fetched starting post via API', ['host' => $matches[2], 'topic' => $matches[1]]); + Logger::info('Fetched starting post via API (message-id mode)', ['host' => $matches[2], 'topic' => $matches[1]]); return; } @@ -70,10 +106,11 @@ function discourse_email_getmessage(App $a, &$message) if (empty($message['item']['plink']) || !preg_match('=(http.*)/t/.*/(.*\d)/(.*\d)=', $message['item']['plink'], $matches)) { Logger::info('This is no Discourse post'); + return; } if (discourse_fetch_topic_from_api($message, $matches[1], $matches[2], $matches[3])) { - Logger::info('Fetched post from via API', ['host' => $matches[1], 'topic' => $matches[2], 'id' => $matches[3]]); + Logger::info('Fetched post via API (plink mode)', ['host' => $matches[1], 'topic' => $matches[2], 'id' => $matches[3]]); return; } From f95160866961da0b434247e340cc16b0a180fc80 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 25 Nov 2019 23:09:18 +0000 Subject: [PATCH 5/9] Added logging value --- discourse/discourse.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discourse/discourse.php b/discourse/discourse.php index d8bb95f2..c171f7ad 100644 --- a/discourse/discourse.php +++ b/discourse/discourse.php @@ -114,7 +114,7 @@ function discourse_email_getmessage(App $a, &$message) return; } - Logger::info('Fallback mode'); + Logger::info('Fallback mode', ['plink' => $message['item']['plink']]); // Search in the HTML part for the discourse entry and the author profile if (!empty($message['html'])) { $message = discourse_get_html($message); From 57f689faf1b1b2eff6fa09b8ad7d15cb3f39552d Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 26 Nov 2019 07:21:49 +0000 Subject: [PATCH 6/9] To-Do added, code fragment removed --- discourse/discourse.php | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/discourse/discourse.php b/discourse/discourse.php index c171f7ad..15081cd1 100644 --- a/discourse/discourse.php +++ b/discourse/discourse.php @@ -24,6 +24,15 @@ use Friendica\Util\Network; use Friendica\Util\Strings; Use Friendica\Util\DateTimeFormat; +/* Todo: + * - Obtaining API tokens to be able to read non public posts as well + * - Handling duplicates (possibly using some non visible marker) + * - Fetching missing posts + * - Fetch topic information + * - Support mail free mode when write tokens are available + * - Fix incomplete (relative) links (hosts are missing) +*/ + function discourse_install() { Hook::register('email_getmessage', __FILE__, 'discourse_email_getmessage'); @@ -94,7 +103,6 @@ function discourse_email_getmessage(App $a, &$message) if (preg_match('=topic/(.*\d)@(.*)=', $message['item']['uri'], $matches) && discourse_fetch_topic_from_api($message, 'https://' . $matches[2], $matches[1], 1)) { - discourse_fetch_post_from_api($message, $matches[2], $matches[3]); Logger::info('Fetched starting post via API (message-id mode)', ['host' => $matches[2], 'topic' => $matches[1]]); return; } From fae6bbe92c7bd83a0b6a54ff528b25e3ccff30d4 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 26 Nov 2019 18:59:50 +0000 Subject: [PATCH 7/9] Some fixes / subject cleaning --- discourse/README | 2 +- discourse/discourse.php | 15 ++++++--------- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/discourse/README b/discourse/README index 65c54d20..338b10bf 100644 --- a/discourse/README +++ b/discourse/README @@ -16,7 +16,7 @@ The mailing list mode in Discourse knows two different options: 2. Don't get your own posts. Then you will missing all your posts that you made directly on Discourse. Since you cannot create a new post via this connector (only comments are possible) - this is not a goog choice either. + this is not a good choice either. Known problems -------------- diff --git a/discourse/discourse.php b/discourse/discourse.php index 15081cd1..2fce970d 100644 --- a/discourse/discourse.php +++ b/discourse/discourse.php @@ -7,8 +7,6 @@ * Author: Michael Vogel * */ -//use DOMDocument; -//use DOMXPath; use Friendica\App; use Friendica\Core\Hook; use Friendica\Core\L10n; @@ -40,13 +38,6 @@ function discourse_install() Hook::register('connector_settings_post', __FILE__, 'discourse_settings_post'); } -function discourse_uninstall() -{ - Hook::unregister('email_getmessage', __FILE__, 'discourse_email_getmessage'); - Hook::unregister('connector_settings', __FILE__, 'discourse_settings'); - Hook::unregister('connector_settings_post', __FILE__, 'discourse_settings_post'); -} - function discourse_settings(App $a, &$s) { if (!local_user()) { @@ -249,6 +240,12 @@ function discourse_process_post($message, $post, $hostaddr) if ($post['post_number'] == 1) { $message['item']['parent-uri'] = $message['item']['uri'] = 'topic/' . $post['topic_id'] . '@' . $host; + + // Remove the Discourse forum name from the subject + $pattern = '=\[.*\].*\s(\[.*\].*)='; + if (preg_match($pattern, $message['item']['title'])) { + $message['item']['title'] = preg_replace($pattern, '$1', $message['item']['title']); + } /// @ToDo Fetch thread information } else { $message['item']['uri'] = 'topic/' . $post['topic_id'] . '/' . $post['id'] . '@' . $host; From 561ae12b029e44bf3170db70dc33cc01ec9ed26e Mon Sep 17 00:00:00 2001 From: Michael Date: Wed, 27 Nov 2019 05:54:26 +0000 Subject: [PATCH 8/9] Settings had been moved into a template --- discourse/discourse.php | 23 ++++++----------------- discourse/templates/settings.tpl | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 17 deletions(-) create mode 100644 discourse/templates/settings.tpl diff --git a/discourse/discourse.php b/discourse/discourse.php index 2fce970d..6ee40943 100644 --- a/discourse/discourse.php +++ b/discourse/discourse.php @@ -45,25 +45,14 @@ function discourse_settings(App $a, &$s) } $enabled = intval(PConfig::get(local_user(), 'discourse', 'enabled')); - $css = ($enabled ? '' : '-disabled'); - $s .= ''; - $s .= '

' . L10n::t('Discourse') . '

'; - $s .= ''; - $s .= ''; } function discourse_settings_post(App $a) diff --git a/discourse/templates/settings.tpl b/discourse/templates/settings.tpl new file mode 100644 index 00000000..dc6ef2f0 --- /dev/null +++ b/discourse/templates/settings.tpl @@ -0,0 +1,15 @@ + +

{{$title}}

+ + From c45df0b96420d35b9f7f9bab6b153def3f1fae3a Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 28 Nov 2019 04:57:39 +0000 Subject: [PATCH 9/9] Improved template --- discourse/discourse.php | 1 - discourse/templates/settings.tpl | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/discourse/discourse.php b/discourse/discourse.php index 6ee40943..262ee554 100644 --- a/discourse/discourse.php +++ b/discourse/discourse.php @@ -49,7 +49,6 @@ function discourse_settings(App $a, &$s) $t = Renderer::getMarkupTemplate('settings.tpl', 'addon/discourse/'); $s .= Renderer::replaceMacros($t, [ '$title' => L10n::t('Discourse'), - '$css' => ($enabled ? '' : '-disabled'), '$enabled' => ['enabled', L10n::t('Enable processing of Discourse mailing list mails'), $enabled, L10n::t('If enabled, incoming mails from Discourse will be improved so they look much better. To make it work, you have to configure the e-mail settings in Friendica. You also have to enable the mailing list mode in Discourse. Then you have to add the Discourse mail account as contact.')], '$submit' => L10n::t('Save Settings'), ]); diff --git a/discourse/templates/settings.tpl b/discourse/templates/settings.tpl index dc6ef2f0..d4b0bf8c 100644 --- a/discourse/templates/settings.tpl +++ b/discourse/templates/settings.tpl @@ -1,9 +1,9 @@ -

{{$title}}

+

{{$title}}