Merge pull request #7474 from MrPetovan/bug/7457-html-bbcode-plaintext

Fix HTML/BBCode::toPlaintext

include/api.php

@@ -614,7 +614,7 @@ function api_get_user(App $a, $contact_id = null)
 				'name' => $contact["name"],
 				'screen_name' => (($contact['nick']) ? $contact['nick'] : $contact['name']),
 				'location' => ($contact["location"] != "") ? $contact["location"] : ContactSelector::networkToName($contact['network'], $contact['url']),
-				'description' => HTML::toPlaintext(BBCode::toPlaintext($contact["about"])),
+				'description' => BBCode::toPlaintext($contact["about"]),
 				'profile_image_url' => $contact["micro"],
 				'profile_image_url_https' => $contact["micro"],
 				'profile_image_url_profile_size' => $contact["thumb"],
@@ -693,7 +693,7 @@ function api_get_user(App $a, $contact_id = null)
 		'name' => (($uinfo[0]['name']) ? $uinfo[0]['name'] : $uinfo[0]['nick']),
 		'screen_name' => (($uinfo[0]['nick']) ? $uinfo[0]['nick'] : $uinfo[0]['name']),
 		'location' => $location,
-		'description' => HTML::toPlaintext(BBCode::toPlaintext($description)),
+		'description' => BBCode::toPlaintext($description),
 		'profile_image_url' => $uinfo[0]['micro'],
 		'profile_image_url_https' => $uinfo[0]['micro'],
 		'profile_image_url_profile_size' => $uinfo[0]["thumb"],

src/Content/Text/BBCode.php

@@ -357,10 +357,7 @@ class BBCode extends BaseObject
 	 */
 	public static function toPlaintext($text, $keep_urls = true)
 	{
-		$naked_text = preg_replace('/\[.+?\]/','', $text);
+		$naked_text = HTML::toPlaintext(BBCode::convert($text, false, 0, true), 0, !$keep_urls);
-		if (!$keep_urls) {
-			$naked_text = preg_replace('#https?\://[^\s<]+[^\s\.\)]#i', '', $naked_text);
-		}
 
 		return $naked_text;
 	}

src/Content/Text/HTML.php

@@ -56,6 +56,7 @@ class HTML
 
 		$xpath = new DOMXPath($doc);
 
+		/** @var \DOMNode[] $list */
 		$list = $xpath->query("//" . $tag);
 		foreach ($list as $node) {
 			$attr = [];
@@ -98,9 +99,12 @@ class HTML
 				$node->parentNode->insertBefore($StartCode, $node);
 
 				if ($node->hasChildNodes()) {
+					/** @var \DOMNode $child */
 					foreach ($node->childNodes as $child) {
-						$newNode = $child->cloneNode(true);
+						if (trim($child->nodeValue)) {
-						$node->parentNode->insertBefore($newNode, $node);
+							$newNode = $child->cloneNode(true);
+							$node->parentNode->insertBefore($newNode, $node);
+						}
 					}
 				}
 
@@ -560,6 +564,8 @@ class HTML
 				$ignore = false;
 			}
 
+			$ignore = $ignore || strpos($treffer[1], '#') === 0;
+
 			if (!$ignore) {
 				$urls[$treffer[1]] = $treffer[1];
 			}
@@ -583,7 +589,7 @@ class HTML
 
 		$message = mb_convert_encoding($message, 'HTML-ENTITIES', "UTF-8");
 
-		@$doc->loadHTML($message);
+		@$doc->loadHTML($message, LIBXML_HTML_NODEFDTD | LIBXML_NOBLANKS);
 
 		$message = $doc->saveHTML();
 		// Remove eventual UTF-8 BOM
@@ -592,7 +598,7 @@ class HTML
 		// Collecting all links
 		$urls = self::collectURLs($message);
 
-		@$doc->loadHTML($message);
+		@$doc->loadHTML($message, LIBXML_HTML_NODEFDTD | LIBXML_NOBLANKS);
 
 		self::tagToBBCode($doc, 'html', [], '', '');
 		self::tagToBBCode($doc, 'body', [], '', '');

src/Module/Debug/Babel.php

@@ -148,6 +148,12 @@ class Babel extends BaseModule
 						'content' => htmlspecialchars($html2)
 					];
 
+					$bbcode2plain = Text\BBCode::toPlaintext($bbcode);
+					$results[] = [
+						'title'   => L10n::t('HTML::toBBCode => BBCode::toPlaintext'),
+						'content' => '<pre>' . $bbcode2plain . '</pre>'
+					];
+
 					$markdown = Text\HTML::toMarkdown($html);
 					$results[] = [
 						'title'   => L10n::t('HTML::toMarkdown'),
@@ -162,7 +168,7 @@ class Babel extends BaseModule
 
 					$text = Text\HTML::toPlaintext($html, 0, true);
 					$results[] = [
-						'title'   => L10n::t('HTML::toPlaintext'),
+						'title'   => L10n::t('HTML::toPlaintext (compact)'),
 						'content' => '<pre>' . $text . '</pre>'
 					];
 			}

tests/datasets/content/text/html/bug-7457.html

@@ -0,0 +1 @@
+<h2 data-sourcepos="1:1-1:23" dir="auto">&#x000A;<a id="user-content-104-2019-08-01" class="anchor" href="#104-2019-08-01" aria-hidden="true"></a>[1.0.4] - 2019-08-01</h2>&#x000A;<h3 data-sourcepos="2:1-2:9" dir="auto">&#x000A;<a id="user-content-fixed" class="anchor" href="#fixed" aria-hidden="true"></a>Fixed</h3>&#x000A;<ul data-sourcepos="3:1-3:116" dir="auto">&#x000A;<li data-sourcepos="3:1-3:116">Invalid SemVer version generation, when the current branch does not have commits ahead of tag/checked out on a tag</li>&#x000A;</ul>

tests/datasets/content/text/html/bug-7457.txt

@@ -0,0 +1,5 @@
+*[1.0.4] - 2019-08-01*
+
+*Fixed*
+
+* Invalid SemVer version generation, when the current branch does not have commits ahead of tag/checked out on a tag

tests/src/Content/Text/HTMLTest.php

@@ -0,0 +1,53 @@
+<?php
+
+namespace Friendica\Test\src\Content\Text;
+
+use Friendica\Content\Text\HTML;
+use Friendica\Test\MockedTest;
+use Friendica\Test\Util\AppMockTrait;
+use Friendica\Test\Util\VFSTrait;
+
+class HTMLTest extends MockedTest
+{
+	use VFSTrait;
+	use AppMockTrait;
+
+	protected function setUp()
+	{
+		parent::setUp();
+		$this->setUpVfsDir();
+		$this->mockApp($this->root);
+	}
+
+	public function dataHTML()
+	{
+		$inputFiles = glob(__DIR__ . '/../../../datasets/content/text/html/*.html');
+
+		$data = [];
+
+		foreach ($inputFiles as $file) {
+			$data[str_replace('.html', '', $file)] = [
+				'input'    => file_get_contents($file),
+				'expected' => file_get_contents(str_replace('.html', '.txt', $file))
+			];
+		}
+
+		return $data;
+	}
+
+	/**
+	 * Test convert different input Markdown text into HTML
+	 *
+	 * @dataProvider dataHTML
+	 *
+	 * @param string $input    The Markdown text to test
+	 * @param string $expected The expected HTML output
+	 * @throws \Exception
+	 */
+	public function testToPlaintext($input, $expected)
+	{
+		$output = HTML::toPlaintext($input, 0);
+
+		$this->assertEquals($expected, $output);
+	}
+}

tests/src/Content/Text/MarkdownTest.php

@@ -1,52 +1,52 @@
-<?php
+<?php
-
+
-namespace Friendica\Test\src\Content\Text;
+namespace Friendica\Test\src\Content\Text;
-
+
-use Friendica\Content\Text\Markdown;
+use Friendica\Content\Text\Markdown;
-use Friendica\Test\MockedTest;
+use Friendica\Test\MockedTest;
-use Friendica\Test\Util\AppMockTrait;
+use Friendica\Test\Util\AppMockTrait;
-use Friendica\Test\Util\VFSTrait;
+use Friendica\Test\Util\VFSTrait;
-
+
-class MarkdownTest extends MockedTest
+class MarkdownTest extends MockedTest
-{
+{
-	use VFSTrait;
+	use VFSTrait;
-	use AppMockTrait;
+	use AppMockTrait;
-
+
-	protected function setUp()
+	protected function setUp()
-	{
+	{
-		parent::setUp();
+		parent::setUp();
-		$this->setUpVfsDir();
+		$this->setUpVfsDir();
-		$this->mockApp($this->root);
+		$this->mockApp($this->root);
-	}
+	}
-
+
-	public function dataMarkdown()
+	public function dataMarkdown()
-	{
+	{
-		$inputFiles = glob(__DIR__ . '/../../../datasets/content/text/markdown/*.md');
+		$inputFiles = glob(__DIR__ . '/../../../datasets/content/text/markdown/*.md');
-
+
-		$data = [];
+		$data = [];
-
+
-		foreach ($inputFiles as $file) {
+		foreach ($inputFiles as $file) {
-			$data[str_replace('.md', '', $file)] = [
+			$data[str_replace('.md', '', $file)] = [
-				'input'    => file_get_contents($file),
+				'input'    => file_get_contents($file),
-				'expected' => file_get_contents(str_replace('.md', '.html', $file))
+				'expected' => file_get_contents(str_replace('.md', '.html', $file))
-			];
+			];
-		}
+		}
-
+
-		return $data;
+		return $data;
-	}
+	}
-
+
-	/**
+	/**
-	 * Test convert different input Markdown text into HTML
+	 * Test convert different input Markdown text into HTML
-	 * @dataProvider dataMarkdown
+	 * @dataProvider dataMarkdown
-	 *
+	 *
-	 * @param string $input    The Markdown text to test
+	 * @param string $input    The Markdown text to test
-	 * @param string $expected The expected HTML output
+	 * @param string $expected The expected HTML output
-	 * @throws \Exception
+	 * @throws \Exception
-	 */
+	 */
-	public function testConvert($input, $expected)
+	public function testConvert($input, $expected)
-	{
+	{
-		$output = Markdown::convert($input);
+		$output = Markdown::convert($input);
-
+
-		$this->assertEquals($expected, $output);
+		$this->assertEquals($expected, $output);
-	}
+	}
-}
+}