Add HTML Purify to BBCode::convert

- Replace previous HTML cleanup using DOMDocument
- Remove obsolete HTML::sanitizeCSS method
- Use Strings::autoLinkRegEx for PageInfo::getRelevantUrlFromBody
- Update previous tests expectations
- Add new specific XSS tests

src/Content/PageInfo.php

@@ -265,7 +265,7 @@ class PageInfo
 		}
 
 		if (!$matches && $searchNakedUrls) {
-			preg_match('~(?<=\W|^)(?<![=\]])(https?://.+)$~is', $body, $matches);
+			preg_match(Strings::autoLinkRegEx(), $body, $matches);
 			if ($matches && !Strings::endsWith($body, $matches[1])) {
 				unset($matches);
 			}

src/Content/Text/BBCode.php

@@ -671,9 +671,9 @@ class BBCode
 					$return .= sprintf('<a href="%s" target="_blank" rel="noopener noreferrer"><img src="%s" alt="" title="%s" class="attachment-image" /></a>', $data['url'], self::proxyUrl($data['image'], $simplehtml), $data['title']);
 				} else {
 					if (!empty($data['image'])) {
-						$return .= sprintf('<a href="%s" target="_blank" rel="noopener noreferrer"><img src="%s" alt="" title="%s" class="attachment-image" /></a><br />', $data['url'], self::proxyUrl($data['image'], $simplehtml), $data['title']);
+						$return .= sprintf('<a href="%s" target="_blank" rel="noopener noreferrer"><img src="%s" alt="" title="%s" class="attachment-image" /></a><br>', $data['url'], self::proxyUrl($data['image'], $simplehtml), $data['title']);
 					} elseif (!empty($data['preview'])) {
-						$return .= sprintf('<a href="%s" target="_blank" rel="noopener noreferrer"><img src="%s" alt="" title="%s" class="attachment-preview" /></a><br />', $data['url'], self::proxyUrl($data['preview'], $simplehtml), $data['title']);
+						$return .= sprintf('<a href="%s" target="_blank" rel="noopener noreferrer"><img src="%s" alt="" title="%s" class="attachment-preview" /></a><br>', $data['url'], self::proxyUrl($data['preview'], $simplehtml), $data['title']);
 					}
 					$return .= sprintf('<h4><a href="%s">%s</a></h4>', $data['url'], $data['title']);
 				}
@@ -1033,7 +1033,7 @@ class BBCode
 
 		switch ($simplehtml) {
 			case self::API:
-				$text = ($is_quote_share? '<br />' : '') . '<p>' . html_entity_decode('&#x2672; ', ENT_QUOTES, 'UTF-8') . ' ' . $author_contact['addr'] . ': </p>' . "\n" . $content;
+				$text = ($is_quote_share? '<br>' : '') . '<p>' . html_entity_decode('&#x2672; ', ENT_QUOTES, 'UTF-8') . ' ' . $author_contact['addr'] . ': </p>' . "\n" . $content;
 				break;
 			case self::DIASPORA:
 				if (stripos(Strings::normaliseLink($attributes['link']), 'http://twitter.com/') === 0) {
@@ -1062,7 +1062,7 @@ class BBCode
 
 				break;
 			case self::OSTATUS:
-				$text = ($is_quote_share? '<br />' : '') . '<p>' . html_entity_decode('&#x2672; ', ENT_QUOTES, 'UTF-8') . ' @' . $author_contact['addr'] . ': ' . $content . '</p>' . "\n";
+				$text = ($is_quote_share? '<br>' : '') . '<p>' . html_entity_decode('&#x2672; ', ENT_QUOTES, 'UTF-8') . ' @' . $author_contact['addr'] . ': ' . $content . '</p>' . "\n";
 				break;
 			case self::ACTIVITYPUB:
 				$author = '@<span class="vcard"><a href="' . $author_contact['url'] . '" class="url u-url mention" title="' . $author_contact['addr'] . '"><span class="fn nickname mention">' . $author_contact['addr'] . '</span></a>:</span>';
@@ -1275,6 +1275,8 @@ class BBCode
 			return '';
 		}
 
+		Hook::callAll('bbcode', $text);
+
 		$a = DI::app();
 
 		$text = self::performWithEscapedTags($text, ['code'], function ($text) use ($try_oembed, $simple_html, $for_plaintext, $a) {
@@ -1300,11 +1302,12 @@ class BBCode
 					return $return;
 				};
 
-
-
 				// Remove the abstract element. It is a non visible element.
 				$text = self::stripAbstract($text);
 
+				// Line ending normalisation
+				$text = str_replace("\r\n", "\n", $text);
+
 				// Move new lines outside of tags
 				$text = preg_replace("#\[(\w*)](\n*)#ism", '$2[$1]', $text);
 				$text = preg_replace("#(\n*)\[/(\w*)]#ism", '[/$2]$1', $text);
@@ -1338,16 +1341,6 @@ class BBCode
 					$text = preg_replace("/\[share(.*?)avatar\s?=\s?'.*?'\s?(.*?)\]\s?(.*?)\s?\[\/share\]\s?/ism", "\n[share$1$2]$3[/share]", $text);
 				}
 
-				// Convert new line chars to html <br /> tags
-
-				// nlbr seems to be hopelessly messed up
-				//	$Text = nl2br($Text);
-
-				// We'll emulate it.
-
-				$text = trim($text);
-				$text = str_replace("\r\n", "\n", $text);
-
 				// Remove linefeeds inside of the table elements. See issue #6799
 				$search = ["\n[th]", "[th]\n", " [th]", "\n[/th]", "[/th]\n", "[/th] ",
 					"\n[td]", "[td]\n", " [td]", "\n[/td]", "[/td]\n", "[/td] ",
@@ -1367,11 +1360,14 @@ class BBCode
 				$replace = ["[table]", "[/table]"];
 				$text = str_replace($search, $replace, $text);
 
+				// Trim new lines regardless of the system.remove_multiplicated_lines config value
+				$text = trim($text, "\n");
+
 				// removing multiplicated newlines
 				if (DI::config()->get('system', 'remove_multiplicated_lines')) {
-					$search = ["\n\n\n", "\n ", " \n", "[/quote]\n\n", "\n[/quote]", "[/li]\n", "\n[li]", "\n[ul]", "[/ul]\n", "\n\n[share ", "[/attachment]\n",
+					$search = ["\n\n\n", "\n ", " \n", "[/quote]\n\n", "\n[/quote]", "[/li]\n", "\n[li]", "\n[*]", "\n[ul]", "[/ul]\n", "\n\n[share ", "[/attachment]\n",
 							"\n[h1]", "[/h1]\n", "\n[h2]", "[/h2]\n", "\n[h3]", "[/h3]\n", "\n[h4]", "[/h4]\n", "\n[h5]", "[/h5]\n", "\n[h6]", "[/h6]\n"];
-					$replace = ["\n\n", "\n", "\n", "[/quote]\n", "[/quote]", "[/li]", "[li]", "[ul]", "[/ul]", "\n[share ", "[/attachment]",
+					$replace = ["\n\n", "\n", "\n", "[/quote]\n", "[/quote]", "[/li]", "[li]", "[*]", "[ul]", "[/ul]", "\n[share ", "[/attachment]",
 							"[h1]", "[/h1]", "[h2]", "[/h2]", "[h3]", "[/h3]", "[h4]", "[/h4]", "[h5]", "[/h5]", "[h6]", "[/h6]"];
 					do {
 						$oldtext = $text;
@@ -1447,8 +1443,8 @@ class BBCode
 				// Check for sized text
 				// [size=50] --> font-size: 50px (with the unit).
 				if ($simple_html != self::DIASPORA) {
-					$text = preg_replace("(\[size=(\d*?)\](.*?)\[\/size\])ism", "<span style=\"font-size: $1px; line-height: initial;\">$2</span>", $text);
+					$text = preg_replace("(\[size=(\d*?)\](.*?)\[\/size\])ism", '<span style="font-size:$1px;line-height:normal;">$2</span>', $text);
-					$text = preg_replace("(\[size=(.*?)\](.*?)\[\/size\])ism", "<span style=\"font-size: $1; line-height: initial;\">$2</span>", $text);
+					$text = preg_replace("(\[size=(.*?)\](.*?)\[\/size\])ism", '<span style="font-size:$1;line-height:normal;">$2</span>', $text);
 				} else {
 					// Issue 2199: Diaspora doesn't interpret the construct above, nor the <small> or <big> element
 					$text = preg_replace("(\[size=(.*?)\](.*?)\[\/size\])ism", "$2", $text);
@@ -1456,28 +1452,16 @@ class BBCode
 
 
 				// Check for centered text
-				$text = preg_replace("(\[center\](.*?)\[\/center\])ism", "<div style=\"text-align:center;\">$1</div>", $text);
+				$text = preg_replace("(\[center\](.*?)\[\/center\])ism", '<div style="text-align:center;">$1</div>', $text);
 
 				// Check for list text
 				$text = str_replace("[*]", "<li>", $text);
 
 				// Check for style sheet commands
-				$text = preg_replace_callback(
+				$text = preg_replace("(\[style=(.*?)\](.*?)\[\/style\])ism", '<span style="$1">$2</span>', $text);
-					"(\[style=(.*?)\](.*?)\[\/style\])ism",
-					function ($match) {
-						return "<span style=\"" . HTML::sanitizeCSS($match[1]) . ";\">" . $match[2] . "</span>";
-					},
-					$text
-				);
 
 				// Check for CSS classes
-				$text = preg_replace_callback(
+				$text = preg_replace("(\[class=(.*?)\](.*?)\[\/class\])ism", '<span style="$1">$2</span>', $text);
-					"(\[class=(.*?)\](.*?)\[\/class\])ism",
-					function ($match) {
-						return "<span class=\"" . HTML::sanitizeCSS($match[1]) . "\">" . $match[2] . "</span>";
-					},
-					$text
-				);
 
 				// handle nested lists
 				$endlessloop = 0;
@@ -1608,20 +1592,20 @@ class BBCode
 				$text = preg_replace("/\[img\](.*?)\[\/img\]/ism", '<img src="$1" alt="' . DI::l10n()->t('Image/photo') . '" />', $text);
 				$text = preg_replace("/\[zmg\](.*?)\[\/zmg\]/ism", '<img src="$1" alt="' . DI::l10n()->t('Image/photo') . '" />', $text);
 
-				$text = preg_replace("/\[crypt\](.*?)\[\/crypt\]/ism", '<br/><img src="' .DI::baseUrl() . '/images/lock_icon.gif" alt="' . DI::l10n()->t('Encrypted content') . '" title="' . DI::l10n()->t('Encrypted content') . '" /><br />', $text);
+				$text = preg_replace("/\[crypt\](.*?)\[\/crypt\]/ism", '<br><img src="' .DI::baseUrl() . '/images/lock_icon.gif" alt="' . DI::l10n()->t('Encrypted content') . '" title="' . DI::l10n()->t('Encrypted content') . '" /><br>', $text);
-				$text = preg_replace("/\[crypt(.*?)\](.*?)\[\/crypt\]/ism", '<br/><img src="' .DI::baseUrl() . '/images/lock_icon.gif" alt="' . DI::l10n()->t('Encrypted content') . '" title="' . '$1' . ' ' . DI::l10n()->t('Encrypted content') . '" /><br />', $text);
+				$text = preg_replace("/\[crypt(.*?)\](.*?)\[\/crypt\]/ism", '<br><img src="' .DI::baseUrl() . '/images/lock_icon.gif" alt="' . DI::l10n()->t('Encrypted content') . '" title="' . '$1' . ' ' . DI::l10n()->t('Encrypted content') . '" /><br>', $text);
-				//$Text = preg_replace("/\[crypt=(.*?)\](.*?)\[\/crypt\]/ism", '<br/><img src="' .DI::baseUrl() . '/images/lock_icon.gif" alt="' . DI::l10n()->t('Encrypted content') . '" title="' . '$1' . ' ' . DI::l10n()->t('Encrypted content') . '" /><br />', $Text);
+				//$Text = preg_replace("/\[crypt=(.*?)\](.*?)\[\/crypt\]/ism", '<br><img src="' .DI::baseUrl() . '/images/lock_icon.gif" alt="' . DI::l10n()->t('Encrypted content') . '" title="' . '$1' . ' ' . DI::l10n()->t('Encrypted content') . '" /><br>', $Text);
 
 				// Simplify "video" element
 				$text = preg_replace('(\[video.*?\ssrc\s?=\s?([^\s\]]+).*?\].*?\[/video\])ism', '[video]$1[/video]', $text);
 
-				// Try to Oembed
 				if ($try_oembed) {
+					// html5 video and audio
 					$text = preg_replace("/\[video\](.*?\.(ogg|ogv|oga|ogm|webm|mp4).*?)\[\/video\]/ism",
-						'<video src="$1" controls="controls" width="' . $a->videowidth . '" height="' . $a->videoheight . '" loop="true"><a href="$1">$1</a></video>', $text);
+						'<video src="$1" controls width="' . $a->videowidth . '" height="' . $a->videoheight . '" loop="true"><a href="$1">$1</a></video>', $text);
 					$text = preg_replace("/\[video\](.*?)\[\/video\]/ism",
 						'<a href="$1" target="_blank" rel="noopener noreferrer">$1</a>', $text);
-					$text = preg_replace("/\[audio\](.*?)\[\/audio\]/ism", '<audio src="$1" controls="controls"><a href="$1">$1</a></audio>', $text);
+					$text = preg_replace("/\[audio\](.*?)\[\/audio\]/ism", '<audio src="$1" controls><a href="$1">$1</a></audio>', $text);
 
 					$text = preg_replace_callback("/\[video\](.*?)\[\/video\]/ism", $try_oembed_callback, $text);
 					$text = preg_replace_callback("/\[audio\](.*?)\[\/audio\]/ism", $try_oembed_callback, $text);
@@ -1632,9 +1616,6 @@ class BBCode
 						'<a href="$1" target="_blank" rel="noopener noreferrer">$1</a>', $text);
 				}
 
-				// html5 video and audio
-
-
 				if ($try_oembed) {
 					$text = preg_replace("/\[iframe\](.*?)\[\/iframe\]/ism", '<iframe src="$1" width="' . $a->videowidth . '" height="' . $a->videoheight . '"><a href="$1">$1</a></iframe>', $text);
 				} else {
@@ -1678,7 +1659,7 @@ class BBCode
 				$text = OEmbed::BBCode2HTML($text);
 
 				// Avoid triple linefeeds through oembed
-				$text = str_replace("<br style='clear:left'></span><br /><br />", "<br style='clear:left'></span><br />", $text);
+				$text = str_replace("<br style='clear:left'></span><br><br>", "<br style='clear:left'></span><br>", $text);
 
 				// If we found an event earlier, strip out all the event code and replace with a reformatted version.
 				// Replace the event-start section with the entire formatted event. The other bbcode is stripped.
@@ -1708,7 +1689,7 @@ class BBCode
 					$conv = html_entity_decode(str_replace([' ', "\n", "\r"], '', $text));
 					// Emojis are always 4 byte Unicode characters
 					if (!empty($conv) && (strlen($conv) / mb_strlen($conv) == 4)) {
-						$text = '<span style="font-size: xx-large; line-height: initial;">' . $text . '</span>';
+						$text = '<span style="font-size: xx-large; line-height: normal;">' . $text . '</span>';
 					}
 				}
 
@@ -1722,8 +1703,6 @@ class BBCode
 					$text = preg_replace_callback("&\[url=([^\[\]]*)\]\[img\](.*)\[\/img\]\[\/url\]&Usi", 'self::removePictureLinksCallback', $text);
 				}
 
-				$text = str_replace(["\r","\n"], ['<br />', '<br />'], $text);
-
 				// Remove all hashtag addresses
 				if ($simple_html && !in_array($simple_html, [self::DIASPORA, self::OSTATUS, self::ACTIVITYPUB])) {
 					$text = preg_replace("/([#@!])\[url\=(.*?)\](.*?)\[\/url\]/ism", '$1$3', $text);
@@ -1872,18 +1851,21 @@ class BBCode
 
 			// Additionally, [pre] tags preserve spaces
 			$text = preg_replace_callback("/\[pre\](.*?)\[\/pre\]/ism", function ($match) {
-				return str_replace(' ', '&nbsp;', $match[1]);
+				return str_replace(' ', '&nbsp;', htmlentities($match[1], ENT_NOQUOTES,'UTF-8'));
 			}, $text);
 
+			// Add HTML new lines
+			$text = str_replace("\n", '<br>', $text);
+
 			return $text;
 		}); // Escaped code
 
 		$text = preg_replace_callback("#\[code(?:=([^\]]*))?\](.*?)\[\/code\]#ism",
 			function ($matches) {
 				if (strpos($matches[2], "\n") !== false) {
-					$return = '<pre><code class="language-' . trim($matches[1]) . '">' . htmlspecialchars(trim($matches[2], "\n\r"), ENT_NOQUOTES, 'UTF-8') . '</code></pre>';
+					$return = '<pre><code class="language-' . trim($matches[1]) . '">' . htmlentities(trim($matches[2], "\n\r"), ENT_NOQUOTES, 'UTF-8') . '</code></pre>';
 				} else {
-					$return = '<code>' . htmlspecialchars($matches[2], ENT_NOQUOTES, 'UTF-8') . '</code>';
+					$return = '<code>' . htmlentities($matches[2], ENT_NOQUOTES, 'UTF-8') . '</code>';
 				}
 
 				return $return;
@@ -1891,37 +1873,20 @@ class BBCode
 			$text
 		);
 
-		// Clean up the HTML by loading and saving the HTML with the DOM.
+		$config = \HTMLPurifier_HTML5Config::createDefault();
-		// Bad structured html can break a whole page.
+		$config->set('HTML.Doctype', 'HTML5');
-		// For performance reasons do it only with activated item cache or at export.
+		$config->set('Attr.AllowedRel', [
-		if (!$try_oembed || (get_itemcachepath() != '')) {
+			'noreferrer' => true,
-			$doc = new DOMDocument();
+			'noopener' => true,
-			$doc->preserveWhiteSpace = false;
+		]);
+		$config->set('Attr.AllowedFrameTargets', [
+			'_blank' => true,
+		]);
 
-			$text = mb_convert_encoding($text, 'HTML-ENTITIES', "UTF-8");
+		$HTMLPurifier = new \HTMLPurifier($config);
+		$text = $HTMLPurifier->purify($text);
 
-			$doctype = '<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">';
+		return $text;
-			$encoding = '<?xml encoding="UTF-8">';
-			@$doc->loadHTML($encoding . $doctype . '<html><body>' . $text . '</body></html>');
-			$doc->encoding = 'UTF-8';
-			$text = $doc->saveHTML();
-			$text = str_replace(['<html><body>', '</body></html>', $doctype, $encoding], ['', '', '', ''], $text);
-
-			$text = str_replace('<br></li>', '</li>', $text);
-
-			//$Text = mb_convert_encoding($Text, "UTF-8", 'HTML-ENTITIES');
-		}
-
-		// Clean up some useless linebreaks in lists
-		//$Text = str_replace('<br /><ul', '<ul ', $Text);
-		//$Text = str_replace('</ul><br />', '</ul>', $Text);
-		//$Text = str_replace('</li><br />', '</li>', $Text);
-		//$Text = str_replace('<br /><li>', '<li>', $Text);
-		//$Text = str_replace('<br /><ul', '<ul ', $Text);
-
-		Hook::callAll('bbcode', $text);
-
-		return trim($text);
 	}
 
 	/**

src/Content/Text/HTML.php

@@ -36,27 +36,6 @@ use League\HTMLToMarkdown\HtmlConverter;
 
 class HTML
 {
-	public static function sanitizeCSS($input)
-	{
-		$cleaned = "";
-
-		$input = strtolower($input);
-
-		for ($i = 0; $i < strlen($input); $i++) {
-			$char = substr($input, $i, 1);
-
-			if (($char >= "a") && ($char <= "z")) {
-				$cleaned .= $char;
-			}
-
-			if (!(strpos(" #;:0123456789-_.%", $char) === false)) {
-				$cleaned .= $char;
-			}
-		}
-
-		return $cleaned;
-	}
-
 	/**
 	 * Search all instances of a specific HTML tag node in the provided DOM document and replaces them with BBCode text nodes.
 	 *

tests/src/Content/Text/BBCodeTest.php

@@ -75,6 +75,18 @@ class BBCodeTest extends MockedTest
 		$this->dice->shouldReceive('create')
 		           ->with(BaseURL::class)
 		           ->andReturn($baseUrlMock);
+
+		$config = \HTMLPurifier_HTML5Config::createDefault();
+		$config->set('HTML.Doctype', 'HTML5');
+		$config->set('Attr.AllowedRel', [
+			'noreferrer' => true,
+			'noopener' => true,
+		]);
+		$config->set('Attr.AllowedFrameTargets', [
+			'_blank' => true,
+		]);
+
+		$this->HTMLPurifier = new \HTMLPurifier($config);
 	}
 
 	public function dataLinks()
@@ -171,7 +183,7 @@ class BBCodeTest extends MockedTest
 	public function testAutoLinking(string $data, bool $assertHTML)
 	{
 		$output = BBCode::convert($data);
-		$assert = '<a href="' . $data . '" target="_blank" rel="noopener noreferrer">' . $data . '</a>';
+		$assert = $this->HTMLPurifier->purify('<a href="' . $data . '" target="_blank" rel="noopener noreferrer">' . $data . '</a>');
 		if ($assertHTML) {
 			self::assertEquals($assert, $output);
 		} else {
@@ -203,11 +215,11 @@ class BBCodeTest extends MockedTest
 [/ul]',
 			],
 			'bug-2199-named-size' => [
-				'expectedHtml' => '<span style="font-size: xx-large; line-height: initial;">Test text</span>',
+				'expectedHtml' => '<span style="font-size:xx-large;line-height:normal;">Test text</span>',
 				'text' => '[size=xx-large]Test text[/size]',
 			],
 			'bug-2199-numeric-size' => [
-				'expectedHtml' => '<span style="font-size: 24px; line-height: initial;">Test text</span>',
+				'expectedHtml' => '<span style="font-size:24px;line-height:normal;">Test text</span>',
 				'text' => '[size=24]Test text[/size]',
 			],
 			'bug-2199-diaspora-no-named-size' => [
@@ -225,7 +237,7 @@ class BBCodeTest extends MockedTest
 				'simpleHtml' => 3,
 			],
 			'bug-7665-audio-tag' => [
-				'expectedHtml' => '<audio src="http://www.cendrones.fr/colloque2017/jonathanbocquet.mp3" controls="controls"><a href="http://www.cendrones.fr/colloque2017/jonathanbocquet.mp3">http://www.cendrones.fr/colloque2017/jonathanbocquet.mp3</a></audio>',
+				'expectedHtml' => '<audio src="http://www.cendrones.fr/colloque2017/jonathanbocquet.mp3" controls><a href="http://www.cendrones.fr/colloque2017/jonathanbocquet.mp3">http://www.cendrones.fr/colloque2017/jonathanbocquet.mp3</a></audio>',
 				'text' => '[audio]http://www.cendrones.fr/colloque2017/jonathanbocquet.mp3[/audio]',
 				'try_oembed' => true,
 			],
@@ -246,9 +258,25 @@ class BBCodeTest extends MockedTest
 				'text' => '[test] Space',
 			],
 			'task-8800-pre-spaces' => [
-				'expectedHtml' => '&nbsp;&nbsp;&nbsp;&nbsp;Spaces',
+				'expectedHtml' => '    Spaces',
 				'text' => '[pre]    Spaces[/pre]',
 			],
+			'bug-9611-purify-xss-nobb' => [
+				'expectedHTML' => '<span>dare to move your mouse here</span>',
+				'text' => '[nobb]<span onmouseover="alert(0)">dare to move your mouse here</span>[/nobb]'
+			],
+			'bug-9611-purify-xss-noparse' => [
+				'expectedHTML' => '<span>dare to move your mouse here</span>',
+				'text' => '[noparse]<span onmouseover="alert(0)">dare to move your mouse here</span>[/noparse]'
+			],
+			'bug-9611-purify-xss-attributes' => [
+				'expectedHTML' => '<span>dare to move your mouse here</span>',
+				'text' => '[color="onmouseover=alert(0) style="]dare to move your mouse here[/color]'
+			],
+			'bug-9611-purify-attributes-correct' => [
+				'expectedHTML' => '<span style="color:#FFFFFF;">dare to move your mouse here</span>',
+				'text' => '[color=FFFFFF]dare to move your mouse here[/color]'
+			],
 		];
 	}