From 424106fd4ee72a1983a760d71458f9bd73ea990b Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Fri, 20 Dec 2019 23:19:06 -0500 Subject: [PATCH 1/5] Add BBCode to Markdown to HTML raw result in Module\Babel - Display HTML entities for BBCode to Markdown output --- src/Module/Debug/Babel.php | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Module/Debug/Babel.php b/src/Module/Debug/Babel.php index 17187a37c..edd897be8 100644 --- a/src/Module/Debug/Babel.php +++ b/src/Module/Debug/Babel.php @@ -59,10 +59,14 @@ class Babel extends BaseModule $markdown = Text\BBCode::toMarkdown($bbcode); $results[] = [ 'title' => L10n::t('BBCode::toMarkdown'), - 'content' => visible_whitespace($markdown) + 'content' => visible_whitespace(htmlspecialchars($markdown)) ]; $html2 = Text\Markdown::convert($markdown); + $results[] = [ + 'title' => L10n::t('BBCode::toMarkdown => Markdown::convert (raw HTML)'), + 'content' => visible_whitespace(htmlspecialchars($html2)) + ]; $results[] = [ 'title' => L10n::t('BBCode::toMarkdown => Markdown::convert'), 'content' => $html2 From 887b0f035f931815653a93e88652a4a91f711c16 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Fri, 27 Dec 2019 21:41:48 -0500 Subject: [PATCH 2/5] [composer] Update league/html-to-markdown to version 4.9.1 --- composer.lock | 99 +++++++++------------------------------------------ 1 file changed, 16 insertions(+), 83 deletions(-) diff --git a/composer.lock b/composer.lock index 8e55db88d..3a4670b20 100644 --- a/composer.lock +++ b/composer.lock @@ -759,16 +759,16 @@ }, { "name": "league/html-to-markdown", - "version": "4.8.2", + "version": "4.9.1", "source": { "type": "git", "url": "https://github.com/thephpleague/html-to-markdown.git", - "reference": "e747489191f8e9144a7270eb61f8b9516e99e413" + "reference": "1dcd0f85de786f46a7f224a27cc3d709ddd2a68c" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/html-to-markdown/zipball/e747489191f8e9144a7270eb61f8b9516e99e413", - "reference": "e747489191f8e9144a7270eb61f8b9516e99e413", + "url": "https://api.github.com/repos/thephpleague/html-to-markdown/zipball/1dcd0f85de786f46a7f224a27cc3d709ddd2a68c", + "reference": "1dcd0f85de786f46a7f224a27cc3d709ddd2a68c", "shasum": "" }, "require": { @@ -778,7 +778,7 @@ }, "require-dev": { "mikehaertl/php-shellcommand": "~1.1.0", - "phpunit/phpunit": "4.*", + "phpunit/phpunit": "^4.8|^5.7", "scrutinizer/ocular": "~1.1" }, "bin": [ @@ -787,7 +787,7 @@ "type": "library", "extra": { "branch-alias": { - "dev-master": "4.9-dev" + "dev-master": "4.10-dev" } }, "autoload": { @@ -819,7 +819,7 @@ "html", "markdown" ], - "time": "2019-08-02T11:57:39+00:00" + "time": "2019-12-28T01:32:28+00:00" }, { "name": "level-2/dice", @@ -1128,7 +1128,6 @@ "dist": { "type": "tar", "url": "https://registry.npmjs.org/cropperjs/-/cropperjs-1.2.2.tgz", - "reference": null, "shasum": "30dc7a7ce872155b23a33bd10ad4c76c0d613f55" }, "require-dev": { @@ -1222,7 +1221,6 @@ "dist": { "type": "tar", "url": "https://registry.npmjs.org/ev-emitter/-/ev-emitter-1.1.1.tgz", - "reference": null, "shasum": "8f18b0ce5c76a5d18017f71c0a795c65b9138f2a" }, "type": "npm-asset-library", @@ -1265,7 +1263,6 @@ "dist": { "type": "tar", "url": "https://registry.npmjs.org/fullcalendar/-/fullcalendar-3.10.1.tgz", - "reference": null, "shasum": "cca3f9a2656a7e978a3f3facb7f35934a91185db" }, "type": "npm-asset-library", @@ -1312,28 +1309,11 @@ "dist": { "type": "tar", "url": "https://registry.npmjs.org/imagesloaded/-/imagesloaded-4.1.4.tgz", - "reference": null, "shasum": "1376efcd162bb768c34c3727ac89cc04051f3cc7" }, "require": { "npm-asset/ev-emitter": ">=1.0.0,<2.0.0" }, - "require-dev": { - "npm-asset/chalk": ">=1.1.1,<2.0.0", - "npm-asset/cheerio": ">=0.19.0,<0.20.0", - "npm-asset/gulp": ">=3.9.0,<4.0.0", - "npm-asset/gulp-jshint": ">=1.11.2,<2.0.0", - "npm-asset/gulp-json-lint": ">=0.1.0,<0.2.0", - "npm-asset/gulp-rename": ">=1.2.2,<2.0.0", - "npm-asset/gulp-replace": ">=0.5.4,<0.6.0", - "npm-asset/gulp-requirejs-optimize": "dev-github:metafizzy/gulp-requirejs-optimize", - "npm-asset/gulp-uglify": ">=1.4.2,<2.0.0", - "npm-asset/gulp-util": ">=3.0.7,<4.0.0", - "npm-asset/highlight.js": ">=8.9.1,<9.0.0", - "npm-asset/marked": ">=0.3.5,<0.4.0", - "npm-asset/minimist": ">=1.2.0,<2.0.0", - "npm-asset/transfob": ">=1.0.0,<2.0.0" - }, "type": "npm-asset-library", "extra": { "npm-asset-bugs": { @@ -1376,17 +1356,8 @@ "dist": { "type": "tar", "url": "https://registry.npmjs.org/jgrowl/-/jgrowl-1.4.6.tgz", - "reference": null, "shasum": "2736e332aaee73ccf0a14a5f0066391a0a13f4a3" }, - "require-dev": { - "npm-asset/grunt": "~0.4.2", - "npm-asset/grunt-contrib-cssmin": "~0.9.0", - "npm-asset/grunt-contrib-jshint": "~0.6.3", - "npm-asset/grunt-contrib-less": "~0.11.0", - "npm-asset/grunt-contrib-uglify": "~0.4.0", - "npm-asset/grunt-contrib-watch": "~0.6.1" - }, "type": "npm-asset-library", "extra": { "npm-asset-bugs": { @@ -1417,35 +1388,8 @@ "dist": { "type": "tar", "url": "https://registry.npmjs.org/jquery/-/jquery-2.2.4.tgz", - "reference": null, "shasum": "2c89d6889b5eac522a7eea32c14521559c6cbf02" }, - "require-dev": { - "npm-asset/commitplease": "2.0.0", - "npm-asset/core-js": "0.9.17", - "npm-asset/grunt": "0.4.5", - "npm-asset/grunt-babel": "5.0.1", - "npm-asset/grunt-cli": "0.1.13", - "npm-asset/grunt-compare-size": "0.4.0", - "npm-asset/grunt-contrib-jshint": "0.11.2", - "npm-asset/grunt-contrib-uglify": "0.9.2", - "npm-asset/grunt-contrib-watch": "0.6.1", - "npm-asset/grunt-git-authors": "2.0.1", - "npm-asset/grunt-jscs": "2.1.0", - "npm-asset/grunt-jsonlint": "1.0.4", - "npm-asset/grunt-npmcopy": "0.1.0", - "npm-asset/gzip-js": "0.3.2", - "npm-asset/jsdom": "5.6.1", - "npm-asset/load-grunt-tasks": "1.0.0", - "npm-asset/qunit-assert-step": "1.0.3", - "npm-asset/qunitjs": "1.17.1", - "npm-asset/requirejs": "2.1.17", - "npm-asset/sinon": "1.10.3", - "npm-asset/sizzle": "2.2.1", - "npm-asset/strip-json-comments": "1.0.3", - "npm-asset/testswarm": "1.1.0", - "npm-asset/win-spawn": "2.0.0" - }, "type": "npm-asset-library", "extra": { "npm-asset-bugs": { @@ -1488,7 +1432,6 @@ "dist": { "type": "tar", "url": "https://registry.npmjs.org/jquery-colorbox/-/jquery-colorbox-1.6.4.tgz", - "reference": null, "shasum": "799452523a6c494839224ef702e807deb9c06cc5" }, "require": { @@ -1535,7 +1478,6 @@ "dist": { "type": "tar", "url": "https://registry.npmjs.org/jquery-datetimepicker/-/jquery-datetimepicker-2.5.21.tgz", - "reference": null, "shasum": "00c388a78df2732fedfdb5c6529b6e84d53e0235" }, "require": { @@ -1593,15 +1535,8 @@ "dist": { "type": "tar", "url": "https://registry.npmjs.org/jquery-mousewheel/-/jquery-mousewheel-3.1.13.tgz", - "reference": null, "shasum": "06f0335f16e353a695e7206bf50503cb523a6ee5" }, - "require-dev": { - "npm-asset/grunt": "~0.4.1", - "npm-asset/grunt-contrib-connect": "~0.5.0", - "npm-asset/grunt-contrib-jshint": "~0.7.1", - "npm-asset/grunt-contrib-uglify": "~0.2.7" - }, "type": "npm-asset-library", "extra": { "npm-asset-bugs": { @@ -1648,7 +1583,6 @@ "dist": { "type": "tar", "url": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", - "reference": null, "shasum": "0d055d53f5052aa653c9f6eb68bb5d12bf5c2b5b" }, "type": "npm-asset-library", @@ -1765,7 +1699,6 @@ "dist": { "type": "tar", "url": "https://registry.npmjs.org/typeahead.js/-/typeahead.js-0.11.1.tgz", - "reference": null, "shasum": "4e64e671b22310a8606f4aec805924ba84b015b8" }, "require": { @@ -3402,8 +3335,8 @@ "authors": [ { "name": "Sebastian Bergmann", - "email": "sb@sebastian-bergmann.de", - "role": "lead" + "role": "lead", + "email": "sb@sebastian-bergmann.de" } ], "description": "Library that provides collection, processing, and rendering functionality for PHP code coverage information.", @@ -3670,8 +3603,8 @@ "authors": [ { "name": "Sebastian Bergmann", - "email": "sebastian@phpunit.de", - "role": "lead" + "role": "lead", + "email": "sebastian@phpunit.de" } ], "description": "The PHP Unit Testing framework.", @@ -3844,7 +3777,7 @@ } ], "description": "Provides the functionality to compare PHP values for equality", - "homepage": "https://github.com/sebastianbergmann/comparator", + "homepage": "http://www.github.com/sebastianbergmann/comparator", "keywords": [ "comparator", "compare", @@ -3946,7 +3879,7 @@ } ], "description": "Provides functionality to handle HHVM/PHP environments", - "homepage": "https://github.com/sebastianbergmann/environment", + "homepage": "http://www.github.com/sebastianbergmann/environment", "keywords": [ "Xdebug", "environment", @@ -4014,7 +3947,7 @@ } ], "description": "Provides the functionality to export PHP variables for visualization", - "homepage": "https://github.com/sebastianbergmann/exporter", + "homepage": "http://www.github.com/sebastianbergmann/exporter", "keywords": [ "export", "exporter" @@ -4066,7 +3999,7 @@ } ], "description": "Snapshotting of global state", - "homepage": "https://github.com/sebastianbergmann/global-state", + "homepage": "http://www.github.com/sebastianbergmann/global-state", "keywords": [ "global state" ], @@ -4168,7 +4101,7 @@ } ], "description": "Provides functionality to recursively process PHP variables", - "homepage": "https://github.com/sebastianbergmann/recursion-context", + "homepage": "http://www.github.com/sebastianbergmann/recursion-context", "time": "2016-11-19T07:33:16+00:00" }, { From 39cb3e68b9c46fa6c31ef46430df4b4e7f0ce9d5 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Fri, 20 Dec 2019 23:20:09 -0500 Subject: [PATCH 3/5] Remove faulty escape for HTML entities in BBCode::toMarkdown - Mangled Markdown output with chevrons --- src/Content/Text/BBCode.php | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index e5622d14b..a3d26a736 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -2052,9 +2052,6 @@ class BBCode extends BaseObject $text = self::convert($text, false, 4); } - // mask some special HTML chars from conversation to markdown - $text = str_replace(['<', '>', '&'], ['&_lt_;', '&_gt_;', '&_amp_;'], $text); - // If a link is followed by a quote then there should be a newline before it // Maybe we should make this newline at every time before a quote. $text = str_replace(['
'], ['
'], $text); @@ -2064,9 +2061,6 @@ class BBCode extends BaseObject // Now convert HTML to Markdown $text = HTML::toMarkdown($text); - // unmask the special chars back to HTML - $text = str_replace(['&\_lt\_;', '&\_gt\_;', '&\_amp\_;'], ['<', '>', '&'], $text); - $a->getProfiler()->saveTimestamp($stamp1, "parser", System::callstack()); // Libertree has a problem with escaped hashtags. From aa3a85c7272052a743e6d1d1164484a749cc4079 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Fri, 27 Dec 2019 21:40:41 -0500 Subject: [PATCH 4/5] Escape major HTML characters in code blocks in BBCode::convert - HTML sanitization was removing unescaped opening chevrons in code blocks --- src/Content/Text/BBCode.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Content/Text/BBCode.php b/src/Content/Text/BBCode.php index a3d26a736..3617470df 100644 --- a/src/Content/Text/BBCode.php +++ b/src/Content/Text/BBCode.php @@ -1283,9 +1283,9 @@ class BBCode extends BaseObject function ($matches) use (&$codeblocks) { $return = '#codeblock-' . count($codeblocks) . '#'; if (strpos($matches[2], "\n") !== false) { - $codeblocks[] = '
' . trim($matches[2], "\n\r") . '
'; + $codeblocks[] = '
' . htmlspecialchars(trim($matches[2], "\n\r"), ENT_NOQUOTES, 'UTF-8') . '
'; } else { - $codeblocks[] = '' . $matches[2] . ''; + $codeblocks[] = '' . htmlspecialchars($matches[2], ENT_NOQUOTES, 'UTF-8') . ''; } return $return; From 39a537c415fad16c51e1422ff32ab60ac76da6b8 Mon Sep 17 00:00:00 2001 From: Hypolite Petovan Date: Fri, 20 Dec 2019 23:20:41 -0500 Subject: [PATCH 5/5] Added test for BBCode to Markdown chevron issue - See https://github.com/friendica/friendica/issues/7808 Add tests --- tests/src/Content/Text/BBCodeTest.php | 47 +++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/src/Content/Text/BBCodeTest.php b/tests/src/Content/Text/BBCodeTest.php index 1ff653db9..ed33306ed 100644 --- a/tests/src/Content/Text/BBCodeTest.php +++ b/tests/src/Content/Text/BBCodeTest.php @@ -203,6 +203,18 @@ class BBCodeTest extends MockedTest 'text' => '[audio]http://www.cendrones.fr/colloque2017/jonathanbocquet.mp3[/audio]', 'try_oembed' => true, ], + 'bug-7808-code-lt' => [ + 'expectedHtml' => '<', + 'text' => '[code]<[/code]', + ], + 'bug-7808-code-gt' => [ + 'expectedHtml' => '>', + 'text' => '[code]>[/code]', + ], + 'bug-7808-code-amp' => [ + 'expectedHtml' => '&', + 'text' => '[code]&[/code]', + ] ]; } @@ -224,4 +236,39 @@ class BBCodeTest extends MockedTest $this->assertEquals($expectedHtml, $actual); } + + public function dataBBCodesToMarkdown() + { + return [ + 'bug-7808-gt' => [ + 'expected' => '>`>`', + 'text' => '>[code]>[/code]', + ], + 'bug-7808-lt' => [ + 'expected' => '<`<`', + 'text' => '<[code]<[/code]', + ], + 'bug-7808-amp' => [ + 'expected' => '&`&`', + 'text' => '&[code]&[/code]', + ], + ]; + } + + /** + * Test convert bbcodes to Markdown + * + * @dataProvider dataBBCodesToMarkdown + * + * @param string $expected Expected Markdown output + * @param string $text BBCode text + * @param bool $for_diaspora + * @throws \Friendica\Network\HTTPException\InternalServerErrorException + */ + public function testToMarkdown($expected, $text, $for_diaspora = false) + { + $actual = BBCode::toMarkdown($text, $for_diaspora); + + $this->assertEquals($expected, $actual); + } }