diff --git a/js/src/javascript/tokenizer.js b/js/src/javascript/tokenizer.js index 2b2b73fb0..3417b0cbf 100644 --- a/js/src/javascript/tokenizer.js +++ b/js/src/javascript/tokenizer.js @@ -106,7 +106,7 @@ var Tokenizer = function(input_string, options) { BaseTokenizer.call(this, input_string, options); this._patterns.whitespace = this._patterns.whitespace.matching( - /\u00A0\u1680\u180e\u2000-\u200a\u202f\u205f\u3000\ufeff/.source, + /\u00A0\u1680\u2000-\u200a\u202f\u205f\u3000\ufeff/.source, /\u2028\u2029/.source); var pattern_reader = new Pattern(this._input); diff --git a/python/jsbeautifier/javascript/acorn.py b/python/jsbeautifier/javascript/acorn.py index c856d276e..0e1ce4577 100644 --- a/python/jsbeautifier/javascript/acorn.py +++ b/python/jsbeautifier/javascript/acorn.py @@ -67,7 +67,7 @@ + "])+" ) -_nonASCIIwhitespace = re.compile(r"[\u1680\u180e\u2000-\u200a\u202f\u205f\u3000\ufeff]") +_nonASCIIwhitespace = re.compile(r"[\u1680\u2000-\u200a\u202f\u205f\u3000\ufeff]") # Whether a single character denotes a newline. newline = re.compile(r"[\n\r\u2028\u2029]") diff --git a/python/jsbeautifier/javascript/tokenizer.py b/python/jsbeautifier/javascript/tokenizer.py index a50aabf33..f2a35a06a 100644 --- a/python/jsbeautifier/javascript/tokenizer.py +++ b/python/jsbeautifier/javascript/tokenizer.py @@ -131,7 +131,7 @@ def __init__(self, input_scanner, acorn, options): BaseTokenizerPatterns.__init__(self, input_scanner) self.whitespace = self.whitespace.matching( - r"\u00A0\u1680\u180e\u2000-\u200a\u202f\u205f\u3000\ufeff", + r"\u00A0\u1680\u2000-\u200a\u202f\u205f\u3000\ufeff", r"\u2028\u2029", ) diff --git a/test/data/html/tests.js b/test/data/html/tests.js index 0d7e611a9..b6a2ea502 100644 --- a/test/data/html/tests.js +++ b/test/data/html/tests.js @@ -4414,6 +4414,12 @@ exports.test_data = { '' ] }] + }, { + name: "PR #2459 Mongolian Vowel Separator", + description: "", + tests: [{ + unchanged: "

ᠬᠠᠷ᠎ᠠ

" + }] }, { name: "New Test Suite" }]