Escape leading block markers in MarkdownRenderer

When MarkdownRenderer reformats text whose leading punctuation was
backslash-escaped in the source, it dropped the escape and emitted the
bare marker. Re-parsing the result then turned literal text into a new
block, e.g. "\* literal" round-tripped into a list item and
"\# text" into a heading. The same happened on continuation lines of a
paragraph and inside list items.

Re-escape a leading list/heading/block-quote marker on each rendered
line, and keep backticks escaped in text so they are not re-parsed as a
code span. Markers that are genuinely list/heading/quote syntax still
come through their own tokens, and ordinary prose (snake_case, hyphens,
"C#", "a.b.c") is left untouched.

Author: sarathfrancis90

src/mistune/renderers/markdown.py

@@ -8,6 +8,10 @@
 
 fenced_re = re.compile(r"^[`~]+", re.M)
 
+#: leading markers that would be parsed as a new block (list, heading, block
+#: quote) if they appear unescaped at the start of a line.
+_block_prefix_re = re.compile(r"^(\s*)(>|[-+*]|#{1,6}|\d{1,9}[.)])(\s|$)")
+
 
 class MarkdownRenderer(BaseRenderer):
     """A renderer to re-format Markdown text."""
@@ -35,7 +39,9 @@ def render_children(self, token: Dict[str, Any], state: BlockState) -> str:
         return self.render_tokens(children, state)
 
     def text(self, token: Dict[str, Any], state: BlockState) -> str:
-        return cast(str, token["raw"])
+        # a backtick always opens a code span, so it must stay escaped to
+        # survive a re-parse as literal text.
+        return cast(str, token["raw"]).replace("`", "\\`")
 
     def emphasis(self, token: Dict[str, Any], state: BlockState) -> str:
         return "*" + self.render_children(token, state) + "*"
@@ -87,7 +93,7 @@ def inline_html(self, token: Dict[str, Any], state: BlockState) -> str:
 
     def paragraph(self, token: Dict[str, Any], state: BlockState) -> str:
         text = self.render_children(token, state)
-        return text + "\n\n"
+        return _escape_block_prefix(text) + "\n\n"
 
     def heading(self, token: Dict[str, Any], state: BlockState) -> str:
         level = cast(int, token["attrs"]["level"])
@@ -99,7 +105,7 @@ def thematic_break(self, token: Dict[str, Any], state: BlockState) -> str:
         return "***\n\n"
 
     def block_text(self, token: Dict[str, Any], state: BlockState) -> str:
-        return self.render_children(token, state) + "\n"
+        return _escape_block_prefix(self.render_children(token, state)) + "\n"
 
     def block_code(self, token: Dict[str, Any], state: BlockState) -> str:
         attrs = token.get("attrs", {})
@@ -129,6 +135,20 @@ def list(self, token: Dict[str, Any], state: BlockState) -> str:
         return render_list(self, token, state)
 
 
+def _escape_block_prefix(text: str) -> str:
+    """Backslash-escape a leading block marker on each line so that literal
+    text is not re-parsed as a list, heading or block quote."""
+    return "\n".join(_escape_line_prefix(line) for line in text.split("\n"))
+
+
+def _escape_line_prefix(line: str) -> str:
+    m = _block_prefix_re.match(line)
+    if not m:
+        return line
+    indent_, marker = m.group(1), m.group(2)
+    return indent_ + marker[:-1] + "\\" + marker[-1] + line[m.end(2) :]
+
+
 def _get_fenced_marker(code: str) -> str:
     found = fenced_re.findall(code)
     if not found:

tests/test_renderers.py

@@ -1,3 +1,5 @@
+from unittest import TestCase
+
 from mistune import create_markdown
 from mistune.renderers.rst import RSTRenderer
 from mistune.renderers.markdown import MarkdownRenderer
@@ -15,3 +17,51 @@ class TestRenderer(BaseTestCase):
 
 load_renderer(RSTRenderer())
 load_renderer(MarkdownRenderer())
+
+
+class TestMarkdownRendererRoundTrip(TestCase):
+    """Reformatting valid Markdown must not change its meaning: rendering the
+    reformatted source to HTML must match rendering the original source."""
+
+    to_html = create_markdown(escape=False)
+    reformat = create_markdown(renderer=MarkdownRenderer())
+
+    def assert_round_trip(self, text):
+        self.assertEqual(
+            self.to_html(self.reformat(text)),
+            self.to_html(text),
+        )
+
+    def test_escaped_block_markers(self):
+        # an escaped leading marker is literal text, not a new block
+        for marker in (r"\*", r"\-", r"\+", r"\#", r"\##", r"\>", r"1\.", r"3\)"):
+            self.assert_round_trip(marker + " literal\n")
+
+    def test_escaped_marker_on_continuation_line(self):
+        self.assert_round_trip("paragraph\n\\* not a list\n")
+
+    def test_escaped_marker_inside_list_item(self):
+        self.assert_round_trip("- item\n\n  \\# not a heading\n")
+
+    def test_escaped_backtick(self):
+        self.assert_round_trip(r"\`not code\`" + "\n")
+
+    def test_real_markers_preserved(self):
+        for text in (
+            "* bullet\n",
+            "# heading\n",
+            "1. ordered\n",
+            "> quote\n",
+            "`code`\n",
+            "*emphasis*\n",
+        ):
+            self.assert_round_trip(text)
+
+    def test_prose_not_over_escaped(self):
+        for text in (
+            "snake_case_var\n",
+            "use - dashes - here\n",
+            "C# and a.b.c and 1.5\n",
+            "ratio a/b is fine\n",
+        ):
+            self.assert_round_trip(text)