fix(markdown): stop WYSIWYG hard breaks rendering as literal "<br />" (#13076)

The help/FAQ pages (e.g. /help/faq/editing) showed literal `<br />` text
because Open Library has two markdown engines that disagree on hard breaks:

* Display: OLMarkdown (vendored Python-Markdown) turns every newline into an
  injected `<br/>` and has no escape syntax for hard breaks.
* Editing: the Tiptap WYSIWYG component (tiptap-markdown) serializes a hard
  break as CommonMark `\` + newline.

When editor-saved content is displayed, OLMarkdown appends its `<br/>` right
after the editor's trailing `\`; the backslash escapes the `<`, so readers see
the literal text `<br />`. Every multi-line paragraph edited via the editor
accumulated this (and stray backslashes — see the page history).

Fixes both sides:

* Editor: a new OLHardBreak node serializes hard breaks as a bare newline
  (OLMarkdown's dialect) instead of CommonMark `\`. Re-saving a page now also
  cleans up legacy backslash/`<br>` cruft.
* Renderer: LineBreaksPreprocessor strips a lone trailing `\` before injecting
  `<br/>`, so already-corrupted pages display correctly without a re-save, and
  no longer glues a `<br/>` onto the line above a link-reference definition.

Adds JS round-trip tests and Python render tests; both fail without the fix.

Ref: #13074

Author: mekarpeles

openlibrary/components/lit/editor-core.js

@@ -8,6 +8,7 @@ import { Markdown } from 'tiptap-markdown';
 import Placeholder from '@tiptap/extension-placeholder';
 import Image from '@tiptap/extension-image';
 import { HtmlBlock } from './html-block.js';
+import { OLHardBreak } from './hard-break.js';
 
 /**
  * Creates a configured Tiptap editor instance.
@@ -29,8 +30,12 @@ export function createEditor({ element, content, placeholder, onUpdate, onTransa
                 codeBlock: enableCode ? undefined : false,
                 code: enableCode ? undefined : false,
                 link: { openOnClick: false, autolink: true },
-                strike: false
+                strike: false,
+                // Replaced with OLHardBreak below so hard breaks serialize to
+                // OLMarkdown's dialect (a bare newline, not CommonMark "\").
+                hardBreak: false
             }),
+            OLHardBreak,
             Markdown.configure({
                 breaks: true,
                 linkify: true

openlibrary/components/lit/hard-break.js

@@ -0,0 +1,34 @@
+import { HardBreak } from '@tiptap/extension-hard-break';
+
+/**
+ * OLHardBreak – a hard-break node that serializes to OLMarkdown's dialect.
+ *
+ * The server-side display renderer (OLMarkdown, a vendored Python-Markdown)
+ * treats every newline inside a paragraph as a `<br>` and has no escape syntax
+ * for hard breaks. tiptap-markdown's default serializer emits CommonMark's
+ * `\<newline>`; that trailing backslash makes OLMarkdown escape the `<` of the
+ * `<br/>` it injects, so readers see the literal text `<br />`.
+ *
+ * Emitting a bare newline instead keeps the two renderers in agreement, and
+ * re-saving a page through the editor cleans up any legacy backslash/`<br>`
+ * cruft. See internetarchive/openlibrary#13074.
+ */
+export const OLHardBreak = HardBreak.extend({
+    addStorage() {
+        return {
+            markdown: {
+                serialize(state, node, parent, index) {
+                    for (let i = index + 1; i < parent.childCount; i++) {
+                        if (parent.child(i).type !== node.type) {
+                            state.write('\n');
+                            return;
+                        }
+                    }
+                },
+                parse: {
+                    // handled by markdown-it
+                },
+            },
+        };
+    },
+});

openlibrary/core/olmarkdown.py

@@ -64,12 +64,28 @@ def run(self, lines):
 class LineBreaksPreprocessor(markdown.Preprocessor):
     def run(self, lines):
         for i in range(len(lines) - 1):
-            # append <br/> to all lines expect blank lines and the line before blankline.
+            # Only consider non-blank lines followed by another non-blank line,
+            # and never touch indented code (tabbed) lines.
+            if not (lines[i].strip() and lines[i + 1].strip()):
+                continue
+            if markdown.RE.regExp["tabbed"].match(lines[i]):
+                continue
+
+            # A lone trailing backslash is CommonMark's hard-break marker, which
+            # the Tiptap WYSIWYG editor emits between wrapped lines. OLMarkdown
+            # has no such syntax: left alone the backslash either escapes the "<"
+            # of the <br /> we append (so the tag renders as the literal text
+            # "<br />" to readers) or shows as a stray "\". Drop it either way so
+            # the two renderers agree. See issue #13074.
+            lines[i] = re.sub(r"(?<!\\)\\$", "", lines[i])
+
             if (
-                lines[i].strip()
-                and lines[i + 1].strip()
-                and not markdown.RE.regExp["tabbed"].match(lines[i])
-                and not LINK_REFERENCE_RE.match(lines[i])
+                not LINK_REFERENCE_RE.match(lines[i])
+                # Don't glue a hard break onto a line that immediately precedes a
+                # link-reference definition. This runs before REFERENCE_PREPROCESSOR
+                # strips those definitions, so a <br /> appended here gets orphaned
+                # inside the reference block and leaks into the page as literal markup.
+                and not LINK_REFERENCE_RE.match(lines[i + 1])
                 and not lines[i].lstrip().startswith(">")
             ):
                 lines[i] += "<br />"

openlibrary/tests/core/test_olmarkdown.py

@@ -18,6 +18,63 @@ def p(html):
     assert md("a\nb") == p("a<br/>\n   b")
 
 
+def test_olmarkdown_no_br_leak_into_reference_block():
+    """Regression: a hard line break must not be glued onto a line that
+    immediately precedes a markdown link-reference definition.
+
+    LineBreaksPreprocessor runs before REFERENCE_PREPROCESSOR strips the
+    ``[id]: url`` definitions, so a ``<br />`` appended to the line above a
+    reference block gets orphaned inside it and leaks into the rendered page
+    as literal markup. This was visible at the bottom of /help/faq/editing
+    (internetarchive/openlibrary#13074), where a wrapped/malformed footnote
+    block rendered ``...Tortilla<br /> [24]:`` etc.
+    """
+
+    def md(text):
+        return OLMarkdown(text).convert().strip()
+
+    # Text line immediately followed by a reference definition: no <br>.
+    assert md("foo\n  [1]: https://example.com") == "<p>foo\n</p>"
+
+    # A wrapped reference value (continuation line sitting between two
+    # definitions, carrying the editor's "\" hard-break marker) must come out
+    # clean: no leaked <br /> and no stray backslash.
+    body = (
+        "intro\n\n"
+        "  [23]: https://openlibrary.org/works/OL23185W/Short_Novels\n"
+        "(Cannery_Row_Of_Mice_and_Men_Tortilla\\\n"
+        "  [24]: https://openlibrary.org/works/OL14876179W/Adventures\n"
+    )
+    rendered = md(body)
+    assert "<br" not in rendered
+    assert "\\" not in rendered
+
+    # Sanity: an ordinary line break between two plain lines is untouched.
+    assert md("a\nb") == "<p>a<br/>\n   b\n</p>"
+
+
+def test_olmarkdown_commonmark_hard_break_does_not_leak_br():
+    """Regression: a CommonMark hard break (trailing ``\\``) must render as a
+    line break, not the literal text ``<br />``.
+
+    The Tiptap WYSIWYG editor serializes hard breaks as ``\\`` + newline. With
+    no handling, OLMarkdown appends its own ``<br />`` right after the
+    backslash, the backslash escapes the ``<``, and readers see the literal
+    string ``<br />`` (internetarchive/openlibrary#13074).
+    """
+
+    def md(text):
+        return OLMarkdown(text).convert().strip()
+
+    out = md("first line\\\nsecond line")
+    assert "&lt;br" not in out  # no escaped/literal <br /> shown to the reader
+    assert "\\" not in out  # the hard-break backslash is consumed
+    assert out == "<p>first line<br/>\n   second line\n</p>"
+
+    # A genuine escaped backslash (``\\``) is preserved, not swallowed.
+    assert "\\" in md("a\\\\b\nc")
+
+
 def test_fenced_code_preprocessor():
     pre = FencedCodePreprocessor()
 

package-lock.json

@@ -39,6 +39,7 @@
     "@playwright/test": "^1.60.0",
     "@sentry/browser": "^10.60.0",
     "@tiptap/core": "^3.20.4",
+    "@tiptap/extension-hard-break": "^3.26.0",
     "@tiptap/extension-image": "^3.22.1",
     "@tiptap/extension-placeholder": "^3.20.4",
     "@tiptap/pm": "^3.20.4",

package.json

@@ -13,6 +13,7 @@ import StarterKit from '@tiptap/starter-kit';
 import { Markdown } from 'tiptap-markdown';
 import Image from '@tiptap/extension-image';
 import { HtmlBlock } from '../../../openlibrary/components/lit/html-block.js';
+import { OLHardBreak } from '../../../openlibrary/components/lit/hard-break.js';
 
 /* ------------------------------------------------------------------ */
 /*  Helper: create an editor identical to production, roundtrip text   */
@@ -30,8 +31,10 @@ function createTestEditor(markdownContent, { enableCode = false } = {}) {
                 codeBlock: enableCode ? undefined : false,
                 code: enableCode ? undefined : false,
                 link: { openOnClick: false, autolink: true },
-                strike: false
+                strike: false,
+                hardBreak: false
             }),
+            OLHardBreak,
             Markdown.configure({
                 breaks: true,
                 linkify: true
@@ -439,6 +442,26 @@ Visit [the website](https://example.org) for more.
         expect(output.length).toBeGreaterThan(input.length * 0.9);
         expect(output).toContain('books and reading');
     });
+
+    /* ---------- hard breaks (OLMarkdown dialect) ---------- */
+
+    test('hard break serializes as a bare newline, not CommonMark "\\"', () => {
+        // OLMarkdown treats every newline as a <br>. Emitting CommonMark's
+        // "\<newline>" makes the display renderer escape its own injected
+        // <br/> and show a literal "<br />" to readers. Regression for
+        // internetarchive/openlibrary#13074.
+        const output = roundTrip('first line\nsecond line');
+        expect(output).not.toContain('\\');
+        expect(output).not.toContain('<br');
+        expect(normalize(output)).toBe('first line\nsecond line');
+    });
+
+    test('multi-line paragraph round-trips without backslashes', () => {
+        const input = 'line one\nline two\nline three';
+        const output = roundTrip(input);
+        expect(output).not.toContain('\\');
+        expect(normalize(output)).toBe(input);
+    });
 });
 
 /* ================================================================== */