import type { EmojiNode, RichTextDocument, TextNode } from "./ast.js"; import { DocBuilder, type ParseOptions } from "./builder.js"; import { decodeEntities } from "./entities.js"; /* * Converts the Mastodon/PeerTube HTML subset into the rich-text AST. This is * a converter, not an HTML renderer: no tag or attribute survives as markup. * Recognised structure (p, br, a, Mastodon mention/hashtag/invisible/ellipsis * spans) maps to AST nodes; dangerous containers are dropped with their * content; every other tag is unwrapped to its text. The tokenizer is a single * forward scan (no regex backtracking), so huge or malformed input is linear. */ /** Element content parsed as raw text by browsers; skipped up to the matching end tag. */ const RAW_TEXT = new Set(["script", "style", "textarea", "title", "xmp", "iframe", "noembed", "noframes", "noscript"]); /** Elements whose entire content is discarded. */ const DROP_CONTENT = new Set([ "svg", "math", "template", "object", "embed", "applet", "frame", "frameset", "head", "select", "option", "form", "button", "audio", "video", "canvas", "map", "picture", "source", "track", "dialog", "portal", ]); const VOID = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"]); const PARAGRAPH = new Set([ "p", "div", "blockquote", "pre", "ul", "ol", "li", "h1", "h2", "h3", "h4", "h5", "h6", "section", "article", "header", "footer", "aside", "table", "tr", "dl", "dt", "dd", "figure", "figcaption", "hr", ]); type Token = | { kind: "text"; text: string } | { kind: "start"; name: string; attrs: Map; selfClosing: boolean } | { kind: "end"; name: string }; const MAX_ATTRS = 16; function isAlpha(c: number): boolean { return (c >= 65 && c <= 90) || (c >= 97 && c <= 122); } function isSpace(c: number): boolean { return c === 32 || c === 9 || c === 10 || c === 12 || c === 13; } function* tokenize(src: string): Generator { const lower = src.toLowerCase(); const n = src.length; let i = 0; while (i < n) { const lt = src.indexOf("<", i); if (lt === -1) { yield { kind: "text", text: src.slice(i) }; return; } if (lt > i) yield { kind: "text", text: src.slice(i, lt) }; i = lt; const next = src.charCodeAt(i + 1); if (src.startsWith("", i + 4); i = close === -1 ? n : close + 3; continue; } if (next === 33 /* ! */ || next === 63 /* ? */ || src.startsWith("", i)) { // Bogus comment / doctype / processing instruction / "". const close = src.indexOf(">", i + 1); i = close === -1 ? n : close + 1; continue; } const isEnd = next === 47; const nameStart = isEnd ? i + 2 : i + 1; if (!isAlpha(src.charCodeAt(nameStart))) { yield { kind: "text", text: "<" }; i++; continue; } let j = nameStart; while (j < n && !isSpace(src.charCodeAt(j)) && src[j] !== "/" && src[j] !== ">") j++; const name = lower.slice(nameStart, j); // Attributes: name[=value], value quoted or unquoted. const attrs = new Map(); let selfClosing = false; while (j < n && src[j] !== ">") { const c = src.charCodeAt(j); if (isSpace(c)) { j++; continue; } if (src[j] === "/") { selfClosing = true; j++; continue; } selfClosing = false; const an = j; while (j < n && !isSpace(src.charCodeAt(j)) && src[j] !== "/" && src[j] !== ">" && src[j] !== "=") j++; if (j === an) j++; // stray "=": consume to guarantee progress const attrName = lower.slice(an, j); let value = ""; while (j < n && isSpace(src.charCodeAt(j))) j++; if (src[j] === "=") { j++; while (j < n && isSpace(src.charCodeAt(j))) j++; const q = src[j]; if (q === '"' || q === "'") { const close = src.indexOf(q, j + 1); const end = close === -1 ? n : close; value = src.slice(j + 1, end); j = end + 1; } else { const vs = j; while (j < n && !isSpace(src.charCodeAt(j)) && src[j] !== ">") j++; value = src.slice(vs, j); } } if (attrName && !attrs.has(attrName) && attrs.size < MAX_ATTRS) attrs.set(attrName, decodeEntities(value)); } i = j + 1; if (isEnd) { yield { kind: "end", name }; continue; } if (name === "plaintext") return; if (RAW_TEXT.has(name)) { const close = lower.indexOf(" { const gt = src.indexOf(">", close); return gt === -1 ? n : gt + 1; })(); continue; } yield { kind: "start", name, attrs, selfClosing }; } } type Frame = { name: string; role: "drop" | "link" | "invisible" | "ellipsis" | "other" }; type OpenLink = { href: string; classes: Set; rel: Set; label: (TextNode | EmojiNode)[] }; function words(value: string | undefined): Set { return new Set((value ?? "").toLowerCase().split(/\s+/).filter(Boolean)); } /** Mastodon/PeerTube-style HTML → rich-text AST. */ export function parseProviderHtml(input: unknown, options: ParseOptions = {}): RichTextDocument { const builder = new DocBuilder(options); if (typeof input !== "string") return builder.finish(); const src = builder.clip(input); const stack: Frame[] = []; let dropDepth = 0; let invisibleDepth = 0; let link: OpenLink | null = null; const flushLink = () => { if (!link) return; const current = link; link = null; const labelText = current.label.map((n) => (n.type === "text" ? n.text : `:${n.shortcode}:`)).join("").trim(); if (current.classes.has("mention") && (current.classes.has("hashtag") || current.rel.has("tag"))) { builder.hashtag(labelText, current.href); } else if (current.classes.has("mention")) { builder.mention(handleFor(labelText, current.href), current.href); } else { builder.link(current.href, current.label); } }; const emitText = (raw: string) => { if (dropDepth > 0) return; if (link && invisibleDepth > 0) return; const text = decodeEntities(raw).replace(/[\t\n\f\r ]+/g, " "); if (link) link.label.push(...builder.inlineText(text)); else builder.text(text); }; const close = (index: number) => { while (stack.length > index) { const frame = stack.pop() as Frame; if (frame.role === "drop") dropDepth--; else if (frame.role === "invisible") invisibleDepth--; else if (frame.role === "ellipsis" && link && dropDepth === 0) link.label.push({ type: "text", text: "…" }); else if (frame.role === "link") flushLink(); if (PARAGRAPH.has(frame.name) && dropDepth === 0) { flushLink(); builder.endParagraph(); } } }; for (const token of tokenize(src)) { if (token.kind === "text") { emitText(token.text); continue; } if (token.kind === "end") { if (token.name === "br" && dropDepth === 0) builder.lineBreak(); // "
" acts as
in browsers for (let k = stack.length - 1; k >= 0; k--) { const frame = stack[k] as Frame; // Inside dropped content only the dropped elements' own end tags count. if (frame.name === token.name && (dropDepth === 0 || frame.role === "drop")) { close(k); break; } } continue; } const { name, attrs } = token; if (dropDepth > 0) { if (DROP_CONTENT.has(name) && !VOID.has(name) && !token.selfClosing && stack.length < builder.limits.maxDepth) { stack.push({ name, role: "drop" }); dropDepth++; } continue; } if (name === "br") { if (link) link.label.push({ type: "text", text: " " }); else builder.lineBreak(); continue; } if (name === "img") { const alt = attrs.get("alt") ?? ""; if (alt) emitText(alt); continue; } if (PARAGRAPH.has(name)) { flushLink(); builder.endParagraph(); } if (VOID.has(name) || token.selfClosing && !DROP_CONTENT.has(name) && name !== "a" && name !== "span") continue; if (stack.length >= builder.limits.maxDepth) { // Too deep: stop tracking structure, keep text (inert) and flag it. builder.truncated = true; if (DROP_CONTENT.has(name)) { // Can't track the matching end tag any more; drop the rest of the input. break; } continue; } if (DROP_CONTENT.has(name)) { if (token.selfClosing) continue; stack.push({ name, role: "drop" }); dropDepth++; continue; } if (name === "a") { flushLink(); link = { href: attrs.get("href") ?? "", classes: words(attrs.get("class")), rel: words(attrs.get("rel")), label: [] }; stack.push({ name, role: "link" }); continue; } if (name === "span" && link) { const classes = words(attrs.get("class")); if (classes.has("invisible")) { invisibleDepth++; stack.push({ name, role: "invisible" }); continue; } if (classes.has("ellipsis")) { stack.push({ name, role: "ellipsis" }); continue; } } stack.push({ name, role: "other" }); } close(0); flushLink(); return builder.finish(); } /** Mastodon renders only the local username in mention text; qualify it with the href host. */ function handleFor(label: string, href: string): string { const handle = label.replace(/^@+/, ""); if (handle.includes("@")) return handle; try { const url = new URL(href); if (url.protocol === "http:" || url.protocol === "https:") return `${handle}@${url.hostname}`; } catch { // fall through: unqualified handle } return handle; }