diff --git a/__tests__/source-map/code.spec.ts b/__tests__/source-map/code.spec.ts index ed470a7..2031a4e 100644 --- a/__tests__/source-map/code.spec.ts +++ b/__tests__/source-map/code.spec.ts @@ -218,6 +218,17 @@ describe('parseMdWithSourceMap: code.value → raw source', () => { expect(sourceMap.getSourceRange(node, 0, 0).start.offset).toBe(4); }); + test('preserves the empty range before extra closing-fence indentation', () => { + const md = '```\n ```'; + const { ast, sourceMap } = parseMdWithSourceMap(md); + const node = nodesOfType(ast, 'code')[0]; + expect(node.value).toBe(''); + + const range = sourceMap.getSourceRange(node, 0, 0); + expect(range.start.offset).toBe(4); + expect(range.end.offset).toBe(4); + }); + test.each([ ['```', 3], ['~~~', 3], diff --git a/scripts/bench-parse.mjs b/scripts/bench-parse.mjs index 61fe19c..d358f3a 100644 --- a/scripts/bench-parse.mjs +++ b/scripts/bench-parse.mjs @@ -108,8 +108,9 @@ const SMOKE_GROWTH_CHECKS = [ // Child mode and parity mode // --------------------------------------------------------------------------- -function makeState() { +function makeState(source) { return { + source, segments: new WeakMap(), inlineCodeSegments: new WeakMap(), codeSegments: new WeakMap(), @@ -188,7 +189,10 @@ function phaseRunner(bundle, phase) { case 'B2': return (md) => fromMarkdown(md, { extensions: micromarkExtensions, - mdastExtensions: [...fromMarkdownExtensions, recordingExtension(makeState())], + mdastExtensions: [ + ...fromMarkdownExtensions, + recordingExtension(makeState(md)), + ], }); case 'C': return (md) => parseMdWithSourceMap(md).ast; @@ -264,7 +268,10 @@ async function runParity() { })); const b2 = canon(fromMarkdown(md, { extensions: micromarkExtensions, - mdastExtensions: [...fromMarkdownExtensions, recordingExtension(makeState())], + mdastExtensions: [ + ...fromMarkdownExtensions, + recordingExtension(makeState(md)), + ], })); const c = canon(parseMdWithSourceMap(md).ast); return { bytes: Buffer.byteLength(md), aEqC: a === c, bEqC: b === c, b2EqC: b2 === c }; diff --git a/src/source-map/build-source-map.ts b/src/source-map/build-source-map.ts index 019fe03..813244b 100644 --- a/src/source-map/build-source-map.ts +++ b/src/source-map/build-source-map.ts @@ -36,11 +36,13 @@ import type { const { micromarkExtensions, fromMarkdownExtensions } = getParserExtensions(); interface RecordingState { + /** Markdown source used for parser-confirmed indentation spans. */ + source: string /** node -> ordered, gap-free, non-overlapping segments. */ segments: WeakMap /** inlineCode node -> parser-recorded value segments. */ inlineCodeSegments: WeakMap - /** code node -> value segments (see buildCodeSegments). */ + /** code node -> parser-recorded or indented-code value segments. */ codeSegments: WeakMap /** code node -> source point for an empty value. */ emptyCodeOffsets: WeakMap @@ -437,6 +439,7 @@ function resolveSegmentRange(options: SegmentRangeOptions): ParsedPosition { */ export const parseMdWithSourceMap = (md: string): ParsedMarkdownDocument => { const state: RecordingState = { + source: md, segments: new WeakMap(), inlineCodeSegments: new WeakMap(), codeSegments: new WeakMap(), @@ -471,6 +474,7 @@ export const parseMdWithSourceMap = (md: string): ParsedMarkdownDocument => { if ( node.type === 'code' && hasStringField(node, 'value') + && !state.codeSegments.has(node) ) { const mapping = buildCodeSegments(md, node); if (mapping) { diff --git a/src/source-map/code-segments.ts b/src/source-map/code-segments.ts index dc1eda0..cc459a2 100644 --- a/src/source-map/code-segments.ts +++ b/src/source-map/code-segments.ts @@ -68,67 +68,6 @@ function skipBlockQuoteMarkers( return offset; } -function skipIndentation( - md: string, - start: number, - end: number, - columns: number, -): number { - let offset = start; - let removed = 0; - while (offset < end && removed < columns) { - const char = md.charCodeAt(offset); - if (char === 32) { - offset++; - removed++; - } - else if (char === 9) { - offset++; - removed += 4 - (removed % 4); - } - else { - break; - } - } - return offset; -} - -function fencedIndentation( - md: string, - lineStartOffset: number, - fenceStart: number, - quoteDepth: number, -): number { - let offset = lineStartOffset; - if (quoteDepth > 0) { - const afterMarkers = skipBlockQuoteMarkers( - md, - lineStartOffset, - fenceStart, - quoteDepth, - ); - if (afterMarkers === undefined) - return -1; - offset = afterMarkers; - } - let indentation = 0; - while (offset < fenceStart) { - const char = md.charCodeAt(offset); - if (char === 32) { - offset++; - indentation++; - } - else if (char === 9) { - offset++; - indentation += 4 - (indentation % 4); - } - else { - return -1; - } - } - return indentation; -} - function trimTrailingLineEnding(md: string, spans: SourceSpan[]): void { const last = spans[spans.length - 1]; if (!last) @@ -189,88 +128,6 @@ function segmentsFromSpans( return valueOffset === value.length ? segments : undefined; } -function buildFencedCodeSegments( - md: string, - node: { value: string; position?: ParsedPosition }, -): CodeSegments | undefined { - const position = node.position; - if (!position) - return undefined; - const start = position.start.offset; - const end = position.end.offset; - const marker = md.charCodeAt(start); - if (marker !== 96 && marker !== 126) - return undefined; - - let fenceLength = 0; - while (md.charCodeAt(start + fenceLength) === marker) fenceLength++; - if (fenceLength < 3) - return undefined; - - const physicalLineStart = lineStart(md, 0, start); - const quoteDepth = blockQuoteDepth(md, physicalLineStart, start); - const openingIndent = fencedIndentation(md, physicalLineStart, start, quoteDepth); - if (openingIndent < 0) - return undefined; - // A blockquote code node can end before its final physical line ending, so - // derive the opening line boundary from the complete Markdown rather than - // the node's position span. This keeps an unclosed empty fence's insertion - // point at the actual EOF. - const openingLineEnd = lineEnd(md, start, md.length); - let contentEnd = end; - let hasClosingFence = false; - const closingLineStart = lineStart(md, start, end); - let closingFenceStart = openingLineEnd; - if (closingLineStart >= openingLineEnd && closingLineStart < end) { - const closingStart = quoteDepth === 0 - ? closingLineStart - : skipBlockQuoteMarkers(md, closingLineStart, end, quoteDepth); - if (closingStart === undefined) - return undefined; - closingFenceStart = closingStart; - closingFenceStart = skipIndentation( - md, - closingFenceStart, - end, - openingIndent, - ); - const closing = md.slice(closingFenceStart, end); - const closingMatch = /^( {0,3})(`+|~+)[ \t]*$/.exec(closing); - if ( - closingMatch - && closingMatch[2].charCodeAt(0) === marker - && closingMatch[2].length >= fenceLength - ) { - contentEnd = closingLineStart; - hasClosingFence = true; - } - } - - const spans: SourceSpan[] = []; - let offset = openingLineEnd; - while (offset < contentEnd) { - const endOfLine = lineEnd(md, offset, contentEnd); - let contentStart = quoteDepth === 0 - ? offset - : skipBlockQuoteMarkers(md, offset, endOfLine, quoteDepth); - if (contentStart === undefined) - return undefined; - contentStart = skipIndentation(md, contentStart, endOfLine, openingIndent); - spans.push({ start: contentStart, end: endOfLine }); - offset = endOfLine; - } - const emptyOffset = spans[0]?.start - ?? (hasClosingFence ? closingFenceStart : openingLineEnd); - trimTrailingLineEnding(md, spans); - const segments = segmentsFromSpans(md, spans, node.value); - if (!segments) - return undefined; - return { - segments, - emptyOffset, - }; -} - function buildIndentedCodeSegmentsFromIndentation( md: string, node: { value: string; position?: ParsedPosition }, @@ -422,12 +279,5 @@ export function buildCodeSegments( md: string, node: { value: string; position?: ParsedPosition }, ): CodeSegments | undefined { - const position = node.position; - if (!position) - return undefined; - const start = position.start.offset; - const marker = md.charCodeAt(start); - return marker === 96 || marker === 126 - ? buildFencedCodeSegments(md, node) - : buildIndentedCodeSegments(md, node); + return buildIndentedCodeSegments(md, node); } diff --git a/src/source-map/recording-extension.ts b/src/source-map/recording-extension.ts index 22de3b3..fe6a93e 100644 --- a/src/source-map/recording-extension.ts +++ b/src/source-map/recording-extension.ts @@ -4,8 +4,11 @@ import type { ParsedPoint } from '../types'; import type { MarkdownSourceMapSegment, SourceSpan } from './types'; interface RecordingState { + source: string segments: WeakMap inlineCodeSegments: WeakMap + codeSegments: WeakMap + emptyCodeOffsets: WeakMap urlSourceSpans: WeakMap } @@ -20,6 +23,17 @@ interface PendingConstruct { sourceEnd: number } +interface FencedCodeRecording { + segments: MarkdownSourceMapSegment[] + valueLength: number + emptyOffset: number + sawOpeningLineEnding: boolean + sawContentAfterOpening: boolean + sawClosingFence: boolean + openingIndent: number + closingFenceStart?: number +} + const REPLACEMENT_CHARACTER = '�'; const point = (d: { line: number; column: number; offset: number }): ParsedPoint => ({ @@ -30,6 +44,82 @@ const point = (d: { line: number; column: number; offset: number }): ParsedPoint const createText = () => ({ type: 'text', value: '' }); +function sliceLiteralSegments( + segments: MarkdownSourceMapSegment[], + valueStart: number, + valueEnd: number, +): MarkdownSourceMapSegment[] | undefined { + const result: MarkdownSourceMapSegment[] = []; + for (const segment of segments) { + const start = Math.max(segment.valueStart, valueStart); + const end = Math.min(segment.valueEnd, valueEnd); + if (start >= end) + continue; + const sourceStart = segment.sourceStart + start - segment.valueStart; + const sourceEnd = segment.sourceStart + end - segment.valueStart; + const previous = result[result.length - 1]; + if (previous && previous.sourceEnd === sourceStart) { + previous.valueEnd = end - valueStart; + previous.sourceEnd = sourceEnd; + } + else { + result.push({ + valueStart: start - valueStart, + valueEnd: end - valueStart, + sourceStart, + sourceEnd, + kind: 'literal', + }); + } + } + let mappedLength = 0; + for (const segment of result) { + if (segment.valueStart !== mappedLength) + return undefined; + mappedLength = segment.valueEnd; + } + return mappedLength === valueEnd - valueStart ? result : undefined; +} + +function indentationColumns(source: string, start: number, end: number): number { + let columns = 0; + for (let offset = start; offset < end; offset++) { + const char = source.charCodeAt(offset); + if (char === 32) + columns++; + else if (char === 9) + columns += 4 - (columns % 4); + else + return 0; + } + return columns; +} + +function skipIndentationColumns( + source: string, + start: number, + end: number, + columns: number, +): number { + let offset = start; + let removed = 0; + while (offset < end && removed < columns) { + const char = source.charCodeAt(offset); + if (char === 32) { + offset++; + removed++; + } + else if (char === 9) { + offset++; + removed += 4 - (removed % 4); + } + else { + break; + } + } + return offset; +} + interface CompileContext { stack: Array config: { canContainEols: string[] } @@ -45,6 +135,7 @@ interface CompileContext { /** * Build a `mdast` extension that records source mappings during compilation. * Its handlers record mappings for `text` and `inlineCode` values. + * They also record fenced `code` values. * * ⚠️ This couples to `mdast-util-from-markdown` / micromark INTERNALS, not the * public remark API. Upgrading any parser-sensitive dependency is a parser @@ -52,7 +143,8 @@ interface CompileContext { * this relies on are: * * - token event handler names (enter/exit): `data`, `codeText`, `codeTextData`, - * `characterEscape` / + * `codeFenced`, `codeFencedFence`, `codeFencedFenceSequence`, + * `codeFlowValue`, `blockQuotePrefix`, `characterEscape` / * `characterEscapeValue`, `characterReference` / `characterReferenceValue`, * `lineEnding`, `autolinkProtocol`, `autolinkEmail`, * `resourceDestinationString`, `definitionDestinationString`, their literal @@ -72,6 +164,9 @@ interface CompileContext { export function recordingExtension(state: RecordingState) { let pendingConstruct: PendingConstruct | undefined; let inlineCodeSegments: MarkdownSourceMapSegment[] | undefined; + let fencedCodeRecording: FencedCodeRecording | undefined; + let pendingEmptyFencedCode: object | undefined; + let lineIndentStart = 0; const takePendingConstruct = (): PendingConstruct => { if (!pendingConstruct) { @@ -195,7 +290,40 @@ export function recordingExtension(state: RecordingState) { }); }; + const recordFencedCodeSegment = function (this: CompileContext, token: any) { + const recording = fencedCodeRecording; + if (!recording) + return; + const valueLength = this.sliceSerialize(token).length; + const sourceStart = token.start.offset; + // A line-ending token can include the next container prefix in its bounds. + // `sliceSerialize` excludes that prefix from the compiled value. + const sourceEnd = sourceStart + valueLength; + const previous = recording.segments[recording.segments.length - 1]; + if (previous && previous.sourceEnd === sourceStart) { + previous.valueEnd += valueLength; + previous.sourceEnd = sourceEnd; + } + else { + recording.segments.push({ + valueStart: recording.valueLength, + valueEnd: recording.valueLength + valueLength, + sourceStart, + sourceEnd, + kind: 'literal', + }); + } + recording.valueLength += valueLength; + }; + const onexitlineending = function (this: CompileContext, token: any) { + const lineEndingLength = this.sliceSerialize(token).length; + if (pendingEmptyFencedCode) { + const emptyOffset = token.start.offset + lineEndingLength; + state.emptyCodeOffsets.set(pendingEmptyFencedCode, emptyOffset); + pendingEmptyFencedCode = undefined; + } + lineIndentStart = token.start.offset + lineEndingLength; if (this.getData('atHardBreak')) { const tail = this.stack[this.stack.length - 1].children.slice(-1)[0]; tail.position.end = point(token.end); @@ -214,6 +342,21 @@ export function recordingExtension(state: RecordingState) { kind: 'literal', }); recordInlineCodeSegment.call(this, token); + if (fencedCodeRecording) { + if (!fencedCodeRecording.sawOpeningLineEnding) { + const lineEndingEnd = token.start.offset + + lineEndingLength; + fencedCodeRecording.emptyOffset = Math.max( + token.end.offset, + lineEndingEnd, + ); + fencedCodeRecording.sawOpeningLineEnding = true; + } + else { + fencedCodeRecording.sawContentAfterOpening = true; + } + recordFencedCodeSegment.call(this, token); + } } }; @@ -258,6 +401,111 @@ export function recordingExtension(state: RecordingState) { this.exit(token); }; + const onentercodefenced = function (this: CompileContext, token: any) { + if (fencedCodeRecording) { + throw new Error('A fenced code source map is already being recorded'); + } + this.enter({ type: 'code', lang: null, meta: null, value: '' }, token); + fencedCodeRecording = { + segments: [], + valueLength: 0, + emptyOffset: token.start.offset, + sawOpeningLineEnding: false, + sawContentAfterOpening: false, + sawClosingFence: false, + openingIndent: indentationColumns( + state.source, + lineIndentStart, + token.start.offset, + ), + }; + }; + + const onentercodefencedfence = function ( + this: CompileContext, + token: any, + ) { + if (this.getData('flowCodeInside') && fencedCodeRecording) + fencedCodeRecording.closingFenceStart = token.start.offset; + }; + + const onexitcodefencedfence = function (this: CompileContext, token: any) { + if (this.getData('flowCodeInside')) { + if (fencedCodeRecording) + fencedCodeRecording.sawClosingFence = true; + return; + } + this.buffer(); + this.setData('flowCodeInside', true); + if (fencedCodeRecording) + fencedCodeRecording.emptyOffset = token.end.offset; + }; + + const onexitcodefencedfencesequence = function ( + this: CompileContext, + token: any, + ) { + if ( + this.getData('flowCodeInside') + && fencedCodeRecording + && !fencedCodeRecording.sawContentAfterOpening + ) { + const closingStart = fencedCodeRecording.closingFenceStart + ?? token.start.offset; + fencedCodeRecording.emptyOffset = skipIndentationColumns( + state.source, + closingStart, + token.start.offset, + fencedCodeRecording.openingIndent, + ); + } + }; + + const onexitcodeflowvalue = function (this: CompileContext, token: any) { + onexitdata.call(this, token, { + sourceStart: token.start.offset, + sourceEnd: token.end.offset, + kind: 'literal', + }); + if (fencedCodeRecording) + fencedCodeRecording.sawContentAfterOpening = true; + recordFencedCodeSegment.call(this, token); + }; + + const onexitcodefenced = function (this: CompileContext, token: any) { + const data = this.resume(); + const node = this.stack[this.stack.length - 1]; + if (!node || node.type !== 'code') { + throw new Error('Expected code node while recording fenced code'); + } + const value = data.replace(/^(\r?\n|\r)|(\r?\n|\r)$/g, ''); + node.value = value; + const recording = fencedCodeRecording; + fencedCodeRecording = undefined; + if (!recording) { + throw new Error('Missing fenced code source-map recording'); + } + const leadingLineEnding = /^(\r?\n|\r)/.exec(data)?.[0].length ?? 0; + const segments = sliceLiteralSegments( + recording.segments, + leadingLineEnding, + leadingLineEnding + value.length, + ); + if (segments) { + state.codeSegments.set(node, segments); + state.emptyCodeOffsets.set(node, recording.emptyOffset); + } + if ( + value.length === 0 + && !recording.sawOpeningLineEnding + && !recording.sawClosingFence + ) { + pendingEmptyFencedCode = node; + } + this.setData('flowCodeInside'); + this.exit(token); + }; + const onexitautolinkprotocol = function (this: CompileContext, token: any) { onexitdata.call(this, token, { sourceStart: token.start.offset, @@ -329,6 +577,9 @@ export function recordingExtension(state: RecordingState) { return { enter: { + codeFenced: onentercodefenced, + codeFencedFence: onentercodefencedfence, + codeFlowValue: onenterdata, codeText: onentercodetext, codeTextData: onentercodetextdata, data: onenterdata, @@ -340,6 +591,10 @@ export function recordingExtension(state: RecordingState) { resourceDestinationString: onenterUrlDestination, }, exit: { + codeFenced: onexitcodefenced, + codeFencedFence: onexitcodefencedfence, + codeFencedFenceSequence: onexitcodefencedfencesequence, + codeFlowValue: onexitcodeflowvalue, codeText: onexitcodetext, codeTextData: onexitcodetextdata, data(this: CompileContext, token: any) { @@ -357,6 +612,9 @@ export function recordingExtension(state: RecordingState) { }); }, characterReferenceValue: onexitcharacterreferencevalue, + blockQuotePrefix(this: CompileContext, token: any) { + lineIndentStart = token.end.offset; + }, lineEnding: onexitlineending, autolinkProtocol: onexitautolinkprotocol, autolinkEmail: onexitautolinkemail,