Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ jobs:
packages/inbox/decisionMode.test.tsx
packages/inbox/decisionCardContract.test.tsx
packages/inbox/threadFolding.test.tsx
packages/inbox/choiceLabels.test.tsx
packages/inbox/pasteTarget.test.ts
packages/inbox/seenOnSight.test.tsx
packages/inbox/projectList.test.tsx
Expand Down
55 changes: 48 additions & 7 deletions packages/core/inbox-folding.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import { isInboxActionReport, isInboxNote, type InboxMessageWire, type InboxQues
import { QUESTION_ANSWERS_HEADING, type QuestionAnswer } from './question-block';
import { INBOX_STILL_OPEN_LINE, sliceText } from './inbox-questions';
import { withoutInboxSkillsBlock } from './skill-reference-block';
import { stripInlineMarkup } from './inline-markdown';
import { INLINE_DESTINATION, stripInlineMarkup } from './inline-markdown';

/**
* Where "new since you looked" starts: the seq of the person's last look at
Expand Down Expand Up @@ -110,10 +110,14 @@ export function planFolds(messages: readonly InboxMessageWire[], fresh: Readonly
return { folded, hidden };
}

/** The words an answer reads as: "Pagefind", "Guides, API reference", "skipped". */
/**
* The words an answer reads as: "Pagefind", "Guides, API reference", "skipped".
* A picked label shows as its plain words (`plainLabel`); Other and free text
* are the person's own words, shown as typed.
*/
export function answerWords(answer: QuestionAnswer): string {
if (answer.skipped) return 'skipped';
const parts = [...answer.selected];
const parts = answer.selected.map(plainLabel);
if (answer.other) parts.push(answer.other);
if (answer.text) parts.push(answer.text);
return parts.join(', ');
Expand Down Expand Up @@ -233,23 +237,60 @@ export function snippetText(snippet: FoldSnippet): string {
* tags out, and paired emphasis marks (`**x**`, `__x__`, `~~x~~`, `*x*`,
* `_x_`) dropped only where they open and close at word boundaries, so
* `snake_case` and `2 * 3` keep their characters. Inline code keeps its text
* (only the backticks go).
* (only the backticks go). A backslash escape (`\*`, `\+`) reads as its
* character, as the inline renderer draws it. A link's destination may hold
* one level of parentheses (`…/Foo_(bar)`). `keepTags` keeps `<…>` text (a
* choice label: the card draws `Array<string>` literally).
*/
export function cleanInline(text: string): string {
export function cleanInline(text: string, options: { keepTags?: boolean } = {}): string {
const code: string[] = [];
let out = text.replace(/(`+)([^`]|[^`][\s\S]*?[^`])\1(?!`)/g, (_m, _ticks: string, inner: string) => {
code.push(inner.trim());
return `\u0000${code.length - 1}\u0000`;
});
out = stripInlineMarkup(out);
// Escaped characters are set aside so no rule below reads them as marks.
const escaped: string[] = [];
out = out.replace(ESCAPED, (_m, ch: string) => {
escaped.push(ch);
return `\u0001${escaped.length - 1}\u0001`;
});
out = stripInlineMarkup(out, { keepTags: options.keepTags });
// A closing mark ends a word: whitespace, the end, or punctuation that does
// not run into another word (`__init__.py` is a name, not emphasis).
const close = `(?=$|\\s|[,;:!?)\\]"'’]|\\.(?!\\w))`;
for (const mark of ['\\*\\*', '__', '~~', '\\*', '_']) {
const pair = new RegExp(`(^|[^\\w*~])${mark}(?=\\S)([\\s\\S]*?\\S)${mark}${close}`, 'g');
out = out.replace(pair, '$1$2');
}
return out.replace(/\u0000(\d+)\u0000/g, (_m, i: string) => code[Number(i)] ?? '');
return out.replace(/\u0001(\d+)\u0001/g, (_m, i: string) => escaped[Number(i)] ?? '').replace(/\u0000(\d+)\u0000/g, (_m, i: string) => code[Number(i)] ?? '');
}

/** The characters a backslash escapes, as the inline renderer reads them (ui's InlineMarkdown). */
const ESCAPED = /\\([\\*_`[\]~!.()\-#>+|{}&$])/g;

/**
* A picked choice's label as plain words. A label is the agent's inline
* markdown (a link, code, bold): a line or row that draws it as text shows
* "15% off", never `[15% off](forecast-15.html)`. Display only: the stored
* answer and the reply the agent reads keep the label exactly as written, so
* the agent matches it to its choice. The words are the ones the live card
* shows: `<…>` text stays (`Array<string>`), an autolink reads as its
* address, an escape as its character. A label that is only an image reads
* as its alt text, else its file name, so the pick never shows as nothing.
*/
export function plainLabel(label: string): string {
const plain = (text: string) =>
cleanInline(text.replace(/<(https?:\/\/[^<>]+)>/g, '$1').replace(/<([^<@>\s]+@[^<>\s]+)>/g, '$1'), { keepTags: true })
.replace(/\s+/g, ' ')
.trim();
// Bounded like inline-markdown.ts: each pattern stops at the next opener.
const image = /!\[([^\][]*)\]\(\s*<?((?:[^\s()<>]|\([^\s()<>]*\))*)/g;
return (
plain(label) ||
plain(label.replace(new RegExp(String.raw`!\[([^\][]*)\]` + INLINE_DESTINATION, 'g'), '$1')) ||
[...label.matchAll(image)].map((m) => m[2]!.split(/[\\/]/).pop()!.replace(/[?#].*$/, '')).filter(Boolean).join(', ') ||
label.trim()
);
}

/**
Expand Down
17 changes: 13 additions & 4 deletions packages/core/inline-markdown.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
* tags takes most of a second to read (every card render and every send).
*/
import { expect, test } from "bun:test";
import { cleanInline } from "./inbox-folding";
import { cleanInline, plainLabel } from "./inbox-folding";
import { stripInlineMarkup } from "./inline-markdown";

test("links, wiki links, images and tags out, their words kept", () => {
Expand All @@ -21,10 +21,19 @@ test("links, wiki links, images and tags out, their words kept", () => {
expect(stripInlineMarkup("see ![x](a_(1).png)here")).toBe("see here");
});

test("a 20,000-character run of brackets or tags is read in linear time, here and in cleanInline", () => {
for (const run of ["[", "[[", "![", "<a", "[a](", "[[a|", "[a]((", "[a](x(", "![a](("]) {
test("a link label with one level of balanced brackets keeps its words; a second level is not a link", () => {
expect(stripInlineMarkup("[a [nested] label](x.html)")).toBe("a [nested] label");
expect(stripInlineMarkup("[a [b [c] d] e](x.html)")).toBe("[a [b [c] d] e](x.html)");
});

test("tags stay with keepTags (a choice label reads `Array<string>`)", () => {
expect(stripInlineMarkup("[Array<string>](t.html) type", { keepTags: true })).toBe("Array<string> type");
});

test("a 20,000-character run of brackets or tags is read in linear time, here, in cleanInline and in plainLabel", () => {
for (const run of ["[", "[[", "![", "<a", "[a](", "[[a|", "[a]((", "[a](x(", "![a]((", "[a[b]", "[[a]", "a[b]", "<http://", "<a@", "![](", "![](a("]) {
const text = run.repeat(Math.ceil(20_000 / run.length)).slice(0, 20_000);
for (const strip of [stripInlineMarkup, cleanInline]) {
for (const strip of [stripInlineMarkup, cleanInline, plainLabel]) {
const start = performance.now();
strip(text);
expect(performance.now() - start).toBeLessThan(50);
Expand Down
35 changes: 20 additions & 15 deletions packages/core/inline-markdown.ts
Original file line number Diff line number Diff line change
@@ -1,28 +1,33 @@
/** A link's or image's destination: one level of balanced parentheses, bounded at any other `(`. */
const DESTINATION = String.raw`\((?:[^()]|\([^()]*\))*\)`;
const IMAGE_RE = new RegExp(String.raw`!\[[^\][]*\]` + DESTINATION, 'g');
const LINK_RE = new RegExp(String.raw`\[([^\][]*)\]` + DESTINATION, 'g');
export const INLINE_DESTINATION = String.raw`\((?:[^()]|\([^()]*\))*\)`;
const IMAGE_RE = new RegExp(String.raw`!\[[^\][]*\]` + INLINE_DESTINATION, 'g');
// A link's label may hold one level of balanced brackets (`[a [nested] label](x.html)`), as the
// inline renderer reads it. Each nested pair closes at the next `]`, so the label still stops at
// any other `[`: a run of brackets is read in linear time.
const LINK_RE = new RegExp(String.raw`\[((?:[^\][]|\[[^\][]*\])*)\]` + INLINE_DESTINATION, 'g');

/**
* Inline markdown's markup taken out, its words kept: images out, link and
* wiki-link labels kept, HTML tags out. Emphasis and code marks are left to
* the caller (the Inbox's `cleanInline` protects code spans around this; the
* question parser's `normalizeQuestionText` drops the marks).
* wiki-link labels kept, HTML tags out (kept with `keepTags`: a choice label
* the card draws as `Array<string>`). Emphasis and code marks are left to
* the caller (the Inbox's `cleanInline` protects code spans and escapes around
* this; the question parser's `normalizeQuestionText` drops the marks).
*
* Imports nothing, so core/question-block and core/inbox-folding can both use
* it with no cycle (Pi vendors it beside question-block: apps/pi-extension/vendor.sh).
*
* Every pattern stops at the next opener (`[`, `<`; a link's destination at
* a `(` it does not close), so a label of thousands of `[`, `[[`, `![` or
* `<a` is read in linear time, not scanned to the end from each one. A link's
* or image's destination may hold one level of balanced parentheses
* (`[Foo](https://en.wikipedia.org/wiki/Foo_(bar))`, `[page](app/(shop)/page.tsx)`).
* Nested brackets in a label never match.
* a `(` it does not close; a link label at a `[` past its one nested pair),
* so a label of thousands of `[`, `[[`, `![` or `<a` is read in linear time,
* not scanned to the end from each one. A link's or image's destination may
* hold one level of balanced parentheses
* (`[Foo](https://en.wikipedia.org/wiki/Foo_(bar))`, `[page](app/(shop)/page.tsx)`),
* and a link's label one level of balanced brackets (`[a [nested] label](x.html)`).
*/
export function stripInlineMarkup(text: string): string {
return text
export function stripInlineMarkup(text: string, options: { keepTags?: boolean } = {}): string {
const out = text
.replace(IMAGE_RE, '')
.replace(LINK_RE, '$1')
.replace(/\[\[([^\][|]*)(?:\|([^\][]*))?\]\]/g, (_m, target: string, label?: string) => label ?? target)
.replace(/<\/?[A-Za-z][^<>]*>/g, '');
.replace(/\[\[([^\][|]*)(?:\|([^\][]*))?\]\]/g, (_m, target: string, label?: string) => label ?? target);
return options.keepTags ? out : out.replace(/<\/?[A-Za-z][^<>]*>/g, '');
}
Loading
Loading