diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b9f9385..80f5794 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -60,6 +60,9 @@ internal/core/entity ├── Toc ├── Type └── MarkerStart / MarkerEnd + +internal/core/mdsplit +└── Split (cuts a document into chunks the GitHub Markdown API accepts) ``` ## Dependency direction @@ -316,15 +319,25 @@ Controller -> LocalMd.Do -> FileChecker.Exists -> HTMLConverter.Convert - -> RemotePoster.Post + -> RemotePoster.Post (documents up to 384 KB) + -> mdsplit.Split (larger documents) + -> RemotePoster.PostBody (once per chunk) -> GitHub /markdown/raw API - -> RegexpExtractor.Extract + -> RegexpExtractor.Extract (once per chunk) -> Renderer.Render -> entity.Toc ``` When debug mode is enabled, `LocalMd` writes the returned HTML to `.debug.html` through `FileWriter`. +`HTMLConverter.Convert` returns one HTML string per request. A document larger than +384 KB is cut by `mdsplit` at blank lines outside fenced code blocks, HTML comments +and raw HTML blocks, so every chunk parses the way those lines parse inside the whole +document; a chunk forced to end inside such a block closes it and reopens it in the +next chunk. `Generator.Grab` then extracts headings from every chunk and renumbers +duplicate anchors against a document-wide counter, because GitHub numbered duplicates +inside each chunk in isolation. + ### Skip header (`--skip-header`) ```text diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bdc96e..33df259 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ Planned release: 2.1.0. - `token.txt`, read from next to the executable, is now the last fallback for a GitHub token, after `--token` and `GH_TOC_TOKEN`. - A Docker image is published to `ghcr.io/ekalinin/github-markdown-toc.go`. +- Documents larger than GitHub's 400 KB Markdown API limit are converted in chunks + instead of failing, with duplicate anchors numbered across the whole document. + ([#25](https://github.com/ekalinin/github-markdown-toc.go/issues/25)) ### Security diff --git a/README.md b/README.md index a98fb19..7d342f2 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ Table of Contents * [Starting Depth](#starting-depth) * [Depth](#depth) * [No Escape](#no-escape) + * [Large documents](#large-documents) * [Github token](#github-token) * [GitHub Enterprise Server](#github-enterprise-server) * [Bash/ZSH auto\-complete](#bashzsh-auto-complete) @@ -431,6 +432,21 @@ No escape * [Dockerfile.vim](#dockerfilevim) ``` +Large documents +--------------- + +GitHub's `/markdown/raw` API renders at most 400 KB per request. A larger document is +split automatically at blank lines outside code blocks, converted in several requests +and reassembled into one TOC, with duplicate anchors numbered as they would be for +the whole document. + +The only visible difference is the number of API calls: a 2 MB document costs six +requests instead of one, which reaches the rate limit six times faster. Pass a token +(see below) when you work with documents of that size. + +A document is only rejected when a single line is larger than the limit, since there +is no way to split it without changing what it renders to. + GitHub token ------------ diff --git a/e2e-tests/want.md b/e2e-tests/want.md index a73dbb4..6101b77 100644 --- a/e2e-tests/want.md +++ b/e2e-tests/want.md @@ -22,6 +22,7 @@ Table of Contents * [Starting Depth](#starting-depth) * [Depth](#depth) * [No escape](#no-escape) + * [Large documents](#large-documents) * [GitHub token](#github-token) * [GitHub Enterprise Server](#github-enterprise-server) * [Bash/ZSH auto\-complete](#bashzsh-auto-complete) diff --git a/e2e-tests/want3.md b/e2e-tests/want3.md index 03cd94a..a6382b1 100644 --- a/e2e-tests/want3.md +++ b/e2e-tests/want3.md @@ -18,6 +18,7 @@ * [Starting Depth](#starting-depth) * [Depth](#depth) * [No escape](#no-escape) + * [Large documents](#large-documents) * [GitHub token](#github-token) * [GitHub Enterprise Server](#github-enterprise-server) * [Bash/ZSH auto\-complete](#bashzsh-auto-complete) diff --git a/internal/adapters/htmlconverter.go b/internal/adapters/htmlconverter.go index 8d83ff6..1c32811 100644 --- a/internal/adapters/htmlconverter.go +++ b/internal/adapters/htmlconverter.go @@ -5,11 +5,15 @@ import ( "errors" "fmt" "net/http" + "os" "strings" + + "github.com/ekalinin/github-markdown-toc.go/v2/internal/core/mdsplit" ) type remotePoster interface { Post(context.Context, string, string, string) (string, error) + PostBody(context.Context, string, string, []byte) (string, error) } type HTMLConverter struct { @@ -36,16 +40,64 @@ func NewHTMLConverterX(token, url string, poster remotePoster, log logger) *HTML } } -func (c *HTMLConverter) Convert(ctx context.Context, file string) (string, error) { +// maxChunkBytes is the largest payload the GitHub Markdown API renders. GitHub +// documents the limit as 400 KB; staying at 384 KB keeps us below it whether that +// means 400*1024 or 400000 bytes. +const maxChunkBytes = 384 << 10 + +// Convert renders file as HTML. The result holds one string per request: a single +// one for a document GitHub accepts whole, and one per chunk for a larger document. +func (c *HTMLConverter) Convert(ctx context.Context, file string) ([]string, error) { c.log.Info("adapters.HTMLConverter.Convert: start", "file", file) ghURL := c.ghURL + "/markdown/raw" c.log.Info("adapters.HTMLConverter.Convert: sending", "url", ghURL) - html, err := c.poster.Post(ctx, ghURL, c.ghToken, file) + if !exceedsChunkLimit(file) { + html, err := c.poster.Post(ctx, ghURL, c.ghToken, file) + if err != nil { + return nil, withRateLimitHint(err) + } + return []string{html}, nil + } + return c.convertInChunks(ctx, ghURL, file) +} + +// exceedsChunkLimit reports whether file is too large for one request. Only a +// successful stat sends us down the chunked path: when it fails, the single request +// reports the file error the way it always has. +func exceedsChunkLimit(file string) bool { + info, err := os.Stat(file) + return err == nil && info.Size() > maxChunkBytes +} + +// convertInChunks converts a document GitHub would refuse as a whole. Requests are +// sent one after another: the order of the results is the order of the document, and +// a burst of parallel requests would only bring the rate limit closer. +func (c *HTMLConverter) convertInChunks(ctx context.Context, ghURL, file string) ([]string, error) { + content, err := os.ReadFile(file) + if err != nil { + return nil, err + } + chunks, err := mdsplit.Split(content, maxChunkBytes) if err != nil { - return "", withRateLimitHint(err) + return nil, err + } + c.log.Info("adapters.HTMLConverter.Convert: splitting large document", + "file", file, "chunks", len(chunks)) + + result := make([]string, 0, len(chunks)) + for i, chunk := range chunks { + if err := ctx.Err(); err != nil { + return nil, err + } + html, err := c.poster.PostBody(ctx, ghURL, c.ghToken, chunk) + if err != nil { + return nil, fmt.Errorf("convert chunk %d/%d: %w", + i+1, len(chunks), withRateLimitHint(err)) + } + result = append(result, html) } - return html, nil + return result, nil } // rateLimitMarker is what GitHub puts in the body when the API rate limit is hit. diff --git a/internal/adapters/htmlconverter_test.go b/internal/adapters/htmlconverter_test.go index 7985b8e..91e5e6d 100644 --- a/internal/adapters/htmlconverter_test.go +++ b/internal/adapters/htmlconverter_test.go @@ -3,8 +3,14 @@ package adapters import ( "context" "errors" + "fmt" + "io" "net/http" + "net/http/httptest" + "os" + "path/filepath" "strings" + "sync" "testing" ) @@ -12,6 +18,7 @@ type fakePoster struct { gotURL string gotToken string gotPath string + gotBody []byte retBody string retErr error } @@ -23,6 +30,13 @@ func (p *fakePoster) Post(_ context.Context, url, token, path string) (string, e return p.retBody, p.retErr } +func (p *fakePoster) PostBody(_ context.Context, url, token string, body []byte) (string, error) { + p.gotURL = url + p.gotToken = token + p.gotBody = body + return p.retBody, p.retErr +} + func Test_HTMLConverter(t *testing.T) { token, url, path := "xx-token", "gh-url", "html-file" @@ -52,8 +66,8 @@ func Test_HTMLConverter(t *testing.T) { } if !tt.failed { - if got != want { - t.Errorf("Got=%v, want=%v", got, want) + if len(got) != 1 || got[0] != want { + t.Errorf("Got=%v, want=[%v]", got, want) } if got := tt.poster.gotPath; got != path { t.Errorf("Got=%v, want=%v", got, path) @@ -83,6 +97,10 @@ func (s posterStub) Post(context.Context, string, string, string) (string, error return "", s.err } +func (s posterStub) PostBody(context.Context, string, string, []byte) (string, error) { + return "", s.err +} + func TestHTMLConverterRateLimitHint(t *testing.T) { tests := []struct { name string @@ -130,3 +148,59 @@ func TestHTMLConverterRateLimitHint(t *testing.T) { }) } } + +func TestHTMLConverterSplitsLargeDocuments(t *testing.T) { + var mu sync.Mutex + var bodies [][]byte + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + return + } + mu.Lock() + bodies = append(bodies, body) + n := len(bodies) + mu.Unlock() + if _, err := fmt.Fprintf(w, "

chunk %d

", n); err != nil { + t.Error(err) + } + })) + defer srv.Close() + + var doc strings.Builder + for doc.Len() <= 3*maxChunkBytes { + doc.WriteString("# Heading\n\n") + doc.WriteString(strings.Repeat("word ", 200)) + doc.WriteString("\n\n") + } + file := filepath.Join(t.TempDir(), "big.md") + if err := os.WriteFile(file, []byte(doc.String()), 0644); err != nil { + t.Fatal(err) + } + + converter := NewHTMLConverterWithClient("", srv.URL, srv.Client(), NewLogger(false)) + got, err := converter.Convert(context.Background(), file) + if err != nil { + t.Fatal(err) + } + + if len(got) < 4 { + t.Fatalf("got %d chunks, want at least 4", len(got)) + } + mu.Lock() + defer mu.Unlock() + if len(bodies) != len(got) { + t.Fatalf("got %d requests for %d chunks, want one each", len(bodies), len(got)) + } + for i, body := range bodies { + if len(body) > maxChunkBytes { + t.Errorf("request %d carried %d bytes, want at most %d", i, len(body), maxChunkBytes) + } + } + for i, html := range got { + if want := fmt.Sprintf("

chunk %d

", i+1); html != want { + t.Errorf("chunk %d is %q, want %q", i, html, want) + } + } +} diff --git a/internal/adapters/http.go b/internal/adapters/http.go index bfd337e..945e67a 100644 --- a/internal/adapters/http.go +++ b/internal/adapters/http.go @@ -1,6 +1,7 @@ package adapters import ( + "bytes" "context" "errors" "fmt" @@ -119,15 +120,24 @@ func HttpPost(ctx context.Context, client *http.Client, url, path, token string) _ = file.Close() }() - req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, file) + fileInfo, err := file.Stat() if err != nil { return "", err } - fileInfo, err := file.Stat() + return httpPost(ctx, client, url, file, fileInfo.Size(), token) +} + +// HttpPostBody sends an in-memory body in an HTTP POST request. +func HttpPostBody(ctx context.Context, client *http.Client, url string, body []byte, token string) (string, error) { + return httpPost(ctx, client, url, bytes.NewReader(body), int64(len(body)), token) +} + +func httpPost(ctx context.Context, client *http.Client, url string, body io.Reader, size int64, token string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, body) if err != nil { return "", err } - req.ContentLength = fileInfo.Size() + req.ContentLength = size if token != "" { req.Header.Add("Authorization", "token "+token) diff --git a/internal/adapters/http_test.go b/internal/adapters/http_test.go index ab701d7..fd59158 100644 --- a/internal/adapters/http_test.go +++ b/internal/adapters/http_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "net" "net/http" "net/http/httptest" @@ -267,3 +268,36 @@ func Test_doHTTPReq_issue35(t *testing.T) { t.Error("response header should be \"Hello, client\", but got:", resHeader) } } + +func TestHttpPostBody(t *testing.T) { + token := "test-token" + var gotBody []byte + var gotToken string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + return + } + gotBody = body + gotToken = r.Header.Get("Authorization") + if _, err := w.Write([]byte("

ok

")); err != nil { + t.Error(err) + } + })) + defer srv.Close() + + got, err := HttpPostBody(context.Background(), testHTTPClient(), srv.URL, []byte("# Title\n"), token) + if err != nil { + t.Fatal(err) + } + if got != "

ok

" { + t.Errorf("got response %q, want the rendered HTML", got) + } + if string(gotBody) != "# Title\n" { + t.Errorf("got body %q, want the posted content", gotBody) + } + if want := "token " + token; gotToken != want { + t.Errorf("got authorization %q, want %q", gotToken, want) + } +} diff --git a/internal/adapters/remoteposter.go b/internal/adapters/remoteposter.go index 59ec543..17c4ccf 100644 --- a/internal/adapters/remoteposter.go +++ b/internal/adapters/remoteposter.go @@ -23,3 +23,10 @@ func (r *RemotePoster) Post(ctx context.Context, url, token, path string) (strin } return HttpPost(ctx, r.client, url, path, token) } + +func (r *RemotePoster) PostBody(ctx context.Context, url, token string, body []byte) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + return HttpPostBody(ctx, r.client, url, body, token) +} diff --git a/internal/core/mdsplit/mdsplit.go b/internal/core/mdsplit/mdsplit.go new file mode 100644 index 0000000..093011c --- /dev/null +++ b/internal/core/mdsplit/mdsplit.go @@ -0,0 +1,300 @@ +// Package mdsplit cuts a Markdown document into pieces small enough for GitHub's +// Markdown API, which refuses to render a payload larger than 400 KB. +package mdsplit + +import ( + "bytes" + "fmt" + "strings" +) + +// blockKind is the kind of block a line can be inside of. Only blocks that survive a +// blank line are tracked: everything else ends at the blank line we cut on anyway. +type blockKind int + +const ( + blockNone blockKind = iota + blockFence + blockComment + blockRawHTML +) + +// rawHTMLTags are the tags whose content GitHub keeps verbatim across blank lines. +var rawHTMLTags = []string{"pre", "script", "style", "textarea"} + +// state is the block context at the end of the lines processed so far. +type state struct { + kind blockKind + openLine string + fenceChar byte + fenceLen int + tag string +} + +// closing is the line that ends the open block at the end of a chunk. +func (s state) closing() string { + switch s.kind { + case blockFence: + return strings.Repeat(string(s.fenceChar), s.fenceLen) + "\n" + case blockComment: + return "-->\n" + case blockRawHTML: + return "\n" + } + return "" +} + +// reopening is the line that restores the block at the start of the next chunk. +func (s state) reopening() string { + switch s.kind { + case blockFence: + if strings.HasSuffix(s.openLine, "\n") { + return s.openLine + } + return s.openLine + "\n" + case blockComment: + return "") { + return state{} + } + return st + case blockRawHTML: + if strings.Contains(strings.ToLower(line), "") { + return state{} + } + return st + } + + if char, length, ok := opensFence(line); ok { + return state{kind: blockFence, openLine: line, fenceChar: char, fenceLen: length} + } + if rest, ok := opensComment(line); ok { + if strings.Contains(rest, "-->") { + return state{} + } + return state{kind: blockComment} + } + if tag, ok := opensRawHTML(line); ok { + if strings.Contains(strings.ToLower(line), "") { + return state{} + } + return state{kind: blockRawHTML, tag: tag} + } + return state{} +} + +// splitLines cuts content into lines that keep their trailing newline. +func splitLines(content []byte) []string { + if len(content) == 0 { + return nil + } + lines := strings.SplitAfter(string(content), "\n") + if last := len(lines) - 1; lines[last] == "" { + lines = lines[:last] + } + return lines +} + +// trimIndent strips leading spaces and reports how many there were. +func trimIndent(line string) (string, int) { + i := 0 + for i < len(line) && line[i] == ' ' { + i++ + } + return line[i:], i +} + +func isBlank(line string) bool { + return strings.TrimSpace(line) == "" +} + +// opensFence reports the fence character and length when line opens a fenced block. +func opensFence(line string) (byte, int, bool) { + trimmed, indent := trimIndent(line) + if indent > 3 || trimmed == "" { + return 0, 0, false + } + char := trimmed[0] + if char != '`' && char != '~' { + return 0, 0, false + } + length := 0 + for length < len(trimmed) && trimmed[length] == char { + length++ + } + if length < 3 { + return 0, 0, false + } + // A backtick fence cannot carry a backtick in its info string. + if char == '`' && strings.Contains(trimmed[length:], "`") { + return 0, 0, false + } + return char, length, true +} + +// closesFence reports whether line closes a fence opened with char repeated length +// times: the same character, at least as long, and nothing else on the line. +func closesFence(line string, char byte, length int) bool { + trimmed, indent := trimIndent(line) + if indent > 3 { + return false + } + count := 0 + for count < len(trimmed) && trimmed[count] == char { + count++ + } + if count < length { + return false + } + return strings.TrimSpace(trimmed[count:]) == "" +} + +// opensComment reports whether line starts an HTML comment block and returns what +// follows the opening marker, so the caller can see a comment closed on one line. +func opensComment(line string) (string, bool) { + trimmed, indent := trimIndent(line) + if indent > 3 || !strings.HasPrefix(trimmed, "\n\n"}, + {"pre block with hashes", "
\n# not a heading\nmore text\n
\n\n"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + const maxBytes = 64 + doc := strings.Repeat(tt.body, 40) + + chunks, err := Split([]byte(doc), maxBytes) + if err != nil { + t.Fatal(err) + } + if len(chunks) < 2 { + t.Fatalf("got %d chunks, want the document to be split", len(chunks)) + } + for i, chunk := range chunks { + if len(chunk) > maxBytes { + t.Errorf("chunk %d is %d bytes, want at most %d", i, len(chunk), maxBytes) + } + text := string(chunk) + if n := strings.Count(text, "```"); n%2 != 0 { + t.Errorf("chunk %d leaves a fence open: %q", i, text) + } + if strings.Count(text, "") { + t.Errorf("chunk %d leaves an HTML comment open: %q", i, text) + } + if strings.Count(text, "
") != strings.Count(text, "
") { + t.Errorf("chunk %d leaves a pre block open: %q", i, text) + } + } + }) + } +} diff --git a/internal/core/toc/anchors.go b/internal/core/toc/anchors.go new file mode 100644 index 0000000..fb64023 --- /dev/null +++ b/internal/core/toc/anchors.go @@ -0,0 +1,58 @@ +package toc + +import ( + "strconv" + "strings" + + "github.com/ekalinin/github-markdown-toc.go/v2/internal/core/entity" +) + +// renumberAnchors makes anchors unique across chunks that GitHub numbered +// separately. GitHub appends "-N" to the Nth repeat of a slug within one rendered +// document, so a document converted in pieces restarts that numbering in every +// piece. The base slug is recovered with the same counter GitHub used inside the +// chunk, then numbered again against a counter that spans the whole document. +func renumberAnchors(chunks [][]entity.Heading) []entity.Heading { + total := 0 + for _, chunk := range chunks { + total += len(chunk) + } + + result := make([]entity.Heading, 0, total) + global := make(map[string]int) + for _, chunk := range chunks { + local := make(map[string]int) + for _, heading := range chunk { + base := baseAnchor(heading.Anchor, local) + local[base]++ + if n := global[base]; n > 0 { + heading.Anchor = base + "-" + strconv.Itoa(n) + } else { + heading.Anchor = base + } + global[base]++ + result = append(result, heading) + } + } + return result +} + +// baseAnchor strips the duplicate counter GitHub appended inside a chunk. A suffix +// only counts as a counter when it matches the number of times the base was already +// seen in that same chunk, which is exactly when GitHub would have produced it. +func baseAnchor(anchor string, local map[string]int) string { + idx := strings.LastIndexByte(anchor, '-') + if idx <= 0 || idx == len(anchor)-1 { + return anchor + } + suffix := anchor[idx+1:] + n, err := strconv.Atoi(suffix) + if err != nil || n <= 0 || suffix != strconv.Itoa(n) { + return anchor + } + base := anchor[:idx] + if local[base] != n { + return anchor + } + return base +} diff --git a/internal/core/toc/generator.go b/internal/core/toc/generator.go index 530ef35..f016535 100644 --- a/internal/core/toc/generator.go +++ b/internal/core/toc/generator.go @@ -25,12 +25,25 @@ func NewGenerator(extractor HeadingExtractor, renderer *Renderer) *Generator { } } -// Grab extracts headings from input and renders them as a TOC for the document at -// path. The path is only used when the renderer is configured for absolute paths. -func (g *Generator) Grab(ctx context.Context, path, input string) (*entity.Toc, error) { - headings, err := g.extractor.Extract(ctx, input) - if err != nil { - return nil, fmt.Errorf("extract headings: %w", err) +// Grab extracts headings from every input and renders them as a single TOC for the +// document at path. The path is only used when the renderer is configured for +// absolute paths. Several inputs are the chunks of one document that was too large +// to convert in one request, and their anchors are renumbered as one document. +func (g *Generator) Grab(ctx context.Context, path string, inputs ...string) (*entity.Toc, error) { + chunks := make([][]entity.Heading, 0, len(inputs)) + for _, input := range inputs { + headings, err := g.extractor.Extract(ctx, input) + if err != nil { + return nil, fmt.Errorf("extract headings: %w", err) + } + chunks = append(chunks, headings) + } + + var headings []entity.Heading + if len(chunks) == 1 { + headings = chunks[0] + } else { + headings = renumberAnchors(chunks) } result, err := g.renderer.Render(ctx, path, headings) diff --git a/internal/core/toc/generator_test.go b/internal/core/toc/generator_test.go index b8116b7..6fd510a 100644 --- a/internal/core/toc/generator_test.go +++ b/internal/core/toc/generator_test.go @@ -79,3 +79,58 @@ func TestGeneratorPassesPathToRenderer(t *testing.T) { t.Errorf("got TOC %v, want %v", got, want) } } + +type chunkExtractorStub struct { + byInput map[string][]entity.Heading +} + +func (s chunkExtractorStub) Extract(_ context.Context, input string) ([]entity.Heading, error) { + return s.byInput[input], nil +} + +func TestGeneratorRenumbersAnchorsAcrossChunks(t *testing.T) { + extractor := chunkExtractorStub{byInput: map[string][]entity.Heading{ + "chunk-1": {{Level: 1, Text: "Usage", Anchor: "usage"}}, + "chunk-2": { + {Level: 1, Text: "Usage", Anchor: "usage"}, + {Level: 1, Text: "Usage", Anchor: "usage-1"}, + }, + }} + generator := NewGenerator(extractor, NewRenderer(DefaultConfig())) + + got, err := generator.Grab(context.Background(), "", "chunk-1", "chunk-2") + if err != nil { + t.Fatal(err) + } + want := entity.Toc{ + "* [Usage](#usage)", + "* [Usage](#usage-1)", + "* [Usage](#usage-2)", + } + if got == nil || !slices.Equal(*got, want) { + t.Errorf("got TOC %v, want %v", got, want) + } +} + +// A single input is what GitHub already numbered for the whole document, so the +// anchors are passed through untouched, including a heading whose own text ends in a +// number. +func TestGeneratorKeepsAnchorsForASingleInput(t *testing.T) { + extractor := extractorStub{headings: []entity.Heading{ + {Level: 1, Text: "Usage", Anchor: "usage"}, + {Level: 1, Text: "Usage 1", Anchor: "usage-1"}, + }} + generator := NewGenerator(extractor, NewRenderer(DefaultConfig())) + + got, err := generator.Grab(context.Background(), "", "input") + if err != nil { + t.Fatal(err) + } + want := entity.Toc{ + "* [Usage](#usage)", + "* [Usage 1](#usage-1)", + } + if got == nil || !slices.Equal(*got, want) { + t.Errorf("got TOC %v, want %v", got, want) + } +} diff --git a/internal/core/usecase/localmd/localmd.go b/internal/core/usecase/localmd/localmd.go index 2babc17..7fd2a1f 100644 --- a/internal/core/usecase/localmd/localmd.go +++ b/internal/core/usecase/localmd/localmd.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io/fs" + "strings" "github.com/ekalinin/github-markdown-toc.go/v2/internal/core/entity" ) @@ -17,11 +18,11 @@ type fileWriter interface { } type htmlConverter interface { - Convert(context.Context, string) (string, error) + Convert(context.Context, string) ([]string, error) } type tocGrabber interface { - Grab(context.Context, string, string) (*entity.Toc, error) + Grab(context.Context, string, ...string) (*entity.Toc, error) } type logger interface { @@ -93,14 +94,14 @@ func (uc *LocalMd) DoAs(ctx context.Context, file, displayPath string) (entity.T htmlFile := debugTarget + ".debug.html" uc.log.Info("LocalMD: writing html", "file", htmlFile) // TODO: move to port - if err := uc.writer.Write(ctx, htmlFile, []byte(html)); err != nil { + if err := uc.writer.Write(ctx, htmlFile, []byte(strings.Join(html, "\n"))); err != nil { uc.log.Info("writing html file error: %s", err) return nil, fmt.Errorf("write debug HTML for %q: %w", file, err) } } uc.log.Info("LocalMD: grabbing the TOC ...") - toc, err := uc.grabber.Grab(ctx, displayPath, html) + toc, err := uc.grabber.Grab(ctx, displayPath, html...) if err != nil { uc.log.Info("LocalMD: failed to grab TOC: %s", err) return nil, fmt.Errorf("grab TOC from local Markdown %q: %w", file, err) diff --git a/internal/core/usecase/localmd/localmd_test.go b/internal/core/usecase/localmd/localmd_test.go index e618c59..f3fb7de 100644 --- a/internal/core/usecase/localmd/localmd_test.go +++ b/internal/core/usecase/localmd/localmd_test.go @@ -35,8 +35,8 @@ type converterStub struct { err error } -func (s converterStub) Convert(context.Context, string) (string, error) { - return s.html, s.err +func (s converterStub) Convert(context.Context, string) ([]string, error) { + return []string{s.html}, s.err } type grabberStub struct { @@ -45,7 +45,7 @@ type grabberStub struct { gotPath string } -func (s *grabberStub) Grab(_ context.Context, path, _ string) (*entity.Toc, error) { +func (s *grabberStub) Grab(_ context.Context, path string, _ ...string) (*entity.Toc, error) { s.gotPath = path return s.toc, s.err } diff --git a/internal/core/usecase/remotehtml/remotehtml.go b/internal/core/usecase/remotehtml/remotehtml.go index 4f88ad1..bc82175 100644 --- a/internal/core/usecase/remotehtml/remotehtml.go +++ b/internal/core/usecase/remotehtml/remotehtml.go @@ -16,7 +16,7 @@ type remoteGetter interface { } type tocGrabber interface { - Grab(context.Context, string, string) (*entity.Toc, error) + Grab(context.Context, string, ...string) (*entity.Toc, error) } type fileTemper interface { diff --git a/internal/core/usecase/remotehtml/remotehtml_test.go b/internal/core/usecase/remotehtml/remotehtml_test.go index 432d13b..d214ccd 100644 --- a/internal/core/usecase/remotehtml/remotehtml_test.go +++ b/internal/core/usecase/remotehtml/remotehtml_test.go @@ -43,7 +43,7 @@ type grabberStub struct { err error } -func (s grabberStub) Grab(context.Context, string, string) (*entity.Toc, error) { +func (s grabberStub) Grab(context.Context, string, ...string) (*entity.Toc, error) { return s.toc, s.err } diff --git a/internal/core/usecase/remotemd/remotemd_test.go b/internal/core/usecase/remotemd/remotemd_test.go index 3f91682..e01d98d 100644 --- a/internal/core/usecase/remotemd/remotemd_test.go +++ b/internal/core/usecase/remotemd/remotemd_test.go @@ -59,8 +59,8 @@ type converterStub struct { err error } -func (s converterStub) Convert(context.Context, string) (string, error) { - return "

Title

", s.err +func (s converterStub) Convert(context.Context, string) ([]string, error) { + return []string{"

Title

"}, s.err } type grabberStub struct { @@ -69,7 +69,7 @@ type grabberStub struct { gotPath string } -func (s *grabberStub) Grab(_ context.Context, path, _ string) (*entity.Toc, error) { +func (s *grabberStub) Grab(_ context.Context, path string, _ ...string) (*entity.Toc, error) { s.gotPath = path return s.toc, s.err }