fix(adapters): build a TOC for documents larger than 400 KB

ARCHITECTURE.md

@@ -60,6 +60,9 @@ internal/core/entity
 ├── Toc
 ├── Type
 └── MarkerStart / MarkerEnd
+
+internal/core/mdsplit
+└── Split (cuts a document into chunks the GitHub Markdown API accepts)
 ```
 
 ## Dependency direction
@@ -316,15 +319,25 @@ Controller
   -> LocalMd.Do
   -> FileChecker.Exists
   -> HTMLConverter.Convert
-  -> RemotePoster.Post
+  -> RemotePoster.Post          (documents up to 384 KB)
+  -> mdsplit.Split              (larger documents)
+  -> RemotePoster.PostBody      (once per chunk)
   -> GitHub /markdown/raw API
-  -> RegexpExtractor.Extract
+  -> RegexpExtractor.Extract    (once per chunk)
   -> Renderer.Render
   -> entity.Toc
 ```
 
 When debug mode is enabled, `LocalMd` writes the returned HTML to `<input>.debug.html` through `FileWriter`.
 
+`HTMLConverter.Convert` returns one HTML string per request. A document larger than
+384 KB is cut by `mdsplit` at blank lines outside fenced code blocks, HTML comments
+and raw HTML blocks, so every chunk parses the way those lines parse inside the whole
+document; a chunk forced to end inside such a block closes it and reopens it in the
+next chunk. `Generator.Grab` then extracts headings from every chunk and renumbers
+duplicate anchors against a document-wide counter, because GitHub numbered duplicates
+inside each chunk in isolation.
+
 ### Skip header (`--skip-header`)
 
 ```text

CHANGELOG.md

@@ -20,6 +20,9 @@ Planned release: 2.1.0.
 - `token.txt`, read from next to the executable, is now the last fallback for a GitHub
   token, after `--token` and `GH_TOC_TOKEN`.
 - A Docker image is published to `ghcr.io/ekalinin/github-markdown-toc.go`.
+- Documents larger than GitHub's 400 KB Markdown API limit are converted in chunks
+  instead of failing, with duplicate anchors numbered across the whole document.
+  ([#25](https://github.com/ekalinin/github-markdown-toc.go/issues/25))
 
 ### Security
 

README.md

@@ -45,6 +45,7 @@ Table of Contents
     * [Starting Depth](#starting-depth)
     * [Depth](#depth)
     * [No Escape](#no-escape)
+    * [Large documents](#large-documents)
     * [Github token](#github-token)
     * [GitHub Enterprise Server](#github-enterprise-server)
     * [Bash/ZSH auto\-complete](#bashzsh-auto-complete)
@@ -431,6 +432,21 @@ No escape
 * [Dockerfile.vim](#dockerfilevim)
 ```
 
+Large documents
+---------------
+
+GitHub's `/markdown/raw` API renders at most 400 KB per request. A larger document is
+split automatically at blank lines outside code blocks, converted in several requests
+and reassembled into one TOC, with duplicate anchors numbered as they would be for
+the whole document.
+
+The only visible difference is the number of API calls: a 2 MB document costs six
+requests instead of one, which reaches the rate limit six times faster. Pass a token
+(see below) when you work with documents of that size.
+
+A document is only rejected when a single line is larger than the limit, since there
+is no way to split it without changing what it renders to.
+
 GitHub token
 ------------
 

e2e-tests/want.md

@@ -22,6 +22,7 @@ Table of Contents
   * [Starting Depth](#starting-depth)
   * [Depth](#depth)
   * [No escape](#no-escape)
+  * [Large documents](#large-documents)
   * [GitHub token](#github-token)
   * [GitHub Enterprise Server](#github-enterprise-server)
   * [Bash/ZSH auto\-complete](#bashzsh-auto-complete)

e2e-tests/want3.md

@@ -18,6 +18,7 @@
     * [Starting Depth](#starting-depth)
     * [Depth](#depth)
     * [No escape](#no-escape)
+    * [Large documents](#large-documents)
     * [GitHub token](#github-token)
     * [GitHub Enterprise Server](#github-enterprise-server)
     * [Bash/ZSH auto\-complete](#bashzsh-auto-complete)

internal/adapters/htmlconverter.go

@@ -5,11 +5,15 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"os"
 	"strings"
+
+	"github.com/ekalinin/github-markdown-toc.go/v2/internal/core/mdsplit"
 )
 
 type remotePoster interface {
 	Post(context.Context, string, string, string) (string, error)
+	PostBody(context.Context, string, string, []byte) (string, error)
 }
 
 type HTMLConverter struct {
@@ -36,16 +40,64 @@ func NewHTMLConverterX(token, url string, poster remotePoster, log logger) *HTML
 	}
 }
 
-func (c *HTMLConverter) Convert(ctx context.Context, file string) (string, error) {
+// maxChunkBytes is the largest payload the GitHub Markdown API renders. GitHub
+// documents the limit as 400 KB; staying at 384 KB keeps us below it whether that
+// means 400*1024 or 400000 bytes.
+const maxChunkBytes = 384 << 10
+
+// Convert renders file as HTML. The result holds one string per request: a single
+// one for a document GitHub accepts whole, and one per chunk for a larger document.
+func (c *HTMLConverter) Convert(ctx context.Context, file string) ([]string, error) {
 	c.log.Info("adapters.HTMLConverter.Convert: start", "file", file)
 	ghURL := c.ghURL + "/markdown/raw"
 	c.log.Info("adapters.HTMLConverter.Convert: sending", "url", ghURL)
 
-	html, err := c.poster.Post(ctx, ghURL, c.ghToken, file)
+	if !exceedsChunkLimit(file) {
+		html, err := c.poster.Post(ctx, ghURL, c.ghToken, file)
+		if err != nil {
+			return nil, withRateLimitHint(err)
+		}
+		return []string{html}, nil
+	}
+	return c.convertInChunks(ctx, ghURL, file)
+}
+
+// exceedsChunkLimit reports whether file is too large for one request. Only a
+// successful stat sends us down the chunked path: when it fails, the single request
+// reports the file error the way it always has.
+func exceedsChunkLimit(file string) bool {
+	info, err := os.Stat(file)
+	return err == nil && info.Size() > maxChunkBytes
+}
+
+// convertInChunks converts a document GitHub would refuse as a whole. Requests are
+// sent one after another: the order of the results is the order of the document, and
+// a burst of parallel requests would only bring the rate limit closer.
+func (c *HTMLConverter) convertInChunks(ctx context.Context, ghURL, file string) ([]string, error) {
+	content, err := os.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	chunks, err := mdsplit.Split(content, maxChunkBytes)
 	if err != nil {
-		return "", withRateLimitHint(err)
+		return nil, err
+	}
+	c.log.Info("adapters.HTMLConverter.Convert: splitting large document",
+		"file", file, "chunks", len(chunks))
+
+	result := make([]string, 0, len(chunks))
+	for i, chunk := range chunks {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+		html, err := c.poster.PostBody(ctx, ghURL, c.ghToken, chunk)
+		if err != nil {
+			return nil, fmt.Errorf("convert chunk %d/%d: %w",
+				i+1, len(chunks), withRateLimitHint(err))
+		}
+		result = append(result, html)
 	}
-	return html, nil
+	return result, nil
 }
 
 // rateLimitMarker is what GitHub puts in the body when the API rate limit is hit.

internal/adapters/htmlconverter_test.go

@@ -3,15 +3,22 @@ package adapters
 import (
 	"context"
 	"errors"
+	"fmt"
+	"io"
 	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
 	"strings"
+	"sync"
 	"testing"
 )
 
 type fakePoster struct {
 	gotURL   string
 	gotToken string
 	gotPath  string
+	gotBody  []byte
 	retBody  string
 	retErr   error
 }
@@ -23,6 +30,13 @@ func (p *fakePoster) Post(_ context.Context, url, token, path string) (string, e
 	return p.retBody, p.retErr
 }
 
+func (p *fakePoster) PostBody(_ context.Context, url, token string, body []byte) (string, error) {
+	p.gotURL = url
+	p.gotToken = token
+	p.gotBody = body
+	return p.retBody, p.retErr
+}
+
 func Test_HTMLConverter(t *testing.T) {
 
 	token, url, path := "xx-token", "gh-url", "html-file"
@@ -52,8 +66,8 @@ func Test_HTMLConverter(t *testing.T) {
 			}
 
 			if !tt.failed {
-				if got != want {
-					t.Errorf("Got=%v, want=%v", got, want)
+				if len(got) != 1 || got[0] != want {
+					t.Errorf("Got=%v, want=[%v]", got, want)
 				}
 				if got := tt.poster.gotPath; got != path {
 					t.Errorf("Got=%v, want=%v", got, path)
@@ -83,6 +97,10 @@ func (s posterStub) Post(context.Context, string, string, string) (string, error
 	return "", s.err
 }
 
+func (s posterStub) PostBody(context.Context, string, string, []byte) (string, error) {
+	return "", s.err
+}
+
 func TestHTMLConverterRateLimitHint(t *testing.T) {
 	tests := []struct {
 		name     string
@@ -130,3 +148,59 @@ func TestHTMLConverterRateLimitHint(t *testing.T) {
 		})
 	}
 }
+
+func TestHTMLConverterSplitsLargeDocuments(t *testing.T) {
+	var mu sync.Mutex
+	var bodies [][]byte
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		mu.Lock()
+		bodies = append(bodies, body)
+		n := len(bodies)
+		mu.Unlock()
+		if _, err := fmt.Fprintf(w, "<h1>chunk %d</h1>", n); err != nil {
+			t.Error(err)
+		}
+	}))
+	defer srv.Close()
+
+	var doc strings.Builder
+	for doc.Len() <= 3*maxChunkBytes {
+		doc.WriteString("# Heading\n\n")
+		doc.WriteString(strings.Repeat("word ", 200))
+		doc.WriteString("\n\n")
+	}
+	file := filepath.Join(t.TempDir(), "big.md")
+	if err := os.WriteFile(file, []byte(doc.String()), 0644); err != nil {
+		t.Fatal(err)
+	}
+
+	converter := NewHTMLConverterWithClient("", srv.URL, srv.Client(), NewLogger(false))
+	got, err := converter.Convert(context.Background(), file)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(got) < 4 {
+		t.Fatalf("got %d chunks, want at least 4", len(got))
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	if len(bodies) != len(got) {
+		t.Fatalf("got %d requests for %d chunks, want one each", len(bodies), len(got))
+	}
+	for i, body := range bodies {
+		if len(body) > maxChunkBytes {
+			t.Errorf("request %d carried %d bytes, want at most %d", i, len(body), maxChunkBytes)
+		}
+	}
+	for i, html := range got {
+		if want := fmt.Sprintf("<h1>chunk %d</h1>", i+1); html != want {
+			t.Errorf("chunk %d is %q, want %q", i, html, want)
+		}
+	}
+}

internal/adapters/http.go

@@ -1,6 +1,7 @@
 package adapters
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -119,15 +120,24 @@ func HttpPost(ctx context.Context, client *http.Client, url, path, token string)
 		_ = file.Close()
 	}()
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, file)
+	fileInfo, err := file.Stat()
 	if err != nil {
 		return "", err
 	}
-	fileInfo, err := file.Stat()
+	return httpPost(ctx, client, url, file, fileInfo.Size(), token)
+}
+
+// HttpPostBody sends an in-memory body in an HTTP POST request.
+func HttpPostBody(ctx context.Context, client *http.Client, url string, body []byte, token string) (string, error) {
+	return httpPost(ctx, client, url, bytes.NewReader(body), int64(len(body)), token)
+}
+
+func httpPost(ctx context.Context, client *http.Client, url string, body io.Reader, size int64, token string) (string, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, body)
 	if err != nil {
 		return "", err
 	}
-	req.ContentLength = fileInfo.Size()
+	req.ContentLength = size
 
 	if token != "" {
 		req.Header.Add("Authorization", "token "+token)

internal/adapters/http_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -267,3 +268,36 @@ func Test_doHTTPReq_issue35(t *testing.T) {
 		t.Error("response header should be \"Hello, client\", but got:", resHeader)
 	}
 }
+
+func TestHttpPostBody(t *testing.T) {
+	token := "test-token"
+	var gotBody []byte
+	var gotToken string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		gotBody = body
+		gotToken = r.Header.Get("Authorization")
+		if _, err := w.Write([]byte("<h1>ok</h1>")); err != nil {
+			t.Error(err)
+		}
+	}))
+	defer srv.Close()
+
+	got, err := HttpPostBody(context.Background(), testHTTPClient(), srv.URL, []byte("# Title\n"), token)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got != "<h1>ok</h1>" {
+		t.Errorf("got response %q, want the rendered HTML", got)
+	}
+	if string(gotBody) != "# Title\n" {
+		t.Errorf("got body %q, want the posted content", gotBody)
+	}
+	if want := "token " + token; gotToken != want {
+		t.Errorf("got authorization %q, want %q", gotToken, want)
+	}
+}

internal/adapters/remoteposter.go

@@ -23,3 +23,10 @@ func (r *RemotePoster) Post(ctx context.Context, url, token, path string) (strin
 	}
 	return HttpPost(ctx, r.client, url, path, token)
 }
+
+func (r *RemotePoster) PostBody(ctx context.Context, url, token string, body []byte) (string, error) {
+	if err := ctx.Err(); err != nil {
+		return "", err
+	}
+	return HttpPostBody(ctx, r.client, url, body, token)
+}