Add optional shallow submodule checkouts

README.md

@@ -26,6 +26,17 @@ fmt.Println(clone.Head(ctx, dst))
 
 Pass `true` as the final argument for a full clone, including when an existing shallow checkout needs to be unshallowed. Clone and fetch errors are returned as `*clone.UnreachableError`, except when the context was canceled or reached its deadline. `errors.As` retrieves the URL and underlying Git error. `ValidateURL` accepts `https://` URLs, while `ValidateRef` rejects leading hyphens, `..`, and characters outside letters, digits, `.`, `_`, `/`, and `-`.
 
+Use `EnsureWithOptions` to include submodules. Submodules are initialized and updated recursively with depth 1 after both a new clone and an existing checkout update. This step is best-effort because submodules may be large or refer to unavailable URLs; failures leave the parent checkout usable. Context cancellation is still returned.
+
+```go
+err := clone.EnsureWithOptions(ctx, clone.Retry{}, url, dst, "main",
+    clone.EnsureOptions{RecurseSubmodules: true},
+)
+if err != nil {
+    log.Fatal(err)
+}
+```
+
 ## Temporarily check out an exact tag
 
 `CheckoutTag` resolves only a local tag with the given exact name, peels an
@@ -50,11 +61,12 @@ an invalid tag or another local repository error. `CheckoutTag` does not fetch.
 
 ## Persistent cache
 
-`Cache` stores one checkout per URL under `Root`. `Prepare` holds a per-URL lock while updating the shallow checkout, then replaces `dst` with a copy and returns its commit. The destination must be outside `Root`.
+`Cache` stores one checkout per URL under `Root`. `Prepare` holds a per-URL lock while updating the shallow checkout, then replaces `dst` with a copy and returns its commit. The destination must be outside `Root`. Set `RecurseSubmodules` to copy best-effort shallow submodule checkouts into each destination.
 
 ```go
 cache := clone.Cache{
-    Root: "/var/cache/my-tool/repositories",
+    Root:              "/var/cache/my-tool/repositories",
+    RecurseSubmodules: true,
 }
 
 commit, err := cache.Prepare(ctx,

cache.go

@@ -14,9 +14,10 @@ import (
 // Cache keeps one persistent checkout per repository URL. A Cache must not be
 // copied after its first use.
 type Cache struct {
-	Root  string // Parent directory for per-URL checkouts.
-	Retry Retry  // Retry policy for clone and fetch operations.
-	mu    sync.Map
+	Root              string // Parent directory for per-URL checkouts.
+	Retry             Retry  // Retry policy for clone and fetch operations.
+	RecurseSubmodules bool   // Best-effort inclusion of nested shallow submodules.
+	mu                sync.Map
 }
 
 // Dir returns the persistent directory for url under c.Root.
@@ -44,7 +45,8 @@ func (c *Cache) Prepare(ctx context.Context, url, ref, dst string) (string, erro
 		return "", err
 	}
 	cacheSrc := filepath.Join(cacheDir, "src")
-	if err := Ensure(ctx, c.Retry, url, cacheSrc, ref, false); err != nil {
+	options := EnsureOptions{RecurseSubmodules: c.RecurseSubmodules}
+	if err := EnsureWithOptions(ctx, c.Retry, url, cacheSrc, ref, options); err != nil {
 		return "", err
 	}
 	commit := Head(ctx, cacheSrc)

cache_test.go

@@ -76,6 +76,23 @@ func TestCachePrepareUpdatesAndReplacesDestination(t *testing.T) {
 	}
 }
 
+func TestCachePrepareIncludesSubmodules(t *testing.T) {
+	fixture := newSubmoduleOriginFixture(t)
+	cache := Cache{Root: t.TempDir(), RecurseSubmodules: true}
+	dst := filepath.Join(t.TempDir(), "workspace", "src")
+
+	if _, err := cache.Prepare(context.Background(), fixture.origin.url, "", dst); err != nil {
+		t.Fatalf("Prepare: %v", err)
+	}
+	content, err := os.ReadFile(filepath.Join(dst, "vendor", "library", "vendor.c"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(content) != "first\n" {
+		t.Errorf("submodule content = %q, want first revision", content)
+	}
+}
+
 func TestCachePrepareRejectsDestinationOverlappingCache(t *testing.T) {
 	rootParent := t.TempDir()
 	cache := Cache{Root: filepath.Join(rootParent, "cache")}

doc.go

@@ -1,7 +1,7 @@
 // Package clone keeps local checkouts of HTTPS Git repositories. It provides
-// shallow clone-or-fetch, exact temporary tag checkouts, bounded retries for
-// network failures, a persistent cache, and capped reads and content
-// classification for files from commits. Operations shell out to the git
-// binary, which must be on PATH. The github.com/git-pkgs/clone/gogit module
-// provides in-process blob reads.
+// shallow clone-or-fetch, optional shallow submodules, exact temporary tag
+// checkouts, bounded retries for network failures, a persistent cache, and
+// capped reads and content classification for files from commits. Operations
+// shell out to the git binary, which must be on PATH. The
+// github.com/git-pkgs/clone/gogit module provides in-process blob reads.
 package clone

ensure.go

@@ -25,12 +25,26 @@ func (e *UnreachableError) Unwrap() error {
 	return e.Err
 }
 
+// EnsureOptions configures an EnsureWithOptions operation.
+type EnsureOptions struct {
+	Full              bool // Clone full history and unshallow an existing checkout.
+	RecurseSubmodules bool // Initialize and update submodules recursively at depth 1.
+}
+
 // Ensure clones url into dst on its first call, then fetches and resets the
 // checkout on later calls. A shallow clone is used unless full is true. An
 // existing shallow clone is unshallowed when full changes to true. ref may
 // be a branch, tag, commit ID, or empty for the remote's default branch.
 func Ensure(ctx context.Context, retry Retry, url, dst, ref string, full bool) error {
-	err := ensure(ctx, retry, url, dst, ref, full)
+	return EnsureWithOptions(ctx, retry, url, dst, ref, EnsureOptions{Full: full})
+}
+
+// EnsureWithOptions clones or updates a checkout like Ensure. When
+// RecurseSubmodules is enabled, it also makes a best-effort attempt to
+// initialize and update nested submodules with depth 1. A submodule failure
+// does not fail the checkout, but context cancellation still does.
+func EnsureWithOptions(ctx context.Context, retry Retry, url, dst, ref string, options EnsureOptions) error {
+	err := ensure(ctx, retry, url, dst, ref, options)
 	if err == nil {
 		return nil
 	}
@@ -40,15 +54,18 @@ func Ensure(ctx context.Context, retry Retry, url, dst, ref string, full bool) e
 	return &UnreachableError{URL: url, Err: err}
 }
 
-func ensure(ctx context.Context, retry Retry, url, dst, ref string, full bool) error {
+func ensure(ctx context.Context, retry Retry, url, dst, ref string, options EnsureOptions) error {
 	if err := ValidateURL(url); err != nil {
 		return err
 	}
 	if err := ValidateRef(ref); err != nil {
 		return err
 	}
 	if _, err := os.Stat(filepath.Join(dst, ".git")); err == nil {
-		return fetchRef(ctx, retry, url, dst, ref, full)
+		if err := fetchRef(ctx, retry, url, dst, ref, options.Full); err != nil {
+			return err
+		}
+		return updateSubmodules(ctx, retry, dst, options.RecurseSubmodules)
 	}
 	if err := os.MkdirAll(filepath.Dir(dst), dirPerm); err != nil {
 		return err
@@ -60,7 +77,7 @@ func ensure(ctx context.Context, retry Retry, url, dst, ref string, full bool) e
 	// repositories unreachable for callers that authenticate via stored git
 	// credentials rather than embedding a token in the URL.
 	args := []string{"clone", "--quiet"} //nolint:goconst // Git argv is clearer with literal subcommands and flags.
-	if !full {
+	if !options.Full {
 		args = append(args, "--depth", "1")
 	}
 	args = append(args, "--", url, dst)
@@ -74,7 +91,26 @@ func ensure(ctx context.Context, retry Retry, url, dst, ref string, full bool) e
 		return fmt.Errorf("%s: %w", strings.TrimSpace(out), err)
 	}
 	if ref != "" {
-		return fetchRef(ctx, retry, url, dst, ref, full)
+		if err := fetchRef(ctx, retry, url, dst, ref, options.Full); err != nil {
+			return err
+		}
+	}
+	return updateSubmodules(ctx, retry, dst, options.RecurseSubmodules)
+}
+
+func updateSubmodules(ctx context.Context, retry Retry, dst string, enabled bool) error {
+	if !enabled {
+		return nil
+	}
+	if _, err := retry.Do(ctx, Command{
+		Label: "submodule",
+		Dir:   dst,
+		Env:   remoteEnv(),
+		Args:  []string{"submodule", "update", "--init", "--recursive", "--depth", "1"},
+	}); err != nil {
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return ctxErr
+		}
 	}
 	return nil
 }

ensure_test.go

@@ -19,6 +19,11 @@ type originFixture struct {
 	featureSHA string
 }
 
+type submoduleOriginFixture struct {
+	origin       originFixture
+	submoduleDir string
+}
+
 func newOriginFixture(t *testing.T) originFixture {
 	t.Helper()
 	requireGit(t)
@@ -66,6 +71,42 @@ func newOriginFixture(t *testing.T) originFixture {
 	}
 }
 
+func newSubmoduleOriginFixture(t *testing.T) submoduleOriginFixture {
+	t.Helper()
+	requireGit(t)
+
+	submoduleDir := t.TempDir()
+	runGitTest(t, submoduleDir, "init", "--quiet", "-b", "main")
+	if err := os.WriteFile(filepath.Join(submoduleDir, "vendor.c"), []byte("first\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	runGitTest(t, submoduleDir, "add", "vendor.c")
+	runGitTest(t, submoduleDir, "commit", "--quiet", "-m", "first")
+
+	origin := newOriginFixture(t)
+	runGitTest(t, origin.dir, "submodule", "add", "--quiet", "file://"+submoduleDir, "vendor/library")
+	runGitTest(t, origin.dir, "commit", "--quiet", "-m", "add submodule")
+	origin.mainSHA = runGitTest(t, origin.dir, "rev-parse", "HEAD")
+
+	return submoduleOriginFixture{origin: origin, submoduleDir: submoduleDir}
+}
+
+func (f submoduleOriginFixture) update(t *testing.T, content string) {
+	t.Helper()
+
+	if err := os.WriteFile(filepath.Join(f.submoduleDir, "vendor.c"), []byte(content), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	runGitTest(t, f.submoduleDir, "commit", "--quiet", "-am", "update submodule")
+	submoduleSHA := runGitTest(t, f.submoduleDir, "rev-parse", "HEAD")
+
+	checkout := filepath.Join(f.origin.dir, "vendor", "library")
+	runGitTest(t, checkout, "fetch", "--quiet", "origin", "main")
+	runGitTest(t, checkout, "checkout", "--quiet", submoduleSHA)
+	runGitTest(t, f.origin.dir, "add", "vendor/library")
+	runGitTest(t, f.origin.dir, "commit", "--quiet", "-m", "update submodule pointer")
+}
+
 func TestEnsureClonesFetchesRefsAndUnshallows(t *testing.T) {
 	origin := newOriginFixture(t)
 	dst := filepath.Join(t.TempDir(), "nested", "checkout")
@@ -121,6 +162,118 @@ func TestEnsureClonesFetchesRefsAndUnshallows(t *testing.T) {
 	}
 }
 
+func TestEnsureWithOptionsInitializesAndUpdatesShallowSubmodules(t *testing.T) {
+	fixture := newSubmoduleOriginFixture(t)
+	dst := filepath.Join(t.TempDir(), "checkout")
+	ctx := context.Background()
+	contentPath := filepath.Join(dst, "vendor", "library", "vendor.c")
+
+	if err := Ensure(ctx, Retry{}, fixture.origin.url, dst, "", false); err != nil {
+		t.Fatalf("Ensure without submodules: %v", err)
+	}
+	if _, err := os.Stat(contentPath); !os.IsNotExist(err) {
+		t.Fatalf("submodule content exists without opt-in: %v", err)
+	}
+
+	options := EnsureOptions{RecurseSubmodules: true}
+	if err := EnsureWithOptions(ctx, Retry{}, fixture.origin.url, dst, "", options); err != nil {
+		t.Fatalf("EnsureWithOptions: %v", err)
+	}
+	content, err := os.ReadFile(contentPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(content) != "first\n" {
+		t.Errorf("submodule content = %q, want first revision", content)
+	}
+	submoduleCheckout := filepath.Join(dst, "vendor", "library")
+	if got := runGitTest(t, submoduleCheckout, "rev-parse", "--is-shallow-repository"); got != "true" {
+		t.Errorf("submodule shallow = %q, want true", got)
+	}
+
+	fixture.update(t, "updated\n")
+	if err := EnsureWithOptions(ctx, Retry{}, fixture.origin.url, dst, "", options); err != nil {
+		t.Fatalf("update checkout and submodule: %v", err)
+	}
+	content, err = os.ReadFile(contentPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(content) != "updated\n" {
+		t.Errorf("updated submodule content = %q, want updated revision", content)
+	}
+}
+
+func TestEnsureWithOptionsIgnoresSubmoduleFailure(t *testing.T) {
+	dst := filepath.Join(t.TempDir(), "checkout")
+	if err := os.MkdirAll(filepath.Join(dst, ".git"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+
+	var submoduleArgs []string
+	var submoduleEnv []string
+	retry := Retry{
+		Attempts: 1,
+		Run: func(_ context.Context, dir string, env []string, args ...string) (string, error) {
+			switch subcommand(args) {
+			case "fetch", "reset":
+				return "", nil
+			case "submodule":
+				if dir != dst {
+					t.Errorf("submodule dir = %q, want %q", dir, dst)
+				}
+				submoduleArgs = append([]string(nil), args...)
+				submoduleEnv = append([]string(nil), env...)
+				return "fatal: repository not found", errGitExit
+			default:
+				return "", errors.New("unexpected Git command")
+			}
+		},
+	}
+
+	options := EnsureOptions{RecurseSubmodules: true}
+	if err := EnsureWithOptions(
+		context.Background(), retry, "https://example.invalid/repo", dst, "", options,
+	); err != nil {
+		t.Fatalf("EnsureWithOptions: %v", err)
+	}
+	wantArgs := []string{"submodule", "update", "--init", "--recursive", "--depth", "1"}
+	if !slices.Equal(submoduleArgs, wantArgs) {
+		t.Errorf("submodule args = %v, want %v", submoduleArgs, wantArgs)
+	}
+	if !slices.Contains(submoduleEnv, "GIT_PROTOCOL_FROM_USER=0") {
+		t.Errorf("submodule env = %v", submoduleEnv)
+	}
+}
+
+func TestEnsureWithOptionsReturnsSubmoduleCancellation(t *testing.T) {
+	dst := filepath.Join(t.TempDir(), "checkout")
+	if err := os.MkdirAll(filepath.Join(dst, ".git"), 0o755); err != nil {
+		t.Fatal(err)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	retry := Retry{
+		Attempts: 1,
+		Run: func(_ context.Context, _ string, _ []string, args ...string) (string, error) {
+			if subcommand(args) == "submodule" {
+				cancel()
+				return "submodule canceled", errGitExit
+			}
+			return "", nil
+		},
+	}
+	options := EnsureOptions{RecurseSubmodules: true}
+	err := EnsureWithOptions(ctx, retry, "https://example.invalid/repo", dst, "", options)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("error = %v, want context.Canceled", err)
+	}
+	var unreachable *UnreachableError
+	if errors.As(err, &unreachable) {
+		t.Fatalf("cancellation wrapped as UnreachableError: %v", err)
+	}
+}
+
 func TestEnsureRejectsInputBeforeRunningGit(t *testing.T) {
 	for _, test := range []struct {
 		url string