diff --git a/README.md b/README.md index 175c788..03d4bdb 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,17 @@ fmt.Println(clone.Head(ctx, dst)) Pass `true` as the final argument for a full clone, including when an existing shallow checkout needs to be unshallowed. Clone and fetch errors are returned as `*clone.UnreachableError`, except when the context was canceled or reached its deadline. `errors.As` retrieves the URL and underlying Git error. `ValidateURL` accepts `https://` URLs, while `ValidateRef` rejects leading hyphens, `..`, and characters outside letters, digits, `.`, `_`, `/`, and `-`. +Use `EnsureWithOptions` to include submodules. Submodules are initialized and updated recursively with depth 1 after both a new clone and an existing checkout update. This step is best-effort because submodules may be large or refer to unavailable URLs; failures leave the parent checkout usable. Context cancellation is still returned. + +```go +err := clone.EnsureWithOptions(ctx, clone.Retry{}, url, dst, "main", + clone.EnsureOptions{RecurseSubmodules: true}, +) +if err != nil { + log.Fatal(err) +} +``` + ## Temporarily check out an exact tag `CheckoutTag` resolves only a local tag with the given exact name, peels an @@ -50,11 +61,12 @@ an invalid tag or another local repository error. `CheckoutTag` does not fetch. ## Persistent cache -`Cache` stores one checkout per URL under `Root`. `Prepare` holds a per-URL lock while updating the shallow checkout, then replaces `dst` with a copy and returns its commit. The destination must be outside `Root`. +`Cache` stores one checkout per URL under `Root`. `Prepare` holds a per-URL lock while updating the shallow checkout, then replaces `dst` with a copy and returns its commit. The destination must be outside `Root`. Set `RecurseSubmodules` to copy best-effort shallow submodule checkouts into each destination. ```go cache := clone.Cache{ - Root: "/var/cache/my-tool/repositories", + Root: "/var/cache/my-tool/repositories", + RecurseSubmodules: true, } commit, err := cache.Prepare(ctx, diff --git a/cache.go b/cache.go index 53fbcba..61cde78 100644 --- a/cache.go +++ b/cache.go @@ -14,9 +14,10 @@ import ( // Cache keeps one persistent checkout per repository URL. A Cache must not be // copied after its first use. type Cache struct { - Root string // Parent directory for per-URL checkouts. - Retry Retry // Retry policy for clone and fetch operations. - mu sync.Map + Root string // Parent directory for per-URL checkouts. + Retry Retry // Retry policy for clone and fetch operations. + RecurseSubmodules bool // Best-effort inclusion of nested shallow submodules. + mu sync.Map } // Dir returns the persistent directory for url under c.Root. @@ -44,7 +45,8 @@ func (c *Cache) Prepare(ctx context.Context, url, ref, dst string) (string, erro return "", err } cacheSrc := filepath.Join(cacheDir, "src") - if err := Ensure(ctx, c.Retry, url, cacheSrc, ref, false); err != nil { + options := EnsureOptions{RecurseSubmodules: c.RecurseSubmodules} + if err := EnsureWithOptions(ctx, c.Retry, url, cacheSrc, ref, options); err != nil { return "", err } commit := Head(ctx, cacheSrc) diff --git a/cache_test.go b/cache_test.go index d67f754..9d53256 100644 --- a/cache_test.go +++ b/cache_test.go @@ -76,6 +76,23 @@ func TestCachePrepareUpdatesAndReplacesDestination(t *testing.T) { } } +func TestCachePrepareIncludesSubmodules(t *testing.T) { + fixture := newSubmoduleOriginFixture(t) + cache := Cache{Root: t.TempDir(), RecurseSubmodules: true} + dst := filepath.Join(t.TempDir(), "workspace", "src") + + if _, err := cache.Prepare(context.Background(), fixture.origin.url, "", dst); err != nil { + t.Fatalf("Prepare: %v", err) + } + content, err := os.ReadFile(filepath.Join(dst, "vendor", "library", "vendor.c")) + if err != nil { + t.Fatal(err) + } + if string(content) != "first\n" { + t.Errorf("submodule content = %q, want first revision", content) + } +} + func TestCachePrepareRejectsDestinationOverlappingCache(t *testing.T) { rootParent := t.TempDir() cache := Cache{Root: filepath.Join(rootParent, "cache")} diff --git a/doc.go b/doc.go index 52a560e..a3918eb 100644 --- a/doc.go +++ b/doc.go @@ -1,7 +1,7 @@ // Package clone keeps local checkouts of HTTPS Git repositories. It provides -// shallow clone-or-fetch, exact temporary tag checkouts, bounded retries for -// network failures, a persistent cache, and capped reads and content -// classification for files from commits. Operations shell out to the git -// binary, which must be on PATH. The github.com/git-pkgs/clone/gogit module -// provides in-process blob reads. +// shallow clone-or-fetch, optional shallow submodules, exact temporary tag +// checkouts, bounded retries for network failures, a persistent cache, and +// capped reads and content classification for files from commits. Operations +// shell out to the git binary, which must be on PATH. The +// github.com/git-pkgs/clone/gogit module provides in-process blob reads. package clone diff --git a/ensure.go b/ensure.go index ba1772b..38c910f 100644 --- a/ensure.go +++ b/ensure.go @@ -25,12 +25,26 @@ func (e *UnreachableError) Unwrap() error { return e.Err } +// EnsureOptions configures an EnsureWithOptions operation. +type EnsureOptions struct { + Full bool // Clone full history and unshallow an existing checkout. + RecurseSubmodules bool // Initialize and update submodules recursively at depth 1. +} + // Ensure clones url into dst on its first call, then fetches and resets the // checkout on later calls. A shallow clone is used unless full is true. An // existing shallow clone is unshallowed when full changes to true. ref may // be a branch, tag, commit ID, or empty for the remote's default branch. func Ensure(ctx context.Context, retry Retry, url, dst, ref string, full bool) error { - err := ensure(ctx, retry, url, dst, ref, full) + return EnsureWithOptions(ctx, retry, url, dst, ref, EnsureOptions{Full: full}) +} + +// EnsureWithOptions clones or updates a checkout like Ensure. When +// RecurseSubmodules is enabled, it also makes a best-effort attempt to +// initialize and update nested submodules with depth 1. A submodule failure +// does not fail the checkout, but context cancellation still does. +func EnsureWithOptions(ctx context.Context, retry Retry, url, dst, ref string, options EnsureOptions) error { + err := ensure(ctx, retry, url, dst, ref, options) if err == nil { return nil } @@ -40,7 +54,7 @@ func Ensure(ctx context.Context, retry Retry, url, dst, ref string, full bool) e return &UnreachableError{URL: url, Err: err} } -func ensure(ctx context.Context, retry Retry, url, dst, ref string, full bool) error { +func ensure(ctx context.Context, retry Retry, url, dst, ref string, options EnsureOptions) error { if err := ValidateURL(url); err != nil { return err } @@ -48,7 +62,10 @@ func ensure(ctx context.Context, retry Retry, url, dst, ref string, full bool) e return err } if _, err := os.Stat(filepath.Join(dst, ".git")); err == nil { - return fetchRef(ctx, retry, url, dst, ref, full) + if err := fetchRef(ctx, retry, url, dst, ref, options.Full); err != nil { + return err + } + return updateSubmodules(ctx, retry, dst, options.RecurseSubmodules) } if err := os.MkdirAll(filepath.Dir(dst), dirPerm); err != nil { return err @@ -60,7 +77,7 @@ func ensure(ctx context.Context, retry Retry, url, dst, ref string, full bool) e // repositories unreachable for callers that authenticate via stored git // credentials rather than embedding a token in the URL. args := []string{"clone", "--quiet"} //nolint:goconst // Git argv is clearer with literal subcommands and flags. - if !full { + if !options.Full { args = append(args, "--depth", "1") } args = append(args, "--", url, dst) @@ -74,7 +91,26 @@ func ensure(ctx context.Context, retry Retry, url, dst, ref string, full bool) e return fmt.Errorf("%s: %w", strings.TrimSpace(out), err) } if ref != "" { - return fetchRef(ctx, retry, url, dst, ref, full) + if err := fetchRef(ctx, retry, url, dst, ref, options.Full); err != nil { + return err + } + } + return updateSubmodules(ctx, retry, dst, options.RecurseSubmodules) +} + +func updateSubmodules(ctx context.Context, retry Retry, dst string, enabled bool) error { + if !enabled { + return nil + } + if _, err := retry.Do(ctx, Command{ + Label: "submodule", + Dir: dst, + Env: remoteEnv(), + Args: []string{"submodule", "update", "--init", "--recursive", "--depth", "1"}, + }); err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } } return nil } diff --git a/ensure_test.go b/ensure_test.go index a88caeb..fa8a3d9 100644 --- a/ensure_test.go +++ b/ensure_test.go @@ -19,6 +19,11 @@ type originFixture struct { featureSHA string } +type submoduleOriginFixture struct { + origin originFixture + submoduleDir string +} + func newOriginFixture(t *testing.T) originFixture { t.Helper() requireGit(t) @@ -66,6 +71,42 @@ func newOriginFixture(t *testing.T) originFixture { } } +func newSubmoduleOriginFixture(t *testing.T) submoduleOriginFixture { + t.Helper() + requireGit(t) + + submoduleDir := t.TempDir() + runGitTest(t, submoduleDir, "init", "--quiet", "-b", "main") + if err := os.WriteFile(filepath.Join(submoduleDir, "vendor.c"), []byte("first\n"), 0o644); err != nil { + t.Fatal(err) + } + runGitTest(t, submoduleDir, "add", "vendor.c") + runGitTest(t, submoduleDir, "commit", "--quiet", "-m", "first") + + origin := newOriginFixture(t) + runGitTest(t, origin.dir, "submodule", "add", "--quiet", "file://"+submoduleDir, "vendor/library") + runGitTest(t, origin.dir, "commit", "--quiet", "-m", "add submodule") + origin.mainSHA = runGitTest(t, origin.dir, "rev-parse", "HEAD") + + return submoduleOriginFixture{origin: origin, submoduleDir: submoduleDir} +} + +func (f submoduleOriginFixture) update(t *testing.T, content string) { + t.Helper() + + if err := os.WriteFile(filepath.Join(f.submoduleDir, "vendor.c"), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + runGitTest(t, f.submoduleDir, "commit", "--quiet", "-am", "update submodule") + submoduleSHA := runGitTest(t, f.submoduleDir, "rev-parse", "HEAD") + + checkout := filepath.Join(f.origin.dir, "vendor", "library") + runGitTest(t, checkout, "fetch", "--quiet", "origin", "main") + runGitTest(t, checkout, "checkout", "--quiet", submoduleSHA) + runGitTest(t, f.origin.dir, "add", "vendor/library") + runGitTest(t, f.origin.dir, "commit", "--quiet", "-m", "update submodule pointer") +} + func TestEnsureClonesFetchesRefsAndUnshallows(t *testing.T) { origin := newOriginFixture(t) dst := filepath.Join(t.TempDir(), "nested", "checkout") @@ -121,6 +162,118 @@ func TestEnsureClonesFetchesRefsAndUnshallows(t *testing.T) { } } +func TestEnsureWithOptionsInitializesAndUpdatesShallowSubmodules(t *testing.T) { + fixture := newSubmoduleOriginFixture(t) + dst := filepath.Join(t.TempDir(), "checkout") + ctx := context.Background() + contentPath := filepath.Join(dst, "vendor", "library", "vendor.c") + + if err := Ensure(ctx, Retry{}, fixture.origin.url, dst, "", false); err != nil { + t.Fatalf("Ensure without submodules: %v", err) + } + if _, err := os.Stat(contentPath); !os.IsNotExist(err) { + t.Fatalf("submodule content exists without opt-in: %v", err) + } + + options := EnsureOptions{RecurseSubmodules: true} + if err := EnsureWithOptions(ctx, Retry{}, fixture.origin.url, dst, "", options); err != nil { + t.Fatalf("EnsureWithOptions: %v", err) + } + content, err := os.ReadFile(contentPath) + if err != nil { + t.Fatal(err) + } + if string(content) != "first\n" { + t.Errorf("submodule content = %q, want first revision", content) + } + submoduleCheckout := filepath.Join(dst, "vendor", "library") + if got := runGitTest(t, submoduleCheckout, "rev-parse", "--is-shallow-repository"); got != "true" { + t.Errorf("submodule shallow = %q, want true", got) + } + + fixture.update(t, "updated\n") + if err := EnsureWithOptions(ctx, Retry{}, fixture.origin.url, dst, "", options); err != nil { + t.Fatalf("update checkout and submodule: %v", err) + } + content, err = os.ReadFile(contentPath) + if err != nil { + t.Fatal(err) + } + if string(content) != "updated\n" { + t.Errorf("updated submodule content = %q, want updated revision", content) + } +} + +func TestEnsureWithOptionsIgnoresSubmoduleFailure(t *testing.T) { + dst := filepath.Join(t.TempDir(), "checkout") + if err := os.MkdirAll(filepath.Join(dst, ".git"), 0o755); err != nil { + t.Fatal(err) + } + + var submoduleArgs []string + var submoduleEnv []string + retry := Retry{ + Attempts: 1, + Run: func(_ context.Context, dir string, env []string, args ...string) (string, error) { + switch subcommand(args) { + case "fetch", "reset": + return "", nil + case "submodule": + if dir != dst { + t.Errorf("submodule dir = %q, want %q", dir, dst) + } + submoduleArgs = append([]string(nil), args...) + submoduleEnv = append([]string(nil), env...) + return "fatal: repository not found", errGitExit + default: + return "", errors.New("unexpected Git command") + } + }, + } + + options := EnsureOptions{RecurseSubmodules: true} + if err := EnsureWithOptions( + context.Background(), retry, "https://example.invalid/repo", dst, "", options, + ); err != nil { + t.Fatalf("EnsureWithOptions: %v", err) + } + wantArgs := []string{"submodule", "update", "--init", "--recursive", "--depth", "1"} + if !slices.Equal(submoduleArgs, wantArgs) { + t.Errorf("submodule args = %v, want %v", submoduleArgs, wantArgs) + } + if !slices.Contains(submoduleEnv, "GIT_PROTOCOL_FROM_USER=0") { + t.Errorf("submodule env = %v", submoduleEnv) + } +} + +func TestEnsureWithOptionsReturnsSubmoduleCancellation(t *testing.T) { + dst := filepath.Join(t.TempDir(), "checkout") + if err := os.MkdirAll(filepath.Join(dst, ".git"), 0o755); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + retry := Retry{ + Attempts: 1, + Run: func(_ context.Context, _ string, _ []string, args ...string) (string, error) { + if subcommand(args) == "submodule" { + cancel() + return "submodule canceled", errGitExit + } + return "", nil + }, + } + options := EnsureOptions{RecurseSubmodules: true} + err := EnsureWithOptions(ctx, retry, "https://example.invalid/repo", dst, "", options) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context.Canceled", err) + } + var unreachable *UnreachableError + if errors.As(err, &unreachable) { + t.Fatalf("cancellation wrapped as UnreachableError: %v", err) + } +} + func TestEnsureRejectsInputBeforeRunningGit(t *testing.T) { for _, test := range []struct { url string