diff --git a/go/cmd/gitter/git.go b/go/cmd/gitter/git.go new file mode 100644 index 00000000000..0dfebf9dbb4 --- /dev/null +++ b/go/cmd/gitter/git.go @@ -0,0 +1,313 @@ +package main + +import ( + "context" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/google/osv.dev/go/logger" +) + +type FetchOptions struct { + ForceUpdate bool + SkipReqConcurrencySemaphore bool +} + +// prepareCmd prepares the command with context cancellation handled by sending SIGINT. +func prepareCmd(ctx context.Context, dir string, env []string, name string, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, name, args...) + if dir != "" { + cmd.Dir = dir + } + if len(env) > 0 { + cmd.Env = append(os.Environ(), env...) + } + // Use SIGINT instead of SIGKILL for graceful shutdown of subprocesses + cmd.Cancel = func() error { + logger.DebugContext(ctx, "SIGINT sent to command", slog.String("cmd", name), slog.Any("args", args)) + return cmd.Process.Signal(syscall.SIGINT) + } + // Ensure it eventually dies if it ignores SIGINT + cmd.WaitDelay = shutdownTimeout / 2 + + return cmd +} + +// runCmd executes a command with context cancellation handled by sending SIGINT. +// It logs cancellation errors separately as requested. +func runCmd(ctx context.Context, dir string, env []string, name string, args ...string) error { + cmd := prepareCmd(ctx, dir, env, name, args...) + out, err := cmd.CombinedOutput() + + if err != nil { + if ctx.Err() != nil { + // Log separately if cancelled + logger.WarnContext(ctx, "Command cancelled", slog.String("cmd", name), slog.Any("err", ctx.Err())) + return fmt.Errorf("command %s cancelled: %w", name, ctx.Err()) + } + + return fmt.Errorf("command %s failed: %w, output: %s", name, err, out) + } + + logger.DebugContext(ctx, "Git command executed", + slog.String("cmd", name), + slog.Any("args", args), + slog.String("output", string(out)), + ) + + return nil +} + +// cloneRepo clones a git repository into repoPath. +func cloneRepo(ctx context.Context, repoURL string, repoPath string) error { + return runCmd(ctx, "", []string{"GIT_TERMINAL_PROMPT=0"}, "git", "clone", "--", repoURL, repoPath) +} + +func isIndexLockError(err error) bool { + if err == nil { + return false + } + errString := err.Error() + + return strings.Contains(errString, "index.lock") && strings.Contains(errString, "File exists") +} + +func isRefConflictError(err error) bool { + if err == nil { + return false + } + errString := err.Error() + + return strings.Contains(errString, "refname conflict") || + (strings.Contains(errString, "some local refs could not be updated") && strings.Contains(errString, "try running 'git remote prune origin'")) +} + +// Attempt to recover from git fetch + reset errors +// Returns true if recovery was attempted and we should retry fetch + reset +func attemptGitRecovery(ctx context.Context, repoPath string, err error) bool { + if err == nil { + return false + } + + // Refname conflict, likely name conflict between local and remote refs + // We can try removing stale remote-tracking branches and retry + if isRefConflictError(err) { + logger.WarnContext(ctx, "Ref conflict detected, running git remote prune origin") + if err := runCmd(ctx, repoPath, nil, "git", "remote", "prune", "origin"); err != nil { + logger.ErrorContext(ctx, "Failed to prune origin", slog.Any("err", err)) + return false + } + + return true + } + + // index.lock exists, likely a previous git reset got terminated and wasn't cleaned up properly. + // We want to reclone as fallback but log a separate warning (for stats) + if isIndexLockError(err) { + logger.WarnContext(ctx, "index.lock exists, will reclone instead") + } + + return false +} + +// fetchAndResetRepo fetches remote origin and resets origin/HEAD to the remote's default branch +func fetchAndResetRepo(ctx context.Context, repoPath string) error { + err := runCmd(ctx, repoPath, nil, "git", "fetch", "origin") + if err != nil { + return fmt.Errorf("git fetch failed: %w", err) + } + + // Make sure origin/HEAD points to the latest default branch from remotes + err = runCmd(ctx, repoPath, nil, "git", "remote", "set-head", "origin", "--auto") + if err != nil { + logger.WarnContext(ctx, "git remote set-head failed: ", slog.Any("err", err)) + } + + err = runCmd(ctx, repoPath, nil, "git", "reset", "--hard", "origin/HEAD") + if err != nil { + return fmt.Errorf("git reset failed: %w", err) + } + + return nil +} + +// refreshRepo updates the local repository on disk if it is stale or if forceUpdate is true. +// It clones the repository if it doesn't exist, or fetches and resets it with error recovery and fallback recloning. +func refreshRepo(ctx context.Context, repoURL string, forceUpdate bool) error { + logger.DebugContext(ctx, "Refreshing repository on disk") + start := time.Now() + + repoDirName := getRepoDirName(repoURL) + repoPath := filepath.Join(gitStorePath, repoDirName) + + repoLock := GetRepoLock(repoURL) + repoLock.Lock() + defer repoLock.Unlock() + + lastFetchMu.Lock() + accessTime, ok := lastFetch[repoURL] + lastFetchMu.Unlock() + + // Check if we need to fetch + if forceUpdate || !ok || time.Since(accessTime) > fetchTimeout { + if _, err := os.Stat(filepath.Join(repoPath, ".git")); os.IsNotExist(err) { + // Clone + logger.DebugContext(ctx, "Cloning git repository", slog.Duration("sinceAccessTime", time.Since(accessTime))) + if err := cloneRepo(ctx, repoURL, repoPath); err != nil { + return fmt.Errorf("git clone failed: %w", err) + } + } else { + // Fetch and reset + logger.DebugContext(ctx, "Fetching git repository", slog.Duration("sinceAccessTime", time.Since(accessTime))) + err := fetchAndResetRepo(ctx, repoPath) + + // Attempt recovery and fallback + if err != nil { + logger.WarnContext(ctx, "Initial fetch and reset failed, attempting to recover", slog.Any("err", err)) + + // Attempt recovery and retry fetch and reset if successful + if attemptGitRecovery(ctx, repoPath, err) { + logger.InfoContext(ctx, "Retrying fetch and reset after recovery") + err = fetchAndResetRepo(ctx, repoPath) + } + + // If still failing or recovery wasn't attempted, reclone the repo as final fallback + if err != nil { + if isForbiddenError(err) { + logger.WarnContext(ctx, "Fetch failed with 403 Forbidden. Using local repo.", slog.Duration("sinceLastFetch", time.Since(accessTime)), slog.Any("err", err)) + return nil + } + + logger.WarnContext(ctx, "Fetch and reset failed after recovery attempt, deleting repo and recloning", slog.Any("err", err)) + if err := os.RemoveAll(repoPath); err != nil { + return fmt.Errorf("failed to remove repo directory for reclone: %w", err) + } + + logger.InfoContext(ctx, "Cloning git repository after fallback", slog.Duration("sinceAccessTime", time.Since(accessTime))) + if err := cloneRepo(ctx, repoURL, repoPath); err != nil { + return fmt.Errorf("git clone failed after fallback: %w", err) + } + } + } + } + + updateLastFetch(repoURL) + } + + // Double check if the git directory exist + _, err := os.Stat(filepath.Join(repoPath, ".git")) + if err != nil { + if os.IsNotExist(err) { + deleteLastFetch(repoURL) + } + + return fmt.Errorf("failed to read file: %w", err) + } + + logger.InfoContext(ctx, "Repository refresh completed", slog.Duration("duration", time.Since(start))) + + return nil +} + +// SyncRepoOnDisk syncs/updates the repository on disk and returns a Repository struct with repoPath set. +// Skips the expensive LoadRepository (commit graph building and patch ID calculation). +func SyncRepoOnDisk(ctx context.Context, repoURL string, opts FetchOptions) (*Repository, error) { + _, err, _ := gFetch.Do(repoURL, func() (any, error) { + return runWithConcurrencyControl(ctx, opts.SkipReqConcurrencySemaphore, func() (any, error) { + return nil, refreshRepo(ctx, repoURL, opts.ForceUpdate) + }) + }) + if err != nil { + logger.ErrorContext(ctx, "Error syncing repository on disk", slog.Any("error", err)) + return nil, err + } + + return NewRepository(repoURL), nil +} + +// LoadRepo handles fetching and loading of a repository into RAM (commit graph, patch IDs). +// If opts.ForceUpdate is false, it will use the in-memory cache if available. +func LoadRepo(ctx context.Context, repoURL string, opts FetchOptions) (*Repository, error) { + repoDirName := getRepoDirName(repoURL) + repoPath := filepath.Join(gitStorePath, repoDirName) + + if !opts.ForceUpdate { + if repo, ok := repoCache.Get(repoURL); ok { + // repoCache.Get() will not return expired items, so we can safely return the repo + logger.DebugContext(ctx, "Repository already in cache, skipping fetch and load") + return repo, nil + } + } + + if _, err := SyncRepoOnDisk(ctx, repoURL, opts); err != nil { + return nil, err + } + + repoAny, err, _ := gLoad.Do(repoURL, func() (any, error) { + return runWithConcurrencyControl(ctx, opts.SkipReqConcurrencySemaphore, func() (any, error) { + repoLock := GetRepoLock(repoURL) + repoLock.RLock() + defer repoLock.RUnlock() + + return LoadRepository(ctx, repoPath) + }) + }) + if err != nil { + logger.ErrorContext(ctx, "Failed to load repository", slog.Any("error", err)) + return nil, err + } + repo := repoAny.(*Repository) + repoCache.SetWithTTL(repoURL, repo, 0, repoTTL) + + return repo, nil +} + +// ArchiveRepo compresses git repository into a blob +func ArchiveRepo(ctx context.Context, repoURL string) ([]byte, error) { + repoDirName := getRepoDirName(repoURL) + repoPath := filepath.Join(gitStorePath, repoDirName) + archivePath := repoPath + ".zst" + + repoLock := GetRepoLock(repoURL) + repoLock.RLock() + defer repoLock.RUnlock() + + lastFetchMu.Lock() + accessTime := lastFetch[repoURL] + lastFetchMu.Unlock() + + // Check if archive needs update + // We update if archive does not exist OR if it is older than the last fetch + stats, err := os.Stat(archivePath) + if os.IsNotExist(err) || (err == nil && stats.ModTime().Before(accessTime)) { + logger.DebugContext(ctx, "Archiving git blob") + startArchive := time.Now() + // Archive + // tar --zstd -cf -C "/" . + // using -C to archive the relative path so it unzips nicely + err := runCmd(ctx, "", nil, "tar", "--zstd", "-cf", archivePath, "-C", filepath.Join(gitStorePath, repoDirName), ".") + if err != nil { + return nil, fmt.Errorf("tar zstd failed: %w", err) + } + logger.InfoContext(ctx, "Archiving git blob completed", slog.Duration("duration", time.Since(startArchive))) + } + + // If the context is cancelled, still do the fetching stuff, just don't bother returning the result + // As we can still cache the result and reply faster next time. + if ctx.Err() != nil { + return nil, ctx.Err() + } + + fileData, err := os.ReadFile(archivePath) + if err != nil { + return nil, fmt.Errorf("failed to read file: %w", err) + } + + return fileData, nil +} diff --git a/go/cmd/gitter/git_test.go b/go/cmd/gitter/git_test.go new file mode 100644 index 00000000000..8575c63dc15 --- /dev/null +++ b/go/cmd/gitter/git_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "errors" + "testing" +) + +func TestIsIndexLockError(t *testing.T) { + tests := []struct { + err error + expected bool + }{ + {errors.New("fatal: Unable to create '/path/to/repo.git/index.lock': File exists"), true}, + {errors.New("some other error"), false}, + {nil, false}, + } + + for _, tt := range tests { + if result := isIndexLockError(tt.err); result != tt.expected { + t.Errorf("isIndexLockError(%v) = %v, expected %v", tt.err, result, tt.expected) + } + } +} + +func TestIsRefConflictError(t *testing.T) { + tests := []struct { + err error + expected bool + }{ + {errors.New("error: some local refs could not be updated; try running 'git remote prune origin' to remove any old, conflicting branches"), true}, + {errors.New("error: fetching ref refs/remotes/some-ref-name failed: refname conflict"), true}, + {errors.New("some other error"), false}, + {nil, false}, + } + + for _, tt := range tests { + if result := isRefConflictError(tt.err); result != tt.expected { + t.Errorf("isRefConflictError(%v) = %v, expected %v", tt.err, result, tt.expected) + } + } +} + +func TestSyncRepoOnDiskAndLoadRepo(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + setupTest(t) + + url := "https://github.com/oliverchang/osv-test.git" + + // Test SyncRepoOnDisk with SkipReqConcurrencySemaphore: true + // There's no easy way to test the semaphore skipping part, so we just test that it works. + repoDisk, err := SyncRepoOnDisk(t.Context(), url, FetchOptions{ForceUpdate: false, SkipReqConcurrencySemaphore: true}) + if err != nil { + t.Fatalf("SyncRepoOnDisk failed with SkipReqConcurrencySemaphore=true: %v", err) + } + if repoDisk == nil || repoDisk.repoPath == "" { + t.Errorf("SyncRepoOnDisk returned invalid repository struct") + } + + // Test LoadRepo with SkipReqConcurrencySemaphore: false + repoLoaded, err := LoadRepo(t.Context(), url, FetchOptions{ForceUpdate: false, SkipReqConcurrencySemaphore: false}) + if err != nil { + t.Fatalf("LoadRepo failed with SkipReqConcurrencySemaphore=false: %v", err) + } + if repoLoaded == nil || len(repoLoaded.commits) == 0 { + t.Errorf("LoadRepo returned empty repository commits") + } +} diff --git a/go/cmd/gitter/gitter.go b/go/cmd/gitter/gitter.go index 5d1afaab654..918d074fa49 100644 --- a/go/cmd/gitter/gitter.go +++ b/go/cmd/gitter/gitter.go @@ -16,7 +16,6 @@ import ( "net/http" "net/url" "os" - "os/exec" "os/signal" "path" "path/filepath" @@ -55,18 +54,20 @@ var endpointHandlers = map[string]http.HandlerFunc{ "POST /cache": cacheHandler, "GET /tags": tagsHandler, "POST /affected-commits": affectedCommitsHandler, + "POST /file-diffs": fileDiffsHandler, + "POST /file-content": fileContentHandler, } var ( - gFetch singleflight.Group - gArchive singleflight.Group - gLoad singleflight.Group - gLsRemote singleflight.Group - gLocalTags singleflight.Group - persistencePath = filepath.Join(defaultGitterWorkDir, persistenceFileName) - gitStorePath = filepath.Join(defaultGitterWorkDir, gitStoreFileName) - fetchTimeout time.Duration - semaphore chan struct{} // Request concurrency control + gFetch singleflight.Group + gArchive singleflight.Group + gLoad singleflight.Group + gLsRemote singleflight.Group + gLocalTags singleflight.Group + persistencePath = filepath.Join(defaultGitterWorkDir, persistenceFileName) + gitStorePath = filepath.Join(defaultGitterWorkDir, gitStoreFileName) + fetchTimeout time.Duration + reqConcurrencySemaphore chan struct{} // Request concurrency control // LRU cache for recently loaded repositories (key: repo URL) repoCache *ristretto.Cache[string, *Repository] repoTTL time.Duration @@ -202,57 +203,17 @@ func CloseInvalidRepoCache() { } } -// prepareCmd prepares the command with context cancellation handled by sending SIGINT. -func prepareCmd(ctx context.Context, dir string, env []string, name string, args ...string) *exec.Cmd { - cmd := exec.CommandContext(ctx, name, args...) - if dir != "" { - cmd.Dir = dir - } - if len(env) > 0 { - cmd.Env = append(os.Environ(), env...) - } - // Use SIGINT instead of SIGKILL for graceful shutdown of subprocesses - cmd.Cancel = func() error { - logger.DebugContext(ctx, "SIGINT sent to command", slog.String("cmd", name), slog.Any("args", args)) - return cmd.Process.Signal(syscall.SIGINT) - } - // Ensure it eventually dies if it ignores SIGINT - cmd.WaitDelay = shutdownTimeout / 2 - - return cmd -} - -// runCmd executes a command with context cancellation handled by sending SIGINT. -// It logs cancellation errors separately as requested. -func runCmd(ctx context.Context, dir string, env []string, name string, args ...string) error { - cmd := prepareCmd(ctx, dir, env, name, args...) - out, err := cmd.CombinedOutput() - - if err != nil { - if ctx.Err() != nil { - // Log separately if cancelled - logger.WarnContext(ctx, "Command cancelled", slog.String("cmd", name), slog.Any("err", ctx.Err())) - return fmt.Errorf("command %s cancelled: %w", name, ctx.Err()) - } - - return fmt.Errorf("command %s failed: %w, output: %s", name, err, out) +// runWithConcurrencyControl runs function f for request concurrency control. +// If skipReqConcurrencySemaphore is true, it executes f directly without waiting for a semaphore spot. +func runWithConcurrencyControl(ctx context.Context, skipReqConcurrencySemaphore bool, f func() (any, error)) (any, error) { + if skipReqConcurrencySemaphore { + return f() } - logger.DebugContext(ctx, "Git command executed", - slog.String("cmd", name), - slog.Any("args", args), - slog.String("output", string(out)), - ) - - return nil -} - -// runWithSemaphore runs function after waiting at semaphore for concurrency control -func runWithSemaphore(ctx context.Context, f func() (any, error)) (any, error) { select { - case semaphore <- struct{}{}: - defer func() { <-semaphore }() - logger.DebugContext(ctx, "Concurrent requests", slog.Int("count", len(semaphore))) + case reqConcurrencySemaphore <- struct{}{}: + defer func() { <-reqConcurrencySemaphore }() + logger.DebugContext(ctx, "Concurrent requests", slog.Int("count", len(reqConcurrencySemaphore))) return f() case <-ctx.Done(): @@ -261,12 +222,12 @@ func runWithSemaphore(ctx context.Context, f func() (any, error)) (any, error) { } } -func isLocalRequest(r *http.Request) bool { - host, _, err := net.SplitHostPort(r.RemoteAddr) +func isLocalRequest(req *http.Request) bool { + host, _, err := net.SplitHostPort(req.RemoteAddr) if err != nil { // If SplitHostPort fails, it might be a raw IP (though rare in RemoteAddr), // or an empty string. Try parsing the whole string as an IP. - host = r.RemoteAddr + host = req.RemoteAddr } ip := net.ParseIP(host) @@ -278,7 +239,7 @@ func isLocalRequest(r *http.Request) bool { return ip.IsLoopback() } -func prepareURL(r *http.Request, repoURL string) (string, error) { +func prepareURL(req *http.Request, repoURL string) (string, error) { if repoURL == "" { return "", errors.New("missing url parameter") } @@ -311,7 +272,7 @@ func prepareURL(r *http.Request, repoURL string) (string, error) { return mirror, nil } - if !isLocalRequest(r) { + if !isLocalRequest(req) { if u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "git" { return "", fmt.Errorf("unsupported protocol: %s", u.Scheme) } @@ -367,265 +328,64 @@ func isNotFoundError(err error) bool { return strings.Contains(strings.ToLower(errString), "repository") && strings.Contains(strings.ToLower(errString), "not found") } -// Helper function to unmarshal request body based on Content-Type (protobuf or JSON) -func unmarshalRequest(r *http.Request, body proto.Message) error { - data, err := io.ReadAll(r.Body) - if err != nil { - return err - } - defer r.Body.Close() - - contentType := r.Header.Get("Content-Type") - if contentType == "application/json" { - return protojson.Unmarshal(data, body) - } - // Default to protobuf - return proto.Unmarshal(data, body) -} - -// Helper function to marshal response body based on Content-Type (protobuf or JSON) -func marshalResponse(r *http.Request, m proto.Message) ([]byte, error) { - contentType := r.Header.Get("Content-Type") - if contentType == "application/json" { - return protojson.Marshal(m) - } - // Default to protobuf - return proto.Marshal(m) -} - -func doFetch(ctx context.Context, repoURL string, forceUpdate bool) error { - _, err, _ := gFetch.Do(repoURL, func() (any, error) { - return runWithSemaphore(ctx, func() (any, error) { - return nil, FetchRepo(ctx, repoURL, forceUpdate) - }) - }) - if err != nil { - logger.ErrorContext(ctx, "Error fetching blob", slog.Any("error", err)) - return err - } - - return nil -} - -// getFreshRepo handles fetching and loading of a repository -// If forceUpdate is true, it will always refetch and rebuild the repository (commit graph, patch ID, etc) -// Otherwise, it will use a cache if available -func getFreshRepo(ctx context.Context, repoURL string, forceUpdate bool) (*Repository, error) { - repoDirName := getRepoDirName(repoURL) - repoPath := filepath.Join(gitStorePath, repoDirName) - - if !forceUpdate { - if repo, ok := repoCache.Get(repoURL); ok { - // repoCache.Get() will not return expired items, so we can safely return the repo - logger.DebugContext(ctx, "Repository already in cache, skipping fetch and load") - return repo, nil - } - } - - if err := doFetch(ctx, repoURL, forceUpdate); err != nil { - return nil, err - } - - repoAny, err, _ := gLoad.Do(repoURL, func() (any, error) { - return runWithSemaphore(ctx, func() (any, error) { - repoLock := GetRepoLock(repoURL) - repoLock.RLock() - defer repoLock.RUnlock() - - return LoadRepository(ctx, repoPath) - }) - }) - if err != nil { - logger.ErrorContext(ctx, "Failed to load repository", slog.Any("error", err)) - return nil, err - } - repo := repoAny.(*Repository) - repoCache.SetWithTTL(repoURL, repo, 0, repoTTL) - - return repo, nil -} - -func isIndexLockError(err error) bool { +func isRefNotFoundError(err error) bool { if err == nil { return false } - errString := err.Error() + s := strings.ToLower(err.Error()) - return strings.Contains(errString, "index.lock") && strings.Contains(errString, "File exists") + return strings.Contains(s, "not found or invalid") || + strings.Contains(s, "failed to resolve target ref") || + strings.Contains(s, "failed to run git rev-parse") || + strings.Contains(s, "ref cannot be empty") } -func isRefConflictError(err error) bool { +func isFileNotFoundError(err error) bool { if err == nil { return false } - errString := err.Error() - - return strings.Contains(errString, "refname conflict") || - (strings.Contains(errString, "some local refs could not be updated") && strings.Contains(errString, "try running 'git remote prune origin'")) -} - -// Attempt to recover from git fetch + reset errors -// Returns true if recovery was attempted and we should retry fetch + reset -func attemptGitRecovery(ctx context.Context, repoPath string, err error) bool { - if err == nil { - return false - } - - // Refname conflict, likely name conflict between local and remote refs - // We can try removing stale remote-tracking branches and retry - if isRefConflictError(err) { - logger.WarnContext(ctx, "ref conflict detected, running git remote prune origin") - if err := runCmd(ctx, repoPath, nil, "git", "remote", "prune", "origin"); err != nil { - logger.ErrorContext(ctx, "failed to prune origin", slog.Any("err", err)) - return false - } - - return true - } - - // index.lock exists, likely a previous git reset got terminated and wasn't cleaned up properly. - // We want to reclone as fallback but log a separate warning (for stats) - if isIndexLockError(err) { - logger.WarnContext(ctx, "index.lock exists, will reclone instead") - } + s := strings.ToLower(err.Error()) - return false + return strings.Contains(s, "git cat-file failed") || + strings.Contains(s, "invalid object name") || + strings.Contains(s, "does not exist in") } -// Helper function to group git fetch and git reset --hard together -func fetchAndReset(ctx context.Context, repoPath string) error { - err := runCmd(ctx, repoPath, nil, "git", "fetch", "origin") +// Helper function to unmarshal request body based on Content-Type (protobuf or JSON) +func unmarshalRequest(req *http.Request, msg proto.Message) error { + data, err := io.ReadAll(req.Body) if err != nil { - return fmt.Errorf("git fetch failed: %w", err) + return err } + defer req.Body.Close() - err = runCmd(ctx, repoPath, nil, "git", "reset", "--hard", "origin/HEAD") - if err != nil { - return fmt.Errorf("git reset failed: %w", err) + contentType := req.Header.Get("Content-Type") + if contentType == "application/json" { + return protojson.Unmarshal(data, msg) } - - return nil + // Default to protobuf + return proto.Unmarshal(data, msg) } -func FetchRepo(ctx context.Context, repoURL string, forceUpdate bool) error { - logger.DebugContext(ctx, "Starting fetch repo") - start := time.Now() - - repoDirName := getRepoDirName(repoURL) - repoPath := filepath.Join(gitStorePath, repoDirName) - - repoLock := GetRepoLock(repoURL) - repoLock.Lock() - defer repoLock.Unlock() - - lastFetchMu.Lock() - accessTime, ok := lastFetch[repoURL] - lastFetchMu.Unlock() - - // Check if we need to fetch - if forceUpdate || !ok || time.Since(accessTime) > fetchTimeout { - if _, err := os.Stat(filepath.Join(repoPath, ".git")); os.IsNotExist(err) { - // Clone - logger.DebugContext(ctx, "Cloning git repository", slog.Duration("sinceAccessTime", time.Since(accessTime))) - err := runCmd(ctx, "", []string{"GIT_TERMINAL_PROMPT=0"}, "git", "clone", "--", repoURL, repoPath) - if err != nil { - return fmt.Errorf("git clone failed: %w", err) - } - } else { - // Fetch and reset - logger.DebugContext(ctx, "Fetching git repository", slog.Duration("sinceAccessTime", time.Since(accessTime))) - err := fetchAndReset(ctx, repoPath) - - // Attempt recovery and fallback - if err != nil { - logger.WarnContext(ctx, "Initial fetch and reset failed, attempting to recover", slog.Any("err", err)) - - // Attempt recovery and retry fetch and reset if successful - if attemptGitRecovery(ctx, repoPath, err) { - logger.InfoContext(ctx, "Retrying fetch and reset after recovery") - err = fetchAndReset(ctx, repoPath) - } - - // If still failing or recovery wasn't attempted, reclone the repo as final fallback - if err != nil { - if isForbiddenError(err) { - logger.WarnContext(ctx, "Fetch failed with 403 Forbidden. Using local repo.", slog.Duration("sinceLastFetch", time.Since(accessTime)), slog.Any("err", err)) - return nil - } - - logger.WarnContext(ctx, "Fetch and reset failed after recovery attempt, deleting repo and recloning", slog.Any("err", err)) - if err := os.RemoveAll(repoPath); err != nil { - return fmt.Errorf("failed to remove repo directory for reclone: %w", err) - } - - logger.InfoContext(ctx, "Cloning git repository after fallback", slog.Duration("sinceAccessTime", time.Since(accessTime))) - err := runCmd(ctx, "", []string{"GIT_TERMINAL_PROMPT=0"}, "git", "clone", "--", repoURL, repoPath) - if err != nil { - return fmt.Errorf("git clone failed after fallback: %w", err) - } - } - } - } - - updateLastFetch(repoURL) +// Helper function to marshal and write response body based on Content-Type (protobuf or JSON) +func writeResponse(w http.ResponseWriter, req *http.Request, msg proto.Message) error { + contentType := req.Header.Get("Content-Type") + var out []byte + var err error + if contentType == "application/json" { + out, err = protojson.Marshal(msg) + } else { + out, err = proto.Marshal(msg) } - - // Double check if the git directory exist - _, err := os.Stat(filepath.Join(repoPath, ".git")) if err != nil { - if os.IsNotExist(err) { - deleteLastFetch(repoURL) - } - - return fmt.Errorf("failed to read file: %w", err) - } - - logger.InfoContext(ctx, "Fetch completed", slog.Duration("duration", time.Since(start))) - - return nil -} - -func ArchiveRepo(ctx context.Context, repoURL string) ([]byte, error) { - repoDirName := getRepoDirName(repoURL) - repoPath := filepath.Join(gitStorePath, repoDirName) - archivePath := repoPath + ".zst" - - repoLock := GetRepoLock(repoURL) - repoLock.RLock() - defer repoLock.RUnlock() - - lastFetchMu.Lock() - accessTime := lastFetch[repoURL] - lastFetchMu.Unlock() - - // Check if archive needs update - // We update if archive does not exist OR if it is older than the last fetch - stats, err := os.Stat(archivePath) - if os.IsNotExist(err) || (err == nil && stats.ModTime().Before(accessTime)) { - logger.DebugContext(ctx, "Archiving git blob") - startArchive := time.Now() - // Archive - // tar --zstd -cf -C "/" . - // using -C to archive the relative path so it unzips nicely - err := runCmd(ctx, "", nil, "tar", "--zstd", "-cf", archivePath, "-C", filepath.Join(gitStorePath, repoDirName), ".") - if err != nil { - return nil, fmt.Errorf("tar zstd failed: %w", err) - } - logger.InfoContext(ctx, "Archiving git blob completed", slog.Duration("duration", time.Since(startArchive))) - } - - // If the context is cancelled, still do the fetching stuff, just don't bother returning the result - // As we can still cache the result and reply faster next time. - if ctx.Err() != nil { - return nil, ctx.Err() + return err } - fileData, err := os.ReadFile(archivePath) - if err != nil { - return nil, fmt.Errorf("failed to read file: %w", err) - } + w.Header().Set("Content-Type", contentType) + w.WriteHeader(http.StatusOK) + _, err = w.Write(out) - return fileData, nil + return err } func main() { @@ -644,7 +404,7 @@ func main() { flag.Int64Var(&invalidRepoCacheMaxEntries, "invalid-repo-cache-max-entries", 5000, "Invalid repository cache max entries") flag.Parse() - semaphore = make(chan struct{}, *concurrentLimit) + reqConcurrencySemaphore = make(chan struct{}, *concurrentLimit) persistencePath = filepath.Join(*workDir, persistenceFileName) gitStorePath = filepath.Join(*workDir, gitStoreFileName) @@ -722,13 +482,13 @@ func main() { logger.Info("Server exiting") } -func gitHandler(w http.ResponseWriter, r *http.Request) { +func gitHandler(w http.ResponseWriter, req *http.Request) { start := time.Now() statusCode := http.StatusOK - ctx := r.Context() + ctx := req.Context() defer func() { logRequestCompletion(ctx, "/git", start, statusCode) }() - repoURL, err := prepareURL(r, r.URL.Query().Get("url")) + repoURL, err := prepareURL(req, req.URL.Query().Get("url")) if err != nil { statusCode = http.StatusBadRequest http.Error(w, err.Error(), statusCode) @@ -736,15 +496,15 @@ func gitHandler(w http.ResponseWriter, r *http.Request) { return } - forceUpdate := r.URL.Query().Get("force-update") == "true" + forceUpdate := req.URL.Query().Get("force-update") == "true" - refID := r.URL.Query().Get("ref_id") + refID := req.URL.Query().Get("ref_id") ctx = context.WithValue(ctx, urlKey, repoURL) ctx = context.WithValue(ctx, refIDKey, refID) - logger.DebugContext(ctx, "Received request: /git", slog.Bool("forceUpdate", forceUpdate), slog.String("remoteAddr", r.RemoteAddr)) + logger.DebugContext(ctx, "Received request: /git", slog.Bool("forceUpdate", forceUpdate), slog.String("remoteAddr", req.RemoteAddr)) // Fetch repo first - if err := doFetch(ctx, repoURL, forceUpdate); err != nil { + if _, err := SyncRepoOnDisk(ctx, repoURL, FetchOptions{ForceUpdate: forceUpdate, SkipReqConcurrencySemaphore: true}); err != nil { if isAuthError(err) || isForbiddenError(err) { statusCode = http.StatusForbidden } else if isNotFoundError(err) { @@ -759,7 +519,7 @@ func gitHandler(w http.ResponseWriter, r *http.Request) { // Archive repo fileDataAny, err, _ := gArchive.Do(repoURL, func() (any, error) { - return runWithSemaphore(ctx, func() (any, error) { + return runWithConcurrencyControl(ctx, true, func() (any, error) { return ArchiveRepo(ctx, repoURL) }) }) @@ -784,21 +544,21 @@ func gitHandler(w http.ResponseWriter, r *http.Request) { } } -func cacheHandler(w http.ResponseWriter, r *http.Request) { +func cacheHandler(w http.ResponseWriter, req *http.Request) { start := time.Now() statusCode := http.StatusOK - ctx := r.Context() + ctx := req.Context() defer func() { logRequestCompletion(ctx, "/cache", start, statusCode) }() body := &pb.CacheRequest{} - if err := unmarshalRequest(r, body); err != nil { + if err := unmarshalRequest(req, body); err != nil { statusCode = http.StatusBadRequest http.Error(w, fmt.Sprintf("Error unmarshaling request: %v", err), statusCode) return } - repoURL, err := prepareURL(r, body.GetUrl()) + repoURL, err := prepareURL(req, body.GetUrl()) if err != nil { statusCode = http.StatusBadRequest http.Error(w, err.Error(), statusCode) @@ -811,7 +571,7 @@ func cacheHandler(w http.ResponseWriter, r *http.Request) { ctx = context.WithValue(ctx, refIDKey, refID) logger.DebugContext(ctx, "Received request: /cache") - if _, err := getFreshRepo(ctx, repoURL, body.GetForceUpdate()); err != nil { + if _, err := LoadRepo(ctx, repoURL, FetchOptions{ForceUpdate: body.GetForceUpdate(), SkipReqConcurrencySemaphore: true}); err != nil { if isAuthError(err) || isForbiddenError(err) { statusCode = http.StatusForbidden } else if isNotFoundError(err) { @@ -827,21 +587,21 @@ func cacheHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } -func affectedCommitsHandler(w http.ResponseWriter, r *http.Request) { +func affectedCommitsHandler(w http.ResponseWriter, req *http.Request) { start := time.Now() statusCode := http.StatusOK - ctx := r.Context() + ctx := req.Context() defer func() { logRequestCompletion(ctx, "/affected-commits", start, statusCode) }() body := &pb.AffectedCommitsRequest{} - if err := unmarshalRequest(r, body); err != nil { + if err := unmarshalRequest(req, body); err != nil { statusCode = http.StatusBadRequest http.Error(w, fmt.Sprintf("Error unmarshaling request: %v", err), statusCode) return } - repoURL, err := prepareURL(r, body.GetUrl()) + repoURL, err := prepareURL(req, body.GetUrl()) if err != nil { statusCode = http.StatusBadRequest http.Error(w, err.Error(), statusCode) @@ -881,7 +641,7 @@ func affectedCommitsHandler(w http.ResponseWriter, r *http.Request) { slog.Bool("considerAllBranches", considerAllBranches), ) - repo, err := getFreshRepo(ctx, repoURL, body.GetForceUpdate()) + repo, err := LoadRepo(ctx, repoURL, FetchOptions{ForceUpdate: body.GetForceUpdate(), SkipReqConcurrencySemaphore: false}) if err != nil { if isAuthError(err) || isForbiddenError(err) { statusCode = http.StatusForbidden @@ -944,19 +704,10 @@ func affectedCommitsHandler(w http.ResponseWriter, r *http.Request) { } } - out, err := marshalResponse(r, resp) - if err != nil { - logger.ErrorContext(ctx, "Error marshaling affected commits", slog.Any("error", err)) + if err := writeResponse(w, req, resp); err != nil { + logger.ErrorContext(ctx, "Error writing affected commits response", slog.Any("error", err)) statusCode = http.StatusInternalServerError - http.Error(w, fmt.Sprintf("Error marshaling affected commits: %v", err), statusCode) - - return - } - - w.Header().Set("Content-Type", r.Header.Get("Content-Type")) - w.WriteHeader(http.StatusOK) - if _, err := w.Write(out); err != nil { - logger.ErrorContext(ctx, "Error writing response", slog.Any("error", err)) + http.Error(w, fmt.Sprintf("Error writing affected commits response: %v", err), statusCode) } } @@ -974,13 +725,13 @@ func makeTagsResponse(tagsMap map[string]SHA1) *pb.TagsResponse { return resp } -func tagsHandler(w http.ResponseWriter, r *http.Request) { +func tagsHandler(w http.ResponseWriter, req *http.Request) { start := time.Now() statusCode := http.StatusOK - ctx := r.Context() + ctx := req.Context() defer func() { logRequestCompletion(ctx, "/tags", start, statusCode) }() - repoURL, err := prepareURL(r, r.URL.Query().Get("url")) + repoURL, err := prepareURL(req, req.URL.Query().Get("url")) if err != nil { statusCode = http.StatusBadRequest http.Error(w, err.Error(), statusCode) @@ -988,7 +739,7 @@ func tagsHandler(w http.ResponseWriter, r *http.Request) { return } - refID := r.URL.Query().Get("ref_id") + refID := req.URL.Query().Get("ref_id") ctx = context.WithValue(ctx, urlKey, repoURL) ctx = context.WithValue(ctx, refIDKey, refID) logger.DebugContext(ctx, "Received request: /tags") @@ -1019,9 +770,7 @@ func tagsHandler(w http.ResponseWriter, r *http.Request) { // We want to use show-ref instead of ls-remote because it's faster and we don't have to worry about rate limits if repo.repoPath != "" { logger.DebugContext(ctx, "Local repo found, using show-ref") - if _, errFetch, _ := gFetch.Do(repoURL, func() (any, error) { - return nil, FetchRepo(ctx, repoURL, false) - }); errFetch != nil { + if _, errFetch := SyncRepoOnDisk(ctx, repoURL, FetchOptions{ForceUpdate: false, SkipReqConcurrencySemaphore: false}); errFetch != nil { logger.ErrorContext(ctx, "Error fetching repo", slog.Any("error", errFetch)) if isAuthError(errFetch) || isForbiddenError(errFetch) { invalidRepoCache.SetWithTTL(repoURL, http.StatusForbidden, 1, invalidRepoTTL) @@ -1095,18 +844,167 @@ func tagsHandler(w http.ResponseWriter, r *http.Request) { } resp := makeTagsResponse(tagsMap) - out, err := marshalResponse(r, resp) + if err := writeResponse(w, req, resp); err != nil { + logger.ErrorContext(ctx, "Error writing tags response", slog.Any("error", err)) + statusCode = http.StatusInternalServerError + http.Error(w, fmt.Sprintf("Error writing tags response: %v", err), statusCode) + } +} + +func fileDiffsHandler(w http.ResponseWriter, req *http.Request) { + start := time.Now() + statusCode := http.StatusOK + ctx := req.Context() + defer func() { logRequestCompletion(ctx, "/file-diffs", start, statusCode) }() + + body := &pb.FileDiffsRequest{} + if err := unmarshalRequest(req, body); err != nil { + statusCode = http.StatusBadRequest + http.Error(w, fmt.Sprintf("Error unmarshaling request: %v", err), statusCode) + + return + } + + repoURL, err := prepareURL(req, body.GetUrl()) + if err != nil { + statusCode = http.StatusBadRequest + http.Error(w, err.Error(), statusCode) + + return + } + + lastSyncedCommit := body.GetLastSyncedCommit() + branch := body.GetBranch() + + ctx = context.WithValue(ctx, urlKey, repoURL) + logger.DebugContext(ctx, "Received request: /file-diffs", + slog.String("last_synced_commit", lastSyncedCommit), + slog.String("branch", branch), + ) + + repo, err := SyncRepoOnDisk(ctx, repoURL, FetchOptions{ForceUpdate: true, SkipReqConcurrencySemaphore: true}) + if err != nil { + if isAuthError(err) || isForbiddenError(err) { + statusCode = http.StatusForbidden + } else if isNotFoundError(err) { + statusCode = http.StatusNotFound + } else { + statusCode = http.StatusInternalServerError + } + http.Error(w, fmt.Sprintf("Error getting repo: %v", err), statusCode) + + return + } + + latestCommit, changes, err := repo.ListFileDiffs(ctx, lastSyncedCommit, branch) if err != nil { - logger.ErrorContext(ctx, "Error marshaling tags response", slog.Any("error", err)) + if isRefNotFoundError(err) { + statusCode = http.StatusBadRequest + http.Error(w, fmt.Sprintf("Invalid commit or branch reference: %v", err), statusCode) + + return + } + logger.ErrorContext(ctx, "Error listing file diffs", slog.Any("error", err)) statusCode = http.StatusInternalServerError - http.Error(w, fmt.Sprintf("Error marshaling tags response: %v", err), statusCode) + http.Error(w, fmt.Sprintf("Error listing file diffs: %v", err), statusCode) return } - w.Header().Set("Content-Type", r.Header.Get("Content-Type")) - w.WriteHeader(http.StatusOK) - if _, err := w.Write(out); err != nil { - logger.ErrorContext(ctx, "Error writing tags response", slog.Any("error", err)) + pbChanges := make([]*pb.FileChange, 0, len(changes)) + for _, c := range changes { + pbChanges = append(pbChanges, &pb.FileChange{ + FromPath: c.From, + ToPath: c.To, + }) + } + + resp := &pb.FileDiffsResponse{ + LatestCommit: latestCommit, + Changes: pbChanges, + } + + if err := writeResponse(w, req, resp); err != nil { + logger.ErrorContext(ctx, "Error writing diff response", slog.Any("error", err)) + statusCode = http.StatusInternalServerError + http.Error(w, fmt.Sprintf("Error writing diff response: %v", err), statusCode) + } +} + +func fileContentHandler(w http.ResponseWriter, req *http.Request) { + start := time.Now() + statusCode := http.StatusOK + ctx := req.Context() + defer func() { logRequestCompletion(ctx, "/file-content", start, statusCode) }() + + body := &pb.FileContentRequest{} + if err := unmarshalRequest(req, body); err != nil { + statusCode = http.StatusBadRequest + http.Error(w, fmt.Sprintf("Error unmarshaling request: %v", err), statusCode) + + return + } + + repoURL, err := prepareURL(req, body.GetUrl()) + if err != nil { + statusCode = http.StatusBadRequest + http.Error(w, err.Error(), statusCode) + + return + } + + commit := body.GetCommit() + filePath := body.GetPath() + if commit == "" || filePath == "" { + statusCode = http.StatusBadRequest + http.Error(w, "Missing commit or path", statusCode) + + return + } + + ctx = context.WithValue(ctx, urlKey, repoURL) + logger.DebugContext(ctx, "Received request: /file-content", + slog.String("commit", commit), + slog.String("path", filePath), + ) + + repo, err := SyncRepoOnDisk(ctx, repoURL, FetchOptions{ForceUpdate: false, SkipReqConcurrencySemaphore: true}) + if err != nil { + if isAuthError(err) || isForbiddenError(err) { + statusCode = http.StatusForbidden + } else if isNotFoundError(err) { + statusCode = http.StatusNotFound + } else { + statusCode = http.StatusInternalServerError + } + http.Error(w, fmt.Sprintf("Error getting repo: %v", err), statusCode) + + return + } + + content, err := repo.GetFileContent(ctx, commit, filePath) + if err != nil { + if isRefNotFoundError(err) || isFileNotFoundError(err) { + logger.DebugContext(ctx, "File content not found", slog.Any("error", err)) + statusCode = http.StatusNotFound + http.Error(w, fmt.Sprintf("Error getting file content: %v", err), statusCode) + + return + } + logger.ErrorContext(ctx, "Error getting file content", slog.Any("error", err)) + statusCode = http.StatusInternalServerError + http.Error(w, fmt.Sprintf("Error getting file content: %v", err), statusCode) + + return + } + + resp := &pb.FileContentResponse{ + Content: content, + } + + if err := writeResponse(w, req, resp); err != nil { + logger.ErrorContext(ctx, "Error writing file content response", slog.Any("error", err)) + statusCode = http.StatusInternalServerError + http.Error(w, fmt.Sprintf("Error writing file content response: %v", err), statusCode) } } diff --git a/go/cmd/gitter/gitter_test.go b/go/cmd/gitter/gitter_test.go index c4b1a8f8ca5..5d59280d409 100644 --- a/go/cmd/gitter/gitter_test.go +++ b/go/cmd/gitter/gitter_test.go @@ -11,7 +11,9 @@ import ( "github.com/google/go-cmp/cmp" pb "github.com/google/osv.dev/go/internal/gitter/pb/repository" + "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/testing/protocmp" ) func TestGetRepoDirName(t *testing.T) { @@ -193,41 +195,6 @@ func TestIsNotFoundError(t *testing.T) { } } -func TestIsIndexLockError(t *testing.T) { - tests := []struct { - err error - expected bool - }{ - {errors.New("fatal: Unable to create '/path/to/repo.git/index.lock': File exists"), true}, - {errors.New("some other error"), false}, - {nil, false}, - } - - for _, tt := range tests { - if result := isIndexLockError(tt.err); result != tt.expected { - t.Errorf("isIndexLockError(%v) = %v, expected %v", tt.err, result, tt.expected) - } - } -} - -func TestIsRefConflictError(t *testing.T) { - tests := []struct { - err error - expected bool - }{ - {errors.New("error: some local refs could not be updated; try running 'git remote prune origin' to remove any old, conflicting branches"), true}, - {errors.New("error: fetching ref refs/remotes/some-ref-name failed: refname conflict"), true}, - {errors.New("some other error"), false}, - {nil, false}, - } - - for _, tt := range tests { - if result := isRefConflictError(tt.err); result != tt.expected { - t.Errorf("isRefConflictError(%v) = %v, expected %v", tt.err, result, tt.expected) - } - } -} - func TestGitHandler_InvalidURL(t *testing.T) { tests := []struct { url string @@ -282,7 +249,7 @@ func setupTest(t *testing.T) { lastFetchMu.Unlock() // Initialize semaphore for tests - semaphore = make(chan struct{}, 100) + reqConcurrencySemaphore = make(chan struct{}, 100) // Initialize caches for tests repoCacheMaxCostBytes = 1024 * 1024 // 1MB for test @@ -559,3 +526,289 @@ func TestTagsHandler(t *testing.T) { }) } } + +func TestFileDiffsHandler(t *testing.T) { + setupTest(t) + + tests := []struct { + name string + reqBody *pb.FileDiffsRequest + rawBody []byte + contentType string + expectedCode int + expectedResponse *pb.FileDiffsResponse + }{ + { + name: "Missing repo URL", + reqBody: &pb.FileDiffsRequest{ + LastSyncedCommit: "1234567890123456789012345678901234567890", + }, + contentType: "application/json", + expectedCode: http.StatusBadRequest, + }, + { + name: "Forbidden or non-existent repo", + reqBody: &pb.FileDiffsRequest{ + Url: "https://github.com/google/this-repo-does-not-exist-12345.git", + LastSyncedCommit: "ff8cc32ba60ad9cbb3b23f0a82aad96ebe9ff76b", + }, + contentType: "application/json", + expectedCode: http.StatusForbidden, + }, + { + name: "Empty last sync for full tree diff", + reqBody: &pb.FileDiffsRequest{ + Url: "https://github.com/oliverchang/osv-test.git", + LastSyncedCommit: "", + Branch: "branch_1", + }, + contentType: "application/json", + expectedCode: http.StatusOK, + expectedResponse: &pb.FileDiffsResponse{ + LatestCommit: "ff8cc32ba60ad9cbb3b23f0a82aad96ebe9ff76b", + Changes: []*pb.FileChange{ + {ToPath: "abc"}, + {ToPath: "branch"}, + {ToPath: "def"}, + {ToPath: "regress"}, + }, + }, + }, + { + name: "Empty branch name for remote origin HEAD branch", + reqBody: &pb.FileDiffsRequest{ + Url: "https://github.com/oliverchang/osv-test.git", + LastSyncedCommit: "b1c95a196f22d06fcf80df8c6691cd113d8fefff", + Branch: "", + }, + contentType: "application/json", + expectedCode: http.StatusOK, + expectedResponse: &pb.FileDiffsResponse{ + LatestCommit: "b9b3fd4732695b83c3068b7b6a14bb372ec31f98", + Changes: []*pb.FileChange{ + { + ToPath: "branch", + }, + { + FromPath: "regress", + }, + }, + }, + }, + { + name: "Specifying non-default branch and last synced commit", + reqBody: &pb.FileDiffsRequest{ + Url: "https://github.com/oliverchang/osv-test.git", + LastSyncedCommit: "57e58a5d7c2bb3ce0f04f17ec0648b92ee82531f", + Branch: "oss-fuzz", + }, + contentType: "application/json", + expectedCode: http.StatusOK, + expectedResponse: &pb.FileDiffsResponse{ + LatestCommit: "25147a74d8aeb27b43665530ee121a2a1b19dc58", + Changes: []*pb.FileChange{ + { + ToPath: "oss-fuzz-6", + }, + { + ToPath: "oss-fuzz-7", + }, + }, + }, + }, + { + name: "Non-existent branch name", + reqBody: &pb.FileDiffsRequest{ + Url: "https://github.com/oliverchang/osv-test.git", + LastSyncedCommit: "b1c95a196f22d06fcf80df8c6691cd113d8fefff", + Branch: "nonexistent-branch-12345", + }, + contentType: "application/json", + expectedCode: http.StatusBadRequest, + }, + { + name: "Invalid last synced commit", + reqBody: &pb.FileDiffsRequest{ + Url: "https://github.com/oliverchang/osv-test.git", + LastSyncedCommit: "invalidhash123456", + Branch: "branch_1", + }, + contentType: "application/json", + expectedCode: http.StatusBadRequest, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var bodyBytes []byte + if tt.rawBody != nil { + bodyBytes = tt.rawBody + } else if tt.reqBody != nil { + var err error + if tt.contentType == "application/json" { + bodyBytes, err = protojson.Marshal(tt.reqBody) + } else { + bodyBytes, err = proto.Marshal(tt.reqBody) + } + if err != nil { + t.Fatalf("failed to marshal request: %v", err) + } + } + + req, err := http.NewRequest(http.MethodPost, "/file-diffs", bytes.NewReader(bodyBytes)) + if err != nil { + t.Fatal(err) + } + if tt.contentType != "" { + req.Header.Set("Content-Type", tt.contentType) + } + + rr := httptest.NewRecorder() + fileDiffsHandler(rr, req) + + if status := rr.Code; status != tt.expectedCode { + t.Errorf("fileDiffsHandler returned wrong status code: got %v want %v", status, tt.expectedCode) + } + + if tt.expectedResponse != nil { + resp := &pb.FileDiffsResponse{} + var err error + if tt.contentType == "application/json" { + err = protojson.Unmarshal(rr.Body.Bytes(), resp) + } else { + err = proto.Unmarshal(rr.Body.Bytes(), resp) + } + if err != nil { + t.Fatalf("failed to unmarshal response: %v", err) + } + if diff := cmp.Diff(tt.expectedResponse, resp, protocmp.Transform()); diff != "" { + t.Errorf("fileDiffsHandler returned wrong response: -want +got:\n%s", diff) + } + } + }) + } +} + +func TestFileContentHandler(t *testing.T) { + setupTest(t) + + tests := []struct { + name string + reqBody *pb.FileContentRequest + rawBody []byte + contentType string + expectedCode int + expectedResponse *pb.FileContentResponse + }{ + { + name: "Missing repo URL", + reqBody: &pb.FileContentRequest{ + Commit: "1234567890123456789012345678901234567890", + Path: "foo.txt", + }, + contentType: "application/json", + expectedCode: http.StatusBadRequest, + }, + { + name: "Missing commit or path", + reqBody: &pb.FileContentRequest{ + Url: "https://github.com/google/osv.dev.git", + }, + contentType: "application/json", + expectedCode: http.StatusBadRequest, + }, + { + name: "Forbidden or non-existent repo", + reqBody: &pb.FileContentRequest{ + Url: "https://github.com/google/this-repo-does-not-exist-12345.git", + Commit: "1234567890123456789012345678901234567890", + Path: "foo.txt", + }, + contentType: "application/json", + expectedCode: http.StatusForbidden, + }, + { + name: "Commit does not exist in repo", + reqBody: &pb.FileContentRequest{ + Url: "https://github.com/oliverchang/osv-test.git", + Commit: "183107f016fc32d8dee4f1a6b2c98b8d7073aab1", + Path: "foo.txt", + }, + contentType: "application/json", + expectedCode: http.StatusNotFound, + }, + { + name: "File does not exist at commit", + reqBody: &pb.FileContentRequest{ + Url: "https://github.com/oliverchang/osv-test.git", + Commit: "949f182716f037e25394bbb98d39b3295d230a29", + Path: "oss-fuzz-7", + }, + contentType: "application/json", + expectedCode: http.StatusNotFound, + }, + { + name: "Normal request", + reqBody: &pb.FileContentRequest{ + Url: "https://github.com/oliverchang/osv-test.git", + Commit: "b9b3fd4732695b83c3068b7b6a14bb372ec31f98", + Path: "abc", + }, + contentType: "application/json", + expectedCode: http.StatusOK, + expectedResponse: &pb.FileContentResponse{ + Content: []byte("abcd\n"), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var bodyBytes []byte + if tt.rawBody != nil { + bodyBytes = tt.rawBody + } else if tt.reqBody != nil { + var err error + if tt.contentType == "application/json" { + bodyBytes, err = protojson.Marshal(tt.reqBody) + } else { + bodyBytes, err = proto.Marshal(tt.reqBody) + } + if err != nil { + t.Fatalf("failed to marshal request: %v", err) + } + } + + req, err := http.NewRequest(http.MethodPost, "/file-content", bytes.NewReader(bodyBytes)) + if err != nil { + t.Fatal(err) + } + if tt.contentType != "" { + req.Header.Set("Content-Type", tt.contentType) + } + + rr := httptest.NewRecorder() + fileContentHandler(rr, req) + + if status := rr.Code; status != tt.expectedCode { + t.Errorf("fileContentHandler returned wrong status code: got %v want %v", status, tt.expectedCode) + } + + if tt.expectedResponse != nil { + resp := &pb.FileContentResponse{} + var err error + if tt.contentType == "application/json" { + err = protojson.Unmarshal(rr.Body.Bytes(), resp) + } else { + err = proto.Unmarshal(rr.Body.Bytes(), resp) + } + if err != nil { + t.Fatalf("failed to unmarshal response: %v", err) + } + if diff := cmp.Diff(tt.expectedResponse, resp, protocmp.Transform()); diff != "" { + t.Errorf("fileContentHandler returned wrong response: -want +got:\n%s", diff) + } + } + }) + } +} diff --git a/go/cmd/gitter/repository.go b/go/cmd/gitter/repository.go index 1b9e312c0e9..9a7edf91f72 100644 --- a/go/cmd/gitter/repository.go +++ b/go/cmd/gitter/repository.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "sync" "time" @@ -19,6 +20,8 @@ import ( "golang.org/x/sync/errgroup" ) +const emptyTreeSHA1 = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" + type SHA1 [20]byte type Commit struct { @@ -969,3 +972,182 @@ func (r *Repository) GetRemoteTags(ctx context.Context) (map[string]SHA1, error) return r.runAndParseTags(ctx, cmd) } + +// FileChange represents change in a file between two git commits +type FileChange struct { + From string + To string +} + +// resolveCommit resolves a branch or (abbreviated) commit SHA to its raw 20-byte SHA-1. +func (r *Repository) resolveCommit(ctx context.Context, ref string) (string, error) { + if strings.TrimSpace(ref) == "" { + return "", errors.New("ref cannot be empty") + } + cmd := prepareCmd(ctx, r.repoPath, nil, "git", "rev-parse", "--verify", "--quiet", ref+"^{commit}") + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to run git rev-parse on %q: %w, output: %s", ref, err, out) + } + + return strings.TrimSpace(string(out)), nil +} + +// ListFileDiffs lists the files that changed between lastSyncCommit and targetBranch's latest commit. +func (r *Repository) ListFileDiffs(ctx context.Context, lastSyncCommit, targetBranch string) (string, []*FileChange, error) { + logger.DebugContext(ctx, "Starting file diff calculation", slog.String("last_synced_commit", lastSyncCommit), slog.String("branch", targetBranch)) + start := time.Now() + + ref := strings.TrimSpace(targetBranch) + if ref == "" { + // Defaults to the default branch of the repository + ref = "origin/HEAD" + } else if !strings.HasPrefix(ref, "origin/") && !strings.HasPrefix(ref, "refs/") { + // For remote repositories, branch references are usually under refs/remotes/origin/. + // But git revisions resolves by matching refs/heads/, refs/remotes/, etc. + ref = "origin/" + ref + } + + latestCommit, err := r.resolveCommit(ctx, ref) + if err != nil { + // If ref didn't resolve (e.g. in local test repos, tags, or full SHAs), try resolving ref directly + var errDirect error + latestCommit, errDirect = r.resolveCommit(ctx, strings.TrimSpace(targetBranch)) + if errDirect != nil { + return "", nil, fmt.Errorf("failed to resolve target ref %q: %w", targetBranch, err) + } + } + + fromCommit := lastSyncCommit + if fromCommit == "" { + // If lastSyncCommit is empty, diff against empty tree object. + fromCommit = emptyTreeSHA1 + } else { + resolved, err := r.resolveCommit(ctx, fromCommit) + if err != nil { + return "", nil, fmt.Errorf("last_synced_commit %q not found or invalid: %w", fromCommit, err) + } + fromCommit = resolved + } + + // `git diff-tree -r --find-renames --no-commit-id --name-status ` + // -r: recursive + // --find-renames: detect renames + // --no-commit-id: don't print the commit id on a separate line + // --name-status: only show status (A|C|D|M|R|T|U|X|B) and the file path(s) + cmd := prepareCmd(ctx, r.repoPath, nil, "git", "diff-tree", "-r", "--find-renames", "--no-commit-id", "--name-status", fromCommit, latestCommit) + out, err := cmd.CombinedOutput() + if err != nil { + return "", nil, fmt.Errorf("git diff-tree failed: %w, output: %s", err, out) + } + + diffs := parseDiffTree(ctx, out) + + logger.DebugContext(ctx, "File diff calculation completed", slog.Int("changes_count", len(diffs)), slog.Duration("duration", time.Since(start))) + + return latestCommit, diffs, nil +} + +// parseDiffTree parses the output of git diff-tree into list of FileChange. +func parseDiffTree(ctx context.Context, output []byte) []*FileChange { + lines := strings.Split(string(output), "\n") + changes := make([]*FileChange, 0, len(lines)) + + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + change, err := parseNameStatusLine(line) + if err != nil { + logger.WarnContext(ctx, "Failed to parse diff-tree line", slog.String("line", line), slog.Any("error", err)) + continue + } + changes = append(changes, change) + } + + return changes +} + +// parseNameStatusLine parses a single tab-delimited line of git diff-tree --name-status output. +// Format is: \t OR \t\t +func parseNameStatusLine(line string) (*FileChange, error) { + fields := strings.Split(line, "\t") + if len(fields) < 2 || len(fields[0]) == 0 { + return nil, fmt.Errorf("invalid diff-tree line: %q", line) + } + + /** + Reference: https://git-scm.com/docs/diff-format + Possible status letters are: + A: addition of a file + C: copy of a file into a new one + D: deletion of a file + M: modification of the contents or mode of a file + R: renaming of a file + T: change in the type of the file (regular file, symbolic link or submodule) + U: file is unmerged (you must complete the merge before it can be committed) + X: "unknown" change type (most probably a bug, please report it) + + Status letters C and R are always followed by a score (denoting the percentage of similarity between the source and target of the move or copy). + Status letter M may be followed by a score (denoting the percentage of dissimilarity) for file rewrites. + **/ + statusLetter := fields[0][0] // We only want the first char of the status + + switch statusLetter { + case 'A': + return &FileChange{From: "", To: cleanPath(fields[1])}, nil + case 'C': + if len(fields) < 3 { + return nil, fmt.Errorf("copy missing dst path: %q", line) + } + + return &FileChange{From: "", To: cleanPath(fields[2])}, nil + case 'D': + return &FileChange{From: cleanPath(fields[1]), To: ""}, nil + case 'M', 'T': + path := cleanPath(fields[1]) + return &FileChange{From: path, To: path}, nil + case 'R': + if len(fields) < 3 { + return nil, fmt.Errorf("rename missing dst path: %q", line) + } + + return &FileChange{From: cleanPath(fields[1]), To: cleanPath(fields[2])}, nil + default: + // "U" and "X" are not expected, return with error message + return nil, fmt.Errorf("unknown diff status %q: %q", statusLetter, line) + } +} + +// cleanPath unquotes and un-escape paths that Git determines as "unusual" +func cleanPath(p string) string { + p = strings.TrimSpace(p) + // strconv.Unquote conveniently handles everything: + // 1. Remove leading / trailing double quotes + // 2. Unescape C-style backslash escapes (\n, \t, \\, \", \', etc.) + // 3. Unescape bytes >= 0x80 to UTF-8 (handles emojis and special chars) + if unquoted, err := strconv.Unquote(p); err == nil { + return unquoted + } + + return p +} + +// GetFileContent retrieves the raw bytes of a file at a specific commit using git cat-file. +func (r *Repository) GetFileContent(ctx context.Context, ref, path string) ([]byte, error) { + // Resolve ref (branch / (abbreviated) commit hash) to full commit hash + commit, err := r.resolveCommit(ctx, ref) + if err != nil { + return nil, fmt.Errorf("commit %q not found or invalid: %w", ref, err) + } + + cmd := prepareCmd(ctx, r.repoPath, nil, "git", "cat-file", "blob", commit+":"+path) + out, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("git cat-file failed for %s:%s: %w, output: %s", commit, path, err, out) + } + + return out, nil +} diff --git a/go/cmd/gitter/repository_test.go b/go/cmd/gitter/repository_test.go index 4986d283c2d..b721cbb080c 100644 --- a/go/cmd/gitter/repository_test.go +++ b/go/cmd/gitter/repository_test.go @@ -23,7 +23,7 @@ func runGit(t *testing.T, repoPath string, args ...string) { } // A very simple test repository with 3 commits and 2 tags. -func setupTestRepo(t *testing.T, url string) string { +func setupTagsTestRepo(t *testing.T, url string) string { t.Helper() gitStorePath = t.TempDir() repoPath := filepath.Join(gitStorePath, getRepoDirName(url)) @@ -31,7 +31,7 @@ func setupTestRepo(t *testing.T, url string) string { t.Fatalf("failed to create repo path %s: %v", repoPath, err) } - runGit(t, repoPath, "init") + runGit(t, repoPath, "init", "-b", "main") runGit(t, repoPath, "config", "user.email", "test@test.com") runGit(t, repoPath, "config", "user.name", "Test Name") @@ -73,7 +73,7 @@ func setupEmptyTestRepo(t *testing.T, url string) string { t.Fatalf("failed to create repo path %s: %v", repoPath, err) } - runGit(t, repoPath, "init") + runGit(t, repoPath, "init", "-b", "main") runGit(t, repoPath, "config", "user.email", "test@test.com") runGit(t, repoPath, "config", "user.name", "Test Name") @@ -87,8 +87,115 @@ func setupEmptyTestRepo(t *testing.T, url string) string { return url } +// setupFileDiffsTestRepo sets up a comprehensive test git repository containing: +// - Multiple branches (main, feature-branch) and tags (v1.0.0, v2.0.0) +// - Various git change types: addition (A), deletion (D), modification (M), rename (R), copy (C), type change (T) +// - Special character pathnames (spaces, quotes, tabs, Unicode/UTF-8 emojis) to test -z NUL handling +func setupFileDiffsTestRepo(t *testing.T, url string) string { + t.Helper() + gitStorePath = t.TempDir() + repoPath := filepath.Join(gitStorePath, getRepoDirName(url)) + if err := os.MkdirAll(filepath.Join(repoPath, "subfolder"), 0755); err != nil { + t.Fatalf("failed to create repo path %s: %v", repoPath, err) + } + + runGit(t, repoPath, "init", "-b", "main") + runGit(t, repoPath, "config", "user.email", "test@test.com") + runGit(t, repoPath, "config", "user.name", "Test Name") + + // ------------------------------------------------------------------------- + // Commit 1: Baseline setup on main branch + // ------------------------------------------------------------------------- + // Create baseline files that will be modified, deleted, renamed, copied, or type-changed later. + err := os.WriteFile(filepath.Join(repoPath, "modified_file.txt"), []byte("initial modified content"), 0600) + if err != nil { + t.Fatalf("failed to write modified_file.txt: %v", err) + } + err = os.WriteFile(filepath.Join(repoPath, "deleted_file.txt"), []byte("content to be deleted"), 0600) + if err != nil { + t.Fatalf("failed to write deleted_file.txt: %v", err) + } + err = os.WriteFile(filepath.Join(repoPath, "old_name.txt"), []byte("content to be renamed"), 0600) + if err != nil { + t.Fatalf("failed to write old_name.txt: %v", err) + } + err = os.WriteFile(filepath.Join(repoPath, "source_file.txt"), []byte("content to be copied"), 0600) + if err != nil { + t.Fatalf("failed to write source_file.txt: %v", err) + } + err = os.WriteFile(filepath.Join(repoPath, "type_changed.txt"), []byte("regular file to symlink"), 0600) + if err != nil { + t.Fatalf("failed to write type_changed.txt: %v", err) + } + + runGit(t, repoPath, "add", ".") + runGit(t, repoPath, "commit", "-m", "commit 1: baseline") + runGit(t, repoPath, "tag", "v1.0.0") + + // ------------------------------------------------------------------------- + // Commit 2: Switch to feature-branch and perform various git file changes + // ------------------------------------------------------------------------- + runGit(t, repoPath, "checkout", "-b", "feature-branch") + + // 1. Addition (A): Add new files, including special characters (spaces, quotes, tabs, Unicode/UTF-8) + err = os.WriteFile(filepath.Join(repoPath, "added_file.txt"), []byte("newly added content"), 0600) + if err != nil { + t.Fatalf("failed to write added_file.txt: %v", err) + } + err = os.WriteFile(filepath.Join(repoPath, "Spaces &\"quotes\" in name.txt"), []byte("special char content"), 0600) + if err != nil { + t.Fatalf("failed to write special char file: %v", err) + } + err = os.WriteFile(filepath.Join(repoPath, "unicode_filename_utf8_🔥.json"), []byte("{\"status\": \"ok\"}"), 0600) + if err != nil { + t.Fatalf("failed to write unicode file: %v", err) + } + err = os.WriteFile(filepath.Join(repoPath, "subfolder", "nested.txt"), []byte("subfolder file content"), 0600) + if err != nil { + t.Fatalf("failed to write subfolder/nested.txt: %v", err) + } + + // 2. Modification (M): Change content of modified_file.txt + err = os.WriteFile(filepath.Join(repoPath, "modified_file.txt"), []byte("updated modified content"), 0600) + if err != nil { + t.Fatalf("failed to update modified_file.txt: %v", err) + } + + // 3. Deletion (D): Delete deleted_file.txt + runGit(t, repoPath, "rm", "deleted_file.txt") + + // 4. Rename (R): Rename old_name.txt -> new_name.txt + runGit(t, repoPath, "mv", "old_name.txt", "new_name.txt") + + // 5. Copy (C): Duplicate source_file.txt into copied_file.txt + err = os.WriteFile(filepath.Join(repoPath, "copied_file.txt"), []byte("content to be copied"), 0600) + if err != nil { + t.Fatalf("failed to write copied_file.txt: %v", err) + } + + // 6. Type change (T): Convert type_changed.txt into a symlink pointing to modified_file.txt + _ = os.Remove(filepath.Join(repoPath, "type_changed.txt")) + if err := os.Symlink("modified_file.txt", filepath.Join(repoPath, "type_changed.txt")); err != nil { + t.Fatalf("failed to create symlink: %v", err) + } + + runGit(t, repoPath, "add", ".") + runGit(t, repoPath, "commit", "-m", "commit 2: feature changes") + runGit(t, repoPath, "tag", "v2.0.0") + + // ------------------------------------------------------------------------- + // Switch back to main and set HEAD + // ------------------------------------------------------------------------- + runGit(t, repoPath, "checkout", "main") + // Symbolic ref origin/HEAD would have been set to remote's default branch when it is cloned + // But in our local test repo, we need to explicitly set this ref + runGit(t, repoPath, "symbolic-ref", "refs/remotes/origin/HEAD", "refs/heads/main") + + return url +} + func TestBuildCommitGraph(t *testing.T) { - url := setupTestRepo(t, "git://test-repo.git") + url := setupTagsTestRepo(t, "git://test-repo.git") r := NewRepository(url) ctx := context.WithValue(t.Context(), urlKey, url) @@ -113,7 +220,7 @@ func TestBuildCommitGraph(t *testing.T) { } func TestCalculatePatchIDs(t *testing.T) { - url := setupTestRepo(t, "git://test-repo.git") + url := setupTagsTestRepo(t, "git://test-repo.git") r := NewRepository(url) ctx := context.WithValue(t.Context(), urlKey, url) @@ -137,7 +244,7 @@ func TestCalculatePatchIDs(t *testing.T) { } func TestLoadRepository(t *testing.T) { - url := setupTestRepo(t, "git://test-repo.git") + url := setupTagsTestRepo(t, "git://test-repo.git") repoPath := filepath.Join(gitStorePath, getRepoDirName(url)) ctx := context.WithValue(t.Context(), urlKey, url) @@ -1231,7 +1338,7 @@ func TestGetLocalTags(t *testing.T) { }{ { name: "Repo with tags", - setupFunc: setupTestRepo, + setupFunc: setupTagsTestRepo, wantTags: []string{"v1.0.0", "v1.1.0"}, wantCount: 2, }, @@ -1276,7 +1383,7 @@ func TestGetRemoteTags(t *testing.T) { }{ { name: "Repo with tags", - setupFunc: setupTestRepo, + setupFunc: setupTagsTestRepo, wantTags: []string{"v1.0.0", "v1.1.0"}, wantCount: 2, }, @@ -1314,3 +1421,326 @@ func TestGetRemoteTags(t *testing.T) { }) } } + +func TestParseNameStatusLine(t *testing.T) { + tests := []struct { + name string + line string + want *FileChange + wantErr bool + }{ + { + name: "Added file", + line: "A\tpath/to/added.txt", + want: &FileChange{From: "", To: "path/to/added.txt"}, + }, + { + name: "Deleted file", + line: "D\tpath/to/deleted.txt", + want: &FileChange{From: "path/to/deleted.txt", To: ""}, + }, + { + name: "Modified file", + line: "M\tpath/to/modified.txt", + want: &FileChange{From: "path/to/modified.txt", To: "path/to/modified.txt"}, + }, + { + name: "Type changed file", + line: "T\tpath/to/symlink", + want: &FileChange{From: "path/to/symlink", To: "path/to/symlink"}, + }, + { + name: "Renamed file", + line: "R100\told/path.txt\tnew/path.txt", + want: &FileChange{From: "old/path.txt", To: "new/path.txt"}, + }, + { + name: "Copied file", + line: "C90\tsrc/path.txt\tdst/path.txt", + want: &FileChange{From: "", To: "dst/path.txt"}, + }, + { + name: "Quoted path with escaped spaces and quotes", + line: "A\t\"Spaces &\\\"quotes\\\" in name.txt\"", + want: &FileChange{From: "", To: "Spaces &\"quotes\" in name.txt"}, + }, + { + name: "Invalid format - missing tab", + line: "A path/to/file", + wantErr: true, + }, + { + name: "Rename missing dst", + line: "R100\told/path.txt", + wantErr: true, + }, + { + name: "Unknown status letter", + line: "X\tpath/to/file", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseNameStatusLine(tt.line) + if (err != nil) != tt.wantErr { + t.Fatalf("parseNameStatusLine() error = %v, wantErr %v", err, tt.wantErr) + } + if !tt.wantErr { + if diff := cmp.Diff(tt.want, got, cmp.AllowUnexported(FileChange{})); diff != "" { + t.Errorf("parseNameStatusLine() mismatch (-want +got):\n%s", diff) + } + } + }) + } +} + +func TestListFileDiffs(t *testing.T) { + url := setupFileDiffsTestRepo(t, "git://test-repo-file-diffs.git") + repoPath := filepath.Join(gitStorePath, getRepoDirName(url)) + r := NewRepository(url) + ctx := context.WithValue(t.Context(), urlKey, repoPath) + + commit1, err := r.resolveCommit(ctx, "v1.0.0") + if err != nil { + t.Fatalf("resolve commit v1.0.0: %v", err) + } + commit2, err := r.resolveCommit(ctx, "v2.0.0") + if err != nil { + t.Fatalf("resolve commit v2.0.0: %v", err) + } + + // Test ListFileDiffs between commit1 (v1.0.0) and branch target (feature-branch) + latestCommit, changes, err := r.ListFileDiffs(ctx, commit1, "feature-branch") + if err != nil { + t.Fatalf("ListFileDiffs failed: %v", err) + } + + if latestCommit != commit2 { + t.Errorf("expected latestCommit %s, got %s", commit2, latestCommit) + } + + // Expected changes in feature-branch since v1.0.0: + // Added (A): added_file.txt, copied_file.txt, Spaces &"quotes" in name.txt, unicode_filename_utf8_🔥.json, subfolder/nested.txt + // Deleted (D): deleted_file.txt + // Modified (M): modified_file.txt + // Renamed (R): old_name.txt -> new_name.txt + // Type changed (T): type_changed.txt -> type_changed.txt + wantChanges := []*FileChange{ + {From: "", To: "added_file.txt"}, + {From: "", To: "copied_file.txt"}, + {From: "", To: "Spaces &\"quotes\" in name.txt"}, + {From: "", To: "unicode_filename_utf8_🔥.json"}, + {From: "", To: "subfolder/nested.txt"}, + {From: "deleted_file.txt", To: ""}, + {From: "modified_file.txt", To: "modified_file.txt"}, + {From: "old_name.txt", To: "new_name.txt"}, + {From: "type_changed.txt", To: "type_changed.txt"}, + } + + opts := cmpopts.SortSlices(func(a, b *FileChange) bool { + if a.From != b.From { + return a.From < b.From + } + + return a.To < b.To + }) + + if diff := cmp.Diff(wantChanges, changes, opts, cmp.AllowUnexported(FileChange{})); diff != "" { + t.Errorf("ListFileDiffs changes mismatch (-want +got):\n%s", diff) + } + + // Test ListFileDiffs with empty lastSyncCommit (diff against empty tree) + _, emptyCommitChanges, err := r.ListFileDiffs(ctx, "", "feature-branch") + if err != nil { + t.Fatalf("ListFileDiffs with empty lastSyncCommit failed: %v", err) + } + if len(emptyCommitChanges) == 0 { + t.Errorf("expected non-empty changes for empty lastSyncCommit, got 0") + } + for _, c := range emptyCommitChanges { + if c.From != "" { + t.Errorf("expected From path to be empty when diffing against empty tree, got %q", c.From) + } + } + + // Test with invalid lastSyncCommit + _, _, err = r.ListFileDiffs(ctx, "invalidhash123", "feature-branch") + if err == nil { + t.Error("expected error for invalid lastSyncCommit, got nil") + } +} + +func TestGetFileContent(t *testing.T) { + url := setupFileDiffsTestRepo(t, "git://test-repo-content.git") + repoPath := filepath.Join(gitStorePath, getRepoDirName(url)) + r := NewRepository(url) + ctx := context.WithValue(t.Context(), urlKey, repoPath) + + commit1, err := r.resolveCommit(ctx, "v1.0.0") + if err != nil { + t.Fatalf("resolve v1.0.0: %v", err) + } + commit2, err := r.resolveCommit(ctx, "v2.0.0") + if err != nil { + t.Fatalf("resolve v2.0.0: %v", err) + } + + tests := []struct { + name string + ref string + path string + wantContent string + wantErr bool + }{ + { + name: "Fetch file content at v1.0.0 baseline", + ref: commit1, + path: "modified_file.txt", + wantContent: "initial modified content", + wantErr: false, + }, + { + name: "Fetch modified file content at v2.0.0", + ref: commit2, + path: "modified_file.txt", + wantContent: "updated modified content", + wantErr: false, + }, + { + name: "Fetch file with spaces and quotes in name", + ref: commit2, + path: "Spaces &\"quotes\" in name.txt", + wantContent: "special char content", + wantErr: false, + }, + { + name: "Fetch file with Unicode UTF-8 emoji in name", + ref: commit2, + path: "unicode_filename_utf8_🔥.json", + wantContent: "{\"status\": \"ok\"}", + wantErr: false, + }, + { + name: "Fetch nested file in subfolder", + ref: commit2, + path: "subfolder/nested.txt", + wantContent: "subfolder file content", + wantErr: false, + }, + { + name: "Fetch deleted file at v2.0.0 (should fail)", + ref: commit2, + path: "deleted_file.txt", + wantErr: true, + }, + { + name: "Fetch directory instead of file (should fail)", + ref: commit2, + path: "subfolder", + wantErr: true, + }, + { + name: "Fetch with invalid commit ref (should fail)", + ref: "invalid-ref", + path: "modified_file.txt", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := r.GetFileContent(ctx, tt.ref, tt.path) + if (err != nil) != tt.wantErr { + t.Fatalf("GetFileContent(%q, %q) error = %v, wantErr %v", tt.ref, tt.path, err, tt.wantErr) + } + if !tt.wantErr { + if string(got) != tt.wantContent { + t.Errorf("GetFileContent(%q, %q) got %q, want %q", tt.ref, tt.path, string(got), tt.wantContent) + } + } + }) + } +} + +func TestResolveCommit(t *testing.T) { + url := setupFileDiffsTestRepo(t, "git://test-repo-resolve.git") + repoPath := filepath.Join(gitStorePath, getRepoDirName(url)) + r := NewRepository(url) + ctx := context.WithValue(t.Context(), urlKey, repoPath) + + branchSha, err := r.resolveCommit(ctx, "feature-branch") + if err != nil || len(branchSha) != 40 { + t.Fatalf("setup resolve commit feature-branch failed: %v", err) + } + + tests := []struct { + name string + ref string + wantSha string + wantErr bool + }{ + { + name: "Resolve tag", + ref: "v1.0.0", + wantErr: false, + }, + { + name: "Resolve full branch ref", + ref: "refs/heads/feature-branch", + wantSha: branchSha, + wantErr: false, + }, + { + name: "Resolve short branch name", + ref: "feature-branch", + wantSha: branchSha, + wantErr: false, + }, + { + name: "Resolve full 40-char commit SHA", + ref: branchSha, + wantSha: branchSha, + wantErr: false, + }, + { + name: "Resolve abbr commit SHA", + ref: branchSha[:7], + wantSha: branchSha, + wantErr: false, + }, + { + name: "Resolve origin/HEAD", + ref: "origin/HEAD", + wantErr: false, + }, + { + name: "Resolve nonexistent branch name", + ref: "nonexistent-branch", + wantErr: true, + }, + { + name: "Resolve empty ref string", + ref: "", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := r.resolveCommit(ctx, tt.ref) + if (err != nil) != tt.wantErr { + t.Fatalf("ResolveCommit(%q) error = %v, wantErr %v", tt.ref, err, tt.wantErr) + } + if !tt.wantErr { + if len(got) != 40 { + t.Errorf("ResolveCommit(%q) returned SHA length %d, expected 40", tt.ref, len(got)) + } + if tt.wantSha != "" && got != tt.wantSha { + t.Errorf("ResolveCommit(%q) got %q, want %q", tt.ref, got, tt.wantSha) + } + } + }) + } +} diff --git a/go/internal/gitter/pb/repository/repository.pb.go b/go/internal/gitter/pb/repository/repository.pb.go index c8912bcea61..1539c52e532 100644 --- a/go/internal/gitter/pb/repository/repository.pb.go +++ b/go/internal/gitter/pb/repository/repository.pb.go @@ -583,6 +583,274 @@ func (x *AffectedCommitsRequest) GetRefId() string { return "" } +type FileChange struct { + state protoimpl.MessageState `protogen:"open.v1"` + FromPath string `protobuf:"bytes,1,opt,name=from_path,json=fromPath,proto3" json:"from_path,omitempty"` + ToPath string `protobuf:"bytes,2,opt,name=to_path,json=toPath,proto3" json:"to_path,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileChange) Reset() { + *x = FileChange{} + mi := &file_repository_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileChange) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileChange) ProtoMessage() {} + +func (x *FileChange) ProtoReflect() protoreflect.Message { + mi := &file_repository_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileChange.ProtoReflect.Descriptor instead. +func (*FileChange) Descriptor() ([]byte, []int) { + return file_repository_proto_rawDescGZIP(), []int{9} +} + +func (x *FileChange) GetFromPath() string { + if x != nil { + return x.FromPath + } + return "" +} + +func (x *FileChange) GetToPath() string { + if x != nil { + return x.ToPath + } + return "" +} + +type FileDiffsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` + LastSyncedCommit string `protobuf:"bytes,2,opt,name=last_synced_commit,json=lastSyncedCommit,proto3" json:"last_synced_commit,omitempty"` + Branch string `protobuf:"bytes,3,opt,name=branch,proto3" json:"branch,omitempty"` // Optional, defaults to remote HEAD (origin/HEAD) if not specified + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileDiffsRequest) Reset() { + *x = FileDiffsRequest{} + mi := &file_repository_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileDiffsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileDiffsRequest) ProtoMessage() {} + +func (x *FileDiffsRequest) ProtoReflect() protoreflect.Message { + mi := &file_repository_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileDiffsRequest.ProtoReflect.Descriptor instead. +func (*FileDiffsRequest) Descriptor() ([]byte, []int) { + return file_repository_proto_rawDescGZIP(), []int{10} +} + +func (x *FileDiffsRequest) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *FileDiffsRequest) GetLastSyncedCommit() string { + if x != nil { + return x.LastSyncedCommit + } + return "" +} + +func (x *FileDiffsRequest) GetBranch() string { + if x != nil { + return x.Branch + } + return "" +} + +type FileDiffsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + LatestCommit string `protobuf:"bytes,1,opt,name=latest_commit,json=latestCommit,proto3" json:"latest_commit,omitempty"` + Changes []*FileChange `protobuf:"bytes,2,rep,name=changes,proto3" json:"changes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileDiffsResponse) Reset() { + *x = FileDiffsResponse{} + mi := &file_repository_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileDiffsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileDiffsResponse) ProtoMessage() {} + +func (x *FileDiffsResponse) ProtoReflect() protoreflect.Message { + mi := &file_repository_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileDiffsResponse.ProtoReflect.Descriptor instead. +func (*FileDiffsResponse) Descriptor() ([]byte, []int) { + return file_repository_proto_rawDescGZIP(), []int{11} +} + +func (x *FileDiffsResponse) GetLatestCommit() string { + if x != nil { + return x.LatestCommit + } + return "" +} + +func (x *FileDiffsResponse) GetChanges() []*FileChange { + if x != nil { + return x.Changes + } + return nil +} + +type FileContentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` + Commit string `protobuf:"bytes,2,opt,name=commit,proto3" json:"commit,omitempty"` + Path string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileContentRequest) Reset() { + *x = FileContentRequest{} + mi := &file_repository_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileContentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileContentRequest) ProtoMessage() {} + +func (x *FileContentRequest) ProtoReflect() protoreflect.Message { + mi := &file_repository_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileContentRequest.ProtoReflect.Descriptor instead. +func (*FileContentRequest) Descriptor() ([]byte, []int) { + return file_repository_proto_rawDescGZIP(), []int{12} +} + +func (x *FileContentRequest) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *FileContentRequest) GetCommit() string { + if x != nil { + return x.Commit + } + return "" +} + +func (x *FileContentRequest) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +type FileContentResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Content []byte `protobuf:"bytes,1,opt,name=content,proto3" json:"content,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileContentResponse) Reset() { + *x = FileContentResponse{} + mi := &file_repository_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileContentResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileContentResponse) ProtoMessage() {} + +func (x *FileContentResponse) ProtoReflect() protoreflect.Message { + mi := &file_repository_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileContentResponse.ProtoReflect.Descriptor instead. +func (*FileContentResponse) Descriptor() ([]byte, []int) { + return file_repository_proto_rawDescGZIP(), []int{13} +} + +func (x *FileContentResponse) GetContent() []byte { + if x != nil { + return x.Content + } + return nil +} + var File_repository_proto protoreflect.FileDescriptor const file_repository_proto_rawDesc = "" + @@ -620,7 +888,24 @@ const file_repository_proto_rawDesc = "" + "\x18detect_cherrypicks_limit\x18\x05 \x01(\bR\x16detectCherrypicksLimit\x12!\n" + "\fforce_update\x18\x06 \x01(\bR\vforceUpdate\x122\n" + "\x15consider_all_branches\x18\a \x01(\bR\x13considerAllBranches\x12\x15\n" + - "\x06ref_id\x18\b \x01(\tR\x05refId*D\n" + + "\x06ref_id\x18\b \x01(\tR\x05refId\"B\n" + + "\n" + + "FileChange\x12\x1b\n" + + "\tfrom_path\x18\x01 \x01(\tR\bfromPath\x12\x17\n" + + "\ato_path\x18\x02 \x01(\tR\x06toPath\"j\n" + + "\x10FileDiffsRequest\x12\x10\n" + + "\x03url\x18\x01 \x01(\tR\x03url\x12,\n" + + "\x12last_synced_commit\x18\x02 \x01(\tR\x10lastSyncedCommit\x12\x16\n" + + "\x06branch\x18\x03 \x01(\tR\x06branch\"f\n" + + "\x11FileDiffsResponse\x12#\n" + + "\rlatest_commit\x18\x01 \x01(\tR\flatestCommit\x12,\n" + + "\achanges\x18\x02 \x03(\v2\x12.gitter.FileChangeR\achanges\"R\n" + + "\x12FileContentRequest\x12\x10\n" + + "\x03url\x18\x01 \x01(\tR\x03url\x12\x16\n" + + "\x06commit\x18\x02 \x01(\tR\x06commit\x12\x12\n" + + "\x04path\x18\x03 \x01(\tR\x04path\"/\n" + + "\x13FileContentResponse\x12\x18\n" + + "\acontent\x18\x01 \x01(\fR\acontent*D\n" + "\tEventType\x12\x0e\n" + "\n" + "INTRODUCED\x10\x00\x12\t\n" + @@ -641,7 +926,7 @@ func file_repository_proto_rawDescGZIP() []byte { } var file_repository_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_repository_proto_msgTypes = make([]protoimpl.MessageInfo, 9) +var file_repository_proto_msgTypes = make([]protoimpl.MessageInfo, 14) var file_repository_proto_goTypes = []any{ (EventType)(0), // 0: gitter.EventType (*CommitDetail)(nil), // 1: gitter.CommitDetail @@ -653,20 +938,26 @@ var file_repository_proto_goTypes = []any{ (*Event)(nil), // 7: gitter.Event (*CacheRequest)(nil), // 8: gitter.CacheRequest (*AffectedCommitsRequest)(nil), // 9: gitter.AffectedCommitsRequest + (*FileChange)(nil), // 10: gitter.FileChange + (*FileDiffsRequest)(nil), // 11: gitter.FileDiffsRequest + (*FileDiffsResponse)(nil), // 12: gitter.FileDiffsResponse + (*FileContentRequest)(nil), // 13: gitter.FileContentRequest + (*FileContentResponse)(nil), // 14: gitter.FileContentResponse } var file_repository_proto_depIdxs = []int32{ - 1, // 0: gitter.RepositoryCache.commits:type_name -> gitter.CommitDetail - 3, // 1: gitter.AffectedCommitsResponse.commits:type_name -> gitter.Commit - 4, // 2: gitter.AffectedCommitsResponse.tags:type_name -> gitter.Ref - 7, // 3: gitter.AffectedCommitsResponse.cherry_picked_events:type_name -> gitter.Event - 4, // 4: gitter.TagsResponse.tags:type_name -> gitter.Ref - 0, // 5: gitter.Event.event_type:type_name -> gitter.EventType - 7, // 6: gitter.AffectedCommitsRequest.events:type_name -> gitter.Event - 7, // [7:7] is the sub-list for method output_type - 7, // [7:7] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name + 1, // 0: gitter.RepositoryCache.commits:type_name -> gitter.CommitDetail + 3, // 1: gitter.AffectedCommitsResponse.commits:type_name -> gitter.Commit + 4, // 2: gitter.AffectedCommitsResponse.tags:type_name -> gitter.Ref + 7, // 3: gitter.AffectedCommitsResponse.cherry_picked_events:type_name -> gitter.Event + 4, // 4: gitter.TagsResponse.tags:type_name -> gitter.Ref + 0, // 5: gitter.Event.event_type:type_name -> gitter.EventType + 7, // 6: gitter.AffectedCommitsRequest.events:type_name -> gitter.Event + 10, // 7: gitter.FileDiffsResponse.changes:type_name -> gitter.FileChange + 8, // [8:8] is the sub-list for method output_type + 8, // [8:8] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name } func init() { file_repository_proto_init() } @@ -680,7 +971,7 @@ func file_repository_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_repository_proto_rawDesc), len(file_repository_proto_rawDesc)), NumEnums: 1, - NumMessages: 9, + NumMessages: 14, NumExtensions: 0, NumServices: 0, }, diff --git a/go/internal/gitter/pb/repository/repository.proto b/go/internal/gitter/pb/repository/repository.proto index e4ec46a18af..b1dd08a5ec0 100644 --- a/go/internal/gitter/pb/repository/repository.proto +++ b/go/internal/gitter/pb/repository/repository.proto @@ -62,3 +62,29 @@ message AffectedCommitsRequest { bool consider_all_branches = 7; string ref_id = 8; // Optional reference ID for tracing } + +message FileChange { + string from_path = 1; + string to_path = 2; +} + +message FileDiffsRequest { + string url = 1; + string last_synced_commit = 2; + string branch = 3; // Optional, defaults to remote HEAD (origin/HEAD) if not specified +} + +message FileDiffsResponse { + string latest_commit = 1; + repeated FileChange changes = 2; +} + +message FileContentRequest { + string url = 1; + string commit = 2; + string path = 3; +} + +message FileContentResponse { + bytes content = 1; +}