From 84a8b96ff6a9cd1ebb4364e463aeb45dab2f4fa2 Mon Sep 17 00:00:00 2001 From: Rob Zolkos Date: Wed, 30 Sep 2026 09:04:48 -0400 Subject: [PATCH 1/5] test: pin operation deadlines after a healthy keyring probe --- credstore/operation_test.go | 130 ++++++++++++++++++++++++++++++++++++ credstore/probe.go | 5 +- credstore/store.go | 15 +++-- 3 files changed, 143 insertions(+), 7 deletions(-) create mode 100644 credstore/operation_test.go diff --git a/credstore/operation_test.go b/credstore/operation_test.go new file mode 100644 index 0000000..af13c70 --- /dev/null +++ b/credstore/operation_test.go @@ -0,0 +1,130 @@ +package credstore + +import ( + "context" + "errors" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func swapKeyringOperations(t *testing.T, get func(string, string) (string, error), set func(string, string, string) error, del func(string, string) error) { + t.Helper() + oldGet, oldSet, oldDelete := keyringGet, keyringSet, keyringDelete + keyringGet, keyringSet, keyringDelete = get, set, del + t.Cleanup(func() { keyringGet, keyringSet, keyringDelete = oldGet, oldSet, oldDelete }) +} + +// A responsive probe does not promise that the next real keyring operation +// will respond. Timing out must not serve a stale file, write plaintext, or +// start another operation alongside a write/delete whose outcome is unknown. +func TestKeyringOperationTimeout(t *testing.T) { + for _, operation := range []string{"load", "save", "delete", "migrate"} { + t.Run(operation, func(t *testing.T) { + dir := t.TempDir() + file := NewStore(StoreOptions{ForceFile: true, FallbackDir: dir}) + require.NoError(t, file.Save("work", []byte(`{"access_token":"stale-file-token"}`))) + before, err := os.ReadFile(file.credentialsPath()) + require.NoError(t, err) + + stubProbe(t, func(string, time.Duration) error { return nil }) + var calls atomic.Int32 + release, finished := make(chan struct{}), make(chan struct{}) + block := func() { + calls.Add(1) + <-release + close(finished) + } + swapKeyringOperations(t, + func(string, string) (string, error) { block(); return "keyring-token", nil }, + func(string, string, string) error { block(); return nil }, + func(string, string) error { block(); return nil }, + ) + var releaseOnce sync.Once + t.Cleanup(func() { + releaseOnce.Do(func() { close(release) }) + if calls.Load() > 0 { + <-finished + } + }) + store := NewStore(StoreOptions{ServiceName: "test", FallbackDir: dir, OperationTimeout: 40 * time.Millisecond}) + result := make(chan error, 1) + go func() { + switch operation { + case "load": + data, err := store.Load("work") + if len(data) != 0 { + err = errors.New("timed-out read returned credential data") + } + result <- err + case "save": + result <- store.Save("work", []byte("new-token")) + case "delete": + result <- store.Delete("work") + case "migrate": + result <- store.MigrateToKeyring() + } + }() + select { + case err := <-result: + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.ErrorContains(t, err, "keyring") + case <-time.After(time.Second): + releaseOnce.Do(func() { close(release) }) + <-result + t.Fatal("keyring operation ignored OperationTimeout") + } + + assert.True(t, store.UsingKeyring(), "a timed-out operation must not switch to plaintext") + assert.Empty(t, store.FallbackWarning()) + _, err = store.Load("work") + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.ErrorIs(t, store.Save("work", []byte("newer-token")), context.DeadlineExceeded) + assert.ErrorIs(t, store.Delete("work"), context.DeadlineExceeded) + assert.EqualValues(t, 1, calls.Load(), "a stalled operation poisons this store, not the process") + after, err := os.ReadFile(file.credentialsPath()) + require.NoError(t, err) + assert.Equal(t, before, after, "timeout must neither change nor remove the fallback file") + }) + } +} + +func TestBoundedKeyringKeepsHealthyResultsAndErrors(t *testing.T) { + stubProbe(t, func(string, time.Duration) error { return nil }) + providerErr := errors.New("keyring locked") + swapKeyringOperations(t, + func(service, key string) (string, error) { + assert.Equal(t, "test", service) + assert.Equal(t, "test::work", key) + return "token", nil + }, + func(string, string, string) error { return providerErr }, + func(string, string) error { return nil }, + ) + store := NewStore(StoreOptions{ServiceName: "test", OperationTimeout: time.Second}) + data, err := store.Load("work") + require.NoError(t, err) + assert.Equal(t, "token", string(data)) + assert.ErrorIs(t, store.Save("work", data), providerErr) + assert.NoError(t, store.Delete("work"), "an ordinary provider error must not poison the store") +} + +func TestZeroOperationTimeoutPreservesUnboundedCalls(t *testing.T) { + stubProbe(t, func(string, time.Duration) error { return nil }) + swapKeyringOperations(t, + func(string, string) (string, error) { return "token", nil }, + func(string, string, string) error { return nil }, + func(string, string) error { return nil }, + ) + store := NewStore(StoreOptions{ServiceName: "test"}) + data, err := store.Load("work") + require.NoError(t, err) + assert.Equal(t, "token", string(data)) + assert.NoError(t, store.Save("work", data)) + assert.NoError(t, store.Delete("work")) +} diff --git a/credstore/probe.go b/credstore/probe.go index 4729ccb..f0bb9cb 100644 --- a/credstore/probe.go +++ b/credstore/probe.go @@ -47,9 +47,10 @@ const ( // probeSeq numbers this process's probes so no two share an account. var probeSeq atomic.Uint64 -// keyring operations, extracted as vars so tests can observe the entry -// probeDirect writes and removes without a live keyring. +// keyring operations, extracted as vars so tests can exercise storage and +// probing without a live keyring. var ( + keyringGet = keyring.Get keyringSet = keyring.Set keyringDelete = keyring.Delete ) diff --git a/credstore/store.go b/credstore/store.go index a8af293..6c8f2c2 100644 --- a/credstore/store.go +++ b/credstore/store.go @@ -4,8 +4,6 @@ import ( "fmt" "os" "time" - - "github.com/zalando/go-keyring" ) // StoreOptions configures credential storage. @@ -51,6 +49,13 @@ type StoreOptions struct { // probe goes through go-keyring and honors the mock) or ForceFile. ProbeTimeout time.Duration + // OperationTimeout bounds keyring reads, writes, and deletes after a + // successful probe. Zero or negative preserves unbounded operations. + // A timeout is an error, never a transition to plaintext file storage. + // go-keyring cannot cancel an operation already started: it may finish + // after the timeout, so the store refuses further keyring operations. + OperationTimeout time.Duration + // FallbackDir is the directory for file-based credential storage. FallbackDir string } @@ -116,7 +121,7 @@ func (s *Store) key(name string) string { // the keyring and only this process could not reach it. func (s *Store) Load(key string) ([]byte, error) { if s.useKeyring { - data, err := keyring.Get(s.serviceName, s.key(key)) + data, err := keyringGet(s.serviceName, s.key(key)) if err != nil { return nil, fmt.Errorf("credentials not found: %w", err) } @@ -133,7 +138,7 @@ func (s *Store) Load(key string) ([]byte, error) { // Save stores credentials for the given key. func (s *Store) Save(key string, data []byte) error { if s.useKeyring { - return keyring.Set(s.serviceName, s.key(key), string(data)) + return keyringSet(s.serviceName, s.key(key), string(data)) } return s.saveToFile(key, data) } @@ -141,7 +146,7 @@ func (s *Store) Save(key string, data []byte) error { // Delete removes credentials for the given key. func (s *Store) Delete(key string) error { if s.useKeyring { - return keyring.Delete(s.serviceName, s.key(key)) + return keyringDelete(s.serviceName, s.key(key)) } return s.deleteFromFile(key) } From 60dca1a97803d3f9b84ab46d41679ca4e472fe6d Mon Sep 17 00:00:00 2001 From: Rob Zolkos Date: Wed, 30 Sep 2026 09:09:31 -0400 Subject: [PATCH 2/5] Bound opted-in keyring operations without changing storage backends --- README.md | 21 +++++++++++++++ credstore/operation.go | 51 +++++++++++++++++++++++++++++++++++++ credstore/operation_test.go | 38 ++++++++++++++++++++++----- credstore/store.go | 32 +++++++++++++++++------ 4 files changed, 128 insertions(+), 14 deletions(-) create mode 100644 credstore/operation.go diff --git a/README.md b/README.md index 92c1431..f43e9d3 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,27 @@ This repo is the shared Go toolkit that powers all three CLIs. It provides reusa | `profile` | `github.com/basecamp/cli/profile` | Named environment profiles for targeting different accounts or environments | | `surface` | `github.com/basecamp/cli/surface` | CLI surface snapshot and compatibility diffing for Cobra command trees | +### Credential-store deadlines + +`credstore.StoreOptions` has two independent, opt-in deadlines: + +- `ProbeTimeout` bounds initialization. A failed or timed-out probe selects the + file fallback; callers should surface `FallbackWarning()` on reads and writes. +- `OperationTimeout` bounds keyring reads, writes, deletes, and each migration + write after a successful probe. A timeout returns an error wrapping + `context.DeadlineExceeded`; it never changes the store to plaintext or reads a + potentially stale fallback file. + +Both default to zero (unbounded), so existing consumers retain their behavior. +The operation deadline includes waiting behind another operation on the same +store. File operations are unaffected. + +`go-keyring` cannot cancel an operation that has already started. A timed-out +write or delete may still complete, and this store refuses subsequent keyring +operations even if the abandoned call later succeeds. There is at most one +abandoned operation per store. A new store or process can try again; callers +should not retry an unknown write automatically or treat it as a missing login. + ## Seed templates The `seed/` directory contains templates for bootstrapping a new 37signals Go CLI. Templates include project scaffolding for auth, commands, output formatting, distribution, skills, and CI. diff --git a/credstore/operation.go b/credstore/operation.go new file mode 100644 index 0000000..4f2bc44 --- /dev/null +++ b/credstore/operation.go @@ -0,0 +1,51 @@ +package credstore + +import ( + "context" + "fmt" +) + +// keyringOperation bounds both the wait behind another operation and the call +// itself. go-keyring has no cancellation API: after a timeout its worker may +// still finish. Refusing subsequent operations avoids overlapping an unknown +// write/delete and bounds abandoned workers to one per store. File storage is +// never substituted for a previously working keyring. +func (s *Store) keyringOperation(operation string, fn func() ([]byte, error)) ([]byte, error) { + if s.operationTimeout <= 0 { + return fn() + } + ctx, cancel := context.WithTimeout(context.Background(), s.operationTimeout) + defer cancel() + timeoutError := func() error { + return fmt.Errorf("keyring %s timed out after %s (the operation may still complete): %w", operation, s.operationTimeout, ctx.Err()) + } + select { + case s.operationGate <- struct{}{}: + defer func() { <-s.operationGate }() + case <-ctx.Done(): + return nil, timeoutError() + } + if s.operationErr != nil { + return nil, fmt.Errorf("keyring unavailable after an earlier timeout: %w", s.operationErr) + } + if ctx.Err() != nil { + return nil, timeoutError() + } + + type result struct { + data []byte + err error + } + done := make(chan result, 1) + go func() { + data, err := fn() + done <- result{data, err} + }() + select { + case r := <-done: + return r.data, r.err + case <-ctx.Done(): + s.operationErr = timeoutError() + return nil, s.operationErr + } +} diff --git a/credstore/operation_test.go b/credstore/operation_test.go index af13c70..d261fa0 100644 --- a/credstore/operation_test.go +++ b/credstore/operation_test.go @@ -57,11 +57,11 @@ func TestKeyringOperationTimeout(t *testing.T) { go func() { switch operation { case "load": - data, err := store.Load("work") + data, loadErr := store.Load("work") if len(data) != 0 { - err = errors.New("timed-out read returned credential data") + loadErr = errors.New("timed-out read returned credential data") } - result <- err + result <- loadErr case "save": result <- store.Save("work", []byte("new-token")) case "delete": @@ -71,9 +71,9 @@ func TestKeyringOperationTimeout(t *testing.T) { } }() select { - case err := <-result: - require.ErrorIs(t, err, context.DeadlineExceeded) - assert.ErrorContains(t, err, "keyring") + case operationErr := <-result: + require.ErrorIs(t, operationErr, context.DeadlineExceeded) + assert.ErrorContains(t, operationErr, "keyring") case <-time.After(time.Second): releaseOnce.Do(func() { close(release) }) <-result @@ -90,6 +90,13 @@ func TestKeyringOperationTimeout(t *testing.T) { after, err := os.ReadFile(file.credentialsPath()) require.NoError(t, err) assert.Equal(t, before, after, "timeout must neither change nor remove the fallback file") + + // A late success is not permission to resume writes whose order + // relative to the timed-out write/delete cannot be established. + releaseOnce.Do(func() { close(release) }) + <-finished + assert.ErrorIs(t, store.Save("work", []byte("late-token")), context.DeadlineExceeded) + assert.EqualValues(t, 1, calls.Load()) }) } } @@ -114,6 +121,25 @@ func TestBoundedKeyringKeepsHealthyResultsAndErrors(t *testing.T) { assert.NoError(t, store.Delete("work"), "an ordinary provider error must not poison the store") } +func TestWaitingForKeyringOperationAlsoHasADeadline(t *testing.T) { + stubProbe(t, func(string, time.Duration) error { return nil }) + var calls atomic.Int32 + swapKeyringOperations(t, + func(string, string) (string, error) { calls.Add(1); return "token", nil }, + func(string, string, string) error { return nil }, + func(string, string) error { return nil }, + ) + store := NewStore(StoreOptions{ServiceName: "test", OperationTimeout: 40 * time.Millisecond}) + store.operationGate <- struct{}{} + _, err := store.Load("work") + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.Zero(t, calls.Load(), "a queued call must not reach the provider after its deadline") + <-store.operationGate + data, err := store.Load("work") + require.NoError(t, err, "timing out before calling the provider must not poison the store") + assert.Equal(t, "token", string(data)) +} + func TestZeroOperationTimeoutPreservesUnboundedCalls(t *testing.T) { stubProbe(t, func(string, time.Duration) error { return nil }) swapKeyringOperations(t, diff --git a/credstore/store.go b/credstore/store.go index 6c8f2c2..740edbc 100644 --- a/credstore/store.go +++ b/credstore/store.go @@ -66,6 +66,10 @@ type Store struct { useKeyring bool fallbackDir string + operationTimeout time.Duration + operationGate chan struct{} + operationErr error // guarded by operationGate + // probeErr is why the keyring probe failed when the store fell back to // file storage against the caller's wishes. Nil when the keyring is in // use and when file storage was requested (ForceFile, DisableEnvVar): @@ -82,10 +86,12 @@ func NewStore(opts StoreOptions) *Store { err := probeKeyring(opts.ServiceName, opts.ProbeTimeout) return &Store{ - serviceName: opts.ServiceName, - useKeyring: err == nil, - fallbackDir: opts.FallbackDir, - probeErr: err, + serviceName: opts.ServiceName, + useKeyring: err == nil, + fallbackDir: opts.FallbackDir, + probeErr: err, + operationTimeout: opts.OperationTimeout, + operationGate: make(chan struct{}, 1), } } @@ -121,11 +127,14 @@ func (s *Store) key(name string) string { // the keyring and only this process could not reach it. func (s *Store) Load(key string) ([]byte, error) { if s.useKeyring { - data, err := keyringGet(s.serviceName, s.key(key)) + data, err := s.keyringOperation("read", func() ([]byte, error) { + secret, err := keyringGet(s.serviceName, s.key(key)) + return []byte(secret), err + }) if err != nil { return nil, fmt.Errorf("credentials not found: %w", err) } - return []byte(data), nil + return data, nil } data, err := s.loadFromFile(key) @@ -138,7 +147,11 @@ func (s *Store) Load(key string) ([]byte, error) { // Save stores credentials for the given key. func (s *Store) Save(key string, data []byte) error { if s.useKeyring { - return keyringSet(s.serviceName, s.key(key), string(data)) + secret := string(data) + _, err := s.keyringOperation("write", func() ([]byte, error) { + return nil, keyringSet(s.serviceName, s.key(key), secret) + }) + return err } return s.saveToFile(key, data) } @@ -146,7 +159,10 @@ func (s *Store) Save(key string, data []byte) error { // Delete removes credentials for the given key. func (s *Store) Delete(key string) error { if s.useKeyring { - return keyringDelete(s.serviceName, s.key(key)) + _, err := s.keyringOperation("delete", func() ([]byte, error) { + return nil, keyringDelete(s.serviceName, s.key(key)) + }) + return err } return s.deleteFromFile(key) } From b3a9b86ed5d64c5499059a85d3966977f3a26b3a Mon Sep 17 00:00:00 2001 From: Rob Zolkos Date: Wed, 30 Sep 2026 09:10:50 -0400 Subject: [PATCH 3/5] no-mistakes(review): Clarify keyring timeout errors for reads and queued operations --- credstore/operation.go | 14 ++++++++------ credstore/operation_test.go | 6 +++++- credstore/store.go | 5 +++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/credstore/operation.go b/credstore/operation.go index 4f2bc44..033e8b9 100644 --- a/credstore/operation.go +++ b/credstore/operation.go @@ -3,6 +3,7 @@ package credstore import ( "context" "fmt" + "time" ) // keyringOperation bounds both the wait behind another operation and the call @@ -16,20 +17,17 @@ func (s *Store) keyringOperation(operation string, fn func() ([]byte, error)) ([ } ctx, cancel := context.WithTimeout(context.Background(), s.operationTimeout) defer cancel() - timeoutError := func() error { - return fmt.Errorf("keyring %s timed out after %s (the operation may still complete): %w", operation, s.operationTimeout, ctx.Err()) - } select { case s.operationGate <- struct{}{}: defer func() { <-s.operationGate }() case <-ctx.Done(): - return nil, timeoutError() + return nil, queueTimeoutError(operation, s.operationTimeout, ctx.Err()) } if s.operationErr != nil { return nil, fmt.Errorf("keyring unavailable after an earlier timeout: %w", s.operationErr) } if ctx.Err() != nil { - return nil, timeoutError() + return nil, queueTimeoutError(operation, s.operationTimeout, ctx.Err()) } type result struct { @@ -45,7 +43,11 @@ func (s *Store) keyringOperation(operation string, fn func() ([]byte, error)) ([ case r := <-done: return r.data, r.err case <-ctx.Done(): - s.operationErr = timeoutError() + s.operationErr = fmt.Errorf("keyring %s timed out after %s (the operation may still complete): %w", operation, s.operationTimeout, ctx.Err()) return nil, s.operationErr } } + +func queueTimeoutError(operation string, timeout time.Duration, err error) error { + return fmt.Errorf("keyring %s timed out after %s waiting for another operation (not attempted): %w", operation, timeout, err) +} diff --git a/credstore/operation_test.go b/credstore/operation_test.go index d261fa0..dc0858c 100644 --- a/credstore/operation_test.go +++ b/credstore/operation_test.go @@ -73,7 +73,8 @@ func TestKeyringOperationTimeout(t *testing.T) { select { case operationErr := <-result: require.ErrorIs(t, operationErr, context.DeadlineExceeded) - assert.ErrorContains(t, operationErr, "keyring") + assert.ErrorContains(t, operationErr, "may still complete") + assert.NotContains(t, operationErr.Error(), "not found", "a hung keyring is not a missing login") case <-time.After(time.Second): releaseOnce.Do(func() { close(release) }) <-result @@ -84,6 +85,7 @@ func TestKeyringOperationTimeout(t *testing.T) { assert.Empty(t, store.FallbackWarning()) _, err = store.Load("work") assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.NotContains(t, err.Error(), "not found", "a hung keyring is not a missing login") assert.ErrorIs(t, store.Save("work", []byte("newer-token")), context.DeadlineExceeded) assert.ErrorIs(t, store.Delete("work"), context.DeadlineExceeded) assert.EqualValues(t, 1, calls.Load(), "a stalled operation poisons this store, not the process") @@ -133,6 +135,8 @@ func TestWaitingForKeyringOperationAlsoHasADeadline(t *testing.T) { store.operationGate <- struct{}{} _, err := store.Load("work") assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.ErrorContains(t, err, "waiting for another operation") + assert.NotContains(t, err.Error(), "may still complete") assert.Zero(t, calls.Load(), "a queued call must not reach the provider after its deadline") <-store.operationGate data, err := store.Load("work") diff --git a/credstore/store.go b/credstore/store.go index 740edbc..3bf5d0a 100644 --- a/credstore/store.go +++ b/credstore/store.go @@ -1,6 +1,8 @@ package credstore import ( + "context" + "errors" "fmt" "os" "time" @@ -131,6 +133,9 @@ func (s *Store) Load(key string) ([]byte, error) { secret, err := keyringGet(s.serviceName, s.key(key)) return []byte(secret), err }) + if errors.Is(err, context.DeadlineExceeded) { + return nil, err + } if err != nil { return nil, fmt.Errorf("credentials not found: %w", err) } From 86242a69567b7735ea1bc157b6988c6cf802a465 Mon Sep 17 00:00:00 2001 From: Rob Zolkos Date: Wed, 30 Sep 2026 09:12:53 -0400 Subject: [PATCH 4/5] no-mistakes(document): Move credstore timeout contract into StoreOptions doc comment --- README.md | 24 ++++++------------------ credstore/store.go | 19 ++++++++++++++----- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index f43e9d3..97d74bc 100644 --- a/README.md +++ b/README.md @@ -25,24 +25,12 @@ This repo is the shared Go toolkit that powers all three CLIs. It provides reusa ### Credential-store deadlines -`credstore.StoreOptions` has two independent, opt-in deadlines: - -- `ProbeTimeout` bounds initialization. A failed or timed-out probe selects the - file fallback; callers should surface `FallbackWarning()` on reads and writes. -- `OperationTimeout` bounds keyring reads, writes, deletes, and each migration - write after a successful probe. A timeout returns an error wrapping - `context.DeadlineExceeded`; it never changes the store to plaintext or reads a - potentially stale fallback file. - -Both default to zero (unbounded), so existing consumers retain their behavior. -The operation deadline includes waiting behind another operation on the same -store. File operations are unaffected. - -`go-keyring` cannot cancel an operation that has already started. A timed-out -write or delete may still complete, and this store refuses subsequent keyring -operations even if the abandoned call later succeeds. There is at most one -abandoned operation per store. A new store or process can try again; callers -should not retry an unknown write automatically or treat it as a missing login. +`credstore.StoreOptions` has two independent, opt-in deadlines, both unbounded +by default: `ProbeTimeout` bounds initialization (a failed probe selects the +file fallback), and `OperationTimeout` bounds keyring operations after a +successful probe (a timeout is an error, never a fallback to plaintext). See +their doc comments in [`credstore/store.go`](credstore/store.go) for the full +contract. ## Seed templates diff --git a/credstore/store.go b/credstore/store.go index 3bf5d0a..d03cfc8 100644 --- a/credstore/store.go +++ b/credstore/store.go @@ -51,11 +51,20 @@ type StoreOptions struct { // probe goes through go-keyring and honors the mock) or ForceFile. ProbeTimeout time.Duration - // OperationTimeout bounds keyring reads, writes, and deletes after a - // successful probe. Zero or negative preserves unbounded operations. - // A timeout is an error, never a transition to plaintext file storage. - // go-keyring cannot cancel an operation already started: it may finish - // after the timeout, so the store refuses further keyring operations. + // OperationTimeout bounds keyring reads, writes, and deletes (including + // each MigrateToKeyring write) after a successful probe, independently of + // ProbeTimeout. Zero or negative preserves unbounded operations. File + // storage is never bounded. The deadline includes waiting behind another + // operation on the same store. + // + // A timeout returns an error wrapping context.DeadlineExceeded — never a + // transition to plaintext file storage or a read of a possibly stale + // fallback file. go-keyring cannot cancel an operation already started: + // a timed-out write or delete may still complete, so the store refuses + // all further keyring operations, even if the abandoned call later + // succeeds. This bounds abandoned operations to one per store. A new + // store or process can try again; callers should not retry an unknown + // write automatically or report the timeout as a missing login. OperationTimeout time.Duration // FallbackDir is the directory for file-based credential storage. From afacf59fa6b6bb636ccdbe19900d9ae01934d3dd Mon Sep 17 00:00:00 2001 From: Rob Zolkos Date: Wed, 30 Sep 2026 12:18:18 -0400 Subject: [PATCH 5/5] Prevent late keyring results from bypassing operation deadlines Check the deadline again before accepting a provider result, including when timer delivery lags. Expired results discard data and keep the store unavailable without switching storage backends. Retain the stalled-provider regression with 64 queued Load/Save/Delete callers, and clarify that an unattempted queue timeout does not itself poison the store. --- credstore/operation.go | 28 ++++++- credstore/operation_test.go | 145 ++++++++++++++++++++++++++++++++++++ credstore/store.go | 5 ++ 3 files changed, 174 insertions(+), 4 deletions(-) diff --git a/credstore/operation.go b/credstore/operation.go index 033e8b9..432689d 100644 --- a/credstore/operation.go +++ b/credstore/operation.go @@ -26,8 +26,8 @@ func (s *Store) keyringOperation(operation string, fn func() ([]byte, error)) ([ if s.operationErr != nil { return nil, fmt.Errorf("keyring unavailable after an earlier timeout: %w", s.operationErr) } - if ctx.Err() != nil { - return nil, queueTimeoutError(operation, s.operationTimeout, ctx.Err()) + if err := operationDeadlineError(ctx); err != nil { + return nil, queueTimeoutError(operation, s.operationTimeout, err) } type result struct { @@ -41,11 +41,31 @@ func (s *Store) keyringOperation(operation string, fn func() ([]byte, error)) ([ }() select { case r := <-done: - return r.data, r.err + return s.keyringOperationResult(ctx, operation, r.data, r.err) case <-ctx.Done(): - s.operationErr = fmt.Errorf("keyring %s timed out after %s (the operation may still complete): %w", operation, s.operationTimeout, ctx.Err()) + return s.keyringOperationResult(ctx, operation, nil, ctx.Err()) + } +} + +// keyringOperationResult is called with operationGate held, after dispatching +// the provider call. Deadline expiry wins even when the result was selected first. +func (s *Store) keyringOperationResult(ctx context.Context, operation string, data []byte, providerErr error) ([]byte, error) { + if err := operationDeadlineError(ctx); err != nil { + s.operationErr = fmt.Errorf("keyring %s timed out after %s (the operation may have completed or may still complete): %w", operation, s.operationTimeout, err) return nil, s.operationErr } + return data, providerErr +} + +func operationDeadlineError(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + // Timer delivery can lag behind the deadline when the scheduler is busy. + if deadline, ok := ctx.Deadline(); ok && !time.Now().Before(deadline) { + return context.DeadlineExceeded + } + return nil } func queueTimeoutError(operation string, timeout time.Duration, err error) error { diff --git a/credstore/operation_test.go b/credstore/operation_test.go index dc0858c..da7b3d4 100644 --- a/credstore/operation_test.go +++ b/credstore/operation_test.go @@ -7,6 +7,7 @@ import ( "sync" "sync/atomic" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -158,3 +159,147 @@ func TestZeroOperationTimeoutPreservesUnboundedCalls(t *testing.T) { assert.NoError(t, store.Save("work", data)) assert.NoError(t, store.Delete("work")) } + +func TestKeyringOperationResultRejectsExpiredDeadline(t *testing.T) { + for _, operation := range []string{"read", "write", "delete"} { + for _, timerDelivered := range []bool{true, false} { + name := operation + "/timer-pending" + if timerDelivered { + name = operation + "/timer-delivered" + } + t.Run(name, func(t *testing.T) { + stubProbe(t, func(string, time.Duration) error { return nil }) + var calls atomic.Int32 + swapKeyringOperations(t, nil, + func(string, string, string) error { calls.Add(1); return nil }, nil) + store := NewStore(StoreOptions{ServiceName: "test", OperationTimeout: time.Second}) + + deadline := time.Now().Add(-time.Second) + var ctx context.Context + if timerDelivered { + var cancel context.CancelFunc + ctx, cancel = context.WithDeadline(context.Background(), deadline) + t.Cleanup(cancel) + require.ErrorIs(t, ctx.Err(), context.DeadlineExceeded) + } else { + parent, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + ctx = pendingDeadlineContext{Context: parent, deadline: deadline} + require.NoError(t, ctx.Err()) + } + + // Model selecting the provider's successful result after expiry, + // whether or not the deadline timer has delivered its signal yet. + store.operationGate <- struct{}{} + data, err := store.keyringOperationResult(ctx, operation, []byte("late-token"), nil) + <-store.operationGate + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Empty(t, data) + assert.ErrorContains(t, err, "may have completed") + assert.ErrorIs(t, store.Save("work", []byte("retry")), context.DeadlineExceeded) + assert.Zero(t, calls.Load(), "a late result must not permit another provider call") + assert.True(t, store.UsingKeyring()) + }) + } + } +} + +// pendingDeadlineContext models an expired deadline whose timer callback has +// not yet closed Done or set Err, as can happen when the scheduler is busy. +type pendingDeadlineContext struct { + context.Context + deadline time.Time +} + +func (c pendingDeadlineContext) Deadline() (time.Time, bool) { + return c.deadline, true +} + +func TestConcurrentKeyringCallsBehindStalledProvider(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + stubProbe(t, func(string, time.Duration) error { return nil }) + file := NewStore(StoreOptions{ForceFile: true, FallbackDir: t.TempDir()}) + require.NoError(t, file.Save("work", []byte(`{"access_token":"stale-file-token"}`))) + before, err := os.ReadFile(file.credentialsPath()) + require.NoError(t, err) + + started, release := make(chan struct{}), make(chan struct{}) + var startedOnce, releaseOnce sync.Once + var calls atomic.Int32 + var providers sync.WaitGroup + block := func() { + providers.Add(1) + defer providers.Done() + calls.Add(1) + startedOnce.Do(func() { close(started) }) + <-release + } + swapKeyringOperations(t, + func(string, string) (string, error) { block(); return "keyring-token", nil }, + func(string, string, string) error { block(); return nil }, + func(string, string) error { block(); return nil }, + ) + t.Cleanup(func() { + releaseOnce.Do(func() { close(release) }) + providers.Wait() + }) + + const queued = 64 + const timeout = 40 * time.Millisecond + store := NewStore(StoreOptions{ServiceName: "test", FallbackDir: file.fallbackDir, OperationTimeout: timeout}) + type result struct { + data []byte + err error + } + results := make(chan result, queued+1) + go func() { results <- result{err: store.Save("work", []byte("new-token"))} }() + <-started + synctest.Wait() + + for i := range queued { + go func() { + var r result + switch i % 3 { + case 0: + r.data, r.err = store.Load("work") + case 1: + r.err = store.Save("work", []byte("queued-token")) + case 2: + r.err = store.Delete("work") + } + results <- r + }() + } + // All queued callers are blocked behind the actual provider call + // before fake time advances to their deadlines. + synctest.Wait() + assert.Empty(t, results) + assert.EqualValues(t, 1, calls.Load()) + time.Sleep(timeout) + synctest.Wait() + require.Len(t, results, queued+1, "every caller must return while the provider is still blocked") + for range queued + 1 { + r := <-results + assert.ErrorIs(t, r.err, context.DeadlineExceeded) + assert.Empty(t, r.data, "a queued read must not return stale fallback credentials") + } + assert.EqualValues(t, 1, calls.Load()) + assert.True(t, store.UsingKeyring()) + assert.Empty(t, store.FallbackWarning()) + + // A late successful write must not revive the store or permit any + // queued read/write/delete to reach the provider afterward. + releaseOnce.Do(func() { close(release) }) + providers.Wait() + synctest.Wait() + data, err := store.Load("work") + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.Empty(t, data) + assert.ErrorIs(t, store.Save("work", []byte("retry-token")), context.DeadlineExceeded) + assert.ErrorIs(t, store.Delete("work"), context.DeadlineExceeded) + assert.EqualValues(t, 1, calls.Load()) + after, err := os.ReadFile(file.credentialsPath()) + require.NoError(t, err) + assert.Equal(t, before, after, "the fallback file must remain untouched") + }) +} diff --git a/credstore/store.go b/credstore/store.go index d03cfc8..4807e03 100644 --- a/credstore/store.go +++ b/credstore/store.go @@ -65,6 +65,11 @@ type StoreOptions struct { // succeeds. This bounds abandoned operations to one per store. A new // store or process can try again; callers should not retry an unknown // write automatically or report the timeout as a missing login. + // + // If the deadline expires while waiting behind another operation, the + // queued call is not attempted and cannot complete later. That queue + // timeout does not itself make the store refuse further operations; + // the already-running provider call may independently time out and do so. OperationTimeout time.Duration // FallbackDir is the directory for file-based credential storage.