diff --git a/.badges/operations.svg b/.badges/operations.svg index 40c1826677..60d4458dca 100644 --- a/.badges/operations.svg +++ b/.badges/operations.svg @@ -1,4 +1,4 @@ - + @@ -12,7 +12,7 @@ PARITY entries PARITY entries - 6513 - 6513 + 6596 + 6596 diff --git a/.badges/parity.svg b/.badges/parity.svg index 4b92d6b4ac..c449cdca97 100644 --- a/.badges/parity.svg +++ b/.badges/parity.svg @@ -1,4 +1,4 @@ - + @@ -12,7 +12,7 @@ parity parity - 160 A · 3 B · 4 C - 160 A · 3 B · 4 C + 160 A · 7 B · 4 C + 160 A · 7 B · 4 C diff --git a/.badges/services.svg b/.badges/services.svg index 81bee2e3a8..f52bed18af 100644 --- a/.badges/services.svg +++ b/.badges/services.svg @@ -1,4 +1,4 @@ - + @@ -12,7 +12,7 @@ AWS services AWS services - 169 - 169 + 173 + 173 diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 00903a4072..bc4cde2d62 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -302,11 +302,14 @@ {"_type":"issue","id":"gopherstack-rnd","title":"EventBridge Pipes AWS-accuracy audit (GH#1818)","status":"closed","priority":1,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-05-29T21:27:34Z","created_by":"mayor","updated_at":"2026-07-26T14:59:24Z","closed_at":"2026-07-26T14:59:24Z","close_reason":"Superseded: pipes audited 2026-07-24 (5d5b2188), overall A. Execution gaps closed; only remaining gap is a proven impossibility (no in-repo Kafka/AMQP broker).","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-o2j","title":"OpenSearch AWS-accuracy audit (GH#1817)","status":"closed","priority":1,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-05-29T21:27:29Z","created_by":"mayor","updated_at":"2026-07-26T14:59:23Z","closed_at":"2026-07-26T14:59:23Z","close_reason":"Superseded: opensearch audited 2026-07-25 (acb2e23f9, parity-4), overall A-, gaps documented. Issue predates the parity campaigns.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-g3y","title":"EC2 batch-4 audit: VPC endpoints, TGW, NACL, Route Tables, NAT Gateway. Real stateful emulation, 2k+ lines.","status":"closed","priority":1,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-05-29T10:49:37Z","created_by":"mayor","updated_at":"2026-07-26T14:59:24Z","closed_at":"2026-07-26T14:59:24Z","close_reason":"Superseded: all five families exist and ec2 audited 2026-07-25 (parity-4) at overall A. Remaining EC2 field-diff work tracked in gopherstack-8pce.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-fwd0g","title":"persistence: Snapshot marshals live tables without per-table locks","description":"services/dynamodb persistence.go Snapshot() holds only db.mu.RLock and json.Marshals every *Table while item/table writes run under table.mu alone, racing periodic snapshots during live traffic. Likely structural across services whose backends use nested per-resource locks. Audit which services snapshot under a lock that does not cover all mutators; fix by copying state under the right locks before marshalling; add -race tests running Snapshot concurrently with writes.","status":"open","priority":2,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-26T21:29:01Z","created_by":"Witness Patrol","updated_at":"2026-09-26T21:29:01Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-k1b28","title":"acm: certificate IDs collide when RequestCertificate is called twice in the same nanosecond","description":"services/acm RequestCertificate builds the cert ID from fmt.Sprintf(\"%x\", time.Now().UnixNano()). Two calls in the same nanosecond (always under synctest's fake clock, possible under fast real clocks or concurrent requests) get the same ARN and the second silently overwrites the first. Use a random or monotonic-counter ID like real ACM's UUID-shaped certificate IDs.","status":"closed","priority":2,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-26T17:06:17Z","created_by":"Witness Patrol","updated_at":"2026-09-26T17:16:49Z","closed_at":"2026-09-26T17:16:49Z","close_reason":"UUID certificate IDs","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-jwr13","title":"rds: TestRealClient_DescribePagination/pending_maintenance_actions flakes on wall-clock reconciler timing","description":"waitForInstanceStatus uses require.Eventually(1s) against the real 250ms instanceTransitionDelay reconciler; under CI load the instance misses 'available' (unit-tests (1), run 36222185682). Drive the lifecycle deterministically (synctest or an injected clock/sweep) instead of polling wall time.","status":"closed","priority":2,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-26T06:34:23Z","created_by":"Witness Patrol","updated_at":"2026-09-26T06:47:30Z","closed_at":"2026-09-26T06:47:30Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-0mji2","title":"s3: ListObjectsV2 scans every object in the bucket regardless of prefix","description":"services/s3/listing.go:103 ranges bucket.Objects and HasPrefix-filters; at 50k objects with a ~1% prefix it is 46% of CPU (BenchmarkListObjectsV2/prefix_delimiter). Needs a sorted key index kept in sync across objects.go, multipart.go, objects_delete.go, janitor_lifecycle.go.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-09-25T01:13:07Z","created_by":"Witness Patrol","updated_at":"2026-09-25T01:42:35Z","closed_at":"2026-09-25T01:42:35Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-9e44r","title":"cloudformation: DeleteStack re-resolves props without the GetAtt stash/type side channel","description":"stackPhysicalIDsSnapshot is rebuilt from {logicalID: PhysicalID} at delete time, so props-based deletes (CodeArtifact Repository/PackageGroup DomainName, etc.) that use Fn::GetAtt resolve to the physical ID. Persist the attribute stash + _Type side channel with the stack.","status":"closed","priority":2,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-25T00:20:40Z","created_by":"Witness Patrol","updated_at":"2026-09-25T01:37:58Z","closed_at":"2026-09-25T01:37:58Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-rwwvt","title":"ec2: DescribeTransitGatewayVpcAttachments ignores Filters","description":"handleDescribeTransitGatewayVpcAttachments (handler_networking1.go:276) only honours TransitGatewayAttachmentIds; state, transit-gateway-id, vpc-id, tag filters are silently dropped, returning every attachment.","status":"closed","priority":2,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-24T18:13:58Z","created_by":"Witness Patrol","updated_at":"2026-09-24T20:19:30Z","closed_at":"2026-09-24T20:19:30Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-yiy60","title":"eks: AssociateEncryptionConfig returns a fabricated, unstored Update ID","description":"Same bug as identity provider config (fixed bc048ddf0): DescribeUpdate on the returned ID gives ResourceNotFoundException. Store a real Update like sibling async ops.","status":"closed","priority":2,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-24T16:32:24Z","created_by":"Witness Patrol","updated_at":"2026-09-24T20:36:10Z","closed_at":"2026-09-24T20:36:10Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"gopherstack-f8qcx","title":"terraform tests: verify steps assume their fixture is alone in the shared emulator","description":"CI shard 7 on 3d4d06aa8 (2026-09-24): TestTerraform_MegaBatch5 read DescribeConnections()[0] and got batch 33's connection; MegaBatch23 asserted exactly 6 DataSync locations and saw 7. Fixed those two in 5c20d9fd7. The terraform package runs all fixtures of a shard against one gopherstack container, so any verify that reads an unfiltered List/Describe result by index or asserts an exact count is order-dependent. grep finds ~365 such reads across test/terraform/mega_batch*_test.go. Sweep: filter by the fixture's own names/tags/ARNs (or GreaterOrEqual + find-by-name), never index 0 of an unfiltered list. Also seen non-fatal in the same shard: SQS delete waiter QueueDoesNotExist, and a CloudFormation apigatewayv2 stack DELETE_FAILED 'Integration: NotFoundException; Route: NotFoundException; Stage: NotFoundException' — CFN delete should treat already-gone child resources as deleted.","notes":"Swept all in-scope test/terraform/*_test.go (excluded mega_batch44/45_test.go + services/ec2, owned by another agent in this branch). Triaged 495 hits of [0]/Len/Empty across terraform_test.go's Test* funcs, all mega_batch*_test.go (except 44/45), parity_*_test.go, import_test.go, drift_test.go, services_parity_test.go. Fixed 6 real order-dependent sites: terraform_test.go TestTerraform_EC2 (DescribeInstances now filtered by instance.group-name), TestTerraform_AppSync (GraphqlApis[0] -\u003e findBy), TestTerraform_TimestreamQuery (assert.Empty on shared list -\u003e check specific ARN absent); mega_batch21_test.go ListAccessGrants (added GranteeIdentifier filter -- collided with nothing currently but AccountId-only filter was unsafe); mega_batch6_test.go Grafana ListWorkspaces + NetworkManager DescribeGlobalNetworks (real collision: batches 6/8/9/49 all create Grafana workspaces, 6/8/34 all create NetworkManager global networks -- batch6 was the only one still doing raw [0] indexing instead of find-by-name); mega_batch5_test.go CleanRooms ListCollaborations + DLM GetLifecyclePolicies (real collision with mega-batch-8.tf, which also creates both resource types -- batch5 was the only one not filtering by name). Added test/terraform/find_helpers_test.go with a generic findBy[T] helper, used in all 4 fixes. Verified several other unfiltered List calls (IoT CA certs/topic rule destinations in batch22, WAF Classic sets in batch33, Transfer certs/connectors/workflows in batch29, GuardDuty/SSO/Organizations singletons) have no colliding sibling fixture in the current fixture set, so left unchanged. mega_batch44_test.go (30 hits) and mega_batch45_test.go (14 hits) still need the same sweep but are out of scope here (owned by another agent). CFN apigatewayv2 DELETE_FAILED item not investigated (test-only scope).","status":"open","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-09-24T06:42:59Z","created_by":"Witness Patrol","updated_at":"2026-09-24T08:11:20Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-f8qcx","title":"terraform tests: verify steps assume their fixture is alone in the shared emulator","description":"CI shard 7 on 3d4d06aa8 (2026-09-24): TestTerraform_MegaBatch5 read DescribeConnections()[0] and got batch 33's connection; MegaBatch23 asserted exactly 6 DataSync locations and saw 7. Fixed those two in 5c20d9fd7. The terraform package runs all fixtures of a shard against one gopherstack container, so any verify that reads an unfiltered List/Describe result by index or asserts an exact count is order-dependent. grep finds ~365 such reads across test/terraform/mega_batch*_test.go. Sweep: filter by the fixture's own names/tags/ARNs (or GreaterOrEqual + find-by-name), never index 0 of an unfiltered list. Also seen non-fatal in the same shard: SQS delete waiter QueueDoesNotExist, and a CloudFormation apigatewayv2 stack DELETE_FAILED 'Integration: NotFoundException; Route: NotFoundException; Stage: NotFoundException' — CFN delete should treat already-gone child resources as deleted.","notes":"Swept all in-scope test/terraform/*_test.go (excluded mega_batch44/45_test.go + services/ec2, owned by another agent in this branch). Triaged 495 hits of [0]/Len/Empty across terraform_test.go's Test* funcs, all mega_batch*_test.go (except 44/45), parity_*_test.go, import_test.go, drift_test.go, services_parity_test.go. Fixed 6 real order-dependent sites: terraform_test.go TestTerraform_EC2 (DescribeInstances now filtered by instance.group-name), TestTerraform_AppSync (GraphqlApis[0] -\u003e findBy), TestTerraform_TimestreamQuery (assert.Empty on shared list -\u003e check specific ARN absent); mega_batch21_test.go ListAccessGrants (added GranteeIdentifier filter -- collided with nothing currently but AccountId-only filter was unsafe); mega_batch6_test.go Grafana ListWorkspaces + NetworkManager DescribeGlobalNetworks (real collision: batches 6/8/9/49 all create Grafana workspaces, 6/8/34 all create NetworkManager global networks -- batch6 was the only one still doing raw [0] indexing instead of find-by-name); mega_batch5_test.go CleanRooms ListCollaborations + DLM GetLifecyclePolicies (real collision with mega-batch-8.tf, which also creates both resource types -- batch5 was the only one not filtering by name). Added test/terraform/find_helpers_test.go with a generic findBy[T] helper, used in all 4 fixes. Verified several other unfiltered List calls (IoT CA certs/topic rule destinations in batch22, WAF Classic sets in batch33, Transfer certs/connectors/workflows in batch29, GuardDuty/SSO/Organizations singletons) have no colliding sibling fixture in the current fixture set, so left unchanged. mega_batch44_test.go (30 hits) and mega_batch45_test.go (14 hits) still need the same sweep but are out of scope here (owned by another agent). CFN apigatewayv2 DELETE_FAILED item not investigated (test-only scope).","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-09-24T06:42:59Z","created_by":"Witness Patrol","updated_at":"2026-09-26T20:16:50Z","closed_at":"2026-09-26T20:16:50Z","close_reason":"Generic NotFound-on-delete in CFN; terraform isolation already applied by 54869319e","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-6u8p4","title":"kms: AWS-managed keys and aliases (alias/aws/\u003cservice\u003e) are never provisioned","description":"Found by terraform mega-batch-35 (2026-09-20): aws_dynamodb_table_replica read fails with 'reading KMS Key (alias/aws/dynamodb): couldn't find resource' because the provider resolves the default encryption key alias. Real AWS creates alias/aws/\u003cservice\u003e keys lazily on first use per account/region (dynamodb, s3, ebs, rds, lambda, secretsmanager, ssm, sns, sqs, kinesis, ...). Provision them on demand in services/kms (DescribeKey/ListAliases by alias/aws/*), with KeyManager=AWS and the real restrictions (no ScheduleKeyDeletion, no policy changes), and have the services that default to them (dynamodb SSEDescription, s3 SSE-KMS default, ebs default key) reference the real alias.","status":"closed","priority":2,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-09-20T10:47:38Z","created_by":"Witness Patrol","updated_at":"2026-09-24T03:34:47Z","closed_at":"2026-09-24T03:34:47Z","close_reason":"Implemented lazy AWS-managed-key provisioning (alias/aws/\u003cservice\u003e) in services/kms: DescribeKey/Encrypt/Decrypt/GenerateDataKey*/GetKeyPolicy provision on first reference with KeyManager=AWS; ScheduleKeyDeletion/DisableKey/PutKeyPolicy/UpdateAlias/DeleteAlias reject KeyManager=AWS keys with the real declared error codes. CreateAlias's alias/aws/ rejection was already correct. See aws_managed_keys.go / aws_managed_keys_test.go, PARITY.md 2026-09-23 entry.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-jtf4s","title":"terraform: FSx delete waiters hang the harness (lustre test 11 min, windows \u003e4m41s) although DeleteFileSystem hard-removes the record","description":"CI terraform shard 6 on cd027034c (2026-09-20) timed out at 15m with TestTerraform_FSxLustre/lustre running 11m1s: apply 30s, verify failed on cross-test pollution (fixed in the next commit), then the non-fatal destroy hung. Batch-32's agent saw the same on aws_fsx_windows_file_system (\u003e4m41s destroying) and worked around it with timeouts { delete = \"5s\" } in test/terraform/fixtures/mega-batch-32.tf. services/fsx DeleteFileSystem hard-deletes, so the provider's first DescribeFileSystems poll should be NotFound and the waiter should return; find out what the provider (v5.100.0 internal/service/fsx waitFileSystemDeleted / findFileSystemByID) actually sees with TF_LOG=trace and fix the emulator (real AWS: DELETING lifecycle then NotFound). Suspect: DescribeFileSystems by id returning 200 with an empty list instead of FileSystemNotFound, or the delete waiter's Delay. The harness 15m shard budget cannot absorb this.","status":"closed","priority":2,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-20T10:32:19Z","created_by":"Witness Patrol","updated_at":"2026-09-24T03:04:14Z","closed_at":"2026-09-24T03:04:14Z","close_reason":"Not a gopherstack bug: services/fsx DeleteFileSystem already hard-deletes and DescribeFileSystems immediately returns a real typed FileSystemNotFound (confirmed live + TestDeleteFileSystem_ThenDescribeIsTypedNotFound, all 4 fs types). Root cause is terraform-provider-aws v5.100.0's waitFileSystemDeleted (internal/service/fsx/lustre_file_system.go), which has a hardcoded Delay: 10*time.Minute before its first poll, shared by lustre/windows/ontap/openzfs delete -- irreducible, confirmed against terraform-plugin-sdk retry.StateChangeConf semantics. Applied mega-batch-32.tf's existing timeouts{delete=\"5s\"} workaround to fixtures/fsx/lustre.tf too (it lacked it, hence the CI timeout). Verified: TestTerraform_FSxLustre/lustre 65.96s, TestTerraform_MegaBatch32/success 122.47s, both well under the 15m shard budget.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-ku5hx","title":"flake: s3 TestLifecycle_TagFilter races the wall-clock janitor in CI","description":"CI unit-tests (0) on #2471 run 35494xxxxx (2026-09-20): TestLifecycle_TagFilter/tag_filter_evicts_only_tagged_objects failed 'expected key \"tagged-key\" to be evicted' after 0.67s. services/s3/janitor_test.go starts newFastJanitor + go j.Run and polls with require.Eventually; under CI load the eviction misses the window. Fix: drive one janitor sweep deterministically (call the sweep function directly, or testing/synctest with the fake clock) instead of a real ticker + Eventually; no time.Sleep. Same class as the kinesis SubscribeToShard idle-close race (gopherstack-j60e) which also failed this branch's CI on 2026-09-20.","notes":"2026-09-20: Fixed by removing wall-clock racing entirely. TestLifecycle_TagFilter and its siblings (TestS3Janitor_LifecycleExpiry incl. a literal time.Sleep(50ms), TestS3Janitor_NoncurrentVersionExpiration, and TestS3Janitor_BucketDeletion's 4 subtests) all replaced `go newFastJanitor(b).Run(ctx)` + require.Eventually/time.Sleep polling with a single deterministic sweep call: SweepOnce(ctx) (already existed, already used by every other s3 lifecycle test file) for lifecycle/multipart tests, and a new exported test helper Janitor.DrainPendingBucketsOnce(ctx) (services/s3/export_test.go, mirrors sweepAndDrain's bucket-selection but calls processBucket synchronously, no goroutines/semaphore) for the bucket-deletion drain tests. Assertions kept identical. Verified: go test -race -run 'Janitor|Lifecycle' -count=50 ./services/s3/... passed; also passed -count=100 under synthetic CPU starvation (GOMAXPROCS=2 + 4x `yes` stress) that reliably reproduces this class of flake elsewhere in this session. golangci-lint clean (0 issues) after fixing one govet shadow finding introduced by the rewrite.","status":"closed","priority":2,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-20T06:29:21Z","created_by":"Witness Patrol","updated_at":"2026-09-20T07:45:53Z","closed_at":"2026-09-20T07:45:53Z","close_reason":"Fixed: converted TestLifecycle_TagFilter and sibling janitor tests from go Run()+require.Eventually/time.Sleep wall-clock polling to deterministic SweepOnce/DrainPendingBucketsOnce calls. Verified with -race -count=50 and -count=100 under synthetic CPU-starvation stress; golangci-lint clean.","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -731,7 +734,7 @@ {"_type":"issue","id":"gopherstack-nb10","title":"firehose: Reset() leaves Kinesis-source poller goroutines running against deleted streams","status":"closed","priority":2,"issue_type":"bug","created_at":"2026-09-04T05:19:00Z","updated_at":"2026-09-04T05:48:53Z","closed_at":"2026-09-04T05:48:52Z","close_reason":"fixed and regression-tested on chore/parity-sweep-2026-09-03","labels":["parity-campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-pe7x","title":"firehose: CloudWatchLoggingOptions never writes events to the CloudWatch Logs backend","description":"REAL / small (triaged 2026-09-06). services/firehose/flush.go:517 logDeliveryIssue() only logs; CloudWatchLoggingOptions is validated and stored but no delivery error or record ever reaches a CloudWatch Logs stream.\n\nSmallest of the cross-service delivery cluster because the exact hook shape already exists and is reusable verbatim: services/lambda/store.go:74-78 defines CWLogsBackend{EnsureLogGroupAndStream, PutLogLines}, cli.go:5823 has cwLogsAdapter implementing it, and cli.go:5760 wireLambdaCWLogs is the wiring precedent. Unwired backend must stay a silent no-op.","status":"closed","priority":2,"issue_type":"bug","assignee":"Witness Patrol","created_at":"2026-09-04T05:19:00Z","updated_at":"2026-09-06T14:24:36Z","started_at":"2026-09-06T14:07:53Z","closed_at":"2026-09-06T14:24:36Z","close_reason":"Closed","labels":["parity-campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-o4ny","title":"cli.go: Firehose KinesisStreamAsSource is never wired; SetKinesisBackend has no production call site so such streams silently ingest nothing","status":"closed","priority":2,"issue_type":"task","created_at":"2026-09-04T05:18:59Z","updated_at":"2026-09-04T05:48:51Z","closed_at":"2026-09-04T05:48:51Z","close_reason":"fixed: cli.go now wires SetKinesisBackend; verified end-to-end (record Kinesis-\u003eFirehose-\u003eS3), fails before fix","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"gopherstack-i8q7","title":"kinesis: TestSubscribeToShard_RoundTrip flake reproduced once in 1500+ runs; no isolable defect found, not fixed","description":"RE-ATTEMPTED 2026-09-06, NOT reproducible in this environment. Leaving open; do not close as fixed and do not add a retry or sleep to quiet it.\n\nThe CPU-contention technique that cracked the sibling flake gopherstack-nn94 was applied here and escalated well past that recipe: background busy-loops up to 34 processes on an 8-core box (~4x oversubscription), -race -count=40 -parallel=200, -cpu varied 1/2/4/8, the test process pinned with taskset to 2 cores and then 1 core while all 8 were saturated, plus whole-package runs so real neighbour tests contend. Roughly 540 executions of the target test. Zero failures, zero race reports anywhere in services/kinesis.\n\nSo unlike nn94 -- where the same technique turned 1-in-1500 into 3-in-4 and the error named the cause outright -- contention alone does not surface this one.\n\nCode reading found no obvious ordering bug: PutRecord completes synchronously before the client opens the subscribe stream, and handleSubscribeToShardHTTP does one immediate advanceShardCursor poll before the ticker starts, so the record should land in that first poll rather than waiting a 200ms tick. The test's wait is 5 seconds (subscribe_roundtrip_test.go:69), a wide margin at the load levels reachable here. Locking is a single stream.mu held consistently across reads and writes, with no package-level mutable state.\n\nWorking hypothesis for why it did not reproduce: the original 1-in-1500 may need conditions this sandbox cannot create -- a smaller CI runner, memory or disk pressure, or a specific interleaving with other packages running concurrently in the same CI job, rather than one package hammered in isolation.\n\nNext attempt should target resource constraint rather than more iterations: cap memory and cores (a container with 1-2 cores and a low memory limit), or reproduce inside a full-repo CI-shaped run rather than a single-package loop.","status":"closed","priority":2,"issue_type":"bug","assignee":"Witness Patrol","created_at":"2026-09-04T05:12:13Z","updated_at":"2026-09-13T11:44:53Z","started_at":"2026-09-06T19:27:59Z","closed_at":"2026-09-13T11:44:53Z","close_reason":"Root cause found: net/http Transport writeLoop closes a reused keep-alive conn while the SDK event-stream reader is mid-read; reproduced 3-8/200 under -race GOMAXPROCS=2 shuffled load, 250/250 clean with DisableKeepAlives on the test client.","labels":["parity-campaign"],"comments":[{"id":"01a09a3e-bec1-7a83-9c5d-6210985fcb4e","issue_id":"gopherstack-i8q7","author":"Witness Patrol","text":"2026-09-13: second instance of the streaming-close flake class on CI (PR #2467 run 34750198623 unit-tests(1)): TestSubscribeToShard_IdleCloseIsGraceful got 'read tcp ...: use of closed network connection' from stream.Err() instead of clean EOF at the deadline close. Not reproducible locally (-race -count=30, package -shuffle x3). bedrockruntime's InvokeModelWithResponseStream test hit the identical error text once on run 34737227162 (filed separately today). Same shape both times: SDK event-stream reader still reading when the server ends the response under heavy -race shard load. Worth one focused investigation across both: does the handler end the response via return (clean chunked terminator) or via a Hijack/SetWriteDeadline/conn close; does echo's server WriteTimeout or the httptest listener close race the final flush; does the SDK reader's Close on ctx race the last read.","created_at":"2026-09-13T10:09:58Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} +{"_type":"issue","id":"gopherstack-i8q7","title":"kinesis: TestSubscribeToShard_RoundTrip flake reproduced once in 1500+ runs; no isolable defect found, not fixed","description":"RE-ATTEMPTED 2026-09-06, NOT reproducible in this environment. Leaving open; do not close as fixed and do not add a retry or sleep to quiet it.\n\nThe CPU-contention technique that cracked the sibling flake gopherstack-nn94 was applied here and escalated well past that recipe: background busy-loops up to 34 processes on an 8-core box (~4x oversubscription), -race -count=40 -parallel=200, -cpu varied 1/2/4/8, the test process pinned with taskset to 2 cores and then 1 core while all 8 were saturated, plus whole-package runs so real neighbour tests contend. Roughly 540 executions of the target test. Zero failures, zero race reports anywhere in services/kinesis.\n\nSo unlike nn94 -- where the same technique turned 1-in-1500 into 3-in-4 and the error named the cause outright -- contention alone does not surface this one.\n\nCode reading found no obvious ordering bug: PutRecord completes synchronously before the client opens the subscribe stream, and handleSubscribeToShardHTTP does one immediate advanceShardCursor poll before the ticker starts, so the record should land in that first poll rather than waiting a 200ms tick. The test's wait is 5 seconds (subscribe_roundtrip_test.go:69), a wide margin at the load levels reachable here. Locking is a single stream.mu held consistently across reads and writes, with no package-level mutable state.\n\nWorking hypothesis for why it did not reproduce: the original 1-in-1500 may need conditions this sandbox cannot create -- a smaller CI runner, memory or disk pressure, or a specific interleaving with other packages running concurrently in the same CI job, rather than one package hammered in isolation.\n\nNext attempt should target resource constraint rather than more iterations: cap memory and cores (a container with 1-2 cores and a low memory limit), or reproduce inside a full-repo CI-shaped run rather than a single-package loop.","status":"in_progress","priority":2,"issue_type":"bug","assignee":"Witness Patrol","created_at":"2026-09-04T05:12:13Z","updated_at":"2026-09-06T20:06:04Z","started_at":"2026-09-06T19:27:59Z","labels":["parity-campaign"],"comments":[{"id":"01a09a3e-bec1-7a83-9c5d-6210985fcb4e","issue_id":"gopherstack-i8q7","author":"Witness Patrol","text":"2026-09-13: second instance of the streaming-close flake class on CI (PR #2467 run 34750198623 unit-tests(1)): TestSubscribeToShard_IdleCloseIsGraceful got 'read tcp ...: use of closed network connection' from stream.Err() instead of clean EOF at the deadline close. Not reproducible locally (-race -count=30, package -shuffle x3). bedrockruntime's InvokeModelWithResponseStream test hit the identical error text once on run 34737227162 (filed separately today). Same shape both times: SDK event-stream reader still reading when the server ends the response under heavy -race shard load. Worth one focused investigation across both: does the handler end the response via return (clean chunked terminator) or via a Hijack/SetWriteDeadline/conn close; does echo's server WriteTimeout or the httptest listener close race the final flush; does the SDK reader's Close on ctx race the last read.","created_at":"2026-09-13T10:09:58Z"}],"dependency_count":0,"dependent_count":0,"comment_count":1} {"_type":"issue","id":"gopherstack-qowd","title":"cli.go: kinesisReaderAdapter uses context.Background(), so Kinesis-to-Lambda event source mappings always resolve the default region","status":"closed","priority":2,"issue_type":"bug","assignee":"Witness Patrol","created_at":"2026-09-04T05:12:13Z","updated_at":"2026-09-06T14:06:13Z","started_at":"2026-09-06T13:07:55Z","closed_at":"2026-09-06T14:06:13Z","close_reason":"Closed","labels":["parity-campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-0hju","title":"kinesis: GetRecords chained NextShardIterator has zero CreatedAt, so it never expires and ExpiredIteratorException is unreachable","status":"closed","priority":2,"issue_type":"bug","created_at":"2026-09-04T05:12:12Z","updated_at":"2026-09-04T05:12:32Z","closed_at":"2026-09-04T05:12:32Z","close_reason":"fixed and regression-tested on chore/parity-sweep-2026-09-03","labels":["parity-campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-tnp9","title":"kinesis: Enable/DisableEnhancedMonitoring responses omit StreamARN, which the SDK output type declares","status":"closed","priority":2,"issue_type":"bug","created_at":"2026-09-04T05:12:12Z","updated_at":"2026-09-04T05:12:33Z","closed_at":"2026-09-04T05:12:33Z","close_reason":"fixed and regression-tested on chore/parity-sweep-2026-09-03","labels":["parity-campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} @@ -1453,11 +1456,15 @@ {"_type":"issue","id":"go-hwb.106","title":"S3 Control: ~48 missing ops (access points/grants/batch jobs/MRAP)","description":"## S3 Control — Service Deep Dive\n\nAudit of [services/s3control/](services/s3control/) and UI in [ui/src/routes/s3control/](ui/src/routes/s3control/).\n\n### 1. Missing SDK Operations\n~48 missing ([sdk_completeness_test.go#L21](services/s3control/sdk_completeness_test.go#L21)): `DeleteAccessGrant`, `DeleteBucket`, `GetAccessPoint`, `ListAccessPoints`, `PutAccessPointPolicy`, Access Grants, Access Points, Batch Jobs, MRAP, Storage Lens Group. Only 13 supported (public access block + partial).\n\n### 2. Missing UI / Dashboard Features\nPublic access block display only. Missing: access points mgmt, access grants, batch job UI, MRAP, storage lens groups, Object Lambda.\n\n### 3. Goroutine / Resource / Lock Leaks\nClean. Map cloning on snapshot ([persistence.go#L44](services/s3control/persistence.go#L44)).\n\n### 4. Performance Optimizations\n1. 10+ separate maps — consolidate with typed keys to reduce Reset cost.\n2. Atomic counter for IDs ([backend.go#L178](services/s3control/backend.go#L178)) good.\n\n### Suggested Order\n1. Access Points (Create/Get/List/Put policy)\n2. Access Grants (Create/Delete/List)\n3. Batch Jobs + MRAP + Storage Lens Group\n4. Consolidate map structure\n\n\n---\n**Source:** https://github.com/BlackbirdWorks/gopherstack/issues/1223\n","status":"closed","priority":2,"issue_type":"task","owner":"andrew.bishop9625@gmail.com","created_at":"2026-05-02T18:28:35Z","created_by":"mayor","updated_at":"2026-07-30T16:59:49Z","closed_at":"2026-07-30T16:59:49Z","close_reason":"STALE (parity-5 verification sweep): verified against code, not PARITY.md prose. The authoritative reflective TestSDKCompleteness test passes for this service with an empty notImplemented list, i.e. zero unaccounted SDK operations - refuting the ticket's missing-ops claim. UI route page exists and is substantial. Ticket was auto-generated 2026-05-02, before the parity-3/4/5 campaigns did this work.","external_ref":"gh-1223","labels":["ai-queue"],"dependencies":[{"issue_id":"go-hwb.106","depends_on_id":"go-hwb","type":"parent-child","created_at":"2026-05-02T13:28:35Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"go-hwb.102","title":"Timestream Write: SDK complete; per-table WriteRecords locks","description":"## Timestream Write — Service Deep Dive\n\nAudit of [services/timestreamwrite/](services/timestreamwrite/) and shared UI in [ui/src/routes/timestream/](ui/src/routes/timestream/).\n\n### 1. Missing SDK Operations\n**0 missing.** 20 ops implemented including `CreateDatabase`, `CreateTable`, `WriteRecords`, `CreateBatchLoadTask`, `ResumeBatchLoadTask`, tags.\n\n### 2. Missing UI / Dashboard Features\nShared UI covers DBs + tables + scheduled queries. Full CRUD. Batch load UI could be enhanced.\n\n### 3. Goroutine / Resource / Lock Leaks\nClean. 4 nested maps under single `lockmetrics.RWMutex` ([backend.go#L159](services/timestreamwrite/backend.go#L159)).\n\n### 4. Performance Optimizations\n1. **Single mutex serializes WriteRecords across tables** — partition by table-ARN for ~10x throughput.\n2. Dispatch pre-built ([handler.go#L62](services/timestreamwrite/handler.go#L62)).\n\n### Suggested Order\n1. Per-table-ARN partition locks for `WriteRecords`\n2. Batch load UI polish\n\n\n---\n**Source:** https://github.com/BlackbirdWorks/gopherstack/issues/1227\n","status":"closed","priority":2,"issue_type":"task","owner":"andrew.bishop9625@gmail.com","created_at":"2026-05-02T18:28:34Z","created_by":"mayor","updated_at":"2026-07-30T16:59:48Z","closed_at":"2026-07-30T16:59:48Z","close_reason":"STALE (parity-5 verification sweep): verified against code, not PARITY.md prose. The authoritative reflective TestSDKCompleteness test passes for this service with an empty notImplemented list, i.e. zero unaccounted SDK operations - refuting the ticket's missing-ops claim. UI route page exists and is substantial. Ticket was auto-generated 2026-05-02, before the parity-3/4/5 campaigns did this work.","external_ref":"gh-1227","labels":["ai-queue"],"dependencies":[{"issue_id":"go-hwb.102","depends_on_id":"go-hwb","type":"parent-child","created_at":"2026-05-02T13:28:34Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"go-hwb","title":"Epic: ai-queue from BlackbirdWorks/gopherstack","description":"Autonomous grinding of GitHub issues labeled 'ai-queue' from BlackbirdWorks/gopherstack. Each child bead corresponds to one GitHub issue (external-ref gh-N). Launched via gt mountain for wave-based dispatch with Witness failure tracking and merge-on-CI-pass via Refinery.","status":"open","priority":2,"issue_type":"epic","owner":"andrew.bishop9625@gmail.com","created_at":"2026-05-02T18:27:46Z","created_by":"mayor","updated_at":"2026-05-02T18:27:46Z","labels":["ai-queue"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-yhgs9","title":"terraform fixtures: two fixtures each create the account's single GuardDuty detector","description":"test/terraform/fixtures/guardduty/success.tf and guardduty-and-securityhub.tf both declare aws_guardduty_detector; CreateDetector rejects a second detector per account/region (matches AWS). If both land in the same shard's shared emulator, the second apply fails. Share one detector or isolate by account/region.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-26T20:16:55Z","created_by":"Witness Patrol","updated_at":"2026-09-26T20:42:34Z","closed_at":"2026-09-26T20:42:34Z","close_reason":"Serialised GuardDuty, Security Hub, Macie2, Config recorder and Detective fixtures via per-singleton mutexes (organizationsMu pattern)","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-ru9la","title":"lockmetrics: activeReadersLock gauge is orphaned after Close","description":"RWMutex.New curries activeReadersLock once; Close deletes the series but never refreshes the handle, so RLock/RUnlock after Close (or a same-name recreate racing Close) write to a deleted series and vanish from /metrics. Pre-existing. Fixing it conflicts with TestRWMutex_CloseRemovesLabelValues' zero-series invariant; decide the use-after-Close contract.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-26T19:35:05Z","created_by":"Witness Patrol","updated_at":"2026-09-26T20:53:56Z","closed_at":"2026-09-26T20:53:56Z","close_reason":"activeReadersLock lazily re-curried after Close","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-lr8qu","title":"secretsmanager: RotationRules.Duration rotation window is ignored","description":"AWS rotates a scheduled secret at any time within RotationRules.Duration (rotate-secrets_schedule.html). gopherstack stores Duration but rotation.go fires exactly at the cron/rate boundary. Model the window (deterministically, e.g. at window start, or configurable) and document.","status":"closed","priority":3,"issue_type":"feature","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-09-26T17:59:40Z","created_by":"Witness Patrol","updated_at":"2026-09-26T18:41:47Z","started_at":"2026-09-26T18:28:50Z","closed_at":"2026-09-26T18:41:47Z","close_reason":"Implemented rotation window semantics: rate() schedules align window start to midnight UTC (days) / top of hour (hours) per docs; cron() already aligned. Duration validated (format [0-9]+h len 2-3, must not extend past window/UTC day); AutomaticallyAfterDays/ScheduleExpression made mutually exclusive per API_RotationRulesType.html. Fires at window start (documented-valid choice). See services/secretsmanager/rotation.go, rotation_window_test.go, PARITY.md 2026-09-26 note.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-erj2j","title":"cloudformation: cfnattrgen table coverage drifts with repo literal counts","description":"cmd/cfnattrgen skips any type whose attribute names would push a string literal past goconst's threshold, so regenerating against the same spec dropped AWS::EC2::PrefixList and AWS::SageMaker::ImageVersion once other code added literals. Emit attribute names through generated consts or exclude the generated file from goconst counting so coverage is stable.","status":"closed","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-09-26T05:43:56Z","created_by":"Witness Patrol","updated_at":"2026-09-26T12:23:55Z","closed_at":"2026-09-26T12:23:55Z","close_reason":"cfn_attributes.json embed + drift test; 204 types covered","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-p7pvq","title":"cloudformation: Fn::GetAtt on an unknown attribute silently returns the physical ID","description":"Real CloudFormation fails validation with 'Template error: resource X does not support attribute type Y in Fn::GetAtt'. getResourceAttribute/getExtraResourceAttribute fall back to physID for Outputs and properties.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-25T00:20:44Z","created_by":"Witness Patrol","updated_at":"2026-09-25T01:37:59Z","closed_at":"2026-09-25T01:37:59Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"gopherstack-z2w1a","title":"s3: S3 Express One Zone CreateSession / sigv4-s3express auth not implemented","description":"aws_s3_directory_bucket fails 403 SignatureDoesNotMatch; directory buckets need CreateSession and the s3express session-token signing flow.","status":"open","priority":3,"issue_type":"feature","owner":"blackbird7181@gmail.com","created_at":"2026-09-24T16:32:30Z","created_by":"Witness Patrol","updated_at":"2026-09-24T16:32:30Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-z2w1a","title":"s3: S3 Express One Zone CreateSession / sigv4-s3express auth not implemented","description":"aws_s3_directory_bucket fails 403 SignatureDoesNotMatch; directory buckets need CreateSession and the s3express session-token signing flow.","status":"closed","priority":3,"issue_type":"feature","owner":"blackbird7181@gmail.com","created_at":"2026-09-24T16:32:30Z","created_by":"Witness Patrol","updated_at":"2026-09-26T05:52:23Z","closed_at":"2026-09-26T05:52:23Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-h8cej","title":"terraform: aws_transcribe_medical_vocabulary destroy waiter errors though GetMedicalVocabulary is 404","description":"Provider delete waiter doesn't treat our 404 as gone; check error code/shape (likely NotFoundException vs BadRequestException) against the SDK.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-24T16:32:29Z","created_by":"Witness Patrol","updated_at":"2026-09-24T19:58:35Z","closed_at":"2026-09-24T19:58:35Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-zfrof","title":"terraform: aws_ec2_transit_gateway_connect_peer create waiter never finds the peer","description":"DescribeTransitGatewayConnectPeers returns the peer with state=available on every poll, yet provider v5.100 reports couldn't find resource for the whole retry budget (batch 53 dropped it). Needs TF_LOG=trace + provider source read.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-24T16:32:28Z","created_by":"Witness Patrol","updated_at":"2026-09-25T03:28:44Z","closed_at":"2026-09-25T03:28:44Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"gopherstack-rbrz6","title":"perf: audit pgoload/pprof hot paths (Part C of goroutine-leak sweep)","description":"Split off from gopherstack-1x2u0's Part C, which was never started: profile hot paths with pgoload/pprof across services and address findings. Unrelated to the goroutine-leak retrofit (rds/secretsmanager/sqs/pipes/ec2/sns), which is done as of 2026-09-24.","status":"open","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-09-24T11:02:06Z","created_by":"Witness Patrol","updated_at":"2026-09-24T11:02:06Z","dependencies":[{"issue_id":"gopherstack-rbrz6","depends_on_id":"gopherstack-1x2u0","type":"discovered-from","created_at":"2026-09-24T06:02:05Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-rbrz6","title":"perf: audit pgoload/pprof hot paths (Part C of goroutine-leak sweep)","description":"Split off from gopherstack-1x2u0's Part C, which was never started: profile hot paths with pgoload/pprof across services and address findings. Unrelated to the goroutine-leak retrofit (rds/secretsmanager/sqs/pipes/ec2/sns), which is done as of 2026-09-24.","status":"closed","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-09-24T11:02:06Z","created_by":"Witness Patrol","updated_at":"2026-09-26T19:35:00Z","closed_at":"2026-09-26T19:35:00Z","close_reason":"pgoload profile: cloudtrail trim realloc, capture buffering, lockmetrics label hashing fixed","dependencies":[{"issue_id":"gopherstack-rbrz6","depends_on_id":"gopherstack-1x2u0","type":"discovered-from","created_at":"2026-09-24T06:02:05Z","created_by":"Witness Patrol","metadata":"{}"}],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-z07y0","title":"Leak sweep: remaining low-traffic delete-tombstone/terminal-state candidates","description":"Follow-up to the 2026-09-24 leak sweep (medialive gopherstack-f9w3k + ec2\nc254cd795 + ecs 3fa9337a8 + ram + acmpca + quicksight, all fixed this pass:\nDELETED/terminal resources kept in their store forever instead of being\nevicted after a bounded window).\n\nSwept all of services/ for the same class (grep for State/Status assignments\nto DELETED/INACTIVE/CANCELLED/TERMINATED-like terminal values, cross-checked\nagainst a following `.Delete(` call and any existing janitor/TTL). Most hits\nturned out to be false positives: either the record IS removed from its\ntable right after the terminal-state assignment (appmesh, apprunner x6,\nvpclattice x2, grafana, rds automated-backups x2, opensearch, quicksight\nVPCConnection -- returned copy only reflects the terminal state for the\nresponse), or a real janitor/lazy-sweep already exists (batch job\ndefinitions, ec2, ecs, opensearch serverless collections).\n\nRemaining candidates NOT fixed this pass (low priority / low terraform\ntraffic / not a clear \"delete but tombstone kept\" case -- worth a second\nlook but none looked urgent):\n\n- organizations/handshakes.go: expireStaleHandshakesLocked transitions\n OPEN-\u003eEXPIRED but never removes the row from b.handshakes. Low volume\n (account governance handshakes), no AWS DeleteHandshake op exists either\n (AWS itself has no delete for handshakes) -- plausibly class b (bounded by\n natural usage) but not verified against an AWS-documented retention.\n- ram/share_invitations.go: expireInvitationLocked transitions PENDING-\u003e\n EXPIRED but never removes the row from b.invitations. Same shape as the\n resource-share leak just fixed, but AWS has no DeleteInvitation op either\n (invitations are meant to persist as history) -- likely class b but not\n verified.\n- swf/workflow_executions.go, swf/decision_tasks.go: workflow executions\n terminate/cancel but are kept -- SWF has a real documented\n workflowExecutionRetentionPeriodInDays concept (execution history\n retained N days after close, configurable per domain, like the\n Step Functions 90-day precedent already cited in PARITY conventions).\n Plausibly class b but the retention period itself is not modeled/enforced\n here, so worth checking whether it should be.\n- emr clusters/sessions/steps (terminateSingle/terminateSessionInPlace/\n cancelStep): TERMINATED clusters/sessions stay listed forever. No\n DeleteCluster-equivalent op exists in real EMR either (clusters just\n terminate and remain describable) -- likely class b but unverified\n against any AWS-documented cap.\n- athena/iot/ssm/cloudwatchlogs/omics/eventbridge/outposts/guardduty/\n managedblockchain/lightsail/amplify/redshift/route53/mediaconvert/\n accessanalyzer/eks: various Cancel*/Stop*/Terminate* ops on jobs,\n sessions, executions, queries, tasks that transition to a terminal status\n but were never \"deleted\" by any op in the first place (no delete-waiter\n tombstone pattern involved) -- out of scope for this bug class, flagged\n only because the grep matched; not reviewed in depth beyond confirming no\n Delete() call exists nearby. If any of these grow unbounded in practice\n it's a different (pre-existing, not delete-tombstone) unbounded-history\n problem, worth its own audit.\n\nGrep starting point (services/ minus ec2/ecs/medialive/ram/acmpca/\nquicksight, already fixed):\n grep -rnE '\\.(State|Status)\\s*=\\s*[A-Za-z]' services --include='*.go' | grep -v _test | grep -iE '(delet|inactiv|cancel|terminat|expir)'","notes":"Triaged all listed candidates. Fixed 3 (highest-confidence real leaks, cited);\nrecorded classification for the rest -- most turned out to be already handled\n(existing janitor/cap) or genuinely out of scope for this bug class.\n\nFIXED (1h class-c or documented-retention class-b eviction added):\n\n- organizations/handshakes.go: AcceptHandshake/CancelHandshake/DeclineHandshake/\n expireStaleHandshakesLocked never removed a non-OPEN handshake from\n b.handshakes. AWS docs (API_Handshake.html): \"Handshakes that are CANCELED,\n ACCEPTED, DECLINED, or EXPIRED show up in lists for only 30 days after\n entering that state. After that they are deleted.\" Class b. Added\n Handshake.StateChangedAt (new persisted field, additive, no version bump)\n + pruneStaleHandshakesLocked, 30d TTL. Tests: handshake_expiry_test.go.\n\n- ram/share_invitations.go: AcceptResourceShareInvitation/\n RejectResourceShareInvitation/expireInvitationLocked never removed a\n terminal invitation from b.invitations. No AWS-documented retention, no\n DeleteInvitation op. Class c. Added pruneTerminalInvitationsLocked reusing\n existing LastUpdatedTime, 1h TTL (same convention as ramDeletedShareTTL from\n the prior ram fix). Tests: invitation_expiry_test.go.\n\n- eventbridge/replays.go: CancelReplay/scheduleReplayWorker never removed a\n COMPLETED/CANCELLED replay from b.replays. No AWS-documented retention, no\n DeleteReplay op. Class c. Added pruneStaleReplaysLocked reusing existing\n ReplayEndTime, 1h TTL. Tests: replay_expiry_test.go.\n\nCLASSIFIED, NOT FIXED (already fine or out of scope):\n\n- swf/workflow_executions.go: class d. Already bounded by a 10,000-execution\n LRU cache (maxWorkflowExecutions, registerExecutionOrderLocked/\n evictExecutionLocked) that evicts regardless of terminal state. Domain\n workflowExecutionRetentionPeriodInDays is stored but not enforced against\n ListClosedWorkflowExecutions visibility -- an accuracy gap, not a leak;\n worth its own ticket if it matters.\n\n- emr clusters/sessions/steps: class a. services/emr/janitor.go's\n sweepTerminatedClusters already evicts TERMINATED/TERMINATED_WITH_ERRORS\n clusters via a configurable TTL (default 1h, EMR_TERMINATED_TTL), using an\n existing terminatedAt field already carried through persistence\n (clusterDTO.TerminatedAt). AWS itself documents ~2 months\n (docs.aws.amazon.com .../emr-manage-view-clusters.html: \"Amazon EMR saves\n metadata about terminated clusters for your reference for two months\"),\n but the 1h default is an intentional memory-bound tradeoff already made for\n this emulator, not a bug -- raising it to 2mo would reintroduce the\n unbounded-growth risk this sweep exists to prevent. Sessions/steps live\n embedded in the Cluster struct, so they're evicted along with their parent\n cluster automatically; no separate leak.\n\n- athena (StopQueryExecution/TerminateSession/StopCalculationExecution):\n class a. services/athena/janitor.go already sweeps terminal query\n executions/sessions/calculations on a 24h TTL. CancelCapacityReservation\n not covered by the janitor but is a low-cardinality config resource, not\n reviewed further.\n\n- cloudwatchlogs (CancelExportTask/CancelImportTask/StopQuery): class d.\n exportTasks/importTasks are capped (maxExportTasks/maxImportTasks reject\n new creates once full); queries are bounded by an LRU (b.maxQueries +\n b.queriesOrder). All three already bounded, no fix needed.\n\n- iot (CancelJob and friends), lightsail (Stop*), redshift (CancelResize),\n guardduty (StopMonitoringMembers), managedblockchain (accessors/proposals):\n reviewed -- each has either an explicit separate Delete op the AWS caller\n must invoke (iot DeleteJob requires terminal state or force, matching real\n AWS's own \"kept until deleted\" behavior) or the terminal-state record isn't\n actually a deleted resource (guardduty member stays associated, just\n Disabled; lightsail/redshift ops act on a still-existing resource). Not the\n delete-tombstone leak class. managedblockchain accessors: AWS's own doc\n comment says PENDING_DELETION accessors are NOT removed from\n GetAccessor/ListAccessors (already correctly modeled, matches real AWS\n forever-visible design, low cardinality in practice) -- proposals similar,\n no documented retention found, low churn, not fixed this pass.\n\n- ssm (StopAutomationExecution, CancelMaintenanceWindowExecution): NOT FIXED,\n needs follow-up. StopAutomationExecution transitions automationExecutions'\n Status but nothing evicts it (ssm/janitor.go covers commands/sessions/\n parameters but not automation executions). AWS docs strongly suggest a\n ~30-day automation execution history window but no single clean API-level\n citation was found this pass (only console/OpsCenter-adjacent hints) --\n didn't want to guess wrong and break parity by evicting too early. Worth\n a dedicated follow-up once a firm citation is found.\n CancelMaintenanceWindowExecution looks like a stub (no execution store\n backing it at all, just echoes the input ID) -- separate bug class\n (no-stub violation), not a leak; flag for its own fix.\n\n- omics/outposts/accessanalyzer/eks/amplify/mediaconvert not exhaustively\n re-triaged this pass (budget); each has a Cancel/Stop op whose target\n record is embedded in a parent resource already bounded some other way\n (job/task lists per parent) rather than a standalone unbounded map, spot\n checked without finding an obvious standalone leak, but not proven clean.\n Lower priority than the above given the bd issue's own note that most of\n this list \"was flagged only because the grep matched.\"\n\nGates (all 3 fixed services): gofmt -l clean, go vet clean, go test -race\n-count=1 passes, golangci-lint run 0 issues, go test ./pkgs/persistence/\npasses (1 additive field, golden updated, no version bump),\ngo run ./cmd/parityfmtcheck -dir services clean, git diff --stat go.mod\ngo.sum empty, go build ./... clean.","status":"closed","priority":3,"issue_type":"task","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-09-24T10:17:01Z","created_by":"Witness Patrol","updated_at":"2026-09-24T10:36:05Z","started_at":"2026-09-24T10:19:37Z","closed_at":"2026-09-24T10:36:05Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-f9w3k","title":"medialive: DELETED input security groups and multiplexes are kept forever","description":"Commit after 6d259bd81 (2026-09-24) keeps them describable as DELETED for the provider's delete waiter, but never evicts them — same unbounded-growth class as the ec2 tombstones fixed in c254cd795. Add a retention window (lazy prune on read/delete, 1h like ec2/ecs) and a synctest expiry test.","status":"closed","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-09-24T09:55:16Z","created_by":"Witness Patrol","updated_at":"2026-09-24T10:00:58Z","closed_at":"2026-09-24T10:00:58Z","close_reason":"Fixed: DeletedAt stamp + lazy prune on read/write, 1h TTL matching ec2/ecs. Tests + PARITY.md updated.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-54bv0","title":"ec2: DeleteVpnConnection should tombstone like transit gateway route tables and fleets","description":"Found by terraform mega-batch-34 (2026-09-20): DeleteVpnConnection hard-removes the record, so the provider's delete waiter polls DescribeVpnConnections for state 'deleted' until its ~5 min timeout on every destroy (non-fatal, but 5 min of dead time per test). Apply the describeWithTombstones pattern from services/ec2/describe_helpers.go (commit bfdb308be) to VPN connections. Also: aws_ec2_transit_gateway_connect create waiter reports 'couldn't find resource' despite DescribeTransitGatewayConnects returning state=available on every poll — investigate before covering transit_gateway_connect_peer_association. wafregional: no SDK v2 module exists; services/waf only routes AWSWAF_20150824 targets.","status":"closed","priority":3,"issue_type":"task","owner":"blackbird7181@gmail.com","created_at":"2026-09-20T09:04:31Z","created_by":"Witness Patrol","updated_at":"2026-09-24T03:34:51Z","closed_at":"2026-09-24T03:34:51Z","close_reason":"Applied describeWithTombstones to DeleteVpnConnection/DescribeVpnConnections (services/ec2/vpn_connections.go), mirroring the TGW route table/fleet tombstone pattern: a just-deleted VPN connection stays describable by id in state=deleted while unfiltered DescribeVpnConnections omits it. DeleteCustomerGateway/DeleteVpnGateway audited against terraform-provider-aws v5.100.0's find.go/wait.go: both already return the correct NotFoundException error code on hard delete, and both waiters' own findByID helpers treat state=deleted the same as NotFound, so no waiter delay exists there today -- no tombstone needed. Test: vpn_connection_tombstone_test.go. TestDescribeInstances_WireOutputUnchanged still green.","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -1467,11 +1474,11 @@ {"_type":"issue","id":"gopherstack-frq01","title":"[decision] ses: real SMTP delivery option beside the mailbox simulator","description":"SES accepts sends and emits simulator bounce/complaint events but never delivers; LocalStack routes mail to a local SMTP. Option: SES_SMTP_HOST config to relay via net/smtp when set (default off), plus a built-in capture endpoint under /_gopherstack/ses/messages for tests. Needs a decision on whether outbound network from the emulator is acceptable by default.","status":"open","priority":3,"issue_type":"feature","owner":"blackbird7181@gmail.com","created_at":"2026-09-19T14:29:03Z","created_by":"Witness Patrol","updated_at":"2026-09-19T14:29:03Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-7neth","title":"[decision] eks: real cluster provider (kind/k3s in docker) behind the control plane","description":"docs/migration.md: no EKS real containers; LocalStack Pro provisions a real k8s. Option: EKS_PROVIDER=docker runs k3s/kind via the docker runner, returns a real kubeconfig from DescribeCluster endpoint/certificate. Default off. Needs a decision on the runtime dependency and CI coverage before code.","status":"open","priority":3,"issue_type":"feature","owner":"blackbird7181@gmail.com","created_at":"2026-09-19T14:29:01Z","created_by":"Witness Patrol","updated_at":"2026-09-19T14:29:01Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-cpztm","title":"[bug] cmd/enumcheck resolves enums by shared wire key, not by the emitting field's type: 102 of 104 findings on eks/sagemaker/glue were false positives","description":"Measured 2026-09-18 on chore/parity-sweep-2026-09-17: enumcheck flagged eks 38 / sagemaker 33 / glue 33 rows. Hand verification against each field's real struct in types.go found 2 real bugs (sagemaker AutoML 'validation:accuracy' -\u003e 'Accuracy'; a phantom PipelineExecutionStep.StepType) and 102 false positives: the tool builds a candidate set from every enum whose members appear under the same key name ('status', 'type', 'state') and reports a value 'not a member of every candidate enum', instead of resolving the emitting struct field to its one real enum. It also treats plain *string fields (ItemError.Code, DevEndpoint.Status, BatchDescribeModelPackageError.ErrorCode) as enums, and misses enums absent from its candidate list (EksAnywhereSubscriptionStatus). Fix: resolve the Go struct field's json tag -\u003e SDK output type -\u003e member type via the pinned module's types.go before comparing; skip *string members. Remaining repo-wide rows (~600 across 67 services) are not worth sweeping until then.","status":"open","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-18T17:47:40Z","created_by":"Witness Patrol","updated_at":"2026-09-18T17:47:40Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"gopherstack-7r6bz","title":"dsql: emulate Aurora DSQL (the one LocalStack-documented service gopherstack lacks)","description":"Web-verified 2026-09-17: gopherstack's AWS surface is a superset of LocalStack's documented service list except Aurora DSQL (LocalStack Pro-only). Scope: new services/dsql with the control plane (CreateCluster/GetCluster/UpdateCluster/DeleteCluster/ListClusters, multi-region peering fields, tags, DbConnect token generation) per the pinned aws-sdk-go-v2/service/dsql module (needs a go.mod add — decide), and a decision on the data plane (DSQL speaks Postgres wire protocol; embedded engine vs metadata-only like rds). Own PR; do not fold into a parity sweep.","status":"open","priority":3,"issue_type":"feature","owner":"blackbird7181@gmail.com","created_at":"2026-09-18T15:05:45Z","created_by":"Witness Patrol","updated_at":"2026-09-18T15:05:45Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-7r6bz","title":"dsql: emulate Aurora DSQL (the one LocalStack-documented service gopherstack lacks)","description":"Web-verified 2026-09-17: gopherstack's AWS surface is a superset of LocalStack's documented service list except Aurora DSQL (LocalStack Pro-only). Scope: new services/dsql with the control plane (CreateCluster/GetCluster/UpdateCluster/DeleteCluster/ListClusters, multi-region peering fields, tags, DbConnect token generation) per the pinned aws-sdk-go-v2/service/dsql module (needs a go.mod add — decide), and a decision on the data plane (DSQL speaks Postgres wire protocol; embedded engine vs metadata-only like rds). Own PR; do not fold into a parity sweep.","status":"closed","priority":3,"issue_type":"feature","owner":"blackbird7181@gmail.com","created_at":"2026-09-18T15:05:45Z","created_by":"Witness Patrol","updated_at":"2026-09-26T06:36:59Z","closed_at":"2026-09-26T06:36:59Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-yf2hu","title":"outposts: TestPersistence_SnapshotRestoreRoundTrip_MidFlightCapacityTaskTransition flakes on CI (IN_PROGRESS window missed)","description":"PR #2467 run 34744381812 unit-tests(3): capacity_tasks_test.go:32 require.Eventually 'capacity task never reached status IN_PROGRESS' after 10s. Test predates the branch (8955a7e56, 2026-08-26). The backend advances REQUESTED→IN_PROGRESS→COMPLETED on real timers, so under -race -shuffle load the 10ms poller can miss the IN_PROGRESS window entirely. Fix per repo rule (no wall-clock waits): drive the transition through testing/synctest or expose a clock seam on the backend and step it, then assert each status deterministically; same for the sibling MidFlightOrderTransition test.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-13T07:22:08Z","created_by":"Witness Patrol","updated_at":"2026-09-13T07:40:37Z","closed_at":"2026-09-13T07:40:37Z","close_reason":"Mid-flight transition tests moved onto synctest's clock against the backend directly; -race -count=20 -shuffle clean.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-4o9gw","title":"bedrockruntime: TestInvokeModelWithResponseStream_SDKRoundTrip flaked once in CI (use of closed network connection)","description":"PR #2467 run 34737227162 unit-tests(2): wire_sdk_roundtrip_test.go:99 stream.Err() = 'read tcp ...: use of closed network connection'. Not reproducible locally (-race -count=40, and -shuffle on the package x5). Test predates the branch (only PARITY.md + typed slice 15 file added to the service). Suspect: httptest server or transport closed while the eventstream reader is mid-read under CI load; check whether handleInvokeModelWithResponseStream returns before the client drains, and whether t.Cleanup ordering (srv.Close registered in the helper before stream.Close) matters under shuffle.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-13T04:27:51Z","created_by":"Witness Patrol","updated_at":"2026-09-13T11:44:59Z","closed_at":"2026-09-13T11:44:59Z","close_reason":"Same mechanism as i8q7 (Transport keep-alive reuse race); DisableKeepAlives applied to newTestBedrockRuntimeSDKClient defensively — not independently reproduced for this service.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-aqgw0","title":"clientcoverage misattributes AgentsHandler-family ops from services/bedrock to services/bedrockagent","description":"Found while driving typed_slice22_realclient_test.go (services/bedrock, gopherstack-n3zi slice 22).\n\nservices/bedrock's AgentsHandler implements the same bedrock-agent op names\n(CreateAgent, CreateFlow, CreatePrompt, AssociateAgentCollaborator, etc.) as\nthe separate, actually-live services/bedrockagent package -- both use the\nsame real aws-sdk-go-v2/service/bedrockagent SDK module. cmd/opcensus already\nknows about this pairing (bedrock's \"chased\" sdkModules list includes both\n\"bedrock\" and \"bedrockagent\") but cmd/clientcoverage's resolveOwner requires\na SINGLE owning service per (module, op) pair.\n\nMeasured directly: with typed_slice22_realclient_test.go present (72\nAgentsHandler ops driven through a real bedrockagent client, asserted\nagainst services/bedrock's own AgentsHandler test harness --\nnewTestBedrockRegistryServer, NOT services/bedrockagent), the census shows:\n bedrock: 105/179 -\u003e 108/179 (+3, only the EnforcedGuardrailConfiguration\n trio, which is NOT name-ambiguous)\n bedrockagent: 38/75 -\u003e 75/75 (+37, entirely from slice 22's calls)\n\nWith the test file removed: bedrock 105/179, bedrockagent 38/75 (its\npre-slice-22 baseline). Confirms every client.CreateAgent/.../ call using a\n*bedrockagentsdk.Client anywhere in the repo's tests is attributed wholesale\nto \"bedrockagent\", never \"bedrock\", regardless of which backend the httptest\nserver actually points at.\n\nNet effect: bedrock's AgentsHandler family (the \"AgentsHandler is dead code\nin production, shadowed by services/bedrockagent's higher route priority\"\nfinding already on record in services/bedrock/PARITY.md, gopherstack-y1zn)\ncan NEVER show up as covered in bedrock's own census number no matter how\nmuch typed-client testing targets it directly -- the tool structurally\ncredits it to the unrelated, higher-priority sibling service instead.\n\nNot fixed this pass (tooling change, out of scope for a coverage-sweep\nslice). Options for a future pass: teach resolveOwner to disambiguate by\nwhich service's own test-file/package the call site lives in (not just SDK\nimport), or accept the ambiguity and stop reporting bedrock/bedrockagent as\nseparately measurable services in this census.","status":"closed","priority":3,"issue_type":"chore","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-09-12T16:54:09Z","created_by":"Witness Patrol","updated_at":"2026-09-18T04:51:55Z","started_at":"2026-09-18T04:12:41Z","closed_at":"2026-09-18T04:51:55Z","close_reason":"AgentsHandler deleted in 07d713826; bedrock 108/108 declared once, clientcoverage 97.2%","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"gopherstack-hgjl8","title":"lambda: DurableExecution.FunctionARN never assigned — ListDurableExecutionsByFunction always empty","description":"Found 2026-09-12 (typed slice 21). services/lambda durable_execution.go: the only creation path (CheckpointDurableExecution) never threads the function identity into DurableExecution.FunctionARN, so ListDurableExecutionsByFunction returns zero results for every function. Determine the real entry point that creates a durable execution for a function (Invoke with a durable config? StartDurableExecution?) per lambda@v1.107.0 api_op_*DurableExecution*.go, thread the function ARN, and add a real-client test listing by function. Recorded in lambda PARITY.md items_still_open.","status":"open","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-12T16:44:46Z","created_by":"Witness Patrol","updated_at":"2026-09-12T16:44:46Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-hgjl8","title":"lambda: DurableExecution.FunctionARN never assigned — ListDurableExecutionsByFunction always empty","description":"Found 2026-09-12 (typed slice 21). services/lambda durable_execution.go: the only creation path (CheckpointDurableExecution) never threads the function identity into DurableExecution.FunctionARN, so ListDurableExecutionsByFunction returns zero results for every function. Determine the real entry point that creates a durable execution for a function (Invoke with a durable config? StartDurableExecution?) per lambda@v1.107.0 api_op_*DurableExecution*.go, thread the function ARN, and add a real-client test listing by function. Recorded in lambda PARITY.md items_still_open.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-12T16:44:46Z","created_by":"Witness Patrol","updated_at":"2026-09-26T18:47:00Z","closed_at":"2026-09-26T18:47:00Z","close_reason":"Fixed in d18a31a4c: Invoke assigns FunctionArn/Version; ListDurableExecutionsByFunction FunctionName/Qualifier filters covered by TestRealClient_DurableInvoke","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-pulu9","title":"dms: Describe* filters OR-match only Values[0] (extractFilterValue truncation across ~40 call sites)","description":"Found 2026-09-12 by gopherstack-0vh7l. services/dms/handler.go:433-441 extractFilterValue returns filters[i].Values[0]; types.Filter.Values is plural and AWS OR-matches all values. Most call sites pass the value to Backend.DescribeX(ctx, identifier string) — fix requires widening ~15 backend Describe signatures to accept []string (or a filter struct), matching any value, with real-client tests per op (DescribeReplicationInstances/Tasks/Endpoints/Connections/Certificates/EventSubscriptions/...). Recorded in dms PARITY.md items_still_open.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-12T12:59:58Z","created_by":"Witness Patrol","updated_at":"2026-09-12T13:28:11Z","closed_at":"2026-09-12T13:28:11Z","close_reason":"DescribeFilters model: OR within a filter, AND across; 10 backend signatures widened, ~20 ops converted; real-client tests.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-0vh7l","title":"sweep: documented-plural request members truncated to [0] (iot/ec2/opsworks class)","description":"Recurring class found by the typed-client slices 2026-09-12: a handler reads a plural SDK request member (LayerIds, SecurityGroupIds, SubnetIds, ...) and keeps only the first element, or reads a singular where the wire is plural. Seen in iot, ec2 (twice) and opsworks. Sweep every service: for each Input member whose SDK type is a slice ([]string / []Type) in api_op_*.go, find the handler read site and confirm it is consumed as a list (stored/emitted fully, filters applied to all); fix truncations with real-client tests; record per-service in PARITY.md. Method notes in the session scratchpad typed/NOTES.md (slice 13).","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-12T12:31:31Z","created_by":"Witness Patrol","updated_at":"2026-09-12T12:59:52Z","closed_at":"2026-09-12T12:59:52Z","close_reason":"Swept 149 services / 3,959 slice members; all candidates hand-read; only dms Filters.Values[0] survives (filed separately). Known instances already fixed.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-ggu4a","title":"ec2: Describe-by-id ops silently omit unknown ids — dedicated sweep","description":"Found 2026-09-12 by the typed-client slice: DescribeInternetGateways/DhcpOptions/RouteTables/Snapshots/NetworkAcls/CustomerGateways dropped an explicitly requested nonexistent id instead of returning the InvalidXxx.NotFound error real EC2 returns; fixed for those six via a shared helper (services/ec2/describe_helpers.go). The same pattern likely exists in most other ec2 Describe* ops that accept an id list (subnets? security groups? volumes? images? key pairs? launch templates? transit gateways? ...). Sweep every Describe* with an \u003cResource\u003eIds/Names input: verify the error code per op in ec2@v1.329.0 (api_op doc + errors overview), apply the helper, real-client tests.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-12T05:47:03Z","created_by":"Witness Patrol","updated_at":"2026-09-12T06:49:23Z","closed_at":"2026-09-12T06:49:23Z","close_reason":"~20 Describe-by-id ops now fail with the real NotFound codes via shared helpers; 52 codes added to errCodeLookup; 3 fabricated/miscased codes corrected; soft-filter ops documented.","dependency_count":0,"dependent_count":0,"comment_count":0} @@ -1500,7 +1507,7 @@ {"_type":"issue","id":"gopherstack-g4wur","title":"lightsail PARITY.md overstates its collision verdict as byte-identical","description":"Found while verifying gopherstack-id70's merged audit. Documentation accuracy, not a code defect.\n\nservices/lightsail/PARITY.md's Handler-collision determinism section claims the pre-fix reqfielddiff output was 'byte-identical across all 5 old runs and HEAD... zero damage'. Reproduction with the matched-snapshot method shows that is not literally true:\n\n- The pre-fix tool's aggregate declared-field count flickers between 1244 and 1250. This is a benign package-wide count unrelated to any specific finding, and the same class is already documented for cleanrooms.\n- Three fields shift confidence tier between pre- and post-fix runs: GetOperation.OperationId, and SetupInstanceHttps.CertificateProvider and .DomainNames, all moving tier3 to tier4.\n\nThe SUBSTANTIVE verdict is unchanged in both: all three are flagged as undeclared either way. So lightsail's bottom line - no real bug, no verdict flip - stands. Only the literal 'byte-identical' phrasing is wrong.\n\nFix: soften that sentence to match what was actually observed, and note the tier shift so a future reader re-running the comparison does not think they have found a regression. Relevant because PARITY.md has already been wrong in eighteen distinct ways and is treated as corroboration by covledger.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-11T00:37:37Z","created_by":"Witness Patrol","updated_at":"2026-09-11T01:31:06Z","closed_at":"2026-09-11T01:31:06Z","close_reason":"Fixed in the same commit. services/lightsail/PARITY.md's Handler-collision section now states the observed reality instead of byte-identical: aggregate declared-field count flickered 1244-1250 pre-fix (benign, same class as cleanrooms), three fields (GetOperation.OperationId, SetupInstanceHttps.CertificateProvider, SetupInstanceHttps.DomainNames) shifted tier3 to tier4 pre/post, substantive verdict unchanged since all three stayed flagged undeclared in every run. Phrasing mirrors cleanrooms' fr30 section so the two are consistent. Section and conclusion preserved.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-c7blx","title":"ssm DescribeMaintenanceWindowExecutions and ExecutionTaskInvocations never read their real Filters","description":"Found while fixing gopherstack-tz6z (223269022). These two operations were NOT named in that issue, so they were left alone.\n\nBoth carry real Filters members in the SDK that gopherstack never reads, the same defect tz6z described for their three sibling operations.\n\nFix the same way tz6z was fixed: type the filter to the closed key set the operation's own SDK doc comment documents, apply before pagination, and follow instanceInformationAttr's accept-and-echo precedent for unrecognized keys.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-10T23:34:20Z","created_by":"Witness Patrol","updated_at":"2026-09-11T01:44:50Z","closed_at":"2026-09-11T01:44:50Z","close_reason":"Fixed in c925b4923. Closed key sets verified per operation from ssm v1.77.0: DescribeMaintenanceWindowExecutions.Filters supports ExecutedBefore/ExecutedAfter (api_op:39-40); DescribeMaintenanceWindowExecutionTaskInvocations.Filters supports only STATUS (api_op:42-43). Added Filters []MaintenanceWindowFilter to both inputs reusing the existing type. filterWindowExecutions/matchesExecutionFilters reuse sessionTimestampCompare from sessions.go (the same ISO-8601-vs-Unix-seconds comparison InvokedBefore/InvokedAfter use); filterExecutionTaskInvocations mirrors filterExecutionTasks. Unrecognized keys match everything per instanceInformationAttr's precedent; ssm's paginateSlice convention kept. STATUS test uses the corrected mwExecutionStatusSuccess (SUCCESS, from fb9beca5a) asserted through the real typed client. Narrowing subtests fail against unfixed code; matches-everything subtests pass either way as expected.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-tx8a5","title":"lambda Invoke does not thread Qualifier into scaling-config enforcement","description":"Narrowed deliberately while fixing gopherstack-gjn1 (a244a8c0b), disclosed here rather than left silent.\n\nfunctionScalingConfigs is now correctly keyed by (function, qualifier), so a version or alias can carry its own MaxExecutionEnvironments. But the two invoke-time enforcement lookups in services/lambda/invocation.go:548 and :586 hardcode versionLatest, because Invoke does not thread its Qualifier through to that call.\n\nConsequence: invoking a specific version or alias is enforced against $LATEST's scaling config, not its own. For an unqualified invoke this matches AWS ('no Qualifier means $LATEST'); for a qualified one it is wrong whenever the two configs differ.\n\nFix: thread the invoke's resolved qualifier down to the enforcement path and look up permissionMapKey(name, resolvedQualifier). Check how activeConcurrencies is keyed while there - it is keyed by function name alone, which may have the same collapse.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-10T22:50:34Z","created_by":"Witness Patrol","updated_at":"2026-09-11T01:44:49Z","closed_at":"2026-09-11T01:44:49Z","close_reason":"Fixed in c925b4923. acquireConcurrencySlot now takes the resolved qualifier and looks up permissionMapKey(functionName, qualifier) for both scaling-config checks, replacing the hardcoded versionLatest. Call site passes fn.Version, which resolveQualifier (qualifiers.go:83) already resolves - an alias becomes its target version via versionToFn (versions_aliases.go:360-379) - matching PutFunctionScalingConfig's doc (lambda v1.107.0 api_op_PutFunctionScalingConfig.go:37-38). activeConcurrencies/functionConcurrencies VERIFIED CORRECT AS-IS and left alone: PutFunctionConcurrency's doc says reserved concurrency 'applies to the function as a whole, including all published versions and the unpublished version' (api_op_PutFunctionConcurrency.go:13-14), so the per-function key is AWS semantics, unlike the per-qualifier scaling config. Disclosing comments removed. TestInvoke_ScalingConfig_EnforcedPerResolvedQualifier: unqualified invoke blocked by $LATEST's limit, alias invoke uses its own version-scoped limit; the alias case fails against the hardcoded lookup with a false TooManyRequestsException.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"gopherstack-j60e","title":"[bug] TestIntegration_Kinesis_EnhancedFanOut: SubscribeToShard stream dies with 'use of closed network connection'","description":"INVESTIGATED 2026-09-10, NOT reproducible, NOT fixed. Left open deliberately. Do not close as fixed and do not add a retry, sleep, or error tolerance to quiet it.\n\nORIGINAL FAILURE (run 34455260407, integration-tests (1), commit c92eb3383):\n test/integration/kinesis_test.go:369\n read tcp [::1]:54726-\u003e[::1]:32770: use of closed network connection\n --- FAIL: TestIntegration_Kinesis_EnhancedFanOut\n\nFINDINGS, verified by the main thread against the code and the SDK:\n- The handler bounds a subscription two ways (handler_consumers.go:279-288): a 5-minute hard deadline, which matches the SDK's own documented \"for up to 5 minutes\" (api_op_SubscribeToShard.go:22), and an idle-close after 3 consecutive empty 200ms polls.\n- The idle path returns nil from the handler. That finishes the chunked body normally, and the SDK's generated event-stream reader treats it as io.EOF, closing the channel with NO error set. \"use of closed network connection\" is a net.OpError from reading an already-closed socket, which requires something more abrupt than an ordinary handler return.\n- The polling emulation is already disclosed: services/kinesis/PARITY.md lists \"Enhanced fan-out SubscribeToShard real streaming cadence / HTTP2 push semantics beyond the polling emulation\" under deferred. It is also load-bearing -- consumers_test.go:86 TestSubscribeToShard_StreamClosesAfterIdle drives the handler synchronously through a ResponseRecorder and would hang if the idle-close were removed.\n- No commit in range reaches this path. c56c2ffc8, 724ce9b40 and 6a695630b touch Reset defaults and Lambda ESM ARN parsing, not handler_consumers.go. The shutdown-lifecycle work (3de7de086, cc4dae62f, 9596bc160) does not touch services/kinesis at all; kinesis has no worker.Group and appeared in neither Shutdowner list. The container serves continuously during the test, so teardown timing is moot.\n\nWHAT WAS ADDED: services/kinesis/subscribe_idle_close_test.go (commit 8c521bb57) drives a real AWS SDK client over real TCP (httptest.NewServer, not the ResponseRecorder most kinesis tests use) through the same sequence as the failing integration test and asserts stream.Err() is nil after the idle close. 8x plain, 25x under -race, plus whole-package runs under GOMAXPROCS=2: zero failures. That is coverage of the graceful path, NOT a reproduction.\n\nUNVERIFIED: no Docker on the investigating machine, so the containerised network path where the failure actually occurred was never exercised. The abrupt-close mechanism is inferred (most plausibly container/NAT-layer interference under many parallel shards hitting one container), not observed.\n\nRELATED, NOT CONFIRMED DUPLICATE: gopherstack-i8q7 is an open kinesis SubscribeToShard flake, reproduced once in 1500+ runs and still unreproduced after ~540 further executions under heavy contention. Same operation and same area, but the SYMPTOMS DIFFER -- i8q7 is a missing record, this is a connection error. Treat them as siblings, and do not merge them without evidence.\n\nNEXT STEP IF IT RECURS: get a rate first. A second occurrence with a measurable frequency would justify revisiting the idle-close budget (widening subscribeToShardMaxIdlePolls) as a real fix. A single further occurrence justifies nothing but another data point. Per i8q7's own conclusion, the promising direction is resource constraint -- a container with 1-2 cores and a low memory limit, or a full-repo CI-shaped run -- not more iterations of a single-package loop.","notes":"2026-09-11 data point: TestSubscribeToShard_IdleCloseIsGraceful failed once in ~200 loaded in-process iterations (-race -count=100 with ec2+s3 -count=3 -p 8 concurrently), 0 in 300 unloaded — 'use of closed network connection' instead of io.EOF. First non-container reproduction; rate ≈1/200 under load. Not fixed per this issue's own instructions.","status":"open","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-10T09:08:43Z","created_by":"Witness Patrol","updated_at":"2026-09-12T02:21:48Z","labels":["parity-campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-j60e","title":"[bug] TestIntegration_Kinesis_EnhancedFanOut: SubscribeToShard stream dies with 'use of closed network connection'","description":"INVESTIGATED 2026-09-10, NOT reproducible, NOT fixed. Left open deliberately. Do not close as fixed and do not add a retry, sleep, or error tolerance to quiet it.\n\nORIGINAL FAILURE (run 34455260407, integration-tests (1), commit c92eb3383):\n test/integration/kinesis_test.go:369\n read tcp [::1]:54726-\u003e[::1]:32770: use of closed network connection\n --- FAIL: TestIntegration_Kinesis_EnhancedFanOut\n\nFINDINGS, verified by the main thread against the code and the SDK:\n- The handler bounds a subscription two ways (handler_consumers.go:279-288): a 5-minute hard deadline, which matches the SDK's own documented \"for up to 5 minutes\" (api_op_SubscribeToShard.go:22), and an idle-close after 3 consecutive empty 200ms polls.\n- The idle path returns nil from the handler. That finishes the chunked body normally, and the SDK's generated event-stream reader treats it as io.EOF, closing the channel with NO error set. \"use of closed network connection\" is a net.OpError from reading an already-closed socket, which requires something more abrupt than an ordinary handler return.\n- The polling emulation is already disclosed: services/kinesis/PARITY.md lists \"Enhanced fan-out SubscribeToShard real streaming cadence / HTTP2 push semantics beyond the polling emulation\" under deferred. It is also load-bearing -- consumers_test.go:86 TestSubscribeToShard_StreamClosesAfterIdle drives the handler synchronously through a ResponseRecorder and would hang if the idle-close were removed.\n- No commit in range reaches this path. c56c2ffc8, 724ce9b40 and 6a695630b touch Reset defaults and Lambda ESM ARN parsing, not handler_consumers.go. The shutdown-lifecycle work (3de7de086, cc4dae62f, 9596bc160) does not touch services/kinesis at all; kinesis has no worker.Group and appeared in neither Shutdowner list. The container serves continuously during the test, so teardown timing is moot.\n\nWHAT WAS ADDED: services/kinesis/subscribe_idle_close_test.go (commit 8c521bb57) drives a real AWS SDK client over real TCP (httptest.NewServer, not the ResponseRecorder most kinesis tests use) through the same sequence as the failing integration test and asserts stream.Err() is nil after the idle close. 8x plain, 25x under -race, plus whole-package runs under GOMAXPROCS=2: zero failures. That is coverage of the graceful path, NOT a reproduction.\n\nUNVERIFIED: no Docker on the investigating machine, so the containerised network path where the failure actually occurred was never exercised. The abrupt-close mechanism is inferred (most plausibly container/NAT-layer interference under many parallel shards hitting one container), not observed.\n\nRELATED, NOT CONFIRMED DUPLICATE: gopherstack-i8q7 is an open kinesis SubscribeToShard flake, reproduced once in 1500+ runs and still unreproduced after ~540 further executions under heavy contention. Same operation and same area, but the SYMPTOMS DIFFER -- i8q7 is a missing record, this is a connection error. Treat them as siblings, and do not merge them without evidence.\n\nNEXT STEP IF IT RECURS: get a rate first. A second occurrence with a measurable frequency would justify revisiting the idle-close budget (widening subscribeToShardMaxIdlePolls) as a real fix. A single further occurrence justifies nothing but another data point. Per i8q7's own conclusion, the promising direction is resource constraint -- a container with 1-2 cores and a low memory limit, or a full-repo CI-shaped run -- not more iterations of a single-package loop.","notes":"2026-09-26: reproduced locally once (TestSubscribeToShard_IdleCloseIsGraceful: 'read tcp ...: use of closed network connection') while running go test -race -cpu=1,4 -count=2 on dynamodb,s3,sqs,kinesis concurrently (heavy CPU load). 0/60 in isolation right after. Supports the resource-contention hypothesis; the unit test hits it too, so it is not container/NAT-specific.","status":"open","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-10T09:08:43Z","created_by":"Witness Patrol","updated_at":"2026-09-26T22:09:03Z","labels":["parity-campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-1o31","title":"[bug] latent: eks/fsx/mwaa/resiliencehub integration tests use fabricated subnet IDs and will break as validation lands","description":"FOUND while verifying gopherstack-3vif. Not currently failing -- filed so it is not rediscovered as a mystery later.\n\ngopherstack-3vif fixed four integration tests that asserted against resources they never created, once branch-new cross-service validation started rejecting fabricated identifiers. The same fabricated-ID pattern still exists elsewhere and is only passing because those services do not validate yet:\n\n test/integration/eks_test.go:30,95,118 SubnetIds: []string{\"subnet-12345678\"}\n test/integration/fsx_test.go:56,109 SubnetIds: []string{\"subnet-12345678\"}\n test/integration/mwaa_test.go:68 SubnetIds: []string{\"subnet-12345678\",\"subnet-87654321\"}\n test/integration/resiliencehub_test.go:896,1071 SubnetIds: []string{\"subnet-12345678\"}\n\nThe moment eks/fsx/mwaa/resiliencehub gain an EC2Resolver SubnetExists check -- exactly what efs just got -- these fail identically, and the failure will again look like a cross-service wiring regression rather than a stale fixture. That misreading cost real time on 3vif.\n\nFIX: have each create a real VPC + subnet via ec2Client and use the returned SubnetId, following the pattern now in test/integration/efs_test.go. createEC2Client(t) already exists in test/integration/main_test.go.\n\nAlso worth grepping for other fabricated identifier shapes beyond subnets (vpc-, i-, sg-, ami-, arn:aws:... literals) in test/integration/ and test/terraform/, and reporting the full list even if not all are fixed now.\n\nTHE UNDERLYING DEFECT, worth stating in whatever you write: this repo's parity fixes have repeatedly updated unit tests while leaving the integration and terraform suites stale, because those run in separate CI jobs that per-package gates never exercise. Seven integration failures in this branch all traced to that. A checklist or CI-level reminder when a service gains a cross-service validation would prevent the next round.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-10T04:48:11Z","created_by":"Witness Patrol","updated_at":"2026-09-10T05:02:28Z","closed_at":"2026-09-10T05:02:28Z","close_reason":"Closed","labels":["parity-campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-tajh","title":"main_test TestMultipleServersStartupAndShutdown: pre-existing port TOCTOU, same class as the closed pkgs/dns flakes","description":"TestMultipleServersStartupAndShutdown/server_startup_without_DEMO failed in CI run 34225360772 at main_test.go:128 with \"failed to reach server on :46795\" / \"Condition never satisfied\".\n\nNOT BRANCH-INTRODUCED: `git diff origin/main...HEAD -- main_test.go` is empty. The test is byte-identical to main.\n\nMECHANISM, established from the code by the triage agent:\n- freeTCPPort (main_test.go:184-192) opens net.Listen(\"tcp\",\"127.0.0.1:0\"), reads the OS-assigned port, then `defer l.Close()` releases it immediately.\n- The real bind happens much later: startServerOnPort -\u003e run(ctx, cli) -\u003e startServer (cli.go:11474), only after run()'s init chain (cli.go:1982-2088) does port-allocator setup, AWS config, client init, persistence init, initializeServices, persistence wiring, echo build, chaos/registry setup and background workers.\n- That is a TOCTOU window, and an unusually wide one -- anything else requesting an ephemeral port in between can take it.\n- On bind failure the error goes to a buffered errChan that nothing reads until after the require.Eventually loop, so a bind failure and a merely-slow start are indistinguishable from the reported message. Both surface as \"Condition never satisfied\".\n- The root package has 80+ test files, many calling initializeServices with t.Parallel(), so under -race on a loaded runner the 10s Eventually budget is also plausibly tight -- compounding, not competing, with the TOCTOU.\n\nPRECEDENT: gopherstack-nn94 (pkgs/dns TestServer_Stop) and gopherstack-7tbt document the identical pick-port, close, bind-later pattern flaking under parallel load, with a documented fix direction -- a retry helper rather than close-and-race. Both are closed P3s in this campaign. This is the same class in a different package.\n\nMEASUREMENT INCOMPLETE: only 1 of a planned 10 local runs finished before the triage agent reported (it passed). Each cold -race build of the root package takes roughly 4.5 minutes, so a rate needs a deliberate run. Do not quote a rate that has not been measured.\n\nFIX DIRECTION: follow nn94's retry-helper precedent rather than widening the timeout, which would only make the flake rarer and slower to diagnose. Note this repo BANS time.Sleep in tests; use require.Eventually with the package's established intervals or testing/synctest. Also consider surfacing the errChan bind error into the failure message so the two failure modes stop being indistinguishable -- that alone would make the next occurrence diagnosable.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-08T13:29:46Z","created_by":"Witness Patrol","updated_at":"2026-09-10T03:29:03Z","closed_at":"2026-09-10T03:29:03Z","close_reason":"Closed","labels":["parity-campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-neiq","title":"services/lambda goleak fails intermittently (28% on main, 60% on this branch): shared http.DefaultTransport keep-alive goroutines","description":"services/lambda's package-level goleak check fails intermittently, and it fails MORE OFTEN on this branch than on main.\n\nMEASURED by the main thread's triage agent, 25 runs each of `GOTOOLCHAIN=go1.27.0 go test -race -count=1 ./services/lambda/...`:\n origin/main (clean checkout via git archive): 7/25 failed (28%)\n this branch: 15/25 failed (60%)\n\nLEAKING GOROUTINES IDENTIFIED: always net/http.(*persistConn).readLoop and net/http.(*persistConn).writeLoop, created by net/http.(*Transport).dialConn. Origins: http.DefaultClient.Do calls in iam_enforcement_test.go:166 and handler_runtime_test.go:290,442,471,489,895, plus \u0026http.Client{Timeout: ...} values in store_test.go:696,787,854 -- those have a zero-value Transport field so they share http.DefaultTransport's connection pool with DefaultClient. Response bodies ARE closed correctly, so the transport keeps the connection as an idle keep-alive; the parked readLoop/writeLoop are what goleak.VerifyTestMain catches when it samples before they exit after server teardown. Inherently timing-dependent, which is why it is intermittent.\n\nWHY THE BRANCH RATE IS HIGHER -- flagged as the likely explanation, NOT proven: the branch's diff to services/lambda non-test code (functions.go, containers.go, event_source_poller.go, store.go, crossservice.go, lifecycle.go) is all business logic and touches no HTTP client or server lifecycle. The branch does add several hundred lines of new tests, which lengthens the binary's run and widens the sampling window. Someone should confirm or refute this rather than inheriting it as fact.\n\nNOTE: services/lambda/PARITY.md already carries a 2026-09-06 entry documenting this and recommending a CloseIdleConnections or goleak ignore-list follow-up. This issue is that follow-up.\n\nLIKELY FIX DIRECTION: give the tests a client whose transport is theirs to close, and call CloseIdleConnections at teardown, rather than sharing http.DefaultTransport's pool across the whole package. An httptest.Server's own Client() is the idiomatic choice where a server is already in play. A goleak ignore-list entry is the weaker fallback -- it hides a real (if benign) leak and would mask a future genuine one in the same package.\n\nVERIFY any fix by running the package at least 25 times under -race and reporting the failure rate, not once. A single green run proves nothing at a 60% failure rate.","status":"closed","priority":3,"issue_type":"bug","owner":"blackbird7181@gmail.com","created_at":"2026-09-08T13:29:44Z","created_by":"Witness Patrol","updated_at":"2026-09-10T03:11:24Z","closed_at":"2026-09-10T03:11:24Z","close_reason":"Closed","labels":["parity-campaign"],"dependency_count":0,"dependent_count":0,"comment_count":0} @@ -1531,7 +1538,7 @@ {"_type":"issue","id":"gopherstack-kx95","title":"stepfunctions: DELETING is never observable, so StateMachineDeleting can never be returned","description":"Structural finding from gopherstack-2hdk, not flagged by the tool.\n\nReal AWS models state-machine deletion as asynchronous and declares StateMachineDeleting on CreateStateMachine, StartExecution and StartSyncExecution -- verified in the pinned sfn deserializers. This backend has no ErrStateMachineDeleting sentinel anywhere and no classifyError row for the code.\n\nDeleteStateMachine sets Status = statusDeleting and then deletes the record inside the same locked region, so DELETING is never externally observable and the code can never be emitted. A consequence worth noting: CreateStateMachine's duplicate-name guard tests sm.Status != statusDeleting, which is therefore dead code.\n\nMaking this reachable means modelling asynchronous deletion, which is a lifecycle change rather than an error-mapping fix. Related: gopherstack-9ojs records the same class of gap in ram, where reaching FAILED needs a completion signal the backend does not have.","status":"closed","priority":3,"issue_type":"bug","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-09-07T13:58:09Z","created_by":"Witness Patrol","updated_at":"2026-09-08T07:24:45Z","started_at":"2026-09-08T06:48:11Z","closed_at":"2026-09-08T07:24:45Z","close_reason":"Real defect, not a modelling gap: botocore documents DeleteStateMachine as asynchronous. DELETING window now modelled via the existing janitor (immediate delete preserved when nothing is running); all 8 declaring ops gated on the new sentinel. Neuter-verified; latent same-ARN collision on recreate also closed.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-s9zy","title":"stepfunctions: DescribeStateMachineForExecution returns StateMachineDoesNotExist, which it does not declare","description":"The one real finding from gopherstack-2hdk, left unfixed because both candidate remedies need their own evidence pass.\n\nDescribeStateMachineForExecution declares ExecutionDoesNotExist, InvalidArn, KmsAccessDeniedException, KmsInvalidStateException, KmsThrottlingException. Verified by extraction against the pinned sfn module. It returns ErrStateMachineDoesNotExist from the !hasSnapshot fallback in executions.go, and no declared code fits the actual condition -- the execution exists and its state machine does not, which is not ExecutionDoesNotExist.\n\nThe branch fires after a restore, because executionDefinitions is deliberately excluded from persistence under a documented Phase-3.3 boundary, so a restored execution has no definition snapshot.\n\nTwo remedies, neither free. Persisting executionDefinitions fixes the root cause but is a persistence-shape change and would bump sfnSnapshotVersion, which discards user snapshots on restore -- see gopherstack-c8sa for why that matters. Alternatively the branch could return a synthetic 200 like its sibling three lines below, which answers the identical condition that way already; that sibling is strong precedent but converting an error to success silently needs evidence of its own, per the trap recorded on gopherstack-jkma.\n\nA landmine comment at the site names both candidates.","status":"closed","priority":3,"issue_type":"bug","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-09-07T13:58:07Z","created_by":"Witness Patrol","updated_at":"2026-09-08T09:57:32Z","started_at":"2026-09-08T09:47:49Z","closed_at":"2026-09-08T09:57:32Z","close_reason":"No declared code fits (dispatch declares ExecutionDoesNotExist/InvalidArn/Kms* only). New counter-evidence against the synthetic-200 remedy: Definition is min length 1, so the !hasSnapshot branch would return a schema-invalid empty definition. Left as-is, reasoning recorded.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-l81f","title":"codedeploy: five delete and deregister ops emit undeclared not-found codes","description":"Five of the six class A findings from gopherstack-3pz8, confirmed against the pinned codedeploy SDK and left unfixed because no remedy is evidenced. Each site carries a landmine comment naming the mismatch and candidates.\n\n DeleteApplication emits ApplicationDoesNotExistException; declares ApplicationNameRequiredException, InvalidApplicationNameException, InvalidRoleException -- no not-found code at all\n DeleteDeploymentGroup emits ApplicationDoesNotExistException AND DeploymentGroupDoesNotExistException; declares only name-required and invalid-name codes plus InvalidRoleException\n DeleteDeploymentConfig emits DeploymentConfigDoesNotExistException; declares DeploymentConfigInUseException, DeploymentConfigNameRequiredException, InvalidDeploymentConfigNameException, InvalidOperationException -- InvalidOperationException is the only candidate and it is a stretch\n DeregisterOnPremisesInstance emits InstanceDoesNotExistException; declares InstanceNameRequiredException, InvalidInstanceNameException -- no not-found code\n\nThis is the workmail shape: a code with no home in the model. Workmail's remedy was idempotent success, justified by an explicit doc sentence. 3pz8 fetched the live API reference for all five and found none carries such a sentence -- only the generic 'If the action is successful, the service sends back an HTTP 200 response with an empty HTTP body' boilerplate, which codepipeline's DisableStageTransition proves is not idempotency evidence since it declares PipelineNotFoundException and errors on a missing resource.\n\nNote the asymmetry inside this service: GetOnPremisesInstance DOES declare InstanceNotRegisteredException and was fixed under 3pz8, while DeregisterOnPremisesInstance declares nothing usable. Same resource, same lookup, different models.","status":"closed","priority":3,"issue_type":"bug","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-09-07T13:38:16Z","created_by":"Witness Patrol","updated_at":"2026-09-07T20:58:51Z","started_at":"2026-09-07T20:51:14Z","closed_at":"2026-09-07T20:58:51Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"gopherstack-wlab","title":"codepipeline: seven ops emit undeclared not-found and structure codes","description":"All seven class A findings from gopherstack-3djp, confirmed against codepipeline@v1.49.4 and left unfixed because no safe remedy is evidenced. Each site now carries a landmine comment naming the mismatch and the candidate codes.\n\n CreateCustomActionType emits InvalidStructureException; declares ConcurrentModificationException, InvalidTagsException, LimitExceededException, TooManyTagsException, ValidationException\n DeleteCustomActionType emits ActionTypeNotFoundException; declares ConcurrentModificationException, ValidationException\n DeletePipeline emits PipelineNotFoundException; declares ConcurrentModificationException, ValidationException\n UpdatePipeline emits PipelineNotFoundException; declares InvalidActionDeclarationException, InvalidBlockerDeclarationException, InvalidStageDeclarationException, InvalidStructureException, LimitExceededException, ValidationException\n OverrideStageCondition / RetryStageExecution / StopPipelineExecution emit PipelineExecutionNotFoundException, which none declares\n\nThe three delete-shaped ops look like the workmail idempotent-success shape, and 3djp implemented that fix before reverting it. The reason matters for anyone picking this up: the live docs sentence 'If the action is successful, the service sends back an HTTP 200 response with an empty HTTP body' appears on DeletePipeline and DeleteCustomActionType, but it ALSO appears on DisableStageTransition, which declares PipelineNotFoundException and does error on a missing pipeline. It is generic response-shape boilerplate and says nothing about not-found semantics. Do not treat it as evidence.\n\nWorkmail's sentence was different and genuinely semantic: 'Deleting already deleted and non-existing rules does not produce an error.' No codepipeline op has one.\n\nFor the four state-transition ops, silently succeeding would be actively misleading rather than merely unevidenced, so they need a declared code chosen deliberately -- candidates are named in the comments at each site.","notes":"2026-09-18: re-verified on #2470 (commit above) — codepipeline SDK v1.54.0 is schema-based (no deserializeOpError\u003cOp\u003e), the seven undeclared codes still type via errors.As (undeclared_error_codes_test.go); no declared substitute fits any of the seven. Recorded as a single tightened PARITY entry. Recommend closing as decided.","status":"in_progress","priority":3,"issue_type":"bug","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-09-07T13:26:59Z","created_by":"Witness Patrol","updated_at":"2026-09-18T15:21:40Z","started_at":"2026-09-08T09:47:47Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"gopherstack-wlab","title":"codepipeline: seven ops emit undeclared not-found and structure codes","description":"All seven class A findings from gopherstack-3djp, confirmed against codepipeline@v1.49.4 and left unfixed because no safe remedy is evidenced. Each site now carries a landmine comment naming the mismatch and the candidate codes.\n\n CreateCustomActionType emits InvalidStructureException; declares ConcurrentModificationException, InvalidTagsException, LimitExceededException, TooManyTagsException, ValidationException\n DeleteCustomActionType emits ActionTypeNotFoundException; declares ConcurrentModificationException, ValidationException\n DeletePipeline emits PipelineNotFoundException; declares ConcurrentModificationException, ValidationException\n UpdatePipeline emits PipelineNotFoundException; declares InvalidActionDeclarationException, InvalidBlockerDeclarationException, InvalidStageDeclarationException, InvalidStructureException, LimitExceededException, ValidationException\n OverrideStageCondition / RetryStageExecution / StopPipelineExecution emit PipelineExecutionNotFoundException, which none declares\n\nThe three delete-shaped ops look like the workmail idempotent-success shape, and 3djp implemented that fix before reverting it. The reason matters for anyone picking this up: the live docs sentence 'If the action is successful, the service sends back an HTTP 200 response with an empty HTTP body' appears on DeletePipeline and DeleteCustomActionType, but it ALSO appears on DisableStageTransition, which declares PipelineNotFoundException and does error on a missing pipeline. It is generic response-shape boilerplate and says nothing about not-found semantics. Do not treat it as evidence.\n\nWorkmail's sentence was different and genuinely semantic: 'Deleting already deleted and non-existing rules does not produce an error.' No codepipeline op has one.\n\nFor the four state-transition ops, silently succeeding would be actively misleading rather than merely unevidenced, so they need a declared code chosen deliberately -- candidates are named in the comments at each site.","status":"in_progress","priority":3,"issue_type":"bug","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-09-07T13:26:59Z","created_by":"Witness Patrol","updated_at":"2026-09-08T09:47:47Z","started_at":"2026-09-08T09:47:47Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-y3om","title":"iot: four Delete ops return ResourceNotFoundException, which none of them declare","description":"Confirmed finding from gopherstack-yr88, deliberately left unfixed twice now.\n\nDeleteCommand, DeleteCommandExecution, DeletePackage and DeletePackageVersion each return ResourceNotFoundException for a missing resource. None declares any not-found-capable code: DeleteCommand and DeleteCommandExecution declare ConflictException, InternalServerException, ThrottlingException, ValidationException; DeletePackage and DeletePackageVersion declare InternalServerException, ThrottlingException, ValidationException. Verified by extraction against iot@v1.77.4.\n\nThis is the workmail shape -- a code with no home in the model -- where the answer there was idempotent success. But workmail had an explicit doc sentence saying so, and these four do not. A 2026-08-31 pass already investigated these exact four ops with the same evidence and declined for that reason; yr88 re-verified independently, implemented and fully neuter-tested the idempotent-success fix, then reverted it on finding no new evidence beyond what the prior pass weighed.\n\nOne asymmetry worth noting: DeletePackage and DeletePackageVersion carry a clientToken idempotency parameter and the two Command ops do not, so an idempotency argument does not apply uniformly across the four.\n\nDeciding needs evidence the SDK does not carry -- real AWS behaviour on a repeated delete. Two pre-existing tests pin the status quo and would need correcting if the decision goes the other way: handler_commands_test.go's unknown_execution_404 subtest and TestDeleteCommandExecution_EmptyExecutionID.","status":"closed","priority":3,"issue_type":"bug","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-09-07T13:21:01Z","created_by":"Witness Patrol","updated_at":"2026-09-08T10:14:02Z","started_at":"2026-09-08T10:08:00Z","closed_at":"2026-09-08T10:14:02Z","close_reason":"Documentation divergence, not a client-breaking defect: iot is schema-based (no deserializers.go), so errors.As unwraps ResourceNotFoundException correctly for all four ops -- verified with a real SDK client. No declared code fits and no doc supports idempotent success. Test-only, neuter-verified via respondNotFound.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-l20u","title":"rds: DescribeDBClusterEndpoints never validates DBClusterIdentifier, returning empty instead of DBClusterNotFoundFault","description":"Noticed while fixing gopherstack-33jc RC5, and deliberately not widened into.\n\nDescribeDBClusterEndpoints declares exactly one error: DBClusterNotFoundFault. 33jc removed a wrong not-found branch on the optional DBClusterEndpointIdentifier, which is filter-like and correctly yields an empty list for an unknown value. But the DBClusterIdentifier filter is a different matter -- the op's one declared error exists precisely for an unknown cluster, and this backend silently returns an empty list instead.\n\nFix is to validate DBClusterIdentifier when supplied and return ErrClusterNotFound, leaving DBClusterEndpointIdentifier as the filter 33jc made it. Pin both halves in one test so the distinction does not get collapsed again.","status":"closed","priority":3,"issue_type":"bug","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-09-07T13:02:42Z","created_by":"Witness Patrol","updated_at":"2026-09-07T16:57:07Z","started_at":"2026-09-07T16:47:51Z","closed_at":"2026-09-07T16:57:07Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"gopherstack-fm1e","title":"rds: ModifyActivityStream emits DBClusterNotFoundFault; DBInstanceNotFound and ResourceNotFoundFault are both declared","description":"Confirmed finding from gopherstack-33jc, left unfixed because two declared codes both plausibly fit.\n\nModifyActivityStream's declared set in the pinned rds SDK is DBInstanceNotFound, InvalidDBInstanceState, ResourceNotFoundFault. It currently emits DBClusterNotFoundFault, which is not among them. Verified by extraction.\n\nThe op targets a DB instance by ResourceArn, so DBInstanceNotFound reads natural, but ResourceNotFoundFault is also declared and is what the sibling ApplyPendingMaintenanceAction uses for the same arn-shaped lookup (fixed that way under 33jc). Pick deliberately rather than by analogy -- check whether the emulator resolves the ARN to an instance or treats it opaquely.","status":"closed","priority":3,"issue_type":"bug","assignee":"Witness Patrol","owner":"blackbird7181@gmail.com","created_at":"2026-09-07T13:02:40Z","created_by":"Witness Patrol","updated_at":"2026-09-07T20:58:51Z","started_at":"2026-09-07T20:51:13Z","closed_at":"2026-09-07T20:58:51Z","close_reason":"Closed","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/Makefile b/Makefile index 266cc5cb0f..016f59372d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build build-check ui-install ui-lint ui-check ui-lint-fix ui-fmt ui-fmt-fix ui-test ui-build install-deps install-tofu lint lint-changed lint-fix test integration-test terraform-test e2e e2e-test total-coverage clean demo all dev-mcp-install dev-mcp-check pgo docs check-pins bd-audit parity-lint +.PHONY: build build-check ui-install ui-lint ui-check ui-lint-fix ui-fmt ui-fmt-fix ui-test ui-build install-deps install-tofu lint lint-changed lint-fix test integration-test terraform-test e2e e2e-test total-coverage clean demo all dev-mcp-install dev-mcp-check pgo docs check-pins bd-audit parity-lint cfn-attrs-gen cfn-attrs-spec-refresh BINARY_NAME=gopherstack VERSION_PKG=github.com/blackbirdworks/gopherstack/pkgs/version @@ -240,6 +240,20 @@ bd-audit: parity-lint: go run ./cmd/paritylint +# Regenerate services/cloudformation/cfn_attributes.json (Fn::GetAtt +# attribute table) from the committed spec fixture and the current +# resTypeXxx constants. See cmd/cfnattrgen. +cfn-attrs-gen: + go run ./cmd/cfnattrgen -spec cmd/cfnattrgen/testdata/cfn_resource_spec.json -src services/cloudformation -out services/cloudformation/cfn_attributes.json + +# Refresh cmd/cfnattrgen/testdata/cfn_resource_spec.json from a fresh +# download of the full CloudFormation resource specification. Run this +# occasionally, then `make cfn-attrs-gen` and commit both files. +cfn-attrs-spec-refresh: + curl -sL https://d1uauaxba7bl26.cloudfront.net/latest/gzip/CloudFormationResourceSpecification.json | gunzip > /tmp/cfn_full_spec.json + go run ./cmd/cfnattrgen -spec /tmp/cfn_full_spec.json -trimspec-out cmd/cfnattrgen/testdata/cfn_resource_spec.json + rm -f /tmp/cfn_full_spec.json + demo: ui-build docker compose down docker compose build diff --git a/README.md b/README.md index 40695e88c3..078f9ad8d1 100644 --- a/README.md +++ b/README.md @@ -470,7 +470,7 @@ Every service links to its own page with a coverage breakdown — audited operat | [Batch](services/batch/README.md) | A | 45 | 8 gaps | | [EC2](services/ec2/README.md) | A | — | 22 families; 16 gaps; 2 structural gaps; 8 deferred | | [Elastic Beanstalk](services/elasticbeanstalk/README.md) | A | 47 | 13 gaps | -| [Lambda](services/lambda/README.md) | A | — | 10 families; 2 gaps | +| [Lambda](services/lambda/README.md) | A | — | 10 families | ### Containers @@ -488,7 +488,7 @@ Every service links to its own page with a coverage breakdown — audited operat | [Data Lifecycle Manager](services/dlm/README.md) | A | 8 | clean | | [EFS](services/efs/README.md) | A | 31 | 4 gaps; 2 deferred | | [FSx](services/fsx/README.md) | A | — | 13 families; 12 gaps | -| [S3](services/s3/README.md) | A | 24 | 8 gaps | +| [S3](services/s3/README.md) | A | 26 | 8 gaps | | [S3 Control](services/s3control/README.md) | A | 44 | 4 gaps; 3 deferred | | [S3 Glacier](services/glacier/README.md) | A | 33 | 2 gaps | | [S3 Tables](services/s3tables/README.md) | A | 49 | 1 gap | @@ -499,7 +499,7 @@ Every service links to its own page with a coverage breakdown — audited operat |---|---|---|---| | [DAX](services/dax/README.md) | A | 21 | 1 gap; 1 deferred | | [DocumentDB](services/docdb/README.md) | A | 55 | 10 gaps; 1 deferred | -| [DynamoDB](services/dynamodb/README.md) | A | — | 15 families; 8 gaps; 2 deferred | +| [DynamoDB](services/dynamodb/README.md) | A | — | 15 families; 6 gaps; 2 deferred | | [DynamoDB Streams](services/dynamodbstreams/README.md) | A | 4 | clean | | [ElastiCache](services/elasticache/README.md) | A | 75 | 3 gaps; 2 deferred | | [MemoryDB](services/memorydb/README.md) | A | 45 | 5 gaps; 3 deferred | @@ -517,7 +517,7 @@ Every service links to its own page with a coverage breakdown — audited operat | Service | Parity | PARITY Entries | Notes | |---|---|---|---| -| [API Gateway](services/apigateway/README.md) | A | 123 | 3 gaps; 1 deferred | +| [API Gateway](services/apigateway/README.md) | A | 123 | 2 gaps; 1 deferred | | [API Gateway Management API](services/apigatewaymanagementapi/README.md) | A | 3 | 1 gap; 2 deferred | | [API Gateway v2](services/apigatewayv2/README.md) | A | 77 | 4 gaps; 6 deferred | | [App Mesh](services/appmesh/README.md) | A | 38 | 2 gaps | @@ -543,7 +543,7 @@ Every service links to its own page with a coverage breakdown — audited operat | [SES](services/ses/README.md) | A | 71 | 6 gaps; 1 deferred | | [SES v2](services/sesv2/README.md) | A | 112 | 3 gaps | | [SNS](services/sns/README.md) | A | 34 | 2 gaps; 2 deferred | -| [SQS](services/sqs/README.md) | A | 20 | 4 gaps; 4 deferred | +| [SQS](services/sqs/README.md) | A | 20 | 3 gaps; 4 deferred | | [SWF](services/swf/README.md) | A | 39 | 4 gaps | | [Step Functions](services/stepfunctions/README.md) | A | 37 | 9 gaps | | [WorkMail](services/workmail/README.md) | A | 92 | 5 gaps | @@ -594,7 +594,7 @@ Every service links to its own page with a coverage breakdown — audited operat | [Cognito Identity](services/cognitoidentity/README.md) | A | 23 | 2 gaps; 4 deferred | | [Cognito Identity Provider](services/cognitoidp/README.md) | A | 68 | 2 gaps | | [Directory Service](services/directoryservice/README.md) | A | 80 | 10 gaps; 2 deferred | -| [IAM](services/iam/README.md) | A | 37 | 8 gaps | +| [IAM](services/iam/README.md) | A | 38 | 5 gaps | | [IAM Access Analyzer](services/accessanalyzer/README.md) | A | 39 | 6 gaps; 1 deferred | | [IAM Identity Center (SSO)](services/ssoadmin/README.md) | A | 56 | 4 gaps | | [IAM Roles Anywhere](services/rolesanywhere/README.md) | A | 30 | 5 gaps | @@ -622,7 +622,7 @@ Every service links to its own page with a coverage breakdown — audited operat | [Resource Access Manager](services/ram/README.md) | A | 36 | 5 gaps; 3 deferred | | [Resource Groups](services/resourcegroups/README.md) | A | 23 | 3 gaps | | [Resource Groups Tagging API](services/resourcegroupstaggingapi/README.md) | A | 9 | 3 gaps; 1 deferred | -| [Systems Manager](services/ssm/README.md) | A | 105 | 31 gaps | +| [Systems Manager](services/ssm/README.md) | A | 105 | 29 gaps | ### Developer Tools @@ -704,8 +704,12 @@ Every service links to its own page with a coverage breakdown — audited operat | [Azurestoragevhost](services/azurestoragevhost/README.md) | B | 2 | 2 gaps; 1 deferred | | [Cloudfrontkeyvaluestore](services/cloudfrontkeyvaluestore/README.md) | A | 6 | 2 structural gaps | | [Directconnect](services/directconnect/README.md) | A | 64 | 4 gaps; 8 structural gaps; 1 deferred | +| [Dsql](services/dsql/README.md) | B | 16 | 5 gaps | +| [Ecrpublic](services/ecrpublic/README.md) | B | 23 | 5 gaps | | [Grafana](services/grafana/README.md) | A | 25 | 2 gaps; 1 structural gap | | [HealthOmics](services/omics/README.md) | A | — | 25 families; 4 gaps; 1 deferred | +| [Kafkaconnect](services/kafkaconnect/README.md) | B | 19 | 3 gaps | +| [Kinesisvideo](services/kinesisvideo/README.md) | B | 22 | 3 gaps | | [Lightsail](services/lightsail/README.md) | A | — | 28 families; 18 gaps; 2 deferred | | [Managed Blockchain](services/managedblockchain/README.md) | A | 27 | 4 gaps | | [Mgn](services/mgn/README.md) | A | 95 | 3 gaps; 5 structural gaps; 1 deferred | diff --git a/cli.go b/cli.go index b8db9b2264..3ac15039f9 100644 --- a/cli.go +++ b/cli.go @@ -122,11 +122,13 @@ import ( dlmbackend "github.com/blackbirdworks/gopherstack/services/dlm" dmsbackend "github.com/blackbirdworks/gopherstack/services/dms" docdbbackend "github.com/blackbirdworks/gopherstack/services/docdb" + dsqlbackend "github.com/blackbirdworks/gopherstack/services/dsql" ddbbackend "github.com/blackbirdworks/gopherstack/services/dynamodb" ddbmodels "github.com/blackbirdworks/gopherstack/services/dynamodb/models" dynamodbstreamsbackend "github.com/blackbirdworks/gopherstack/services/dynamodbstreams" ec2backend "github.com/blackbirdworks/gopherstack/services/ec2" ecrbackend "github.com/blackbirdworks/gopherstack/services/ecr" + ecrpublicbackend "github.com/blackbirdworks/gopherstack/services/ecrpublic" ecsbackend "github.com/blackbirdworks/gopherstack/services/ecs" efsbackend "github.com/blackbirdworks/gopherstack/services/efs" eksbackend "github.com/blackbirdworks/gopherstack/services/eks" @@ -154,9 +156,11 @@ import ( iotdataplanebackend "github.com/blackbirdworks/gopherstack/services/iotdataplane" iotwirelessbackend "github.com/blackbirdworks/gopherstack/services/iotwireless" kafkabackend "github.com/blackbirdworks/gopherstack/services/kafka" + kafkaconnectbackend "github.com/blackbirdworks/gopherstack/services/kafkaconnect" kinesisbackend "github.com/blackbirdworks/gopherstack/services/kinesis" kinesisanalyticsbackend "github.com/blackbirdworks/gopherstack/services/kinesisanalytics" kinesisanalyticsv2backend "github.com/blackbirdworks/gopherstack/services/kinesisanalyticsv2" + kinesisvideobackend "github.com/blackbirdworks/gopherstack/services/kinesisvideo" kmsbackend "github.com/blackbirdworks/gopherstack/services/kms" lakeformationbackend "github.com/blackbirdworks/gopherstack/services/lakeformation" lambdabackend "github.com/blackbirdworks/gopherstack/services/lambda" @@ -358,8 +362,10 @@ type CLI struct { codeStarConnectionsHandler service.Registerable dynamodbStreamsHandler service.Registerable docdbHandler service.Registerable + dsqlHandler service.Registerable elasticbeanstalkHandler service.Registerable ecrHandler service.Registerable + ecrPublicHandler service.Registerable ecsHandler service.Registerable efsHandler service.Registerable eksHandler service.Registerable @@ -381,7 +387,9 @@ type CLI struct { inspector2Handler service.Registerable iotanalyticsHandler service.Registerable kafkaHandler service.Registerable + kafkaconnectHandler service.Registerable kinesisanalyticsv2Handler service.Registerable + kinesisvideoHandler service.Registerable managedblockchainHandler service.Registerable mediaconvertHandler service.Registerable mqHandler service.Registerable @@ -1302,6 +1310,11 @@ func (c *CLI) GetSupportHandler() service.Registerable { return c.supportHandler //nolint:ireturn // architecturally required to return interface func (c *CLI) GetECRHandler() service.Registerable { return c.ecrHandler } +// GetECRPublicHandler returns the ECR Public handler (dashboard.AWSSDKProvider). +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetECRPublicHandler() service.Registerable { return c.ecrPublicHandler } + // GetECSHandler returns the ECS handler (dashboard.AWSSDKProvider). // //nolint:ireturn // architecturally required to return interface @@ -1367,6 +1380,11 @@ func (c *CLI) GetInspector2Handler() service.Registerable { return c.inspector2H //nolint:ireturn // architecturally required to return interface func (c *CLI) GetKafkaHandler() service.Registerable { return c.kafkaHandler } +// GetKafkaConnectHandler returns the MSK Connect handler (dashboard.AWSSDKProvider). +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetKafkaConnectHandler() service.Registerable { return c.kafkaconnectHandler } + // GetKinesisAnalyticsV2Handler returns the Kinesis Data Analytics v2 handler (dashboard.AWSSDKProvider). // //nolint:ireturn // architecturally required to return interface @@ -1374,6 +1392,11 @@ func (c *CLI) GetKinesisAnalyticsV2Handler() service.Registerable { return c.kinesisanalyticsv2Handler } +// GetKinesisVideoHandler returns the Kinesis Video Streams handler (dashboard.AWSSDKProvider). +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetKinesisVideoHandler() service.Registerable { return c.kinesisvideoHandler } + // GetManagedBlockchainHandler returns the Managed Blockchain handler (dashboard.AWSSDKProvider). // //nolint:ireturn // architecturally required to return interface @@ -1736,6 +1759,11 @@ func (c *CLI) GetElasticbeanstalkHandler() service.Registerable { return c.elast //nolint:ireturn // architecturally required to return interface func (c *CLI) GetDocDBHandler() service.Registerable { return c.docdbHandler } +// GetDSQLHandler returns the Aurora DSQL handler (dashboard.AWSSDKProvider). +// +//nolint:ireturn // architecturally required to return interface +func (c *CLI) GetDSQLHandler() service.Registerable { return c.dsqlHandler } + // GetFISHandler returns the FIS handler (dashboard.AWSSDKProvider). // //nolint:ireturn // architecturally required to return interface @@ -2778,6 +2806,7 @@ func storeCLIExtendedHandlers(cli *CLI, byName map[string]service.Registerable) cli.bedrockHandler = byName["Bedrock"] cli.bedrockruntimeHandler = byName["BedrockRuntime"] cli.ecrHandler = byName["ECR"] + cli.ecrPublicHandler = byName["ECRPublic"] cli.ecsHandler = byName["ECS"] cli.iotHandler = byName["IoT"] cli.cognitoIDPHandler = byName["CognitoIDP"] @@ -2819,7 +2848,9 @@ func storeCLILatestHandlers(cli *CLI, byName map[string]service.Registerable) { cli.inspector2Handler = byName["Inspector2"] cli.iotanalyticsHandler = byName["IoTAnalytics"] cli.kafkaHandler = byName["Kafka"] + cli.kafkaconnectHandler = byName["KafkaConnect"] cli.kinesisanalyticsv2Handler = byName["KinesisAnalyticsV2"] + cli.kinesisvideoHandler = byName["KinesisVideo"] cli.managedblockchainHandler = byName["ManagedBlockchain"] cli.mediaconvertHandler = byName["MediaConvert"] cli.mqHandler = byName["MQ"] @@ -2835,6 +2866,7 @@ func storeCLINewestHandlers(cli *CLI, byName map[string]service.Registerable) { cli.mwaaHandler = byName["MWAA"] cli.neptuneHandler = byName["Neptune"] cli.docdbHandler = byName["DocDB"] + cli.dsqlHandler = byName["DSQL"] cli.pinpointHandler = byName["Pinpoint"] cli.pipesHandler = byName["Pipes"] cli.rdsdataHandler = byName["RDSData"] @@ -3448,6 +3480,30 @@ type elbCertificateResolverAdapter struct { iamBackend *iambackend.InMemoryBackend } +// elbResolverServerCertMatches walks every page of IAM server certificates +// looking for certARN, since ListServerCertificates now paginates. +func elbResolverServerCertMatches(b *iambackend.InMemoryBackend, certARN string) bool { + marker := "" + for { + p, err := b.ListServerCertificates("", marker, 0) + if err != nil { + return false + } + + for _, c := range p.Data { + if c.Arn == certARN { + return true + } + } + + if p.Next == "" { + return false + } + + marker = p.Next + } +} + func (a *elbCertificateResolverAdapter) ResolveCertificate(ctx context.Context, certARN string) bool { if a.acmBackend != nil { if _, err := a.acmBackend.DescribeCertificate(ctx, certARN); err == nil { @@ -3455,15 +3511,8 @@ func (a *elbCertificateResolverAdapter) ResolveCertificate(ctx context.Context, } } - if a.iamBackend != nil { - certs, err := a.iamBackend.ListServerCertificates("") - if err == nil { - for _, c := range certs { - if c.Arn == certARN { - return true - } - } - } + if a.iamBackend != nil && elbResolverServerCertMatches(a.iamBackend, certARN) { + return true } return false @@ -3519,15 +3568,8 @@ func (a *elbv2CertificateResolverAdapter) ResolveCertificate(certARN string) boo } } - if a.iamBackend != nil { - certs, err := a.iamBackend.ListServerCertificates("") - if err == nil { - for _, c := range certs { - if c.Arn == certARN { - return true - } - } - } + if a.iamBackend != nil && elbResolverServerCertMatches(a.iamBackend, certARN) { + return true } return false @@ -4052,12 +4094,16 @@ func getRemainingServiceProviders() []service.Provider { &guarddutybackend.Provider{}, &inspector2backend.Provider{}, &docdbbackend.Provider{}, + &dsqlbackend.Provider{}, &glacierbackend.Provider{}, &iotanalyticsbackend.Provider{}, &iotwirelessbackend.Provider{}, &kinesisanalyticsbackend.Provider{}, &kafkabackend.Provider{}, + &kafkaconnectbackend.Provider{}, &kinesisanalyticsv2backend.Provider{}, + &kinesisvideobackend.Provider{}, + &ecrpublicbackend.Provider{}, &lakeformationbackend.Provider{}, &managedblockchainbackend.Provider{}, &mediaconvertbackend.Provider{}, diff --git a/cli_applicationautoscaling_dynamodb_wiring_test.go b/cli_applicationautoscaling_dynamodb_wiring_test.go new file mode 100644 index 0000000000..ed4bf32a0b --- /dev/null +++ b/cli_applicationautoscaling_dynamodb_wiring_test.go @@ -0,0 +1,114 @@ +package main + +import ( + "log/slog" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + sdkddb "github.com/aws/aws-sdk-go-v2/service/dynamodb" + sdkddbtypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/chaos" + "github.com/blackbirdworks/gopherstack/pkgs/service" + aasbackend "github.com/blackbirdworks/gopherstack/services/applicationautoscaling" + ddbbackend "github.com/blackbirdworks/gopherstack/services/dynamodb" +) + +// Drives the real composition root, not hand-wired backends, so deleting +// applicationautoscaling's SetAppConfig call breaks this test. +func TestInitializeServices_ApplicationAutoscalingDynamoDBWiring(t *testing.T) { + t.Parallel() + + cli := &CLI{AccountID: "000000000000", Region: "us-east-1"} + appCtx := &service.AppContext{ + Logger: slog.Default(), + Config: cli, + JanitorCtx: t.Context(), + } + cli.faultStore = chaos.NewFaultStore() + + services, err := initializeServices(appCtx) + require.NoError(t, err) + + byName := serviceByName(services) + + ddbH, ok := byName["DynamoDB"].(*ddbbackend.DynamoDBHandler) + require.True(t, ok, "DynamoDB handler must be registered") + + aasH, ok := byName["ApplicationAutoscaling"].(*aasbackend.Handler) + require.True(t, ok, "ApplicationAutoscaling handler must be registered") + + ctx := t.Context() + + tableName := "aas-ddb-wiring-table" + _, err = ddbH.Backend.CreateTable(ctx, &sdkddb.CreateTableInput{ + TableName: aws.String(tableName), + KeySchema: []sdkddbtypes.KeySchemaElement{ + {AttributeName: aws.String("pk"), KeyType: sdkddbtypes.KeyTypeHash}, + }, + AttributeDefinitions: []sdkddbtypes.AttributeDefinition{ + {AttributeName: aws.String("pk"), AttributeType: sdkddbtypes.ScalarAttributeTypeS}, + }, + BillingMode: sdkddbtypes.BillingModeProvisioned, + ProvisionedThroughput: &sdkddbtypes.ProvisionedThroughput{ + ReadCapacityUnits: aws.Int64(5), + WriteCapacityUnits: aws.Int64(5), + }, + }) + require.NoError(t, err) + + _, err = aasH.Backend.RegisterScalableTarget( + "dynamodb", "table/"+tableName, "dynamodb:table:WriteCapacityUnits", + aws.Int32(5), aws.Int32(500), nil, "", nil, + ) + require.NoError(t, err) + + desc, err := ddbH.Backend.DescribeTableReplicaAutoScaling(ctx, &sdkddb.DescribeTableReplicaAutoScalingInput{ + TableName: aws.String(tableName), + }) + require.NoError(t, err) + require.Len(t, desc.TableAutoScalingDescription.Replicas, 1) + + settings := desc.TableAutoScalingDescription.Replicas[0].ReplicaProvisionedWriteCapacityAutoScalingSettings + require.NotNil( + t, + settings, + "RegisterScalableTarget must have pushed capacity into DynamoDB's own autoscaling state", + ) + require.Equal(t, int64(5), aws.ToInt64(settings.MinimumUnits)) + require.Equal(t, int64(500), aws.ToInt64(settings.MaximumUnits)) + + // Reverse direction: a DynamoDB-native update shows up on DescribeScalableTargets. + _, err = ddbH.Backend.UpdateTableReplicaAutoScaling(ctx, &sdkddb.UpdateTableReplicaAutoScalingInput{ + TableName: aws.String(tableName), + ReplicaUpdates: []sdkddbtypes.ReplicaAutoScalingUpdate{ + { + RegionName: aws.String("us-east-1"), + ReplicaProvisionedReadCapacityAutoScalingUpdate: &sdkddbtypes.AutoScalingSettingsUpdate{ + MinimumUnits: aws.Int64(2), + MaximumUnits: aws.Int64(200), + }, + }, + }, + }) + require.NoError(t, err) + + targets, _, err := aasH.Backend.DescribeScalableTargets(aasbackend.DescribeScalableTargetsFilter{ + ServiceNamespace: "dynamodb", + ResourceIDs: []string{"table/" + tableName}, + }) + require.NoError(t, err) + + var found *aasbackend.ScalableTarget + + for _, tgt := range targets { + if tgt.ScalableDimension == "dynamodb:table:ReadCapacityUnits" { + found = tgt + } + } + + require.NotNil(t, found, "UpdateTableReplicaAutoScaling must be visible via DescribeScalableTargets") + require.Equal(t, int32(2), found.MinCapacity) + require.Equal(t, int32(200), found.MaxCapacity) +} diff --git a/cli_dynamodb_kinesis_wiring_test.go b/cli_dynamodb_kinesis_wiring_test.go index 872ff0f720..fb9e678b5f 100644 --- a/cli_dynamodb_kinesis_wiring_test.go +++ b/cli_dynamodb_kinesis_wiring_test.go @@ -55,6 +55,8 @@ func TestInitializeServices_DynamoDBKinesisWiring(t *testing.T) { kinesisBk, ok := kinesisH.Backend.(*kinesisbackend.InMemoryBackend) require.True(t, ok, "Kinesis backend must be an InMemoryBackend") + kinesisClock := newKinesisFakeClock(time.Now()) + kinesisBk.WithClock(kinesisClock.Now) ctx := t.Context() @@ -63,6 +65,7 @@ func TestInitializeServices_DynamoDBKinesisWiring(t *testing.T) { StreamName: streamName, ShardCount: 1, })) + kinesisClock.Advance(kinesisStreamSettleWait) tableName := "dynamodb-kinesis-wiring-table" _, err = ddbBk.CreateTable(ctx, &sdkddb.CreateTableInput{ diff --git a/cli_firehose_kinesis_wiring_test.go b/cli_firehose_kinesis_wiring_test.go index c114d6f61a..769566c57e 100644 --- a/cli_firehose_kinesis_wiring_test.go +++ b/cli_firehose_kinesis_wiring_test.go @@ -63,6 +63,8 @@ func TestInitializeServices_FirehoseKinesisSourceWiring(t *testing.T) { kinesisBk, ok := kinesisH.Backend.(*kinesisbackend.InMemoryBackend) require.True(t, ok, "Kinesis backend must be an InMemoryBackend") + kinesisClock := newKinesisFakeClock(time.Now()) + kinesisBk.WithClock(kinesisClock.Now) s3H, ok := byName["S3"].(*s3backend.S3Handler) require.True(t, ok, "S3 handler must be registered") @@ -102,6 +104,7 @@ func TestInitializeServices_FirehoseKinesisSourceWiring(t *testing.T) { StreamName: streamName, ShardCount: 1, })) + kinesisClock.Advance(kinesisStreamSettleWait) roleARN := "arn:aws:iam::000000000000:role/role" streamARN := "arn:aws:kinesis:us-east-1:000000000000:stream/" + streamName diff --git a/cli_kinesis_channel_s3_delivery_wiring_test.go b/cli_kinesis_channel_s3_delivery_wiring_test.go index 1ed9c7da70..fa360765b4 100644 --- a/cli_kinesis_channel_s3_delivery_wiring_test.go +++ b/cli_kinesis_channel_s3_delivery_wiring_test.go @@ -56,6 +56,8 @@ func TestInitializeServices_KinesisChannelS3DeliveryWiring(t *testing.T) { kinesisBk, ok := kinesisH.Backend.(*kinesisbackend.InMemoryBackend) require.True(t, ok, "Kinesis backend must be an InMemoryBackend") + kinesisClock := newKinesisFakeClock(time.Now()) + kinesisBk.WithClock(kinesisClock.Now) s3H, ok := byName["S3"].(*s3backend.S3Handler) require.True(t, ok, "S3 handler must be registered") @@ -73,6 +75,7 @@ func TestInitializeServices_KinesisChannelS3DeliveryWiring(t *testing.T) { StreamName: "kinesis-channel-wiring-stream", StreamMode: kinesisbackend.StreamModeOnDemand, })) + kinesisClock.Advance(kinesisStreamSettleWait) created, err := kinesisBk.CreateChannel(ctx, &kinesisbackend.CreateChannelInput{ ChannelName: "kinesis-channel-wiring-channel", diff --git a/cli_kinesisanalytics_kinesis_s3_wiring_test.go b/cli_kinesisanalytics_kinesis_s3_wiring_test.go index 8722df37a3..e7cb05ac12 100644 --- a/cli_kinesisanalytics_kinesis_s3_wiring_test.go +++ b/cli_kinesisanalytics_kinesis_s3_wiring_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" "github.com/aws/aws-sdk-go-v2/aws" awscfg "github.com/aws/aws-sdk-go-v2/config" @@ -69,6 +70,8 @@ func TestInitializeServices_KinesisAnalyticsKinesisS3Wiring(t *testing.T) { kinesisBk, ok := kinesisH.Backend.(*kinesisbackend.InMemoryBackend) require.True(t, ok, "Kinesis backend must be an InMemoryBackend") + kinesisClock := newKinesisFakeClock(time.Now()) + kinesisBk.WithClock(kinesisClock.Now) s3H, ok := byName["S3"].(*s3backend.S3Handler) require.True(t, ok, "S3 handler must be registered") @@ -156,6 +159,7 @@ func TestInitializeServices_KinesisAnalyticsKinesisS3Wiring(t *testing.T) { ShardCount: 1, }) require.NoError(t, createErr) + kinesisClock.Advance(kinesisStreamSettleWait) for _, data := range []string{`{"id":1,"name":"a"}`, `{"id":2,"name":"bb"}`} { _, putErr := kinesisBk.PutRecord(ctx, &kinesisbackend.PutRecordInput{ diff --git a/cli_pipes_wiring_test.go b/cli_pipes_wiring_test.go index cfae124c59..5fd071ed1f 100644 --- a/cli_pipes_wiring_test.go +++ b/cli_pipes_wiring_test.go @@ -37,16 +37,17 @@ import ( // wiring -- not just the pipes package's own unit-tested Runner logic against // fakes -- delivers end to end. type pipesWiringRig struct { - pipesBk *pipesbackend.InMemoryBackend - sqsBk *sqsbackend.InMemoryBackend - lambdaBk *lambdabackend.InMemoryBackend - snsBk *snsbackend.InMemoryBackend - kinesisBk *kinesisbackend.InMemoryBackend - ebBk *ebbackend.InMemoryBackend - cwlogsBk *cwlogsbackend.InMemoryBackend - firehoseBk *firehosebackend.InMemoryBackend - ddbBk *ddbbackend.InMemoryDB - runner *pipesbackend.Runner + pipesBk *pipesbackend.InMemoryBackend + sqsBk *sqsbackend.InMemoryBackend + lambdaBk *lambdabackend.InMemoryBackend + snsBk *snsbackend.InMemoryBackend + kinesisBk *kinesisbackend.InMemoryBackend + kinesisClock *kinesisFakeClock + ebBk *ebbackend.InMemoryBackend + cwlogsBk *cwlogsbackend.InMemoryBackend + firehoseBk *firehosebackend.InMemoryBackend + ddbBk *ddbbackend.InMemoryDB + runner *pipesbackend.Runner } func newPipesWiringRig(t *testing.T) *pipesWiringRig { @@ -66,7 +67,8 @@ func newPipesWiringRig(t *testing.T) *pipesWiringRig { snsBk := snsbackend.NewInMemoryBackend() snsH := snsbackend.NewHandler(snsBk) - kinesisBk := kinesisbackend.NewInMemoryBackend() + kinesisClock := newKinesisFakeClock(time.Now()) + kinesisBk := kinesisbackend.NewInMemoryBackend().WithClock(kinesisClock.Now) kinesisH := kinesisbackend.NewHandler(kinesisBk) ebBk := ebbackend.NewInMemoryBackend() @@ -102,16 +104,17 @@ func newPipesWiringRig(t *testing.T) *pipesWiringRig { }) return &pipesWiringRig{ - pipesBk: pipesBk, - sqsBk: sqsBk, - lambdaBk: lambdaBk, - snsBk: snsBk, - kinesisBk: kinesisBk, - ebBk: ebBk, - cwlogsBk: cwlogsBk, - firehoseBk: firehoseBk, - ddbBk: ddbBk, - runner: runner, + pipesBk: pipesBk, + sqsBk: sqsBk, + lambdaBk: lambdaBk, + snsBk: snsBk, + kinesisBk: kinesisBk, + kinesisClock: kinesisClock, + ebBk: ebBk, + cwlogsBk: cwlogsBk, + firehoseBk: firehoseBk, + ddbBk: ddbBk, + runner: runner, } } @@ -240,6 +243,7 @@ func TestWirePipesRunner_SQSSourceTargets(t *testing.T) { StreamName: "pipes-kinesis-target", ShardCount: 1, })) + rig.kinesisClock.Advance(kinesisStreamSettleWait) targetARN := arn.Build("kinesis", config.DefaultRegion, config.DefaultAccountID, "stream/pipes-kinesis-target") qURL := rig.createSQSSourcedPipe(t, "kinesis-target-pipe", targetARN) @@ -377,6 +381,7 @@ func TestWirePipesRunner_KinesisSource(t *testing.T) { StreamName: "pipes-kinesis-source", ShardCount: 1, })) + rig.kinesisClock.Advance(kinesisStreamSettleWait) sourceARN := arn.Build("kinesis", config.DefaultRegion, config.DefaultAccountID, "stream/pipes-kinesis-source") targetOut, err := rig.sqsBk.CreateQueue(&sqsbackend.CreateQueueInput{QueueName: "pipes-kinesis-source-target"}) @@ -501,6 +506,7 @@ func TestWirePipesRunner_DLQDelivery(t *testing.T) { StreamName: "pipes-dlq-source", ShardCount: 1, })) + rig.kinesisClock.Advance(kinesisStreamSettleWait) sourceARN := arn.Build("kinesis", config.DefaultRegion, config.DefaultAccountID, "stream/pipes-dlq-source") _, err = rig.pipesBk.CreatePipe(context.Background(), pipesbackend.CreatePipeInput{ diff --git a/cli_sfn_eb_wiring_test.go b/cli_sfn_eb_wiring_test.go index 092f918130..592460b71a 100644 --- a/cli_sfn_eb_wiring_test.go +++ b/cli_sfn_eb_wiring_test.go @@ -162,7 +162,9 @@ func TestWireEventBridgeDelivery_KinesisFirehoseECSStepFunctionsCloudWatchLogs(t ebBk := ebbackend.NewInMemoryBackendWithConfig(config.DefaultAccountID, config.DefaultRegion) ebH := ebbackend.NewHandler(ebBk) - kinesisBk := kinesisbackend.NewInMemoryBackendWithConfig(config.DefaultAccountID, config.DefaultRegion) + kinesisClock := newKinesisFakeClock(time.Now()) + kinesisBk := kinesisbackend.NewInMemoryBackendWithConfig(config.DefaultAccountID, config.DefaultRegion). + WithClock(kinesisClock.Now) kinesisH := kinesisbackend.NewHandler(kinesisBk) firehoseBk := firehosebackend.NewInMemoryBackend(config.DefaultAccountID, config.DefaultRegion) @@ -188,6 +190,7 @@ func TestWireEventBridgeDelivery_KinesisFirehoseECSStepFunctionsCloudWatchLogs(t // --- Fixtures for each target type. --- require.NoError(t, kinesisBk.CreateStream(ctx, &kinesisbackend.CreateStreamInput{StreamName: "wiring-stream"})) + kinesisClock.Advance(kinesisStreamSettleWait) streamDesc, err := kinesisBk.DescribeStream(ctx, &kinesisbackend.DescribeStreamInput{StreamName: "wiring-stream"}) require.NoError(t, err) require.NotEmpty(t, streamDesc.Shards) diff --git a/cli_test.go b/cli_test.go index 879a03853b..639e40ed94 100644 --- a/cli_test.go +++ b/cli_test.go @@ -113,6 +113,22 @@ import ( // when shutdown itself completes on time (gopherstack-becu). const shutdownWaitTimeout = shutdownTimeout + 3*time.Second +// waitForServerReady polls the health endpoint instead of a fixed sleep, +// since startup time varies under load. +func waitForServerReady(t *testing.T, port int) { + t.Helper() + + require.Eventually(t, func() bool { + resp, err := http.Get(fmt.Sprintf("http://localhost:%d/_gopherstack/health", port)) + if err != nil { + return false + } + resp.Body.Close() + + return resp.StatusCode == http.StatusOK + }, 3*time.Second, 50*time.Millisecond, "server did not become ready") +} + // parseCLI parses the given args (key=value env pairs) into a CLI value // by setting environment variables then parsing an empty argument list. func parseCLI(t *testing.T, envPairs map[string]string) CLI { @@ -305,8 +321,7 @@ func TestServerStartupAndShutdown(t *testing.T) { errCh <- run(ctx, cli) }() - // Wait briefly to let the server start (in a real test you might poll the endpoint) - time.Sleep(200 * time.Millisecond) + waitForServerReady(t, port) // Cancel the context to initiate a graceful shutdown cancel() @@ -384,18 +399,13 @@ func TestServerStartup_WithInitScript(t *testing.T) { errCh <- run(ctx, cli) }() - // Poll for the marker file instead of a fixed sleep to avoid timing flakes. - deadline := time.Now().Add(5 * time.Second) var data []byte - for time.Now().Before(deadline) { + require.Eventually(t, func() bool { var readErr error data, readErr = os.ReadFile(marker) - if readErr == nil { - break - } - time.Sleep(20 * time.Millisecond) - } - require.NotNil(t, data, "init script should have created the marker file within 5s") + + return readErr == nil + }, 5*time.Second, 20*time.Millisecond, "init script should have created the marker file within 5s") assert.Contains(t, string(data), "ran") cancel() @@ -430,7 +440,7 @@ func TestServerStartup_WithDNS(t *testing.T) { errCh <- run(ctx, cli) }() - time.Sleep(300 * time.Millisecond) + waitForServerReady(t, port) cancel() select { @@ -457,7 +467,7 @@ func TestServerStartup_InvalidDNSConfig(t *testing.T) { errCh <- run(ctx, cli) }() - time.Sleep(200 * time.Millisecond) + waitForServerReady(t, port) cancel() select { @@ -484,7 +494,7 @@ func TestServerStartup_InvalidPortRange(t *testing.T) { errCh <- run(ctx, cli) }() - time.Sleep(200 * time.Millisecond) + waitForServerReady(t, port) cancel() select { @@ -512,16 +522,7 @@ func TestHealthCmd_Success(t *testing.T) { errCh <- run(ctx, cli) }() - // Wait for the server to be ready. - require.Eventually(t, func() bool { - resp, err := http.Get(fmt.Sprintf("http://localhost:%d/_gopherstack/health", port)) - if err != nil { - return false - } - resp.Body.Close() - - return resp.StatusCode == http.StatusOK - }, 3*time.Second, 50*time.Millisecond, "server did not become ready") + waitForServerReady(t, port) // Run the health command against the running server. cmd := &HealthCmd{Port: portString} diff --git a/cmd/cfnattrgen/main.go b/cmd/cfnattrgen/main.go index 98de8b5ced..2d793a358f 100644 --- a/cmd/cfnattrgen/main.go +++ b/cmd/cfnattrgen/main.go @@ -5,31 +5,24 @@ // type in Fn::GetAtt"); this table is what lets validateIntrinsics tell // an undocumented attribute from a merely-unmodelled one (gopherstack-p7pvq). // -// Every string this table introduces is checked against goconst's own rule -// (a literal repeated 3+ times across the package should be a constant) -// before being emitted, so this generator never hands golangci-lint new -// goconst violations to fix by hand: +// The table is emitted as JSON (services/cloudformation/cfn_attributes.json), +// not Go source: goconst counts string literals package-wide, and a +// generated .go file's literals still push hand-written files over the +// min-occurrences threshold even when the generated file itself is excluded +// from lint reporting (verified empirically -- see gopherstack-erj2j). A +// resource type is included only when the package declares a resTypeXxx +// constant for it (the provisioner's supported-types surface) and the spec +// documents a non-empty Attributes set for it; every documented attribute is +// emitted, since there's no literal-count reason left to drop any. // -// - A resource type is keyed by its existing resTypeXxx constant -// identifier (found by scanning -src), never re-quoted as a new string -// literal -- the type string already appears at that constant's -// declaration and every switch/case dispatching on it. -// - An attribute name is keyed by its existing attrNameXxx-style constant -// when one exists; otherwise the literal is counted against every -// string literal already in -src (tests included, matching goconst's -// own corpus), and only emitted when doing so keeps that string under -// goconst's min-occurrences threshold. +// Usage: // -// A type with no declared constant, or an attribute that would trip -// goconst, is simply left out of the table -- scoping the table to what the -// package already names/can safely re-quote is also exactly "the types we -// support" (gopherstack-p7pvq's intent), so this is conservative, not -// lossy: anything absent from the table is treated as "not in spec" by the -// validator and falls back to today's behaviour. +// go run ./cmd/cfnattrgen -spec -src -out // -// Usage: +// Or, to refresh the trimmed spec fixture used for -spec from a fresh +// download of the full CloudFormation resource specification: // -// go run ./cmd/cfnattrgen -spec -src -out +// go run ./cmd/cfnattrgen -spec -trimspec-out package main import ( @@ -37,7 +30,6 @@ import ( "flag" "fmt" "go/ast" - "go/format" "go/parser" "go/token" "os" @@ -47,11 +39,6 @@ import ( "strings" ) -// goconstMinOccurrences mirrors this repo's golangci-lint goconst default -// (min-occurrences: 3, unconfigured in .golangci.yml): a literal used at -// least this many times across the package should be a constant instead. -const goconstMinOccurrences = 3 - type resourceTypeSpec struct { Attributes map[string]json.RawMessage `json:"Attributes"` } @@ -62,47 +49,99 @@ type resourceSpec struct { func main() { specPath := flag.String("spec", "", "path to the CloudFormation resource specification JSON") - srcDir := flag.String("src", "", "directory to scan for resTypeXxx constants and existing string literals") - outPath := flag.String("out", "", "output Go file path") - pkgName := flag.String("pkg", "cloudformation", "package name for the generated file") + srcDir := flag.String("src", "", "directory to scan for resTypeXxx constants") + outPath := flag.String("out", "", "output attribute-table JSON file path") + trimSpecOut := flag.String( + "trimspec-out", "", + "write a trimmed (types+attribute names only) copy of -spec here instead of generating the table", + ) flag.Parse() + if *trimSpecOut != "" { + if *specPath == "" { + fmt.Fprintln(os.Stderr, "usage: cfnattrgen -spec -trimspec-out ") + os.Exit(1) + } + + if err := runTrimSpec(*specPath, *trimSpecOut); err != nil { + fmt.Fprintln(os.Stderr, "cfnattrgen:", err) + os.Exit(1) + } + + return + } + if *specPath == "" || *srcDir == "" || *outPath == "" { - fmt.Fprintln(os.Stderr, "usage: cfnattrgen -spec -src -out ") + fmt.Fprintln(os.Stderr, "usage: cfnattrgen -spec -src -out ") os.Exit(1) } - if err := run(*specPath, *srcDir, *outPath, *pkgName); err != nil { + if err := run(*specPath, *srcDir, *outPath); err != nil { fmt.Fprintln(os.Stderr, "cfnattrgen:", err) os.Exit(1) } } -func run(specPath, srcDir, outPath, pkgName string) error { +func run(specPath, srcDir, outPath string) error { spec, err := loadSpec(specPath) if err != nil { return fmt.Errorf("load spec: %w", err) } - // Excludes outPath itself: a stale copy from a prior run must not inflate - // litCounts against itself when the table is regenerated. - files, err := parseDir(srcDir, filepath.Base(outPath)) + files, err := parseDir(srcDir) if err != nil { return fmt.Errorf("parse %s: %w", srcDir, err) } - typeConsts := collectStringConsts(files, false, isResourceTypeConst) - attrConsts := collectStringConsts(files, false, isAttrNameConst) - litCounts := countStringLiterals(files) + supportedTypes := collectResourceTypeConsts(files) - table := buildTable(spec, typeConsts, attrConsts, litCounts) + table := buildTable(spec, supportedTypes) - src, err := renderTable(pkgName, table) + out, err := renderJSON(table) if err != nil { return fmt.Errorf("render: %w", err) } - if writeErr := os.WriteFile(outPath, src, 0o600); writeErr != nil { + if writeErr := os.WriteFile(outPath, out, 0o600); writeErr != nil { + return fmt.Errorf("write %s: %w", outPath, writeErr) + } + + return nil +} + +// runTrimSpec strips the full CloudFormation resource specification down to +// just the ResourceTypes/Attributes this generator reads, so a fixture small +// enough to commit can be refreshed from a fresh download without hand +// editing. +func runTrimSpec(specPath, outPath string) error { + spec, err := loadSpec(specPath) + if err != nil { + return fmt.Errorf("load spec: %w", err) + } + + trimmed := resourceSpec{ResourceTypes: make(map[string]resourceTypeSpec, len(spec.ResourceTypes))} + + for name, rt := range spec.ResourceTypes { + if len(rt.Attributes) == 0 { + continue + } + + attrs := make(map[string]json.RawMessage, len(rt.Attributes)) + for attr := range rt.Attributes { + attrs[attr] = json.RawMessage("{}") + } + + trimmed.ResourceTypes[name] = resourceTypeSpec{Attributes: attrs} + } + + out, err := json.MarshalIndent(trimmed, "", " ") + if err != nil { + return fmt.Errorf("marshal trimmed spec: %w", err) + } + + out = append(out, '\n') + + if writeErr := os.WriteFile(outPath, out, 0o600); writeErr != nil { return fmt.Errorf("write %s: %w", outPath, writeErr) } @@ -123,13 +162,8 @@ func loadSpec(path string) (*resourceSpec, error) { return &spec, nil } -type parsedFile struct { - file *ast.File - isTest bool -} - -// parseDir parses every .go file directly under dir (no recursion), skipping skipName. -func parseDir(dir, skipName string) ([]parsedFile, error) { +// parseDir parses every non-test .go file directly under dir (no recursion). +func parseDir(dir string) ([]*ast.File, error) { entries, err := os.ReadDir(dir) if err != nil { return nil, err @@ -137,11 +171,11 @@ func parseDir(dir, skipName string) ([]parsedFile, error) { fset := token.NewFileSet() - var files []parsedFile + var files []*ast.File for _, e := range entries { name := e.Name() - if e.IsDir() || !strings.HasSuffix(name, ".go") || name == skipName { + if e.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { continue } @@ -150,128 +184,67 @@ func parseDir(dir, skipName string) ([]parsedFile, error) { return nil, fmt.Errorf("parse %s: %w", name, perr) } - files = append(files, parsedFile{file: file, isTest: strings.HasSuffix(name, "_test.go")}) + files = append(files, file) } return files, nil } -// collectStringConsts collects top-level `const` declarations (non-test -// files only) whose (identifier, value) pair passes keep, returning a map -// from that value to the constant's identifier name. The first declaration -// found wins when a value has more than one qualifying constant. -func collectStringConsts(files []parsedFile, includeTests bool, keep func(name, value string) bool) map[string]string { - byValue := make(map[string]string) +// collectResourceTypeConsts returns the set of CloudFormation resource type +// strings named by a top-level resTypeXxx constant anywhere in files -- the +// provisioner's supported-types surface. +func collectResourceTypeConsts(files []*ast.File) map[string]struct{} { + types := make(map[string]struct{}) - for _, pf := range files { - if pf.isTest && !includeTests { - continue - } - - for _, decl := range pf.file.Decls { + for _, file := range files { + for _, decl := range file.Decls { gen, ok := decl.(*ast.GenDecl) if !ok || gen.Tok != token.CONST { continue } for _, spec := range gen.Specs { - valueSpec, isValueSpec := spec.(*ast.ValueSpec) - if !isValueSpec { - continue - } - - collectValueSpecConst(valueSpec, keep, byValue) + collectConstResourceType(spec, types) } } } - return byValue + return types } -func collectValueSpecConst(valueSpec *ast.ValueSpec, keep func(name, value string) bool, byValue map[string]string) { - for i, name := range valueSpec.Names { - if i >= len(valueSpec.Values) { - continue - } +func collectConstResourceType(spec ast.Spec, types map[string]struct{}) { + valueSpec, ok := spec.(*ast.ValueSpec) + if !ok { + return + } - lit, ok := valueSpec.Values[i].(*ast.BasicLit) - if !ok || lit.Kind != token.STRING { + for _, val := range valueSpec.Values { + lit, isBasicLit := val.(*ast.BasicLit) + if !isBasicLit || lit.Kind != token.STRING { continue } - value, unquoteErr := strconv.Unquote(lit.Value) - if unquoteErr != nil || !keep(name.Name, value) { + value, err := strconv.Unquote(lit.Value) + if err != nil || !isResourceTypeValue(value) { continue } - if _, exists := byValue[value]; !exists { - byValue[value] = name.Name - } + types[value] = struct{}{} } } -func isResourceTypeConst(_, value string) bool { +func isResourceTypeValue(value string) bool { return strings.HasPrefix(value, "AWS::") || strings.HasPrefix(value, "Alexa::") || strings.HasPrefix(value, "Custom::") } -// isAttrNameConst matches this package's attrNameXxx naming convention -// (attrNameArn, attrNameName, ...) by identifier, not by value: an -// attribute name has no shared shape to check like a resource type does. -func isAttrNameConst(name, _ string) bool { - return strings.HasPrefix(name, "attrName") -} - -// countStringLiterals tallies every string literal expression across all -// files (tests included, matching this repo's own goconst corpus) so the -// caller can tell whether adding one more occurrence would cross -// goconstMinOccurrences. -func countStringLiterals(files []parsedFile) map[string]int { - counts := make(map[string]int) - - for _, pf := range files { - ast.Inspect(pf.file, func(n ast.Node) bool { - lit, ok := n.(*ast.BasicLit) - if !ok || lit.Kind != token.STRING { - return true - } - - if value, err := strconv.Unquote(lit.Value); err == nil { - counts[value]++ - } - - return true - }) - } - - return counts -} - -// attrTable is one resource type's entry: its resTypeXxx constant -// identifier, and its attributes rendered as ready-to-emit Go map-key -// expressions (either an attrNameXxx identifier or a quoted literal). -type attrTable struct { - constIdent string - attrExprs []string -} +// buildTable keeps every supported type (declared via a resTypeXxx constant) +// that the spec documents a non-empty Attributes set for, with its complete +// documented attribute set. +func buildTable(spec *resourceSpec, supportedTypes map[string]struct{}) map[string][]string { + table := make(map[string][]string, len(supportedTypes)) -// buildTable keeps only types with both a spec-documented, non-empty -// Attributes set and a declared resTypeXxx constant, and where every one of -// that type's documented attributes can be safely emitted (see package -// doc). A type is registered in cfnResourceAttributes only with its COMPLETE -// documented attribute set: emitting a partial set would make validation -// reject a real, documented attribute we merely declined to re-quote here, -// which is worse than not validating the type at all -- so a type with even -// one unsafe attribute is dropped whole, falling back to today's behaviour -// for all of its attributes. -func buildTable( - spec *resourceSpec, - typeConsts, attrConsts map[string]string, - litCounts map[string]int, -) map[string]attrTable { - table := make(map[string]attrTable) - - for value, constIdent := range typeConsts { + for value := range supportedTypes { rt, ok := spec.ResourceTypes[value] if !ok || len(rt.Attributes) == 0 { continue @@ -281,72 +254,19 @@ func buildTable( for a := range rt.Attributes { names = append(names, a) } - sort.Strings(names) - exprs := make([]string, 0, len(names)) - complete := true - - for _, a := range names { - expr, safe := attrExpr(a, attrConsts, litCounts) - if !safe { - complete = false - - break - } - - exprs = append(exprs, expr) - } - - if !complete || len(exprs) == 0 { - continue - } - - table[value] = attrTable{constIdent: constIdent, attrExprs: exprs} + sort.Strings(names) + table[value] = names } return table } -// attrExpr returns the Go expression to use as attribute a's map key, and -// whether it's safe to emit at all. -func attrExpr(a string, attrConsts map[string]string, litCounts map[string]int) (string, bool) { - if constIdent, ok := attrConsts[a]; ok { - return constIdent, true - } - - if litCounts[a] >= goconstMinOccurrences-1 { - return "", false - } - - return strconv.Quote(a), true -} - -func renderTable(pkgName string, table map[string]attrTable) ([]byte, error) { - var b strings.Builder - - fmt.Fprintf( - &b, - "// Code generated by cmd/cfnattrgen from the CloudFormation resource specification; DO NOT EDIT.\n", - ) - fmt.Fprintf(&b, "package %s\n\n", pkgName) - fmt.Fprintf(&b, "//nolint:gochecknoglobals // generated static lookup table\n") - fmt.Fprintf(&b, "var cfnResourceAttributes = map[string]map[string]struct{}{\n") - - values := make([]string, 0, len(table)) - for v := range table { - values = append(values, v) - } - sort.Strings(values) - - for _, v := range values { - entry := table[v] - fmt.Fprintf(&b, "\t%s: {\n", entry.constIdent) - for _, expr := range entry.attrExprs { - fmt.Fprintf(&b, "\t\t%s: {},\n", expr) - } - fmt.Fprintf(&b, "\t},\n") +func renderJSON(table map[string][]string) ([]byte, error) { + out, err := json.MarshalIndent(table, "", " ") + if err != nil { + return nil, err } - fmt.Fprintf(&b, "}\n") - return format.Source([]byte(b.String())) + return append(out, '\n'), nil } diff --git a/cmd/cfnattrgen/regen_test.go b/cmd/cfnattrgen/regen_test.go new file mode 100644 index 0000000000..b1cce9cd66 --- /dev/null +++ b/cmd/cfnattrgen/regen_test.go @@ -0,0 +1,93 @@ +package main + +import ( + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + fixtureSpecPath = "testdata/cfn_resource_spec.json" + cloudformationSrcDir = "../../services/cloudformation" + committedTablePath = "../../services/cloudformation/cfn_attributes.json" +) + +// TestGeneratedTableUpToDate guards gopherstack-erj2j: regenerating from the +// committed spec fixture and the current services/cloudformation source must +// reproduce services/cloudformation/cfn_attributes.json byte-for-byte. A +// mismatch means someone added a resTypeXxx constant (or the fixture +// changed) without running `make cfn-attrs-gen`, so coverage silently +// drifted from what the source now supports. +func TestGeneratedTableUpToDate(t *testing.T) { + t.Parallel() + + spec, err := loadSpec(fixtureSpecPath) + require.NoError(t, err) + + files, err := parseDir(cloudformationSrcDir) + require.NoError(t, err) + + supportedTypes := collectResourceTypeConsts(files) + require.NotEmpty(t, supportedTypes, "expected to find resTypeXxx constants in %s", cloudformationSrcDir) + + table := buildTable(spec, supportedTypes) + + got, err := renderJSON(table) + require.NoError(t, err) + + want, err := os.ReadFile(committedTablePath) + require.NoError(t, err) + + assert.Equal(t, string(want), string(got), + "cfn_attributes.json is stale -- run `make cfn-attrs-gen` and commit the result") +} + +// TestBuildTable_CoversEverySupportedSpecType asserts buildTable never drops +// a type the provisioner supports (a declared resTypeXxx constant) once the +// spec documents a non-empty Attributes set for it -- the exact regression +// class that shrank coverage from 97 to 59 types before this fix. +func TestBuildTable_CoversEverySupportedSpecType(t *testing.T) { + t.Parallel() + + spec, err := loadSpec(fixtureSpecPath) + require.NoError(t, err) + + files, err := parseDir(cloudformationSrcDir) + require.NoError(t, err) + + supportedTypes := collectResourceTypeConsts(files) + table := buildTable(spec, supportedTypes) + + for value := range supportedTypes { + rt, inSpec := spec.ResourceTypes[value] + if !inSpec || len(rt.Attributes) == 0 { + continue + } + + assert.Contains(t, table, value, "supported type documented in the spec must be in the table") + } +} + +func TestRenderJSON_Deterministic(t *testing.T) { + t.Parallel() + + table := map[string][]string{ + "AWS::S3::Bucket": {"Arn", "DomainName"}, + "AWS::SNS::Topic": {"TopicArn"}, + } + + first, err := renderJSON(table) + require.NoError(t, err) + + second, err := renderJSON(table) + require.NoError(t, err) + + assert.Equal(t, first, second) + + var roundTrip map[string][]string + require.NoError(t, json.Unmarshal(first, &roundTrip)) + assert.Equal(t, table, roundTrip) +} diff --git a/cmd/cfnattrgen/testdata/cfn_resource_spec.json b/cmd/cfnattrgen/testdata/cfn_resource_spec.json new file mode 100644 index 0000000000..4d6dfdf1f5 --- /dev/null +++ b/cmd/cfnattrgen/testdata/cfn_resource_spec.json @@ -0,0 +1,13717 @@ +{ + "ResourceTypes": { + "AWS::ACMPCA::Certificate": { + "Attributes": { + "Arn": {}, + "Certificate": {} + } + }, + "AWS::ACMPCA::CertificateAuthority": { + "Attributes": { + "Arn": {}, + "CertificateSigningRequest": {} + } + }, + "AWS::ACMPCA::CertificateAuthorityActivation": { + "Attributes": { + "CompleteCertificateChain": {} + } + }, + "AWS::AIOps::InvestigationGroup": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "CreatedBy": {}, + "LastModifiedAt": {}, + "LastModifiedBy": {} + } + }, + "AWS::APS::AnomalyDetector": { + "Attributes": { + "Arn": {} + } + }, + "AWS::APS::RuleGroupsNamespace": { + "Attributes": { + "Arn": {} + } + }, + "AWS::APS::Scraper": { + "Attributes": { + "Arn": {}, + "RoleArn": {}, + "ScraperId": {} + } + }, + "AWS::APS::Workspace": { + "Attributes": { + "Arn": {}, + "PrometheusEndpoint": {}, + "WorkspaceId": {} + } + }, + "AWS::ARCRegionSwitch::Plan": { + "Attributes": { + "Arn": {}, + "Owner": {}, + "PlanHealthChecks": {}, + "Version": {} + } + }, + "AWS::ARCZonalShift::AutoshiftObserverNotificationStatus": { + "Attributes": { + "AccountId": {}, + "Region": {} + } + }, + "AWS::AWSExternalAnthropic::Workspace": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {} + } + }, + "AWS::AccessAnalyzer::Analyzer": { + "Attributes": { + "Arn": {} + } + }, + "AWS::AccessAnalyzer::ArchiveRule": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "UpdatedAt": {} + } + }, + "AWS::AccountAccess::Application": { + "Attributes": { + "ApplicationArn": {}, + "CreatedAt": {}, + "IdentitySource.IdentityCenter.ApplicationArn": {}, + "Status": {}, + "TenantId": {}, + "UpdatedAt": {} + } + }, + "AWS::AccountAccess::Entitlement": { + "Attributes": { + "CreatedAt": {}, + "Entitlement.PrincipalRole.Account": {}, + "EntitlementId": {} + } + }, + "AWS::AgentRegistry::Registry": { + "Attributes": { + "CreatedAt": {}, + "RegistryArn": {}, + "RegistryId": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::AgentRegistry::RegistryRecord": { + "Attributes": { + "CreatedAt": {}, + "CreatedBy": {}, + "RecordArn": {}, + "RecordId": {}, + "RegistryArn": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::AmazonMQ::Broker": { + "Attributes": { + "AmqpEndpoints": {}, + "Arn": {}, + "ConfigurationId": {}, + "ConfigurationRevision": {}, + "ConsoleURLs": {}, + "EngineVersionCurrent": {}, + "Id": {}, + "IpAddresses": {}, + "MqttEndpoints": {}, + "OpenWireEndpoints": {}, + "StompEndpoints": {}, + "WssEndpoints": {} + } + }, + "AWS::AmazonMQ::Configuration": { + "Attributes": { + "Arn": {}, + "Id": {}, + "Revision": {} + } + }, + "AWS::AmazonMQ::ConfigurationAssociation": { + "Attributes": { + "Id": {} + } + }, + "AWS::Amplify::App": { + "Attributes": { + "AppId": {}, + "AppName": {}, + "Arn": {}, + "DefaultDomain": {} + } + }, + "AWS::Amplify::Branch": { + "Attributes": { + "Arn": {}, + "BranchName": {} + } + }, + "AWS::Amplify::Domain": { + "Attributes": { + "Arn": {}, + "AutoSubDomainCreationPatterns": {}, + "AutoSubDomainIAMRole": {}, + "Certificate": {}, + "Certificate.CertificateArn": {}, + "Certificate.CertificateType": {}, + "Certificate.CertificateVerificationDNSRecord": {}, + "CertificateRecord": {}, + "DomainName": {}, + "DomainStatus": {}, + "EnableAutoSubDomain": {}, + "StatusReason": {}, + "UpdateStatus": {} + } + }, + "AWS::Amplify::Jobs": { + "Attributes": { + "Arn": {}, + "CommitId": {}, + "CommitTime": {}, + "JobId": {}, + "StartTime": {}, + "Status": {} + } + }, + "AWS::Amplify::Webhook": { + "Attributes": { + "Arn": {}, + "WebhookId": {}, + "WebhookUrl": {} + } + }, + "AWS::AmplifyUIBuilder::CodegenJob": { + "Attributes": { + "Arn": {}, + "Id": {}, + "Status": {}, + "Tags": {} + } + }, + "AWS::AmplifyUIBuilder::Component": { + "Attributes": { + "CreatedAt": {}, + "Id": {}, + "ModifiedAt": {} + } + }, + "AWS::AmplifyUIBuilder::Form": { + "Attributes": { + "Id": {} + } + }, + "AWS::AmplifyUIBuilder::Theme": { + "Attributes": { + "CreatedAt": {}, + "Id": {}, + "ModifiedAt": {} + } + }, + "AWS::ApiGateway::Account": { + "Attributes": { + "Id": {} + } + }, + "AWS::ApiGateway::ApiKey": { + "Attributes": { + "APIKeyId": {} + } + }, + "AWS::ApiGateway::Authorizer": { + "Attributes": { + "AuthorizerId": {} + } + }, + "AWS::ApiGateway::BasePathMappingV2": { + "Attributes": { + "BasePathMappingArn": {} + } + }, + "AWS::ApiGateway::ClientCertificate": { + "Attributes": { + "ClientCertificateId": {} + } + }, + "AWS::ApiGateway::Deployment": { + "Attributes": { + "DeploymentId": {} + } + }, + "AWS::ApiGateway::DocumentationPart": { + "Attributes": { + "DocumentationPartId": {} + } + }, + "AWS::ApiGateway::DomainName": { + "Attributes": { + "DistributionDomainName": {}, + "DistributionHostedZoneId": {}, + "DomainNameArn": {}, + "RegionalDomainName": {}, + "RegionalHostedZoneId": {} + } + }, + "AWS::ApiGateway::DomainNameAccessAssociation": { + "Attributes": { + "DomainNameAccessAssociationArn": {} + } + }, + "AWS::ApiGateway::DomainNameV2": { + "Attributes": { + "DomainNameArn": {}, + "DomainNameId": {} + } + }, + "AWS::ApiGateway::GatewayResponse": { + "Attributes": { + "Id": {} + } + }, + "AWS::ApiGateway::RequestValidator": { + "Attributes": { + "RequestValidatorId": {} + } + }, + "AWS::ApiGateway::Resource": { + "Attributes": { + "ResourceId": {} + } + }, + "AWS::ApiGateway::RestApi": { + "Attributes": { + "RestApiId": {}, + "RootResourceId": {} + } + }, + "AWS::ApiGateway::UsagePlan": { + "Attributes": { + "Id": {} + } + }, + "AWS::ApiGateway::UsagePlanKey": { + "Attributes": { + "Id": {} + } + }, + "AWS::ApiGateway::VpcLink": { + "Attributes": { + "VpcLinkId": {} + } + }, + "AWS::ApiGatewayV2::Api": { + "Attributes": { + "ApiEndpoint": {}, + "ApiId": {}, + "ExecuteApiArn": {} + } + }, + "AWS::ApiGatewayV2::ApiGatewayManagedOverrides": { + "Attributes": { + "Id": {} + } + }, + "AWS::ApiGatewayV2::ApiMapping": { + "Attributes": { + "ApiMappingId": {} + } + }, + "AWS::ApiGatewayV2::Authorizer": { + "Attributes": { + "AuthorizerId": {} + } + }, + "AWS::ApiGatewayV2::Deployment": { + "Attributes": { + "DeploymentId": {} + } + }, + "AWS::ApiGatewayV2::DomainName": { + "Attributes": { + "DomainNameArn": {}, + "RegionalDomainName": {}, + "RegionalHostedZoneId": {} + } + }, + "AWS::ApiGatewayV2::Integration": { + "Attributes": { + "IntegrationId": {} + } + }, + "AWS::ApiGatewayV2::IntegrationResponse": { + "Attributes": { + "IntegrationResponseId": {} + } + }, + "AWS::ApiGatewayV2::Model": { + "Attributes": { + "ModelId": {} + } + }, + "AWS::ApiGatewayV2::PortalProduct": { + "Attributes": { + "LastModified": {}, + "PortalProductArn": {}, + "PortalProductId": {} + } + }, + "AWS::ApiGatewayV2::Route": { + "Attributes": { + "RouteId": {} + } + }, + "AWS::ApiGatewayV2::RouteResponse": { + "Attributes": { + "RouteResponseId": {} + } + }, + "AWS::ApiGatewayV2::RoutingRule": { + "Attributes": { + "RoutingRuleArn": {}, + "RoutingRuleId": {} + } + }, + "AWS::ApiGatewayV2::VpcLink": { + "Attributes": { + "VpcLinkId": {} + } + }, + "AWS::AppConfig::Application": { + "Attributes": { + "ApplicationId": {} + } + }, + "AWS::AppConfig::ConfigurationProfile": { + "Attributes": { + "ConfigurationProfileId": {}, + "KmsKeyArn": {} + } + }, + "AWS::AppConfig::Deployment": { + "Attributes": { + "DeploymentNumber": {}, + "State": {} + } + }, + "AWS::AppConfig::DeploymentStrategy": { + "Attributes": { + "Id": {} + } + }, + "AWS::AppConfig::Environment": { + "Attributes": { + "EnvironmentId": {} + } + }, + "AWS::AppConfig::ExperimentDefinition": { + "Attributes": { + "ApplicationId": {}, + "Control.Key": {}, + "CreatedAt": {}, + "Id": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::AppConfig::ExperimentRun": { + "Attributes": { + "ApplicationId": {}, + "ExperimentDefinitionId": {}, + "Run": {}, + "StartedAt": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::AppConfig::Extension": { + "Attributes": { + "Arn": {}, + "Id": {}, + "VersionNumber": {} + } + }, + "AWS::AppConfig::ExtensionAssociation": { + "Attributes": { + "Arn": {}, + "ExtensionArn": {}, + "Id": {}, + "ResourceArn": {} + } + }, + "AWS::AppConfig::HostedConfigurationVersion": { + "Attributes": { + "VersionNumber": {} + } + }, + "AWS::AppFlow::Connector": { + "Attributes": { + "ConnectorArn": {} + } + }, + "AWS::AppFlow::ConnectorProfile": { + "Attributes": { + "ConnectorProfileArn": {}, + "CredentialsArn": {} + } + }, + "AWS::AppFlow::Flow": { + "Attributes": { + "FlowArn": {} + } + }, + "AWS::AppIntegrations::Application": { + "Attributes": { + "ApplicationArn": {}, + "Id": {} + } + }, + "AWS::AppIntegrations::DataIntegration": { + "Attributes": { + "DataIntegrationArn": {}, + "Id": {} + } + }, + "AWS::AppIntegrations::EventIntegration": { + "Attributes": { + "EventIntegrationArn": {} + } + }, + "AWS::AppMesh::GatewayRoute": { + "Attributes": { + "Arn": {}, + "GatewayRouteName": {}, + "MeshName": {}, + "MeshOwner": {}, + "ResourceOwner": {}, + "Uid": {}, + "VirtualGatewayName": {} + } + }, + "AWS::AppMesh::Mesh": { + "Attributes": { + "Arn": {}, + "MeshName": {}, + "MeshOwner": {}, + "ResourceOwner": {}, + "Uid": {} + } + }, + "AWS::AppMesh::Route": { + "Attributes": { + "Arn": {}, + "MeshName": {}, + "MeshOwner": {}, + "ResourceOwner": {}, + "RouteName": {}, + "Uid": {}, + "VirtualRouterName": {} + } + }, + "AWS::AppMesh::VirtualGateway": { + "Attributes": { + "Arn": {}, + "MeshName": {}, + "MeshOwner": {}, + "ResourceOwner": {}, + "Uid": {}, + "VirtualGatewayName": {} + } + }, + "AWS::AppMesh::VirtualNode": { + "Attributes": { + "Arn": {}, + "MeshName": {}, + "MeshOwner": {}, + "ResourceOwner": {}, + "Uid": {}, + "VirtualNodeName": {} + } + }, + "AWS::AppMesh::VirtualRouter": { + "Attributes": { + "Arn": {}, + "MeshName": {}, + "MeshOwner": {}, + "ResourceOwner": {}, + "Uid": {}, + "VirtualRouterName": {} + } + }, + "AWS::AppMesh::VirtualService": { + "Attributes": { + "Arn": {}, + "MeshName": {}, + "MeshOwner": {}, + "ResourceOwner": {}, + "Uid": {}, + "VirtualServiceName": {} + } + }, + "AWS::AppRunner::AutoScalingConfiguration": { + "Attributes": { + "AutoScalingConfigurationArn": {}, + "AutoScalingConfigurationRevision": {}, + "Latest": {} + } + }, + "AWS::AppRunner::ObservabilityConfiguration": { + "Attributes": { + "Latest": {}, + "ObservabilityConfigurationArn": {}, + "ObservabilityConfigurationRevision": {} + } + }, + "AWS::AppRunner::Service": { + "Attributes": { + "ServiceArn": {}, + "ServiceId": {}, + "ServiceUrl": {}, + "Status": {} + } + }, + "AWS::AppRunner::VpcConnector": { + "Attributes": { + "VpcConnectorArn": {}, + "VpcConnectorRevision": {} + } + }, + "AWS::AppRunner::VpcIngressConnection": { + "Attributes": { + "DomainName": {}, + "Status": {}, + "VpcIngressConnectionArn": {} + } + }, + "AWS::AppStream::AppBlock": { + "Attributes": { + "Arn": {}, + "CreatedTime": {} + } + }, + "AWS::AppStream::AppBlockBuilder": { + "Attributes": { + "Arn": {}, + "CreatedTime": {} + } + }, + "AWS::AppStream::Application": { + "Attributes": { + "Arn": {}, + "CreatedTime": {} + } + }, + "AWS::AppStream::Entitlement": { + "Attributes": { + "CreatedTime": {}, + "LastModifiedTime": {} + } + }, + "AWS::AppStream::ImageBuilder": { + "Attributes": { + "StreamingUrl": {} + } + }, + "AWS::AppStream::User": { + "Attributes": { + "Arn": {} + } + }, + "AWS::AppSync::Api": { + "Attributes": { + "ApiArn": {}, + "ApiId": {}, + "Dns": {}, + "Dns.Http": {}, + "Dns.Realtime": {} + } + }, + "AWS::AppSync::ApiKey": { + "Attributes": { + "ApiKey": {}, + "ApiKeyId": {}, + "Arn": {} + } + }, + "AWS::AppSync::ChannelNamespace": { + "Attributes": { + "ChannelNamespaceArn": {} + } + }, + "AWS::AppSync::DataSource": { + "Attributes": { + "DataSourceArn": {}, + "Name": {} + } + }, + "AWS::AppSync::DomainName": { + "Attributes": { + "AppSyncDomainName": {}, + "DomainName": {}, + "DomainNameArn": {}, + "HostedZoneId": {} + } + }, + "AWS::AppSync::DomainNameApiAssociation": { + "Attributes": { + "ApiAssociationIdentifier": {} + } + }, + "AWS::AppSync::FunctionConfiguration": { + "Attributes": { + "DataSourceName": {}, + "FunctionArn": {}, + "FunctionId": {}, + "Name": {} + } + }, + "AWS::AppSync::GraphQLApi": { + "Attributes": { + "ApiId": {}, + "Arn": {}, + "GraphQLDns": {}, + "GraphQLEndpointArn": {}, + "GraphQLUrl": {}, + "RealtimeDns": {}, + "RealtimeUrl": {} + } + }, + "AWS::AppSync::Resolver": { + "Attributes": { + "FieldName": {}, + "ResolverArn": {}, + "TypeName": {} + } + }, + "AWS::AppSync::SourceApiAssociation": { + "Attributes": { + "AssociationArn": {}, + "AssociationId": {}, + "LastSuccessfulMergeDate": {}, + "MergedApiArn": {}, + "MergedApiId": {}, + "SourceApiArn": {}, + "SourceApiAssociationStatus": {}, + "SourceApiAssociationStatusDetail": {}, + "SourceApiId": {} + } + }, + "AWS::AppSync::Type": { + "Attributes": { + "Arn": {}, + "Name": {} + } + }, + "AWS::AppTest::TestCase": { + "Attributes": { + "CreationTime": {}, + "LastUpdateTime": {}, + "LatestVersion": {}, + "LatestVersion.Status": {}, + "LatestVersion.Version": {}, + "Status": {}, + "TestCaseArn": {}, + "TestCaseId": {}, + "TestCaseVersion": {} + } + }, + "AWS::ApplicationAutoScaling::ScalableTarget": { + "Attributes": { + "Id": {} + } + }, + "AWS::ApplicationAutoScaling::ScalingPolicy": { + "Attributes": { + "Arn": {} + } + }, + "AWS::ApplicationInsights::Application": { + "Attributes": { + "ApplicationARN": {} + } + }, + "AWS::ApplicationSignals::Discovery": { + "Attributes": { + "AccountId": {} + } + }, + "AWS::ApplicationSignals::GroupingConfiguration": { + "Attributes": { + "AccountId": {}, + "UpdatedAt": {} + } + }, + "AWS::ApplicationSignals::ServiceLevelObjective": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "EvaluationType": {}, + "LastUpdatedTime": {} + } + }, + "AWS::Artifact::ComplianceInquiry": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "InputSource": {}, + "Name": {}, + "Status": {}, + "StatusMessage": {}, + "SupportMode": {}, + "UpdatedAt": {} + } + }, + "AWS::Artifact::Report": { + "Attributes": { + "AcceptanceType": {}, + "Arn": {}, + "Category": {}, + "CompanyName": {}, + "CreatedAt": {}, + "Description": {}, + "Name": {}, + "PeriodEnd": {}, + "PeriodStart": {}, + "ProductName": {}, + "ReportId": {}, + "SequenceNumber": {}, + "Series": {}, + "State": {}, + "TermArn": {}, + "Version": {} + } + }, + "AWS::Athena::CapacityReservation": { + "Attributes": { + "AllocatedDpus": {}, + "Arn": {}, + "CreationTime": {}, + "LastSuccessfulAllocationTime": {}, + "Status": {} + } + }, + "AWS::Athena::NamedQuery": { + "Attributes": { + "NamedQueryId": {} + } + }, + "AWS::Athena::Session": { + "Attributes": { + "Arn": {}, + "EngineConfiguration.AdditionalConfigs": {}, + "EngineConfiguration.SparkProperties": {}, + "EngineVersion": {}, + "SessionId": {} + } + }, + "AWS::Athena::WorkGroup": { + "Attributes": { + "CreationTime": {}, + "WorkGroupConfiguration.EngineVersion.EffectiveEngineVersion": {} + } + }, + "AWS::AuditManager::Assessment": { + "Attributes": { + "Arn": {}, + "AssessmentId": {}, + "CreationTime": {} + } + }, + "AWS::AuditManager::AssessmentFramework": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "CreatedBy": {}, + "FrameworkId": {}, + "LastUpdatedAt": {}, + "LastUpdatedBy": {}, + "Type": {} + } + }, + "AWS::AutoScaling::AutoScalingGroup": { + "Attributes": { + "AutoScalingGroupARN": {} + } + }, + "AWS::AutoScaling::ScalingPolicy": { + "Attributes": { + "Arn": {}, + "PolicyName": {} + } + }, + "AWS::AutoScaling::ScheduledAction": { + "Attributes": { + "ScheduledActionName": {} + } + }, + "AWS::AutoScalingPlans::ScalingPlan": { + "Attributes": { + "ScalingPlanName": {}, + "ScalingPlanVersion": {} + } + }, + "AWS::B2BI::Capability": { + "Attributes": { + "CapabilityArn": {}, + "CapabilityId": {}, + "CreatedAt": {}, + "ModifiedAt": {} + } + }, + "AWS::B2BI::Partnership": { + "Attributes": { + "CreatedAt": {}, + "ModifiedAt": {}, + "PartnershipArn": {}, + "PartnershipId": {}, + "TradingPartnerId": {} + } + }, + "AWS::B2BI::Profile": { + "Attributes": { + "CreatedAt": {}, + "LogGroupName": {}, + "ModifiedAt": {}, + "ProfileArn": {}, + "ProfileId": {} + } + }, + "AWS::B2BI::Transformer": { + "Attributes": { + "CreatedAt": {}, + "ModifiedAt": {}, + "TransformerArn": {}, + "TransformerId": {} + } + }, + "AWS::BCM::Dashboard": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Type": {}, + "UpdatedAt": {} + } + }, + "AWS::BCMDataExports::Export": { + "Attributes": { + "Export.ExportArn": {}, + "ExportArn": {} + } + }, + "AWS::BCMDataExports::Table": { + "Attributes": { + "Arn": {}, + "Description": {}, + "Schema": {} + } + }, + "AWS::Backup::BackupPlan": { + "Attributes": { + "BackupPlanArn": {}, + "BackupPlanId": {}, + "VersionId": {} + } + }, + "AWS::Backup::BackupSelection": { + "Attributes": { + "BackupPlanId": {}, + "Id": {}, + "SelectionId": {} + } + }, + "AWS::Backup::BackupVault": { + "Attributes": { + "BackupVaultArn": {}, + "BackupVaultName": {} + } + }, + "AWS::Backup::Framework": { + "Attributes": { + "CreationTime": {}, + "DeploymentStatus": {}, + "FrameworkArn": {}, + "FrameworkStatus": {} + } + }, + "AWS::Backup::LegalHold": { + "Attributes": { + "Arn": {}, + "CreationDate": {}, + "LegalHoldId": {}, + "Status": {} + } + }, + "AWS::Backup::LogicallyAirGappedBackupVault": { + "Attributes": { + "BackupVaultArn": {}, + "VaultState": {}, + "VaultType": {} + } + }, + "AWS::Backup::ReportPlan": { + "Attributes": { + "ReportPlanArn": {} + } + }, + "AWS::Backup::RestoreTestingPlan": { + "Attributes": { + "RestoreTestingPlanArn": {} + } + }, + "AWS::Backup::TieringConfiguration": { + "Attributes": { + "CreationTime": {}, + "LastUpdatedTime": {}, + "TieringConfigurationArn": {} + } + }, + "AWS::BackupGateway::Gateway": { + "Attributes": { + "DeprecationDate": {}, + "GatewayArn": {}, + "GatewayId": {}, + "HypervisorId": {}, + "LastSeenTime": {}, + "MaintenanceStartTime": {}, + "MaintenanceStartTime.DayOfMonth": {}, + "MaintenanceStartTime.DayOfWeek": {}, + "MaintenanceStartTime.HourOfDay": {}, + "MaintenanceStartTime.MinuteOfHour": {}, + "NextUpdateAvailabilityTime": {}, + "SoftwareVersion": {}, + "VpcEndpoint": {} + } + }, + "AWS::BackupGateway::Hypervisor": { + "Attributes": { + "HypervisorArn": {} + } + }, + "AWS::BackupSearch::SearchJob": { + "Attributes": { + "CreationTime": {}, + "SearchJobArn": {}, + "SearchJobIdentifier": {}, + "Status": {} + } + }, + "AWS::BackupSearch::SearchResultExportJob": { + "Attributes": { + "CreationTime": {}, + "ExportJobArn": {}, + "ExportJobIdentifier": {}, + "SearchJobArn": {}, + "Status": {} + } + }, + "AWS::Batch::ComputeEnvironment": { + "Attributes": { + "ComputeEnvironmentArn": {} + } + }, + "AWS::Batch::ConsumableResource": { + "Attributes": { + "AvailableQuantity": {}, + "ConsumableResourceArn": {}, + "CreatedAt": {}, + "InUseQuantity": {} + } + }, + "AWS::Batch::JobDefinition": { + "Attributes": { + "JobDefinitionArn": {} + } + }, + "AWS::Batch::JobQueue": { + "Attributes": { + "JobQueueArn": {} + } + }, + "AWS::Batch::QuotaShare": { + "Attributes": { + "QuotaShareArn": {} + } + }, + "AWS::Batch::SchedulingPolicy": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Batch::ServiceEnvironment": { + "Attributes": { + "ServiceEnvironmentArn": {} + } + }, + "AWS::BcmPricingCalculator::BillScenario": { + "Attributes": { + "Arn": {}, + "BillInterval": {}, + "BillInterval.End": {}, + "BillInterval.Start": {}, + "CreatedAt": {}, + "FailureMessage": {}, + "Id": {}, + "Status": {} + } + }, + "AWS::Bedrock::Agent": { + "Attributes": { + "AgentArn": {}, + "AgentId": {}, + "AgentStatus": {}, + "AgentVersion": {}, + "CreatedAt": {}, + "FailureReasons": {}, + "PreparedAt": {}, + "RecommendedActions": {}, + "UpdatedAt": {} + } + }, + "AWS::Bedrock::AgentAlias": { + "Attributes": { + "AgentAliasArn": {}, + "AgentAliasHistoryEvents": {}, + "AgentAliasId": {}, + "AgentAliasStatus": {}, + "CreatedAt": {}, + "UpdatedAt": {} + } + }, + "AWS::Bedrock::ApplicationInferenceProfile": { + "Attributes": { + "CreatedAt": {}, + "InferenceProfileArn": {}, + "InferenceProfileId": {}, + "InferenceProfileIdentifier": {}, + "Models": {}, + "Status": {}, + "Type": {}, + "UpdatedAt": {} + } + }, + "AWS::Bedrock::AsyncInvoke": { + "Attributes": { + "EndTime": {}, + "InvocationArn": {}, + "InvocationId": {}, + "LastModifiedTime": {}, + "ModelArn": {}, + "OutputDataConfig": {}, + "OutputDataConfig.S3OutputDataConfig": {}, + "OutputDataConfig.S3OutputDataConfig.S3Uri": {}, + "Status": {}, + "SubmitTime": {} + } + }, + "AWS::Bedrock::AutomatedReasoningPolicy": { + "Attributes": { + "CreatedAt": {}, + "DefinitionHash": {}, + "KmsKeyArn": {}, + "PolicyArn": {}, + "PolicyId": {}, + "UpdatedAt": {}, + "Version": {} + } + }, + "AWS::Bedrock::AutomatedReasoningPolicyVersion": { + "Attributes": { + "CreatedAt": {}, + "DefinitionHash": {}, + "Description": {}, + "Name": {}, + "PolicyId": {}, + "UpdatedAt": {}, + "Version": {} + } + }, + "AWS::Bedrock::Blueprint": { + "Attributes": { + "BlueprintArn": {}, + "BlueprintStage": {}, + "CreationTime": {}, + "LastModifiedTime": {} + } + }, + "AWS::Bedrock::DataAutomationLibrary": { + "Attributes": { + "CreationTime": {}, + "EntityTypes": {}, + "LibraryArn": {}, + "Status": {} + } + }, + "AWS::Bedrock::DataAutomationProject": { + "Attributes": { + "CreationTime": {}, + "LastModifiedTime": {}, + "ProjectArn": {}, + "ProjectStage": {}, + "Status": {} + } + }, + "AWS::Bedrock::DataSource": { + "Attributes": { + "CreatedAt": {}, + "DataSourceConfiguration.WebConfiguration.CrawlerConfiguration.UserAgentHeader": {}, + "DataSourceId": {}, + "DataSourceStatus": {}, + "FailureReasons": {}, + "UpdatedAt": {} + } + }, + "AWS::Bedrock::DefaultPromptRouter": { + "Attributes": { + "CreatedAt": {}, + "Description": {}, + "FallbackModel": {}, + "FallbackModel.ModelArn": {}, + "Models": {}, + "PromptRouterArn": {}, + "PromptRouterId": {}, + "PromptRouterName": {}, + "RoutingCriteria": {}, + "RoutingCriteria.ResponseQualityDifference": {}, + "Status": {}, + "Type": {}, + "UpdatedAt": {} + } + }, + "AWS::Bedrock::EnforcedGuardrailConfiguration": { + "Attributes": { + "ConfigId": {}, + "CreatedAt": {}, + "CreatedBy": {}, + "GuardrailArn": {}, + "GuardrailId": {}, + "Owner": {}, + "UpdatedAt": {}, + "UpdatedBy": {} + } + }, + "AWS::Bedrock::EvaluationJob": { + "Attributes": { + "ApplicationType": {}, + "CreationTime": {}, + "CustomerEncryptionKeyId": {}, + "EvaluationConfig": {}, + "EvaluationConfig.Automated": {}, + "EvaluationConfig.Automated.DatasetMetricConfigs": {}, + "EvaluationConfig.Automated.EvaluatorModelConfig": {}, + "EvaluationConfig.Automated.EvaluatorModelConfig.BedrockEvaluatorModels": {}, + "EvaluationConfig.Human": {}, + "EvaluationConfig.Human.CustomMetrics": {}, + "EvaluationConfig.Human.DatasetMetricConfigs": {}, + "EvaluationConfig.Human.HumanWorkflowConfig": {}, + "EvaluationConfig.Human.HumanWorkflowConfig.FlowDefinitionArn": {}, + "EvaluationConfig.Human.HumanWorkflowConfig.Instructions": {}, + "InferenceConfig": {}, + "InferenceConfig.Models": {}, + "InferenceConfig.RagConfigs": {}, + "JobArn": {}, + "JobDescription": {}, + "JobName": {}, + "JobType": {}, + "LastModifiedTime": {}, + "OutputDataConfig": {}, + "OutputDataConfig.S3Uri": {}, + "RoleArn": {}, + "Status": {}, + "Tags": {} + } + }, + "AWS::Bedrock::Flow": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "Status": {}, + "UpdatedAt": {}, + "Validations": {}, + "Version": {} + } + }, + "AWS::Bedrock::FlowAlias": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "FlowId": {}, + "Id": {}, + "UpdatedAt": {} + } + }, + "AWS::Bedrock::FlowExecution": { + "Attributes": { + "EndedAt": {}, + "ExecutionArn": {}, + "FlowVersion": {}, + "StartedAt": {}, + "Status": {} + } + }, + "AWS::Bedrock::FlowVersion": { + "Attributes": { + "CreatedAt": {}, + "CustomerEncryptionKeyArn": {}, + "Definition": {}, + "Definition.Connections": {}, + "Definition.Nodes": {}, + "ExecutionRoleArn": {}, + "FlowId": {}, + "Name": {}, + "Status": {}, + "Version": {} + } + }, + "AWS::Bedrock::FoundationModel": { + "Attributes": { + "CustomizationsSupported": {}, + "InferenceTypesSupported": {}, + "InputModalities": {}, + "ModelArn": {}, + "ModelId": {}, + "ModelLifecycle": {}, + "ModelLifecycle.EndOfLifeTime": {}, + "ModelLifecycle.LegacyTime": {}, + "ModelLifecycle.PublicExtendedAccessTime": {}, + "ModelLifecycle.StartOfLifeTime": {}, + "ModelLifecycle.Status": {}, + "ModelName": {}, + "OutputModalities": {}, + "ProviderName": {}, + "ResponseStreamingSupported": {} + } + }, + "AWS::Bedrock::Guardrail": { + "Attributes": { + "CreatedAt": {}, + "FailureRecommendations": {}, + "GuardrailArn": {}, + "GuardrailId": {}, + "Status": {}, + "StatusReasons": {}, + "UpdatedAt": {}, + "Version": {} + } + }, + "AWS::Bedrock::GuardrailVersion": { + "Attributes": { + "GuardrailArn": {}, + "GuardrailId": {}, + "Version": {} + } + }, + "AWS::Bedrock::ImportedModel": { + "Attributes": { + "CreationTime": {}, + "CustomModelUnits": {}, + "CustomModelUnits.CustomModelUnitsPerModelCopy": {}, + "CustomModelUnits.CustomModelUnitsVersion": {}, + "InstructSupported": {}, + "JobArn": {}, + "ModelArchitecture": {}, + "ModelArn": {}, + "ModelKmsKeyArn": {}, + "Tags": {} + } + }, + "AWS::Bedrock::IntelligentPromptRouter": { + "Attributes": { + "CreatedAt": {}, + "PromptRouterArn": {}, + "Status": {}, + "Type": {}, + "UpdatedAt": {} + } + }, + "AWS::Bedrock::KnowledgeBase": { + "Attributes": { + "CreatedAt": {}, + "FailureReasons": {}, + "KnowledgeBaseArn": {}, + "KnowledgeBaseId": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::Bedrock::KnowledgeBasePolicy": { + "Attributes": { + "RevisionId": {} + } + }, + "AWS::Bedrock::ModelImportJob": { + "Attributes": { + "CreationTime": {}, + "EndTime": {}, + "ImportedModelName": {}, + "JobArn": {}, + "JobName": {}, + "LastModifiedTime": {}, + "Status": {} + } + }, + "AWS::Bedrock::ModelInvocationJob": { + "Attributes": { + "InputDataConfig": {}, + "InputDataConfig.S3InputDataConfig": {}, + "InputDataConfig.S3InputDataConfig.S3BucketOwner": {}, + "InputDataConfig.S3InputDataConfig.S3Uri": {}, + "JobArn": {}, + "JobExpirationTime": {}, + "JobName": {}, + "LastModifiedTime": {}, + "ModelId": {}, + "OutputDataConfig": {}, + "OutputDataConfig.S3OutputDataConfig": {}, + "OutputDataConfig.S3OutputDataConfig.S3BucketOwner": {}, + "OutputDataConfig.S3OutputDataConfig.S3EncryptionKeyId": {}, + "OutputDataConfig.S3OutputDataConfig.S3Uri": {}, + "RoleArn": {}, + "Status": {}, + "SubmitTime": {}, + "Tags": {}, + "TimeoutDurationInHours": {}, + "VpcConfig": {}, + "VpcConfig.SecurityGroupIds": {}, + "VpcConfig.SubnetIds": {} + } + }, + "AWS::Bedrock::Prompt": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "UpdatedAt": {}, + "Version": {} + } + }, + "AWS::Bedrock::PromptVersion": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "CustomerEncryptionKeyArn": {}, + "DefaultVariant": {}, + "Name": {}, + "PromptId": {}, + "UpdatedAt": {}, + "Variants": {}, + "Version": {} + } + }, + "AWS::Bedrock::Session": { + "Attributes": { + "CreatedAt": {}, + "LastUpdatedAt": {}, + "SessionArn": {}, + "SessionId": {}, + "SessionStatus": {} + } + }, + "AWS::BedrockAgentCore::ApiKeyCredentialProvider": { + "Attributes": { + "ApiKeySecretArn": {}, + "ApiKeySecretArn.SecretArn": {}, + "ApiKeySecretJsonKey": {}, + "CreatedTime": {}, + "CredentialProviderArn": {}, + "LastUpdatedTime": {} + } + }, + "AWS::BedrockAgentCore::Browser": { + "Attributes": { + "BrowserArn": {}, + "BrowserId": {}, + "Name": {}, + "Status": {} + } + }, + "AWS::BedrockAgentCore::BrowserCustom": { + "Attributes": { + "BrowserArn": {}, + "BrowserId": {}, + "CreatedAt": {}, + "FailureReason": {}, + "LastUpdatedAt": {}, + "Status": {} + } + }, + "AWS::BedrockAgentCore::BrowserProfile": { + "Attributes": { + "CreatedAt": {}, + "LastSavedAt": {}, + "LastSavedBrowserId": {}, + "LastSavedBrowserSessionId": {}, + "LastUpdatedAt": {}, + "ProfileArn": {}, + "ProfileId": {}, + "Status": {} + } + }, + "AWS::BedrockAgentCore::CapacityProvider": { + "Attributes": { + "Arn": {}, + "CapacityProviderId": {}, + "CreatedAt": {}, + "LastUpdatedAt": {}, + "Status": {} + } + }, + "AWS::BedrockAgentCore::CodeInterpreter": { + "Attributes": { + "CodeInterpreterArn": {}, + "CodeInterpreterId": {}, + "Status": {} + } + }, + "AWS::BedrockAgentCore::CodeInterpreterCustom": { + "Attributes": { + "CodeInterpreterArn": {}, + "CodeInterpreterId": {}, + "CreatedAt": {}, + "FailureReason": {}, + "LastUpdatedAt": {}, + "Status": {} + } + }, + "AWS::BedrockAgentCore::ConfigurationBundle": { + "Attributes": { + "BundleArn": {}, + "BundleId": {}, + "CreatedAt": {}, + "LineageMetadata": {}, + "LineageMetadata.BranchName": {}, + "LineageMetadata.CommitMessage": {}, + "LineageMetadata.CreatedBy": {}, + "LineageMetadata.CreatedBy.Arn": {}, + "LineageMetadata.CreatedBy.Name": {}, + "LineageMetadata.ParentVersionIds": {}, + "UpdatedAt": {}, + "VersionId": {} + } + }, + "AWS::BedrockAgentCore::ConfigurationBundleVersion": { + "Attributes": { + "BundleArn": {}, + "BundleId": {}, + "ParentVersionIds": {}, + "Tags": {}, + "VersionCreatedAt": {}, + "VersionId": {} + } + }, + "AWS::BedrockAgentCore::Dataset": { + "Attributes": { + "CreatedAt": {}, + "DatasetArn": {}, + "DatasetId": {}, + "ExampleCount": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::BedrockAgentCore::Evaluator": { + "Attributes": { + "CreatedAt": {}, + "EvaluatorArn": {}, + "EvaluatorId": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::BedrockAgentCore::Gateway": { + "Attributes": { + "CreatedAt": {}, + "GatewayArn": {}, + "GatewayIdentifier": {}, + "GatewayUrl": {}, + "Status": {}, + "StatusReasons": {}, + "UpdatedAt": {}, + "WebAclArn": {}, + "WorkloadIdentityDetails": {}, + "WorkloadIdentityDetails.WorkloadIdentityArn": {} + } + }, + "AWS::BedrockAgentCore::GatewayRateLimit": { + "Attributes": { + "CreatedAt": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::BedrockAgentCore::GatewayRule": { + "Attributes": { + "CreatedAt": {}, + "GatewayArn": {}, + "RuleId": {}, + "Status": {}, + "System": {}, + "System.ManagedBy": {}, + "UpdatedAt": {} + } + }, + "AWS::BedrockAgentCore::GatewayTarget": { + "Attributes": { + "AuthorizationData": {}, + "AuthorizationData.Oauth2": {}, + "AuthorizationData.Oauth2.AuthorizationUrl": {}, + "AuthorizationData.Oauth2.UserId": {}, + "CreatedAt": {}, + "GatewayArn": {}, + "LastSynchronizedAt": {}, + "PrivateEndpointManagedResources": {}, + "ProtocolType": {}, + "Status": {}, + "StatusReasons": {}, + "TargetId": {}, + "UpdatedAt": {} + } + }, + "AWS::BedrockAgentCore::Harness": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Environment.AgentCoreRuntimeEnvironment.AgentRuntimeArn": {}, + "Environment.AgentCoreRuntimeEnvironment.AgentRuntimeId": {}, + "Environment.AgentCoreRuntimeEnvironment.AgentRuntimeName": {}, + "HarnessId": {}, + "Memory.ManagedMemoryConfiguration.Arn": {}, + "Status": {}, + "UpdatedAt": {}, + "Version": {} + } + }, + "AWS::BedrockAgentCore::HarnessEndpoint": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "HarnessName": {}, + "LiveVersion": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::BedrockAgentCore::HarnessVersion": { + "Attributes": { + "CreatedAt": {}, + "HarnessArn": {}, + "HarnessName": {}, + "HarnessVersion": {}, + "Status": {}, + "Tags": {}, + "UpdatedAt": {} + } + }, + "AWS::BedrockAgentCore::Memory": { + "Attributes": { + "CreatedAt": {}, + "FailureReason": {}, + "MemoryArn": {}, + "MemoryId": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::BedrockAgentCore::OAuth2CredentialProvider": { + "Attributes": { + "CallbackUrl": {}, + "ClientSecretArn": {}, + "ClientSecretArn.SecretArn": {}, + "ClientSecretJsonKey": {}, + "ClientSecretSource": {}, + "CreatedTime": {}, + "CredentialProviderArn": {}, + "LastUpdatedTime": {}, + "Oauth2ProviderConfigOutput": {}, + "Oauth2ProviderConfigOutput.ClientAuthenticationMethod": {}, + "Oauth2ProviderConfigOutput.ClientId": {}, + "Oauth2ProviderConfigOutput.OauthDiscovery": {}, + "Oauth2ProviderConfigOutput.OauthDiscovery.AuthorizationServerMetadata": {}, + "Oauth2ProviderConfigOutput.OauthDiscovery.DiscoveryUrl": {}, + "Oauth2ProviderConfigOutput.OnBehalfOfTokenExchangeConfig": {}, + "Oauth2ProviderConfigOutput.OnBehalfOfTokenExchangeConfig.GrantType": {}, + "Oauth2ProviderConfigOutput.OnBehalfOfTokenExchangeConfig.TokenExchangeGrantTypeConfig": {}, + "Oauth2ProviderConfigOutput.PrivateEndpoint": {}, + "Oauth2ProviderConfigOutput.PrivateEndpoint.ManagedVpcResource": {}, + "Oauth2ProviderConfigOutput.PrivateEndpoint.SelfManagedLatticeResource": {}, + "Oauth2ProviderConfigOutput.PrivateEndpointOverrides": {}, + "Oauth2ProviderConfigOutput.PrivateKeyJwtConfig": {}, + "Oauth2ProviderConfigOutput.PrivateKeyJwtConfig.AdditionalHeaderClaims": {}, + "Oauth2ProviderConfigOutput.PrivateKeyJwtConfig.AdditionalPayloadClaims": {}, + "Oauth2ProviderConfigOutput.PrivateKeyJwtConfig.PrivateKeySource": {}, + "Oauth2ProviderConfigOutput.PrivateKeyJwtConfig.SigningAlgorithm": {}, + "Status": {} + } + }, + "AWS::BedrockAgentCore::OnlineEvaluationConfig": { + "Attributes": { + "CreatedAt": {}, + "OnlineEvaluationConfigArn": {}, + "OnlineEvaluationConfigId": {}, + "OutputConfig": {}, + "OutputConfig.CloudWatchConfig": {}, + "OutputConfig.CloudWatchConfig.LogGroupName": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::BedrockAgentCore::PaymentConnector": { + "Attributes": { + "AuthorizationUrl": {}, + "ConnectorCreatedAt": {}, + "ConnectorLastUpdatedAt": {}, + "ConnectorStatus": {}, + "PaymentConnectorArn": {}, + "PaymentConnectorId": {} + } + }, + "AWS::BedrockAgentCore::PaymentCredentialProvider": { + "Attributes": { + "CreatedTime": {}, + "CredentialProviderArn": {}, + "LastUpdatedTime": {}, + "ProviderConfigurationOutput": {}, + "ProviderConfigurationOutput.CoinbaseCdpConfiguration": {}, + "ProviderConfigurationOutput.CoinbaseCdpConfiguration.ApiKeyId": {}, + "ProviderConfigurationOutput.CoinbaseCdpConfiguration.ApiKeySecretArn": {}, + "ProviderConfigurationOutput.CoinbaseCdpConfiguration.ApiKeySecretArn.SecretArn": {}, + "ProviderConfigurationOutput.CoinbaseCdpConfiguration.ApiKeySecretJsonKey": {}, + "ProviderConfigurationOutput.CoinbaseCdpConfiguration.ApiKeySecretSource": {}, + "ProviderConfigurationOutput.CoinbaseCdpConfiguration.WalletSecretArn": {}, + "ProviderConfigurationOutput.CoinbaseCdpConfiguration.WalletSecretArn.SecretArn": {}, + "ProviderConfigurationOutput.CoinbaseCdpConfiguration.WalletSecretJsonKey": {}, + "ProviderConfigurationOutput.CoinbaseCdpConfiguration.WalletSecretSource": {}, + "ProviderConfigurationOutput.StripePrivyConfiguration": {}, + "ProviderConfigurationOutput.StripePrivyConfiguration.AppId": {}, + "ProviderConfigurationOutput.StripePrivyConfiguration.AppSecretArn": {}, + "ProviderConfigurationOutput.StripePrivyConfiguration.AppSecretArn.SecretArn": {}, + "ProviderConfigurationOutput.StripePrivyConfiguration.AppSecretJsonKey": {}, + "ProviderConfigurationOutput.StripePrivyConfiguration.AppSecretSource": {}, + "ProviderConfigurationOutput.StripePrivyConfiguration.AuthorizationId": {}, + "ProviderConfigurationOutput.StripePrivyConfiguration.AuthorizationPrivateKeyArn": {}, + "ProviderConfigurationOutput.StripePrivyConfiguration.AuthorizationPrivateKeyArn.SecretArn": {}, + "ProviderConfigurationOutput.StripePrivyConfiguration.AuthorizationPrivateKeyJsonKey": {}, + "ProviderConfigurationOutput.StripePrivyConfiguration.AuthorizationPrivateKeySource": {} + } + }, + "AWS::BedrockAgentCore::PaymentManager": { + "Attributes": { + "CreatedAt": {}, + "LastUpdatedAt": {}, + "PaymentManagerArn": {}, + "PaymentManagerId": {}, + "Status": {}, + "WorkloadIdentityDetails": {}, + "WorkloadIdentityDetails.WorkloadIdentityArn": {} + } + }, + "AWS::BedrockAgentCore::Policy": { + "Attributes": { + "CreatedAt": {}, + "PolicyArn": {}, + "PolicyId": {}, + "Status": {}, + "StatusReasons": {}, + "UpdatedAt": {} + } + }, + "AWS::BedrockAgentCore::PolicyEngine": { + "Attributes": { + "CreatedAt": {}, + "PolicyEngineArn": {}, + "PolicyEngineId": {}, + "Status": {}, + "StatusReasons": {}, + "UpdatedAt": {} + } + }, + "AWS::BedrockAgentCore::PolicyGeneration": { + "Attributes": { + "CreatedAt": {}, + "PolicyGenerationArn": {}, + "PolicyGenerationId": {}, + "Status": {}, + "StatusReasons": {}, + "UpdatedAt": {} + } + }, + "AWS::BedrockAgentCore::Runtime": { + "Attributes": { + "AgentRuntimeArn": {}, + "AgentRuntimeId": {}, + "AgentRuntimeVersion": {}, + "CreatedAt": {}, + "FailureReason": {}, + "LastUpdatedAt": {}, + "Status": {}, + "WorkloadIdentityDetails": {}, + "WorkloadIdentityDetails.WorkloadIdentityArn": {} + } + }, + "AWS::BedrockAgentCore::RuntimeEndpoint": { + "Attributes": { + "AgentRuntimeArn": {}, + "AgentRuntimeEndpointArn": {}, + "CreatedAt": {}, + "FailureReason": {}, + "Id": {}, + "LastUpdatedAt": {}, + "LiveVersion": {}, + "Status": {}, + "TargetVersion": {} + } + }, + "AWS::BedrockAgentCore::TokenVault": { + "Attributes": { + "Arn": {}, + "KmsConfiguration": {}, + "KmsConfiguration.KeyType": {}, + "KmsConfiguration.KmsKeyArn": {}, + "LastModifiedDate": {}, + "TokenVaultId": {} + } + }, + "AWS::BedrockAgentCore::WorkloadIdentity": { + "Attributes": { + "CreatedTime": {}, + "LastUpdatedTime": {}, + "WorkloadIdentityArn": {} + } + }, + "AWS::BedrockMantle::Project": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {} + } + }, + "AWS::Billing::BillingView": { + "Attributes": { + "Arn": {}, + "BillingViewType": {}, + "CreatedAt": {}, + "OwnerAccountId": {}, + "UpdatedAt": {} + } + }, + "AWS::BillingConductor::BillingGroup": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "LastModifiedTime": {}, + "Size": {}, + "Status": {}, + "StatusReason": {} + } + }, + "AWS::BillingConductor::CustomLineItem": { + "Attributes": { + "Arn": {}, + "AssociationSize": {}, + "CreationTime": {}, + "CurrencyCode": {}, + "LastModifiedTime": {}, + "ProductCode": {} + } + }, + "AWS::BillingConductor::PricingPlan": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "LastModifiedTime": {}, + "Size": {} + } + }, + "AWS::BillingConductor::PricingRule": { + "Attributes": { + "Arn": {}, + "AssociatedPricingPlanCount": {}, + "CreationTime": {}, + "LastModifiedTime": {} + } + }, + "AWS::Braket::Job": { + "Attributes": { + "CreatedAt": {}, + "JobArn": {}, + "Status": {} + } + }, + "AWS::Braket::QuantumTask": { + "Attributes": { + "ActionMetadata": {}, + "ActionMetadata.ActionType": {}, + "CreatedAt": {}, + "OutputS3Directory": {}, + "QuantumTaskArn": {}, + "Status": {} + } + }, + "AWS::Braket::SpendingLimit": { + "Attributes": { + "CreatedAt": {}, + "QueuedSpend": {}, + "SpendingLimitArn": {}, + "TotalSpend": {}, + "UpdatedAt": {} + } + }, + "AWS::Budgets::BudgetsAction": { + "Attributes": { + "ActionId": {} + } + }, + "AWS::CE::AnomalyMonitor": { + "Attributes": { + "CreationDate": {}, + "DimensionalValueCount": {}, + "LastEvaluatedDate": {}, + "LastUpdatedDate": {}, + "MonitorArn": {} + } + }, + "AWS::CE::AnomalySubscription": { + "Attributes": { + "AccountId": {}, + "SubscriptionArn": {} + } + }, + "AWS::CE::CostCategory": { + "Attributes": { + "Arn": {}, + "EffectiveStart": {} + } + }, + "AWS::Cases::Case": { + "Attributes": { + "Arn": {}, + "CaseId": {} + } + }, + "AWS::Cases::CaseRule": { + "Attributes": { + "CaseRuleArn": {}, + "CaseRuleId": {}, + "CreatedTime": {}, + "LastModifiedTime": {} + } + }, + "AWS::Cases::Domain": { + "Attributes": { + "CreatedTime": {}, + "DomainArn": {}, + "DomainId": {}, + "DomainStatus": {} + } + }, + "AWS::Cases::Field": { + "Attributes": { + "CreatedTime": {}, + "FieldArn": {}, + "FieldId": {}, + "LastModifiedTime": {}, + "Namespace": {} + } + }, + "AWS::Cases::Layout": { + "Attributes": { + "CreatedTime": {}, + "LastModifiedTime": {}, + "LayoutArn": {}, + "LayoutId": {} + } + }, + "AWS::Cases::Template": { + "Attributes": { + "CreatedTime": {}, + "LastModifiedTime": {}, + "TemplateArn": {}, + "TemplateId": {} + } + }, + "AWS::Cassandra::Stream": { + "Attributes": { + "Arn": {}, + "CreationRequestDateTime": {}, + "StreamLabel": {}, + "StreamStatus": {} + } + }, + "AWS::Cassandra::Type": { + "Attributes": { + "DirectParentTypes": {}, + "DirectReferringTables": {}, + "KeyspaceArn": {}, + "LastModifiedTimestamp": {}, + "MaxNestingDepth": {} + } + }, + "AWS::CertificateManager::Account": { + "Attributes": { + "AccountId": {} + } + }, + "AWS::CertificateManager::AcmeDomainValidation": { + "Attributes": { + "Arn": {} + } + }, + "AWS::CertificateManager::AcmeEndpoint": { + "Attributes": { + "AcmeEndpointArn": {}, + "EndpointUrl": {} + } + }, + "AWS::CertificateManager::AcmeExternalAccountBinding": { + "Attributes": { + "AcmeExternalAccountBindingArn": {} + } + }, + "AWS::CertificateManager::Certificate": { + "Attributes": { + "CertificateArn": {} + } + }, + "AWS::Chatbot::CustomAction": { + "Attributes": { + "CustomActionArn": {} + } + }, + "AWS::Chatbot::MicrosoftTeamsChannelConfiguration": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Chatbot::SlackChannelConfiguration": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Chime::AppInstance": { + "Attributes": { + "AppInstanceArn": {}, + "CreatedTimestamp": {}, + "LastUpdatedTimestamp": {} + } + }, + "AWS::Chime::AppInstanceBot": { + "Attributes": { + "AppInstanceBotArn": {}, + "CreatedTimestamp": {}, + "LastUpdatedTimestamp": {} + } + }, + "AWS::Chime::AppInstanceUser": { + "Attributes": { + "AppInstanceUserArn": {} + } + }, + "AWS::Chime::ChannelFlow": { + "Attributes": { + "AppInstanceId": {}, + "Arn": {}, + "ChannelFlowId": {}, + "CreatedTimestamp": {}, + "LastUpdatedTimestamp": {} + } + }, + "AWS::Chime::MediaCapturePipeline": { + "Attributes": { + "CreatedTimestamp": {}, + "MediaPipelineArn": {}, + "MediaPipelineId": {}, + "SourceArn": {}, + "Status": {}, + "UpdatedTimestamp": {} + } + }, + "AWS::Chime::MediaConcatenationPipeline": { + "Attributes": { + "CreatedTimestamp": {}, + "MediaPipelineArn": {}, + "MediaPipelineId": {}, + "Status": {}, + "UpdatedTimestamp": {} + } + }, + "AWS::Chime::MediaInsightsPipeline": { + "Attributes": { + "CreatedTimestamp": {}, + "MediaPipelineArn": {}, + "MediaPipelineId": {}, + "Status": {} + } + }, + "AWS::Chime::MediaInsightsPipelineConfiguration": { + "Attributes": { + "CreatedTimestamp": {}, + "MediaInsightsPipelineConfigurationArn": {}, + "MediaInsightsPipelineConfigurationId": {}, + "UpdatedTimestamp": {} + } + }, + "AWS::Chime::MediaLiveConnectorPipeline": { + "Attributes": { + "CreatedTimestamp": {}, + "MediaPipelineArn": {}, + "MediaPipelineId": {}, + "Status": {}, + "UpdatedTimestamp": {} + } + }, + "AWS::Chime::MediaPipelineKinesisVideoStreamPool": { + "Attributes": { + "Arn": {}, + "CreatedTimestamp": {}, + "PoolId": {}, + "PoolStatus": {}, + "UpdatedTimestamp": {} + } + }, + "AWS::Chime::MediaStreamPipeline": { + "Attributes": { + "CreatedTimestamp": {}, + "MediaPipelineArn": {}, + "MediaPipelineId": {}, + "Status": {}, + "UpdatedTimestamp": {} + } + }, + "AWS::Chime::SipMediaApplication": { + "Attributes": { + "CreatedTimestamp": {}, + "SipMediaApplicationArn": {}, + "SipMediaApplicationId": {}, + "UpdatedTimestamp": {} + } + }, + "AWS::Chime::VoiceConnector": { + "Attributes": { + "CreatedTimestamp": {}, + "OutboundHostName": {}, + "UpdatedTimestamp": {}, + "VoiceConnectorArn": {}, + "VoiceConnectorId": {} + } + }, + "AWS::CleanRooms::AnalysisTemplate": { + "Attributes": { + "AnalysisTemplateIdentifier": {}, + "Arn": {}, + "CollaborationArn": {}, + "CollaborationIdentifier": {}, + "MembershipArn": {} + } + }, + "AWS::CleanRooms::Collaboration": { + "Attributes": { + "Arn": {}, + "CollaborationIdentifier": {} + } + }, + "AWS::CleanRooms::ConfiguredTable": { + "Attributes": { + "Arn": {}, + "ConfiguredTableIdentifier": {} + } + }, + "AWS::CleanRooms::ConfiguredTableAssociation": { + "Attributes": { + "Arn": {}, + "ConfiguredTableAssociationIdentifier": {} + } + }, + "AWS::CleanRooms::IdMappingTable": { + "Attributes": { + "Arn": {}, + "CollaborationArn": {}, + "CollaborationIdentifier": {}, + "IdMappingTableIdentifier": {}, + "InputReferenceProperties": {}, + "InputReferenceProperties.IdMappingTableInputSource": {}, + "MembershipArn": {} + } + }, + "AWS::CleanRooms::IdNamespaceAssociation": { + "Attributes": { + "Arn": {}, + "CollaborationArn": {}, + "CollaborationIdentifier": {}, + "IdNamespaceAssociationIdentifier": {}, + "InputReferenceProperties": {}, + "InputReferenceProperties.IdMappingWorkflowsSupported": {}, + "InputReferenceProperties.IdNamespaceType": {}, + "MembershipArn": {} + } + }, + "AWS::CleanRooms::IntermediateTable": { + "Attributes": { + "Arn": {}, + "CollaborationArn": {}, + "CollaborationIdentifier": {}, + "IntermediateTableIdentifier": {}, + "MembershipArn": {}, + "Status": {} + } + }, + "AWS::CleanRooms::Membership": { + "Attributes": { + "Arn": {}, + "CollaborationArn": {}, + "CollaborationCreatorAccountId": {}, + "MembershipIdentifier": {} + } + }, + "AWS::CleanRooms::PrivacyBudgetTemplate": { + "Attributes": { + "Arn": {}, + "CollaborationArn": {}, + "CollaborationIdentifier": {}, + "MembershipArn": {}, + "PrivacyBudgetTemplateIdentifier": {} + } + }, + "AWS::CleanRoomsML::AudienceGenerationJob": { + "Attributes": { + "AudienceGenerationJobArn": {}, + "ConfiguredAudienceModelArn": {}, + "CreateTime": {}, + "Metrics": {}, + "Metrics.RecallMetric": {}, + "Metrics.RelevanceMetrics": {}, + "Name": {}, + "StartedBy": {}, + "Status": {}, + "UpdateTime": {} + } + }, + "AWS::CleanRoomsML::AudienceModel": { + "Attributes": { + "AudienceModelArn": {}, + "CreateTime": {}, + "Description": {}, + "KmsKeyArn": {}, + "Name": {}, + "Status": {}, + "Tags": {}, + "TrainingDataEndTime": {}, + "TrainingDataStartTime": {}, + "TrainingDatasetArn": {}, + "UpdateTime": {} + } + }, + "AWS::CleanRoomsML::ConfiguredAudienceModel": { + "Attributes": { + "AudienceModelArn": {}, + "AudienceSizeConfig": {}, + "AudienceSizeConfig.AudienceSizeBins": {}, + "AudienceSizeConfig.AudienceSizeType": {}, + "ConfiguredAudienceModelArn": {}, + "CreateTime": {}, + "Description": {}, + "MinMatchingSeedSize": {}, + "Name": {}, + "OutputConfig": {}, + "OutputConfig.Destination": {}, + "OutputConfig.Destination.S3Destination": {}, + "OutputConfig.Destination.S3Destination.S3Uri": {}, + "OutputConfig.RoleArn": {}, + "SharedAudienceMetrics": {}, + "Status": {}, + "Tags": {}, + "UpdateTime": {} + } + }, + "AWS::CleanRoomsML::ConfiguredModelAlgorithm": { + "Attributes": { + "ConfiguredModelAlgorithmArn": {} + } + }, + "AWS::CleanRoomsML::ConfiguredModelAlgorithmAssociation": { + "Attributes": { + "CollaborationIdentifier": {}, + "ConfiguredModelAlgorithmAssociationArn": {} + } + }, + "AWS::CleanRoomsML::MLInputChannel": { + "Attributes": { + "CollaborationIdentifier": {}, + "ConfiguredModelAlgorithmAssociations": {}, + "CreateTime": {}, + "Description": {}, + "InputChannel": {}, + "InputChannel.DataSource": {}, + "InputChannel.DataSource.ProtectedQueryInputParameters": {}, + "InputChannel.DataSource.ProtectedQueryInputParameters.ComputeConfiguration": {}, + "InputChannel.DataSource.ProtectedQueryInputParameters.ComputeConfiguration.Worker": {}, + "InputChannel.DataSource.ProtectedQueryInputParameters.ComputeConfiguration.Worker.Number": {}, + "InputChannel.DataSource.ProtectedQueryInputParameters.ComputeConfiguration.Worker.Properties": {}, + "InputChannel.DataSource.ProtectedQueryInputParameters.ComputeConfiguration.Worker.Properties.Spark": {}, + "InputChannel.DataSource.ProtectedQueryInputParameters.ComputeConfiguration.Worker.Type": {}, + "InputChannel.DataSource.ProtectedQueryInputParameters.ResultFormat": {}, + "InputChannel.DataSource.ProtectedQueryInputParameters.SqlParameters": {}, + "InputChannel.DataSource.ProtectedQueryInputParameters.SqlParameters.AnalysisTemplateArn": {}, + "InputChannel.DataSource.ProtectedQueryInputParameters.SqlParameters.Parameters": {}, + "InputChannel.DataSource.ProtectedQueryInputParameters.SqlParameters.QueryString": {}, + "InputChannel.RoleArn": {}, + "KmsKeyArn": {}, + "MembershipIdentifier": {}, + "MlInputChannelArn": {}, + "Name": {}, + "NumberOfFiles": {}, + "NumberOfRecords": {}, + "PrivacyBudgets": {}, + "PrivacyBudgets.AccessBudgets": {}, + "ProtectedQueryIdentifier": {}, + "RetentionInDays": {}, + "SizeInGb": {}, + "Status": {}, + "SyntheticDataConfiguration": {}, + "SyntheticDataConfiguration.SyntheticDataParameters": {}, + "SyntheticDataConfiguration.SyntheticDataParameters.ColumnClassification": {}, + "SyntheticDataConfiguration.SyntheticDataParameters.ColumnClassification.ColumnMapping": {}, + "SyntheticDataConfiguration.SyntheticDataParameters.Epsilon": {}, + "SyntheticDataConfiguration.SyntheticDataParameters.MaxMembershipInferenceAttackScore": {}, + "Tags": {}, + "UpdateTime": {} + } + }, + "AWS::CleanRoomsML::TrainedModelInferenceJob": { + "Attributes": { + "CreateTime": {}, + "Status": {}, + "TrainedModelInferenceJobArn": {}, + "UpdateTime": {} + } + }, + "AWS::CleanRoomsML::TrainingDataset": { + "Attributes": { + "Status": {}, + "TrainingDatasetArn": {} + } + }, + "AWS::Cloud9::EnvironmentEC2": { + "Attributes": { + "Arn": {}, + "Name": {} + } + }, + "AWS::CloudFormation::ChangeSet": { + "Attributes": { + "ChangeSetId": {}, + "CreationTime": {}, + "StackId": {} + } + }, + "AWS::CloudFormation::GeneratedTemplate": { + "Attributes": { + "CreationTime": {}, + "GeneratedTemplateId": {}, + "LastUpdatedTime": {}, + "Progress": {}, + "Progress.ResourcesFailed": {}, + "Progress.ResourcesPending": {}, + "Progress.ResourcesProcessing": {}, + "Progress.ResourcesSucceeded": {}, + "Status": {}, + "TotalWarnings": {} + } + }, + "AWS::CloudFormation::GuardHook": { + "Attributes": { + "HookArn": {} + } + }, + "AWS::CloudFormation::HookDefaultVersion": { + "Attributes": { + "Arn": {} + } + }, + "AWS::CloudFormation::HookTypeConfig": { + "Attributes": { + "ConfigurationArn": {} + } + }, + "AWS::CloudFormation::HookVersion": { + "Attributes": { + "Arn": {}, + "IsDefaultVersion": {}, + "TypeArn": {}, + "VersionId": {}, + "Visibility": {} + } + }, + "AWS::CloudFormation::LambdaHook": { + "Attributes": { + "HookArn": {} + } + }, + "AWS::CloudFormation::ModuleVersion": { + "Attributes": { + "Arn": {}, + "Description": {}, + "DocumentationUrl": {}, + "IsDefaultVersion": {}, + "Schema": {}, + "TimeCreated": {}, + "VersionId": {}, + "Visibility": {} + } + }, + "AWS::CloudFormation::PublicTypeVersion": { + "Attributes": { + "PublicTypeArn": {}, + "PublisherId": {}, + "TypeVersionArn": {} + } + }, + "AWS::CloudFormation::Publisher": { + "Attributes": { + "IdentityProvider": {}, + "PublisherId": {}, + "PublisherProfile": {}, + "PublisherStatus": {} + } + }, + "AWS::CloudFormation::ResourceDefaultVersion": { + "Attributes": { + "Arn": {} + } + }, + "AWS::CloudFormation::ResourceScan": { + "Attributes": { + "PercentageCompleted": {}, + "ResourceScanId": {}, + "ScanId": {}, + "StartTime": {}, + "Status": {} + } + }, + "AWS::CloudFormation::ResourceVersion": { + "Attributes": { + "Arn": {}, + "IsDefaultVersion": {}, + "ProvisioningType": {}, + "TypeArn": {}, + "VersionId": {}, + "Visibility": {} + } + }, + "AWS::CloudFormation::StackSet": { + "Attributes": { + "StackSetId": {} + } + }, + "AWS::CloudFormation::TypeActivation": { + "Attributes": { + "Arn": {} + } + }, + "AWS::CloudFormation::WaitCondition": { + "Attributes": { + "Data": {} + } + }, + "AWS::CloudFront::AnycastIpList": { + "Attributes": { + "AnycastIpList": {}, + "AnycastIpList.AnycastIps": {}, + "AnycastIpList.Arn": {}, + "AnycastIpList.Id": {}, + "AnycastIpList.IpAddressType": {}, + "AnycastIpList.IpCount": {}, + "AnycastIpList.IpamCidrConfigResults": {}, + "AnycastIpList.LastModifiedTime": {}, + "AnycastIpList.Name": {}, + "AnycastIpList.Status": {}, + "ETag": {}, + "Id": {}, + "IpamCidrConfigResults": {} + } + }, + "AWS::CloudFront::CachePolicy": { + "Attributes": { + "Id": {}, + "LastModifiedTime": {} + } + }, + "AWS::CloudFront::CloudFrontOriginAccessIdentity": { + "Attributes": { + "Id": {}, + "S3CanonicalUserId": {} + } + }, + "AWS::CloudFront::ConnectionFunction": { + "Attributes": { + "ConnectionFunctionArn": {}, + "CreatedTime": {}, + "ETag": {}, + "Id": {}, + "LastModifiedTime": {}, + "Stage": {}, + "Status": {} + } + }, + "AWS::CloudFront::ConnectionGroup": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "ETag": {}, + "Id": {}, + "IsDefault": {}, + "LastModifiedTime": {}, + "RoutingEndpoint": {}, + "Status": {} + } + }, + "AWS::CloudFront::ContinuousDeploymentPolicy": { + "Attributes": { + "Id": {}, + "LastModifiedTime": {} + } + }, + "AWS::CloudFront::Distribution": { + "Attributes": { + "DomainName": {}, + "Id": {} + } + }, + "AWS::CloudFront::DistributionTenant": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "DomainResults": {}, + "ETag": {}, + "Id": {}, + "LastModifiedTime": {}, + "Status": {} + } + }, + "AWS::CloudFront::Function": { + "Attributes": { + "FunctionARN": {}, + "FunctionMetadata.FunctionARN": {}, + "Stage": {} + } + }, + "AWS::CloudFront::KeyGroup": { + "Attributes": { + "Id": {}, + "LastModifiedTime": {} + } + }, + "AWS::CloudFront::KeyValueStore": { + "Attributes": { + "Arn": {}, + "Id": {}, + "Status": {} + } + }, + "AWS::CloudFront::OriginAccessControl": { + "Attributes": { + "Id": {} + } + }, + "AWS::CloudFront::OriginRequestPolicy": { + "Attributes": { + "Id": {}, + "LastModifiedTime": {} + } + }, + "AWS::CloudFront::PublicKey": { + "Attributes": { + "CreatedTime": {}, + "Id": {} + } + }, + "AWS::CloudFront::RealtimeLogConfig": { + "Attributes": { + "Arn": {} + } + }, + "AWS::CloudFront::ResponseHeadersPolicy": { + "Attributes": { + "Id": {}, + "LastModifiedTime": {} + } + }, + "AWS::CloudFront::StreamingDistribution": { + "Attributes": { + "DomainName": {} + } + }, + "AWS::CloudFront::TrustStore": { + "Attributes": { + "Arn": {}, + "ETag": {}, + "Id": {}, + "LastModifiedTime": {}, + "NumberOfCaCertificates": {}, + "Status": {} + } + }, + "AWS::CloudFront::VpcOrigin": { + "Attributes": { + "AccountId": {}, + "Arn": {}, + "CreatedTime": {}, + "Id": {}, + "LastModifiedTime": {}, + "Status": {} + } + }, + "AWS::CloudHSM::Cluster": { + "Attributes": { + "Arn": {}, + "BackupPolicy": {}, + "ClusterId": {}, + "SecurityGroup": {}, + "State": {}, + "SubnetMapping": {}, + "VpcId": {} + } + }, + "AWS::CloudHSMV2::Backup": { + "Attributes": { + "Arn": {}, + "BackupId": {}, + "BackupState": {}, + "ClusterId": {}, + "CreateTimestamp": {}, + "NeverExpires": {}, + "Tags": {} + } + }, + "AWS::CloudTrail::Channel": { + "Attributes": { + "ChannelArn": {} + } + }, + "AWS::CloudTrail::Dashboard": { + "Attributes": { + "CreatedTimestamp": {}, + "DashboardArn": {}, + "Status": {}, + "Type": {}, + "UpdatedTimestamp": {} + } + }, + "AWS::CloudTrail::EventDataStore": { + "Attributes": { + "CreatedTimestamp": {}, + "EventDataStoreArn": {}, + "Status": {}, + "UpdatedTimestamp": {} + } + }, + "AWS::CloudTrail::Trail": { + "Attributes": { + "Arn": {}, + "SnsTopicArn": {} + } + }, + "AWS::CloudWatch::Alarm": { + "Attributes": { + "Arn": {} + } + }, + "AWS::CloudWatch::AlarmMuteRule": { + "Attributes": { + "Arn": {}, + "LastUpdatedTimestamp": {}, + "MuteType": {}, + "Status": {} + } + }, + "AWS::CloudWatch::CompositeAlarm": { + "Attributes": { + "Arn": {} + } + }, + "AWS::CloudWatch::InsightRule": { + "Attributes": { + "Arn": {}, + "RuleName": {} + } + }, + "AWS::CloudWatch::LogAlarm": { + "Attributes": { + "Arn": {} + } + }, + "AWS::CloudWatch::MetricStream": { + "Attributes": { + "Arn": {}, + "CreationDate": {}, + "LastUpdateDate": {}, + "State": {} + } + }, + "AWS::CloudWatch::OTelEnrichment": { + "Attributes": { + "AccountId": {}, + "Status": {} + } + }, + "AWS::CodeArtifact::Domain": { + "Attributes": { + "Arn": {}, + "EncryptionKey": {}, + "Name": {}, + "Owner": {} + } + }, + "AWS::CodeArtifact::Package": { + "Attributes": { + "Arn": {}, + "OriginConfiguration": {}, + "OriginConfiguration.Restrictions": {}, + "OriginConfiguration.Restrictions.Publish": {}, + "OriginConfiguration.Restrictions.Upstream": {} + } + }, + "AWS::CodeArtifact::PackageGroup": { + "Attributes": { + "Arn": {} + } + }, + "AWS::CodeArtifact::Repository": { + "Attributes": { + "Arn": {}, + "DomainName": {}, + "DomainOwner": {}, + "Name": {} + } + }, + "AWS::CodeBuild::Build": { + "Attributes": { + "Arn": {}, + "BuildComplete": {}, + "BuildNumber": {}, + "BuildStatus": {}, + "CurrentPhase": {}, + "EncryptionKey": {}, + "EndTime": {}, + "Id": {}, + "Initiator": {}, + "QueuedTimeoutInMinutes": {}, + "ServiceRole": {}, + "StartTime": {}, + "TimeoutInMinutes": {} + } + }, + "AWS::CodeBuild::BuildBatch": { + "Attributes": { + "Arn": {}, + "BuildBatchNumber": {}, + "BuildBatchStatus": {}, + "BuildTimeoutInMinutes": {}, + "Complete": {}, + "CurrentPhase": {}, + "EncryptionKey": {}, + "Id": {}, + "Initiator": {}, + "QueuedTimeoutInMinutes": {}, + "StartTime": {} + } + }, + "AWS::CodeBuild::Fleet": { + "Attributes": { + "Arn": {} + } + }, + "AWS::CodeBuild::Project": { + "Attributes": { + "Arn": {} + } + }, + "AWS::CodeBuild::Report": { + "Attributes": { + "Arn": {}, + "CodeCoverageSummary": {}, + "CodeCoverageSummary.BranchCoveragePercentage": {}, + "CodeCoverageSummary.BranchesCovered": {}, + "CodeCoverageSummary.BranchesMissed": {}, + "CodeCoverageSummary.LineCoveragePercentage": {}, + "CodeCoverageSummary.LinesCovered": {}, + "CodeCoverageSummary.LinesMissed": {}, + "Created": {}, + "ExecutionId": {}, + "Expired": {}, + "ExportConfig": {}, + "ExportConfig.ExportConfigType": {}, + "ExportConfig.S3Destination": {}, + "ExportConfig.S3Destination.Bucket": {}, + "ExportConfig.S3Destination.BucketOwner": {}, + "ExportConfig.S3Destination.EncryptionDisabled": {}, + "ExportConfig.S3Destination.EncryptionKey": {}, + "ExportConfig.S3Destination.Packaging": {}, + "ExportConfig.S3Destination.Path": {}, + "Name": {}, + "ReportGroupArn": {}, + "Status": {}, + "TestSummary": {}, + "TestSummary.DurationInNanoSeconds": {}, + "TestSummary.StatusCounts": {}, + "TestSummary.Total": {}, + "Truncated": {}, + "Type": {} + } + }, + "AWS::CodeBuild::ReportGroup": { + "Attributes": { + "Arn": {} + } + }, + "AWS::CodeBuild::Sandbox": { + "Attributes": { + "Arn": {}, + "EncryptionKey": {}, + "Id": {}, + "QueuedTimeoutInMinutes": {}, + "RequestTime": {}, + "ServiceRole": {}, + "StartTime": {}, + "Status": {}, + "TimeoutInMinutes": {} + } + }, + "AWS::CodeBuild::SourceCredential": { + "Attributes": { + "Arn": {} + } + }, + "AWS::CodeCommit::Repository": { + "Attributes": { + "Arn": {}, + "CloneUrlHttp": {}, + "CloneUrlSsh": {}, + "Name": {}, + "RepositoryId": {} + } + }, + "AWS::CodeConnections::Connection": { + "Attributes": { + "ConnectionArn": {}, + "ConnectionStatus": {}, + "OwnerAccountId": {} + } + }, + "AWS::CodeConnections::Host": { + "Attributes": { + "HostArn": {}, + "HostId": {}, + "Status": {} + } + }, + "AWS::CodeGuruProfiler::ProfilingGroup": { + "Attributes": { + "Arn": {} + } + }, + "AWS::CodeGuruReviewer::RepositoryAssociation": { + "Attributes": { + "AssociationArn": {} + } + }, + "AWS::CodePipeline::Pipeline": { + "Attributes": { + "Arn": {}, + "Version": {} + } + }, + "AWS::CodePipeline::Webhook": { + "Attributes": { + "Id": {}, + "Url": {} + } + }, + "AWS::CodeStarConnections::Connection": { + "Attributes": { + "ConnectionArn": {}, + "ConnectionStatus": {}, + "OwnerAccountId": {} + } + }, + "AWS::CodeStarConnections::RepositoryLink": { + "Attributes": { + "ProviderType": {}, + "RepositoryLinkArn": {}, + "RepositoryLinkId": {} + } + }, + "AWS::CodeStarConnections::SyncConfiguration": { + "Attributes": { + "OwnerId": {}, + "ProviderType": {}, + "RepositoryName": {} + } + }, + "AWS::CodeStarNotifications::NotificationRule": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Cognito::IdentityPool": { + "Attributes": { + "Id": {}, + "Name": {} + } + }, + "AWS::Cognito::IdentityPoolRoleAttachment": { + "Attributes": { + "Id": {} + } + }, + "AWS::Cognito::LogDeliveryConfiguration": { + "Attributes": { + "Id": {} + } + }, + "AWS::Cognito::ManagedLoginBranding": { + "Attributes": { + "ManagedLoginBrandingId": {} + } + }, + "AWS::Cognito::Terms": { + "Attributes": { + "TermsId": {} + } + }, + "AWS::Cognito::UserPool": { + "Attributes": { + "Arn": {}, + "ProviderName": {}, + "ProviderURL": {}, + "UserPoolId": {} + } + }, + "AWS::Cognito::UserPoolClient": { + "Attributes": { + "ClientId": {}, + "ClientSecret": {}, + "Name": {} + } + }, + "AWS::Cognito::UserPoolDomain": { + "Attributes": { + "CloudFrontDistribution": {} + } + }, + "AWS::CognitoSync::Dataset": { + "Attributes": { + "Arn": {}, + "CreationDate": {}, + "DataStorage": {}, + "LastModifiedBy": {}, + "LastModifiedDate": {}, + "NumRecords": {} + } + }, + "AWS::Comprehend::DocumentClassificationJob": { + "Attributes": { + "JobArn": {}, + "JobId": {}, + "JobStatus": {}, + "OutputDataConfig": {}, + "OutputDataConfig.S3Uri": {}, + "SubmitTime": {} + } + }, + "AWS::Comprehend::DocumentClassifier": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Comprehend::DocumentClassifierEndpoint": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "CurrentInferenceUnits": {}, + "LastModifiedTime": {} + } + }, + "AWS::Comprehend::DominantLanguageDetectionJob": { + "Attributes": { + "JobArn": {}, + "JobId": {}, + "JobStatus": {}, + "OutputDataConfig": {}, + "OutputDataConfig.KmsKeyId": {}, + "OutputDataConfig.S3Uri": {} + } + }, + "AWS::Comprehend::EntitiesDetectionJob": { + "Attributes": { + "JobArn": {}, + "JobId": {}, + "JobStatus": {}, + "OutputDataConfig": {}, + "OutputDataConfig.S3Uri": {}, + "SubmitTime": {} + } + }, + "AWS::Comprehend::Flywheel": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Comprehend::FlywheelDataset": { + "Attributes": { + "CreationTime": {}, + "DatasetArn": {}, + "DatasetS3Uri": {}, + "NumberOfDocuments": {}, + "Status": {} + } + }, + "AWS::Comprehend::PiiEntitiesDetectionJob": { + "Attributes": { + "JobArn": {}, + "JobId": {}, + "JobStatus": {}, + "OutputDataConfig": {}, + "OutputDataConfig.S3Uri": {}, + "SubmitTime": {} + } + }, + "AWS::Comprehend::SentimentDetectionJob": { + "Attributes": { + "Arn": {}, + "JobId": {}, + "JobStatus": {} + } + }, + "AWS::Comprehend::TargetedSentimentDetectionJob": { + "Attributes": { + "EndTime": {}, + "JobArn": {}, + "JobId": {}, + "JobStatus": {}, + "OutputDataConfig": {}, + "OutputDataConfig.KmsKeyId": {}, + "OutputDataConfig.S3Uri": {}, + "SubmitTime": {} + } + }, + "AWS::ComputeOptimizer::AutomationRule": { + "Attributes": { + "AccountId": {}, + "CreatedTimestamp": {}, + "LastUpdatedTimestamp": {}, + "RuleArn": {}, + "RuleId": {}, + "RuleRevision": {} + } + }, + "AWS::Config::AggregationAuthorization": { + "Attributes": { + "AggregationAuthorizationArn": {} + } + }, + "AWS::Config::ConfigRule": { + "Attributes": { + "Arn": {}, + "Compliance.Type": {}, + "ConfigRuleId": {} + } + }, + "AWS::Config::ConfigurationAggregator": { + "Attributes": { + "ConfigurationAggregatorArn": {} + } + }, + "AWS::Config::ConformancePack": { + "Attributes": { + "ConformancePackArn": {} + } + }, + "AWS::Config::Connector": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "Name": {} + } + }, + "AWS::Config::OrganizationConformancePack": { + "Attributes": { + "OrganizationConformancePackArn": {} + } + }, + "AWS::Config::StoredQuery": { + "Attributes": { + "QueryArn": {}, + "QueryId": {} + } + }, + "AWS::Connect::AgentStatus": { + "Attributes": { + "AgentStatusArn": {}, + "LastModifiedRegion": {}, + "LastModifiedTime": {} + } + }, + "AWS::Connect::ContactFlow": { + "Attributes": { + "ContactFlowArn": {} + } + }, + "AWS::Connect::ContactFlowModule": { + "Attributes": { + "ContactFlowModuleArn": {}, + "Status": {} + } + }, + "AWS::Connect::ContactFlowModuleAlias": { + "Attributes": { + "AliasId": {}, + "ContactFlowModuleAliasARN": {} + } + }, + "AWS::Connect::ContactFlowModuleVersion": { + "Attributes": { + "ContactFlowModuleVersionARN": {}, + "FlowModuleContentSha256": {}, + "Version": {} + } + }, + "AWS::Connect::ContactFlowVersion": { + "Attributes": { + "ContactFlowVersionARN": {}, + "FlowContentSha256": {}, + "Version": {} + } + }, + "AWS::Connect::DataLakeAssociation": { + "Attributes": { + "ResourceShareArn": {}, + "ResourceShareId": {} + } + }, + "AWS::Connect::DataTable": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastModifiedRegion": {}, + "LastModifiedTime": {}, + "LockVersion": {}, + "LockVersion.DataTable": {} + } + }, + "AWS::Connect::DataTableAttribute": { + "Attributes": { + "AttributeId": {}, + "LastModifiedRegion": {}, + "LastModifiedTime": {}, + "LockVersion": {}, + "LockVersion.Attribute": {}, + "LockVersion.DataTable": {} + } + }, + "AWS::Connect::DataTableRecord": { + "Attributes": { + "RecordId": {} + } + }, + "AWS::Connect::EmailAddress": { + "Attributes": { + "EmailAddressArn": {} + } + }, + "AWS::Connect::EvaluationForm": { + "Attributes": { + "EvaluationFormArn": {} + } + }, + "AWS::Connect::HoursOfOperation": { + "Attributes": { + "HoursOfOperationArn": {} + } + }, + "AWS::Connect::Instance": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "Id": {}, + "InstanceStatus": {}, + "ServiceRole": {} + } + }, + "AWS::Connect::InstanceStorageConfig": { + "Attributes": { + "AssociationId": {} + } + }, + "AWS::Connect::IntegrationAssociation": { + "Attributes": { + "IntegrationAssociationId": {} + } + }, + "AWS::Connect::Metric": { + "Attributes": { + "Category": {}, + "CreatedTime": {}, + "CreatedUser": {}, + "CreatedUser.AWSIdentityArn": {}, + "CreatedUser.ConnectUserArn": {}, + "CreationMethod": {}, + "EffectiveTime": {}, + "Filters": {}, + "Groupings": {}, + "LastModifiedRegion": {}, + "LastModifiedTime": {}, + "LastModifiedUser": {}, + "LastModifiedUser.AWSIdentityArn": {}, + "LastModifiedUser.ConnectUserArn": {}, + "MetricArn": {}, + "PrimaryEventSource": {}, + "PrimaryEventSourceEffectiveTimestampType": {}, + "RefreshRate": {}, + "SupportedStats": {}, + "SupportsCustomCalculation": {}, + "SupportsPreaggregateCalculation": {}, + "Type": {} + } + }, + "AWS::Connect::Notification": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {} + } + }, + "AWS::Connect::PhoneNumber": { + "Attributes": { + "Address": {}, + "PhoneNumberArn": {} + } + }, + "AWS::Connect::PredefinedAttribute": { + "Attributes": { + "LastModifiedRegion": {}, + "LastModifiedTime": {} + } + }, + "AWS::Connect::Prompt": { + "Attributes": { + "PromptArn": {} + } + }, + "AWS::Connect::Queue": { + "Attributes": { + "LastModifiedRegion": {}, + "LastModifiedTime": {}, + "QueueArn": {}, + "Type": {} + } + }, + "AWS::Connect::QuickConnect": { + "Attributes": { + "QuickConnectArn": {}, + "QuickConnectType": {} + } + }, + "AWS::Connect::RoutingProfile": { + "Attributes": { + "RoutingProfileArn": {} + } + }, + "AWS::Connect::Rule": { + "Attributes": { + "RuleArn": {} + } + }, + "AWS::Connect::SecurityKey": { + "Attributes": { + "AssociationId": {} + } + }, + "AWS::Connect::SecurityProfile": { + "Attributes": { + "LastModifiedRegion": {}, + "LastModifiedTime": {}, + "SecurityProfileArn": {} + } + }, + "AWS::Connect::TaskTemplate": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Connect::TestCase": { + "Attributes": { + "LastModifiedRegion": {}, + "LastModifiedTime": {}, + "TestCaseArn": {} + } + }, + "AWS::Connect::TrafficDistributionGroup": { + "Attributes": { + "IsDefault": {}, + "Status": {}, + "TrafficDistributionGroupArn": {} + } + }, + "AWS::Connect::User": { + "Attributes": { + "UserArn": {} + } + }, + "AWS::Connect::UserHierarchyGroup": { + "Attributes": { + "UserHierarchyGroupArn": {} + } + }, + "AWS::Connect::UserHierarchyStructure": { + "Attributes": { + "UserHierarchyStructureArn": {} + } + }, + "AWS::Connect::View": { + "Attributes": { + "ViewArn": {}, + "ViewContentSha256": {}, + "ViewId": {} + } + }, + "AWS::Connect::ViewVersion": { + "Attributes": { + "Version": {}, + "ViewVersionArn": {} + } + }, + "AWS::Connect::Workspace": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::ConnectCampaigns::Campaign": { + "Attributes": { + "Arn": {} + } + }, + "AWS::ConnectCampaignsV2::Campaign": { + "Attributes": { + "Arn": {} + } + }, + "AWS::ControlCatalog::CommonControl": { + "Attributes": { + "Arn": {}, + "CommonControlId": {}, + "CreateTime": {}, + "Description": {}, + "Domain": {}, + "Domain.Arn": {}, + "Domain.Name": {}, + "LastUpdateTime": {}, + "Name": {}, + "Objective": {}, + "Objective.Arn": {}, + "Objective.Name": {} + } + }, + "AWS::ControlCatalog::Control": { + "Attributes": { + "Aliases": {}, + "Arn": {}, + "Behavior": {}, + "ControlId": {}, + "CreateTime": {}, + "Description": {}, + "GovernedResources": {}, + "Implementation": {}, + "Implementation.Identifier": {}, + "Implementation.Type": {}, + "Name": {}, + "RegionConfiguration": {}, + "RegionConfiguration.DeployableRegions": {}, + "RegionConfiguration.Scope": {}, + "Severity": {} + } + }, + "AWS::ControlCatalog::Objective": { + "Attributes": { + "Arn": {}, + "CreateTime": {}, + "Description": {}, + "Domain": {}, + "Domain.Arn": {}, + "Domain.Name": {}, + "LastUpdateTime": {}, + "Name": {}, + "ObjectiveId": {} + } + }, + "AWS::ControlTower::EnabledBaseline": { + "Attributes": { + "EnabledBaselineIdentifier": {} + } + }, + "AWS::ControlTower::LandingZone": { + "Attributes": { + "Arn": {}, + "DriftStatus": {}, + "LandingZoneIdentifier": {}, + "LatestAvailableVersion": {}, + "Status": {} + } + }, + "AWS::CustomerProfiles::CalculatedAttributeDefinition": { + "Attributes": { + "CreatedAt": {}, + "LastUpdatedAt": {}, + "Readiness": {}, + "Readiness.Message": {}, + "Readiness.ProgressPercentage": {}, + "Status": {} + } + }, + "AWS::CustomerProfiles::Domain": { + "Attributes": { + "CreatedAt": {}, + "DataStore.Readiness": {}, + "DataStore.Readiness.Message": {}, + "DataStore.Readiness.ProgressPercentage": {}, + "LastUpdatedAt": {}, + "RuleBasedMatching.Status": {}, + "Stats": {}, + "Stats.MeteringProfileCount": {}, + "Stats.ObjectCount": {}, + "Stats.ProfileCount": {}, + "Stats.TotalSize": {} + } + }, + "AWS::CustomerProfiles::DomainObjectType": { + "Attributes": { + "CreatedAt": {}, + "LastUpdatedAt": {} + } + }, + "AWS::CustomerProfiles::EventStream": { + "Attributes": { + "CreatedAt": {}, + "DestinationDetails": {}, + "DestinationDetails.Status": {}, + "DestinationDetails.Uri": {}, + "EventStreamArn": {}, + "State": {} + } + }, + "AWS::CustomerProfiles::EventTrigger": { + "Attributes": { + "CreatedAt": {}, + "LastUpdatedAt": {} + } + }, + "AWS::CustomerProfiles::Integration": { + "Attributes": { + "CreatedAt": {}, + "LastUpdatedAt": {} + } + }, + "AWS::CustomerProfiles::ObjectType": { + "Attributes": { + "CreatedAt": {}, + "LastUpdatedAt": {}, + "MaxAvailableProfileObjectCount": {} + } + }, + "AWS::CustomerProfiles::Recommender": { + "Attributes": { + "CreatedAt": {}, + "FailureReason": {}, + "LastUpdatedAt": {}, + "LatestRecommenderUpdate": {}, + "LatestRecommenderUpdate.CreationDateTime": {}, + "LatestRecommenderUpdate.FailureReason": {}, + "LatestRecommenderUpdate.LastUpdatedDateTime": {}, + "LatestRecommenderUpdate.RecommenderConfig": {}, + "LatestRecommenderUpdate.RecommenderConfig.EventsConfig": {}, + "LatestRecommenderUpdate.RecommenderConfig.EventsConfig.EventParametersList": {}, + "LatestRecommenderUpdate.Status": {}, + "RecommenderArn": {}, + "Status": {}, + "TrainingMetrics": {} + } + }, + "AWS::CustomerProfiles::SegmentDefinition": { + "Attributes": { + "CreatedAt": {}, + "SegmentDefinitionArn": {}, + "SegmentType": {} + } + }, + "AWS::DAX::Cluster": { + "Attributes": { + "Arn": {}, + "ClusterDiscoveryEndpoint": {}, + "ClusterDiscoveryEndpointURL": {} + } + }, + "AWS::DLM::LifecyclePolicy": { + "Attributes": { + "Arn": {}, + "PolicyId": {} + } + }, + "AWS::DMS::Certificate": { + "Attributes": { + "CertificateArn": {} + } + }, + "AWS::DMS::DataMigration": { + "Attributes": { + "DataMigrationArn": {}, + "DataMigrationCreateTime": {} + } + }, + "AWS::DMS::DataProvider": { + "Attributes": { + "DataProviderArn": {}, + "DataProviderCreationTime": {} + } + }, + "AWS::DMS::Endpoint": { + "Attributes": { + "EndpointArn": {}, + "ExternalId": {} + } + }, + "AWS::DMS::InstanceProfile": { + "Attributes": { + "InstanceProfileArn": {}, + "InstanceProfileCreationTime": {} + } + }, + "AWS::DMS::MigrationProject": { + "Attributes": { + "MigrationProjectArn": {} + } + }, + "AWS::DMS::ReplicationConfig": { + "Attributes": { + "ReplicationConfigArn": {} + } + }, + "AWS::DMS::ReplicationInstance": { + "Attributes": { + "ReplicationInstancePrivateIpAddresses": {}, + "ReplicationInstancePublicIpAddresses": {} + } + }, + "AWS::DMS::ReplicationTask": { + "Attributes": { + "ReplicationTaskArn": {} + } + }, + "AWS::DMS::ReplicationTaskAssessmentRun": { + "Attributes": { + "AssessmentProgress": {}, + "AssessmentProgress.IndividualAssessmentCompletedCount": {}, + "AssessmentProgress.IndividualAssessmentCount": {}, + "AssessmentRunName": {}, + "IsLatestTaskAssessmentRun": {}, + "ReplicationTaskArn": {}, + "ReplicationTaskAssessmentRunArn": {}, + "ReplicationTaskAssessmentRunCreationDate": {}, + "ResultEncryptionMode": {}, + "ResultLocationBucket": {}, + "ResultLocationFolder": {}, + "ServiceAccessRoleArn": {}, + "Status": {} + } + }, + "AWS::DRS::LaunchConfigurationTemplate": { + "Attributes": { + "Arn": {}, + "LaunchConfigurationTemplateID": {} + } + }, + "AWS::DRS::RecoveryInstance": { + "Attributes": { + "Arn": {}, + "EC2InstanceID": {}, + "EC2InstanceState": {}, + "IsDrill": {}, + "JobID": {}, + "OriginAvailabilityZone": {}, + "OriginEnvironment": {}, + "RecoveryInstanceID": {}, + "SourceServerID": {}, + "Tags": {} + } + }, + "AWS::DRS::SourceNetwork": { + "Attributes": { + "Arn": {}, + "SourceNetworkID": {} + } + }, + "AWS::DSQL::Cluster": { + "Attributes": { + "CreationTime": {}, + "EncryptionDetails": {}, + "EncryptionDetails.EncryptionStatus": {}, + "EncryptionDetails.EncryptionType": {}, + "EncryptionDetails.KmsKeyArn": {}, + "Endpoint": {}, + "Identifier": {}, + "PolicyVersion": {}, + "ResourceArn": {}, + "Status": {}, + "VpcEndpoint": {}, + "VpcEndpointServiceName": {} + } + }, + "AWS::DataExchange::Assets": { + "Attributes": { + "Arn": {}, + "AssetId": {}, + "AssetType": {}, + "CreatedAt": {}, + "Tags": {}, + "UpdatedAt": {} + } + }, + "AWS::DataExchange::DataSet": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "Origin": {}, + "UpdatedAt": {} + } + }, + "AWS::DataExchange::EntitledDataSets": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "DataSetId": {}, + "Origin": {}, + "SourceId": {}, + "UpdatedAt": {} + } + }, + "AWS::DataExchange::EventAction": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "EventActionId": {}, + "UpdatedAt": {} + } + }, + "AWS::DataExchange::Job": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "State": {}, + "Type": {}, + "UpdatedAt": {} + } + }, + "AWS::DataPipeline::Pipeline": { + "Attributes": { + "PipelineId": {} + } + }, + "AWS::DataSync::Agent": { + "Attributes": { + "AgentArn": {}, + "EndpointType": {} + } + }, + "AWS::DataSync::LocationAzureBlob": { + "Attributes": { + "CmkSecretConfig.SecretArn": {}, + "LocationArn": {}, + "LocationUri": {}, + "ManagedSecretConfig": {}, + "ManagedSecretConfig.SecretArn": {} + } + }, + "AWS::DataSync::LocationEFS": { + "Attributes": { + "LocationArn": {}, + "LocationUri": {} + } + }, + "AWS::DataSync::LocationFSxLustre": { + "Attributes": { + "LocationArn": {}, + "LocationUri": {} + } + }, + "AWS::DataSync::LocationFSxONTAP": { + "Attributes": { + "FsxFilesystemArn": {}, + "LocationArn": {}, + "LocationUri": {}, + "Protocol.SMB.CmkSecretConfig.SecretArn": {}, + "Protocol.SMB.ManagedSecretConfig": {}, + "Protocol.SMB.ManagedSecretConfig.SecretArn": {} + } + }, + "AWS::DataSync::LocationFSxOpenZFS": { + "Attributes": { + "LocationArn": {}, + "LocationUri": {} + } + }, + "AWS::DataSync::LocationFSxWindows": { + "Attributes": { + "CmkSecretConfig.SecretArn": {}, + "LocationArn": {}, + "LocationUri": {}, + "ManagedSecretConfig": {}, + "ManagedSecretConfig.SecretArn": {} + } + }, + "AWS::DataSync::LocationHDFS": { + "Attributes": { + "CmkSecretConfig.SecretArn": {}, + "LocationArn": {}, + "LocationUri": {}, + "ManagedSecretConfig": {}, + "ManagedSecretConfig.SecretArn": {} + } + }, + "AWS::DataSync::LocationNFS": { + "Attributes": { + "LocationArn": {}, + "LocationUri": {} + } + }, + "AWS::DataSync::LocationObjectStorage": { + "Attributes": { + "CmkSecretConfig.SecretArn": {}, + "LocationArn": {}, + "LocationUri": {}, + "ManagedSecretConfig": {}, + "ManagedSecretConfig.SecretArn": {} + } + }, + "AWS::DataSync::LocationS3": { + "Attributes": { + "LocationArn": {}, + "LocationUri": {} + } + }, + "AWS::DataSync::LocationSMB": { + "Attributes": { + "CmkSecretConfig.SecretArn": {}, + "LocationArn": {}, + "LocationUri": {}, + "ManagedSecretConfig": {}, + "ManagedSecretConfig.SecretArn": {} + } + }, + "AWS::DataSync::Task": { + "Attributes": { + "DestinationNetworkInterfaceArns": {}, + "SourceNetworkInterfaceArns": {}, + "Status": {}, + "TaskArn": {} + } + }, + "AWS::DataSync::TaskExecution": { + "Attributes": { + "BytesCompressed": {}, + "BytesTransferred": {}, + "BytesWritten": {}, + "EstimatedBytesToTransfer": {}, + "EstimatedFilesToDelete": {}, + "EstimatedFilesToTransfer": {}, + "FilesDeleted": {}, + "FilesPrepared": {}, + "FilesSkipped": {}, + "FilesTransferred": {}, + "FilesVerified": {}, + "OverrideOptions": {}, + "OverrideOptions.Atime": {}, + "OverrideOptions.BytesPerSecond": {}, + "OverrideOptions.Gid": {}, + "OverrideOptions.LogLevel": {}, + "OverrideOptions.Mtime": {}, + "OverrideOptions.ObjectTags": {}, + "OverrideOptions.OverwriteMode": {}, + "OverrideOptions.PosixPermissions": {}, + "OverrideOptions.PreserveDeletedFiles": {}, + "OverrideOptions.PreserveDevices": {}, + "OverrideOptions.SecurityDescriptorCopyFlags": {}, + "OverrideOptions.TaskQueueing": {}, + "OverrideOptions.TransferMode": {}, + "OverrideOptions.Uid": {}, + "OverrideOptions.VerifyMode": {}, + "StartTime": {}, + "Status": {}, + "TaskExecutionArn": {}, + "TaskMode": {} + } + }, + "AWS::DataZone::Connection": { + "Attributes": { + "ConnectionId": {}, + "DomainId": {}, + "DomainUnitId": {}, + "EnvironmentId": {}, + "EnvironmentUserRole": {}, + "ProjectId": {}, + "Type": {} + } + }, + "AWS::DataZone::DataSource": { + "Attributes": { + "ConnectionId": {}, + "CreatedAt": {}, + "DomainId": {}, + "EnvironmentId": {}, + "Id": {}, + "LastRunAssetCount": {}, + "LastRunAt": {}, + "LastRunStatus": {}, + "ProjectId": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::DataZone::Domain": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "LastUpdatedAt": {}, + "ManagedAccountId": {}, + "PortalUrl": {}, + "RootDomainUnitId": {}, + "Status": {} + } + }, + "AWS::DataZone::DomainUnit": { + "Attributes": { + "CreatedAt": {}, + "DomainId": {}, + "Id": {}, + "Identifier": {}, + "LastUpdatedAt": {}, + "ParentDomainUnitId": {} + } + }, + "AWS::DataZone::Environment": { + "Attributes": { + "AwsAccountId": {}, + "AwsAccountRegion": {}, + "CreatedAt": {}, + "CreatedBy": {}, + "DomainId": {}, + "EnvironmentBlueprintId": {}, + "EnvironmentProfileId": {}, + "Id": {}, + "ProjectId": {}, + "Provider": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::DataZone::EnvironmentActions": { + "Attributes": { + "DomainId": {}, + "EnvironmentId": {}, + "Id": {} + } + }, + "AWS::DataZone::EnvironmentBlueprintConfiguration": { + "Attributes": { + "CreatedAt": {}, + "DomainId": {}, + "EnvironmentBlueprintId": {}, + "UpdatedAt": {} + } + }, + "AWS::DataZone::EnvironmentProfile": { + "Attributes": { + "CreatedAt": {}, + "CreatedBy": {}, + "DomainId": {}, + "EnvironmentBlueprintId": {}, + "Id": {}, + "ProjectId": {}, + "UpdatedAt": {} + } + }, + "AWS::DataZone::FormType": { + "Attributes": { + "CreatedAt": {}, + "CreatedBy": {}, + "DomainId": {}, + "FormTypeIdentifier": {}, + "OwningProjectId": {}, + "Revision": {} + } + }, + "AWS::DataZone::GroupProfile": { + "Attributes": { + "DomainId": {}, + "GroupName": {}, + "Id": {}, + "RolePrincipalId": {} + } + }, + "AWS::DataZone::Owner": { + "Attributes": { + "OwnerIdentifier": {}, + "OwnerType": {} + } + }, + "AWS::DataZone::PolicyGrant": { + "Attributes": { + "CreatedAt": {}, + "CreatedBy": {}, + "GrantId": {} + } + }, + "AWS::DataZone::Project": { + "Attributes": { + "CreatedAt": {}, + "CreatedBy": {}, + "DomainId": {}, + "Id": {}, + "LastUpdatedAt": {}, + "ProjectStatus": {} + } + }, + "AWS::DataZone::ProjectMembership": { + "Attributes": { + "MemberIdentifier": {}, + "MemberIdentifierType": {} + } + }, + "AWS::DataZone::ProjectProfile": { + "Attributes": { + "CreatedAt": {}, + "CreatedBy": {}, + "DomainId": {}, + "DomainUnitId": {}, + "Id": {}, + "Identifier": {}, + "LastUpdatedAt": {} + } + }, + "AWS::DataZone::SubscriptionTarget": { + "Attributes": { + "CreatedAt": {}, + "CreatedBy": {}, + "DomainId": {}, + "EnvironmentId": {}, + "Id": {}, + "ProjectId": {}, + "UpdatedAt": {}, + "UpdatedBy": {} + } + }, + "AWS::DataZone::UserProfile": { + "Attributes": { + "Details": {}, + "Details.Iam": {}, + "Details.Iam.Arn": {}, + "Details.Iam.GroupProfileId": {}, + "Details.Iam.SessionName": {}, + "Details.Sso": {}, + "Details.Sso.FirstName": {}, + "Details.Sso.LastName": {}, + "Details.Sso.Username": {}, + "DomainId": {}, + "Id": {}, + "Type": {} + } + }, + "AWS::Deadline::Budget": { + "Attributes": { + "Arn": {}, + "BudgetId": {}, + "Status": {} + } + }, + "AWS::Deadline::Farm": { + "Attributes": { + "Arn": {}, + "FarmId": {} + } + }, + "AWS::Deadline::Fleet": { + "Attributes": { + "Arn": {}, + "Capabilities": {}, + "Capabilities.Amounts": {}, + "Capabilities.Attributes": {}, + "FleetId": {}, + "Status": {}, + "StatusMessage": {}, + "WorkerCount": {} + } + }, + "AWS::Deadline::Job": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "CreatedBy": {}, + "JobId": {}, + "LifecycleStatus": {}, + "LifecycleStatusMessage": {}, + "Name": {}, + "TaskRunStatus": {} + } + }, + "AWS::Deadline::LicenseEndpoint": { + "Attributes": { + "Arn": {}, + "DnsName": {}, + "LicenseEndpointId": {}, + "Status": {}, + "StatusMessage": {} + } + }, + "AWS::Deadline::Limit": { + "Attributes": { + "CurrentCount": {}, + "LimitId": {} + } + }, + "AWS::Deadline::MeteredProduct": { + "Attributes": { + "Arn": {}, + "Family": {}, + "Port": {}, + "Vendor": {} + } + }, + "AWS::Deadline::Monitor": { + "Attributes": { + "Arn": {}, + "IdentityCenterApplicationArn": {}, + "MonitorId": {}, + "Url": {} + } + }, + "AWS::Deadline::Queue": { + "Attributes": { + "Arn": {}, + "QueueId": {} + } + }, + "AWS::Deadline::QueueEnvironment": { + "Attributes": { + "Name": {}, + "QueueEnvironmentId": {} + } + }, + "AWS::Deadline::StorageProfile": { + "Attributes": { + "StorageProfileId": {} + } + }, + "AWS::Deadline::Volume": { + "Attributes": { + "Arn": {}, + "AttachedWorkerId": {}, + "AvailabilityZoneId": {}, + "CreatedAt": {}, + "ExpiresAt": {}, + "Iops": {}, + "LastAssignedAt": {}, + "LastReleasedAt": {}, + "SizeGiB": {}, + "State": {}, + "ThroughputMiB": {}, + "VolumeId": {}, + "VolumeType": {} + } + }, + "AWS::Deadline::Worker": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "CreatedBy": {}, + "Status": {}, + "WorkerId": {} + } + }, + "AWS::Detective::Graph": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Detective::OrganizationAdmin": { + "Attributes": { + "GraphArn": {} + } + }, + "AWS::DevOpsAgent::AgentSpace": { + "Attributes": { + "AgentSpaceId": {}, + "Arn": {}, + "CreatedAt": {}, + "OperatorApp.Iam.CreatedAt": {}, + "OperatorApp.Iam.UpdatedAt": {}, + "OperatorApp.Idc.CreatedAt": {}, + "OperatorApp.Idc.IdcApplicationArn": {}, + "OperatorApp.Idc.UpdatedAt": {}, + "UpdatedAt": {} + } + }, + "AWS::DevOpsAgent::Asset": { + "Attributes": { + "Arn": {}, + "AssetId": {}, + "CreatedAt": {}, + "UpdatedAt": {}, + "Version": {} + } + }, + "AWS::DevOpsAgent::Association": { + "Attributes": { + "AssociationId": {}, + "CreatedAt": {}, + "UpdatedAt": {} + } + }, + "AWS::DevOpsAgent::PrivateConnection": { + "Attributes": { + "Arn": {}, + "CertificateExpiryTime": {}, + "Status": {} + } + }, + "AWS::DevOpsAgent::Service": { + "Attributes": { + "AccessibleResources": {}, + "AdditionalServiceDetails": {}, + "AdditionalServiceDetails.AzureIdentity": {}, + "AdditionalServiceDetails.AzureIdentity.ClientId": {}, + "AdditionalServiceDetails.AzureIdentity.TenantId": {}, + "AdditionalServiceDetails.AzureIdentity.WebIdentityRoleArn": {}, + "AdditionalServiceDetails.AzureIdentity.WebIdentityTokenAudiences": {}, + "AdditionalServiceDetails.Dynatrace": {}, + "AdditionalServiceDetails.Dynatrace.AccountUrn": {}, + "AdditionalServiceDetails.GitLab": {}, + "AdditionalServiceDetails.GitLab.GroupId": {}, + "AdditionalServiceDetails.GitLab.TargetUrl": {}, + "AdditionalServiceDetails.GitLab.TokenType": {}, + "AdditionalServiceDetails.MCPServer": {}, + "AdditionalServiceDetails.MCPServer.ApiKeyHeader": {}, + "AdditionalServiceDetails.MCPServer.AuthorizationMethod": {}, + "AdditionalServiceDetails.MCPServer.Description": {}, + "AdditionalServiceDetails.MCPServer.Endpoint": {}, + "AdditionalServiceDetails.MCPServer.Name": {}, + "AdditionalServiceDetails.MCPServerGrafana": {}, + "AdditionalServiceDetails.MCPServerGrafana.AuthorizationMethod": {}, + "AdditionalServiceDetails.MCPServerGrafana.Description": {}, + "AdditionalServiceDetails.MCPServerGrafana.Endpoint": {}, + "AdditionalServiceDetails.MCPServerGrafana.Name": {}, + "AdditionalServiceDetails.MCPServerNewRelic": {}, + "AdditionalServiceDetails.MCPServerNewRelic.AccountId": {}, + "AdditionalServiceDetails.MCPServerNewRelic.Description": {}, + "AdditionalServiceDetails.MCPServerNewRelic.Region": {}, + "AdditionalServiceDetails.MCPServerSigV4": {}, + "AdditionalServiceDetails.MCPServerSigV4.CustomHeaders": {}, + "AdditionalServiceDetails.MCPServerSigV4.Description": {}, + "AdditionalServiceDetails.MCPServerSigV4.Endpoint": {}, + "AdditionalServiceDetails.MCPServerSigV4.McpRoleArn": {}, + "AdditionalServiceDetails.MCPServerSigV4.Name": {}, + "AdditionalServiceDetails.MCPServerSigV4.Region": {}, + "AdditionalServiceDetails.MCPServerSigV4.RoleArn": {}, + "AdditionalServiceDetails.MCPServerSigV4.Service": {}, + "AdditionalServiceDetails.MCPServerSplunk": {}, + "AdditionalServiceDetails.MCPServerSplunk.ApiKeyHeader": {}, + "AdditionalServiceDetails.MCPServerSplunk.AuthorizationMethod": {}, + "AdditionalServiceDetails.MCPServerSplunk.Description": {}, + "AdditionalServiceDetails.MCPServerSplunk.Endpoint": {}, + "AdditionalServiceDetails.MCPServerSplunk.Name": {}, + "AdditionalServiceDetails.PagerDuty": {}, + "AdditionalServiceDetails.PagerDuty.Scopes": {}, + "AdditionalServiceDetails.ServiceNow": {}, + "AdditionalServiceDetails.ServiceNow.InstanceUrl": {}, + "Arn": {}, + "ServiceId": {} + } + }, + "AWS::DevOpsAgent::Trigger": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "TriggerId": {}, + "UpdatedAt": {} + } + }, + "AWS::DevOpsGuru::LogAnomalyDetectionIntegration": { + "Attributes": { + "AccountId": {} + } + }, + "AWS::DevOpsGuru::NotificationChannel": { + "Attributes": { + "Id": {} + } + }, + "AWS::DevOpsGuru::ResourceCollection": { + "Attributes": { + "ResourceCollectionType": {} + } + }, + "AWS::DirectConnect::Connection": { + "Attributes": { + "ConnectionArn": {}, + "ConnectionId": {}, + "ConnectionState": {} + } + }, + "AWS::DirectConnect::DirectConnectGateway": { + "Attributes": { + "DirectConnectGatewayArn": {}, + "DirectConnectGatewayId": {} + } + }, + "AWS::DirectConnect::DirectConnectGatewayAssociation": { + "Attributes": { + "AssociationId": {} + } + }, + "AWS::DirectConnect::Lag": { + "Attributes": { + "LagArn": {}, + "LagId": {}, + "LagState": {} + } + }, + "AWS::DirectConnect::PrivateVirtualInterface": { + "Attributes": { + "VirtualInterfaceArn": {}, + "VirtualInterfaceId": {} + } + }, + "AWS::DirectConnect::PublicVirtualInterface": { + "Attributes": { + "VirtualInterfaceArn": {}, + "VirtualInterfaceId": {} + } + }, + "AWS::DirectConnect::TransitVirtualInterface": { + "Attributes": { + "VirtualInterfaceArn": {}, + "VirtualInterfaceId": {} + } + }, + "AWS::DirectoryService::MicrosoftAD": { + "Attributes": { + "Alias": {}, + "DnsIpAddresses": {} + } + }, + "AWS::DirectoryService::SimpleAD": { + "Attributes": { + "Alias": {}, + "DirectoryId": {}, + "DnsIpAddresses": {} + } + }, + "AWS::DocDB::DBCluster": { + "Attributes": { + "ClusterResourceId": {}, + "Endpoint": {}, + "Port": {}, + "ReadEndpoint": {} + } + }, + "AWS::DocDB::DBInstance": { + "Attributes": { + "Endpoint": {}, + "Port": {} + } + }, + "AWS::DocDB::GlobalCluster": { + "Attributes": { + "GlobalClusterArn": {}, + "GlobalClusterResourceId": {} + } + }, + "AWS::DocDBElastic::Cluster": { + "Attributes": { + "ClusterArn": {}, + "ClusterEndpoint": {} + } + }, + "AWS::DocDBElastic::ClusterSnapshot": { + "Attributes": { + "AdminUserName": {}, + "ClusterCreationTime": {}, + "KmsKeyId": {}, + "SnapshotArn": {}, + "SnapshotCreationTime": {}, + "SnapshotType": {}, + "Status": {}, + "SubnetIds": {}, + "VpcSecurityGroupIds": {} + } + }, + "AWS::DynamoDB::Export": { + "Attributes": { + "BilledSizeBytes": {}, + "EndTime": {}, + "ExportArn": {}, + "ExportId": {}, + "ExportManifest": {}, + "ExportStatus": {}, + "ExportTime": {}, + "ItemCount": {}, + "StartTime": {}, + "TableId": {}, + "TableName": {} + } + }, + "AWS::DynamoDB::GlobalTable": { + "Attributes": { + "Arn": {}, + "StreamArn": {}, + "TableId": {} + } + }, + "AWS::DynamoDB::Stream": { + "Attributes": { + "CreationRequestDateTime": {}, + "KeySchema": {}, + "StreamArn": {}, + "StreamLabel": {}, + "StreamStatus": {} + } + }, + "AWS::DynamoDB::Table": { + "Attributes": { + "Arn": {}, + "StreamArn": {} + } + }, + "AWS::EC2::ApplicationStatusCheck": { + "Attributes": { + "ApplicationStatusCheckId": {}, + "Arn": {}, + "CreationTime": {} + } + }, + "AWS::EC2::CapacityManagerDataExport": { + "Attributes": { + "CapacityManagerDataExportId": {} + } + }, + "AWS::EC2::CapacityReservation": { + "Attributes": { + "AvailabilityZone": {}, + "AvailableInstanceCount": {}, + "CapacityAllocationSet": {}, + "CapacityReservationArn": {}, + "CapacityReservationFleetId": {}, + "CommitmentInfo": {}, + "CommitmentInfo.CommitmentEndDate": {}, + "CommitmentInfo.CommittedInstanceCount": {}, + "CreateDate": {}, + "DeliveryPreference": {}, + "Id": {}, + "InstanceType": {}, + "OwnerId": {}, + "ReservationType": {}, + "StartDate": {}, + "State": {}, + "Tenancy": {}, + "TotalInstanceCount": {} + } + }, + "AWS::EC2::CapacityReservationFleet": { + "Attributes": { + "CapacityReservationFleetId": {} + } + }, + "AWS::EC2::CarrierGateway": { + "Attributes": { + "CarrierGatewayId": {}, + "OwnerId": {}, + "State": {} + } + }, + "AWS::EC2::CustomerGateway": { + "Attributes": { + "CustomerGatewayId": {} + } + }, + "AWS::EC2::DHCPOptions": { + "Attributes": { + "DhcpOptionsId": {} + } + }, + "AWS::EC2::EC2Fleet": { + "Attributes": { + "FleetId": {} + } + }, + "AWS::EC2::EIP": { + "Attributes": { + "AllocationId": {}, + "PublicIp": {} + } + }, + "AWS::EC2::EIPAssociation": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::EgressOnlyInternetGateway": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::EnclaveCertificateIamRoleAssociation": { + "Attributes": { + "CertificateS3BucketName": {}, + "CertificateS3ObjectKey": {}, + "EncryptionKmsKeyId": {} + } + }, + "AWS::EC2::ExportInstanceTask": { + "Attributes": { + "Arn": {}, + "ExportTaskId": {}, + "ExportToS3Task.S3Key": {}, + "State": {} + } + }, + "AWS::EC2::FlowLog": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::FpgaImage": { + "Attributes": { + "Arn": {}, + "CreateTime": {}, + "DataRetentionSupport": {}, + "FpgaImageGlobalId": {}, + "FpgaImageId": {}, + "OwnerId": {}, + "Public": {}, + "State": {}, + "UpdateTime": {} + } + }, + "AWS::EC2::GatewayRouteTableAssociation": { + "Attributes": { + "AssociationId": {} + } + }, + "AWS::EC2::Host": { + "Attributes": { + "HostId": {} + } + }, + "AWS::EC2::IPAM": { + "Attributes": { + "Arn": {}, + "DefaultResourceDiscoveryAssociationId": {}, + "DefaultResourceDiscoveryId": {}, + "IpamId": {}, + "PrivateDefaultScopeId": {}, + "PublicDefaultScopeId": {}, + "ResourceDiscoveryAssociationCount": {}, + "ScopeCount": {} + } + }, + "AWS::EC2::IPAMAllocation": { + "Attributes": { + "IpamPoolAllocationId": {} + } + }, + "AWS::EC2::IPAMPool": { + "Attributes": { + "Arn": {}, + "IpamArn": {}, + "IpamPoolId": {}, + "IpamScopeArn": {}, + "IpamScopeType": {}, + "PoolDepth": {}, + "State": {}, + "StateMessage": {} + } + }, + "AWS::EC2::IPAMPoolCidr": { + "Attributes": { + "IpamPoolCidrId": {}, + "State": {} + } + }, + "AWS::EC2::IPAMPrefixListResolver": { + "Attributes": { + "IpamArn": {}, + "IpamPrefixListResolverArn": {}, + "IpamPrefixListResolverId": {} + } + }, + "AWS::EC2::IPAMPrefixListResolverTarget": { + "Attributes": { + "IpamPrefixListResolverTargetArn": {}, + "IpamPrefixListResolverTargetId": {} + } + }, + "AWS::EC2::IPAMResourceDiscovery": { + "Attributes": { + "IpamResourceDiscoveryArn": {}, + "IpamResourceDiscoveryId": {}, + "IpamResourceDiscoveryRegion": {}, + "IsDefault": {}, + "OwnerId": {}, + "State": {} + } + }, + "AWS::EC2::IPAMResourceDiscoveryAssociation": { + "Attributes": { + "IpamArn": {}, + "IpamRegion": {}, + "IpamResourceDiscoveryAssociationArn": {}, + "IpamResourceDiscoveryAssociationId": {}, + "IsDefault": {}, + "OwnerId": {}, + "ResourceDiscoveryStatus": {}, + "State": {} + } + }, + "AWS::EC2::IPAMScope": { + "Attributes": { + "Arn": {}, + "IpamArn": {}, + "IpamScopeId": {}, + "IpamScopeType": {}, + "IsDefault": {}, + "PoolCount": {} + } + }, + "AWS::EC2::Instance": { + "Attributes": { + "AvailabilityZone": {}, + "InstanceId": {}, + "PrivateDnsName": {}, + "PrivateIp": {}, + "PublicDnsName": {}, + "PublicIp": {}, + "State": {}, + "State.Code": {}, + "State.Name": {}, + "VpcId": {} + } + }, + "AWS::EC2::InstanceConnectEndpoint": { + "Attributes": { + "AvailabilityZone": {}, + "AvailabilityZoneId": {}, + "CreatedAt": {}, + "Id": {}, + "InstanceConnectEndpointArn": {}, + "NetworkInterfaceIds": {}, + "OwnerId": {}, + "PublicDnsNames": {}, + "PublicDnsNames.Dualstack": {}, + "PublicDnsNames.Dualstack.DnsName": {}, + "PublicDnsNames.Dualstack.FipsDnsName": {}, + "PublicDnsNames.Ipv4": {}, + "PublicDnsNames.Ipv4.DnsName": {}, + "PublicDnsNames.Ipv4.FipsDnsName": {}, + "State": {}, + "StateMessage": {}, + "VpcId": {} + } + }, + "AWS::EC2::InternetGateway": { + "Attributes": { + "InternetGatewayId": {} + } + }, + "AWS::EC2::IpPoolRouteTableAssociation": { + "Attributes": { + "AssociationId": {} + } + }, + "AWS::EC2::IpamExternalResourceVerificationToken": { + "Attributes": { + "IpamArn": {}, + "IpamExternalResourceVerificationTokenArn": {}, + "IpamExternalResourceVerificationTokenId": {}, + "IpamRegion": {}, + "NotAfter": {}, + "State": {}, + "Status": {}, + "TokenName": {}, + "TokenValue": {} + } + }, + "AWS::EC2::KeyPair": { + "Attributes": { + "KeyFingerprint": {}, + "KeyPairId": {} + } + }, + "AWS::EC2::LaunchTemplate": { + "Attributes": { + "DefaultVersionNumber": {}, + "LatestVersionNumber": {}, + "LaunchTemplateId": {} + } + }, + "AWS::EC2::LocalGatewayRoute": { + "Attributes": { + "State": {}, + "Type": {} + } + }, + "AWS::EC2::LocalGatewayRouteTable": { + "Attributes": { + "LocalGatewayRouteTableArn": {}, + "LocalGatewayRouteTableId": {}, + "OutpostArn": {}, + "OwnerId": {}, + "State": {} + } + }, + "AWS::EC2::LocalGatewayRouteTableVPCAssociation": { + "Attributes": { + "LocalGatewayId": {}, + "LocalGatewayRouteTableVpcAssociationId": {}, + "State": {} + } + }, + "AWS::EC2::LocalGatewayRouteTableVirtualInterfaceGroupAssociation": { + "Attributes": { + "LocalGatewayId": {}, + "LocalGatewayRouteTableArn": {}, + "LocalGatewayRouteTableVirtualInterfaceGroupAssociationId": {}, + "OwnerId": {}, + "State": {} + } + }, + "AWS::EC2::LocalGatewayVirtualInterface": { + "Attributes": { + "ConfigurationState": {}, + "LocalBgpAsn": {}, + "LocalGatewayId": {}, + "LocalGatewayVirtualInterfaceId": {}, + "OwnerId": {} + } + }, + "AWS::EC2::LocalGatewayVirtualInterfaceGroup": { + "Attributes": { + "ConfigurationState": {}, + "LocalGatewayVirtualInterfaceGroupArn": {}, + "LocalGatewayVirtualInterfaceGroupId": {}, + "LocalGatewayVirtualInterfaceIds": {}, + "OwnerId": {} + } + }, + "AWS::EC2::NatGateway": { + "Attributes": { + "AutoProvisionZones": {}, + "AutoScalingIps": {}, + "EniId": {}, + "NatGatewayId": {}, + "RouteTableId": {} + } + }, + "AWS::EC2::NetworkAcl": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::NetworkAclEntry": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::NetworkInsightsAccessScope": { + "Attributes": { + "CreatedDate": {}, + "NetworkInsightsAccessScopeArn": {}, + "NetworkInsightsAccessScopeId": {}, + "UpdatedDate": {} + } + }, + "AWS::EC2::NetworkInsightsAccessScopeAnalysis": { + "Attributes": { + "AnalyzedEniCount": {}, + "EndDate": {}, + "FindingsFound": {}, + "NetworkInsightsAccessScopeAnalysisArn": {}, + "NetworkInsightsAccessScopeAnalysisId": {}, + "StartDate": {}, + "Status": {}, + "StatusMessage": {} + } + }, + "AWS::EC2::NetworkInsightsAnalysis": { + "Attributes": { + "AlternatePathHints": {}, + "Explanations": {}, + "ForwardPathComponents": {}, + "NetworkInsightsAnalysisArn": {}, + "NetworkInsightsAnalysisId": {}, + "NetworkPathFound": {}, + "ReturnPathComponents": {}, + "StartDate": {}, + "Status": {}, + "StatusMessage": {}, + "SuggestedAccounts": {} + } + }, + "AWS::EC2::NetworkInsightsPath": { + "Attributes": { + "CreatedDate": {}, + "DestinationArn": {}, + "NetworkInsightsPathArn": {}, + "NetworkInsightsPathId": {}, + "SourceArn": {} + } + }, + "AWS::EC2::NetworkInterface": { + "Attributes": { + "Id": {}, + "PrimaryIpv6Address": {}, + "PrimaryPrivateIpAddress": {}, + "PublicIpDnsNameOptions": {}, + "PublicIpDnsNameOptions.DnsHostnameType": {}, + "PublicIpDnsNameOptions.PublicDualStackDnsName": {}, + "PublicIpDnsNameOptions.PublicIpv4DnsName": {}, + "PublicIpDnsNameOptions.PublicIpv6DnsName": {}, + "SecondaryPrivateIpAddresses": {}, + "VpcId": {} + } + }, + "AWS::EC2::NetworkInterfaceAttachment": { + "Attributes": { + "AttachmentId": {} + } + }, + "AWS::EC2::PlacementGroup": { + "Attributes": { + "GroupId": {}, + "GroupName": {} + } + }, + "AWS::EC2::PrefixList": { + "Attributes": { + "Arn": {}, + "OwnerId": {}, + "PrefixListId": {}, + "Version": {} + } + }, + "AWS::EC2::ReplaceRootVolumeTask": { + "Attributes": { + "Arn": {}, + "CompleteTime": {}, + "ReplaceRootVolumeTaskId": {}, + "SnapshotId": {}, + "StartTime": {}, + "TaskState": {} + } + }, + "AWS::EC2::Route": { + "Attributes": { + "CidrBlock": {} + } + }, + "AWS::EC2::RouteServer": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::EC2::RouteServerEndpoint": { + "Attributes": { + "Arn": {}, + "EniAddress": {}, + "EniId": {}, + "Id": {}, + "VpcId": {} + } + }, + "AWS::EC2::RouteServerPeer": { + "Attributes": { + "Arn": {}, + "EndpointEniAddress": {}, + "EndpointEniId": {}, + "Id": {}, + "RouteServerId": {}, + "SubnetId": {}, + "VpcId": {} + } + }, + "AWS::EC2::RouteTable": { + "Attributes": { + "RouteTableId": {} + } + }, + "AWS::EC2::SecurityGroup": { + "Attributes": { + "GroupId": {}, + "Id": {}, + "VpcId": {} + } + }, + "AWS::EC2::SecurityGroupEgress": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::SecurityGroupIngress": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::SecurityGroupVpcAssociation": { + "Attributes": { + "State": {}, + "StateReason": {}, + "VpcOwnerId": {} + } + }, + "AWS::EC2::SnapshotBlockPublicAccess": { + "Attributes": { + "AccountId": {} + } + }, + "AWS::EC2::SpotFleet": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::SqlHaStandbyDetectedInstance": { + "Attributes": { + "HaStatus": {}, + "LastUpdatedTime": {}, + "SqlServerLicenseUsage": {} + } + }, + "AWS::EC2::Subnet": { + "Attributes": { + "AvailabilityZone": {}, + "AvailabilityZoneId": {}, + "BlockPublicAccessStates": {}, + "BlockPublicAccessStates.InternetGatewayBlockMode": {}, + "CidrBlock": {}, + "Ipv6CidrBlocks": {}, + "NetworkAclAssociationId": {}, + "OutpostArn": {}, + "SubnetId": {}, + "VpcId": {} + } + }, + "AWS::EC2::SubnetCidrBlock": { + "Attributes": { + "Id": {}, + "IpSource": {}, + "Ipv6AddressAttribute": {} + } + }, + "AWS::EC2::SubnetNetworkAclAssociation": { + "Attributes": { + "AssociationId": {} + } + }, + "AWS::EC2::SubnetRouteTableAssociation": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::TrafficMirrorFilter": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::TrafficMirrorFilterRule": { + "Attributes": { + "TrafficMirrorFilterRuleId": {} + } + }, + "AWS::EC2::TrafficMirrorSession": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::TrafficMirrorTarget": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::TransitGateway": { + "Attributes": { + "EncryptionSupportState": {}, + "Id": {}, + "TransitGatewayArn": {} + } + }, + "AWS::EC2::TransitGatewayAttachment": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::TransitGatewayConnect": { + "Attributes": { + "CreationTime": {}, + "State": {}, + "TransitGatewayAttachmentId": {}, + "TransitGatewayId": {} + } + }, + "AWS::EC2::TransitGatewayConnectPeer": { + "Attributes": { + "ConnectPeerConfiguration.BgpConfigurations": {}, + "ConnectPeerConfiguration.Protocol": {}, + "CreationTime": {}, + "State": {}, + "TransitGatewayConnectPeerId": {} + } + }, + "AWS::EC2::TransitGatewayMeteringPolicy": { + "Attributes": { + "State": {}, + "TransitGatewayMeteringPolicyId": {}, + "UpdateEffectiveAt": {} + } + }, + "AWS::EC2::TransitGatewayMeteringPolicyEntry": { + "Attributes": { + "State": {}, + "UpdateEffectiveAt": {} + } + }, + "AWS::EC2::TransitGatewayMulticastDomain": { + "Attributes": { + "CreationTime": {}, + "State": {}, + "TransitGatewayMulticastDomainArn": {}, + "TransitGatewayMulticastDomainId": {} + } + }, + "AWS::EC2::TransitGatewayMulticastDomainAssociation": { + "Attributes": { + "ResourceId": {}, + "ResourceType": {}, + "State": {} + } + }, + "AWS::EC2::TransitGatewayMulticastGroupMember": { + "Attributes": { + "GroupMember": {}, + "GroupSource": {}, + "MemberType": {}, + "ResourceId": {}, + "ResourceType": {}, + "SubnetId": {}, + "TransitGatewayAttachmentId": {} + } + }, + "AWS::EC2::TransitGatewayMulticastGroupSource": { + "Attributes": { + "GroupMember": {}, + "GroupSource": {}, + "ResourceId": {}, + "ResourceType": {}, + "SourceType": {}, + "SubnetId": {}, + "TransitGatewayAttachmentId": {} + } + }, + "AWS::EC2::TransitGatewayPeeringAttachment": { + "Attributes": { + "CreationTime": {}, + "State": {}, + "Status": {}, + "Status.Code": {}, + "Status.Message": {}, + "TransitGatewayAttachmentId": {} + } + }, + "AWS::EC2::TransitGatewayPolicyTable": { + "Attributes": { + "CreationTime": {}, + "State": {}, + "TransitGatewayPolicyTableId": {} + } + }, + "AWS::EC2::TransitGatewayPolicyTableAssociation": { + "Attributes": { + "State": {} + } + }, + "AWS::EC2::TransitGatewayPolicyTableEntry": { + "Attributes": { + "State": {} + } + }, + "AWS::EC2::TransitGatewayRouteTable": { + "Attributes": { + "TransitGatewayRouteTableId": {} + } + }, + "AWS::EC2::TransitGatewayVpcAttachment": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::VPC": { + "Attributes": { + "CidrBlock": {}, + "CidrBlockAssociations": {}, + "DefaultNetworkAcl": {}, + "DefaultSecurityGroup": {}, + "Ipv6CidrBlocks": {}, + "VpcEncryptionControl.ResourceExclusions": {}, + "VpcEncryptionControl.ResourceExclusions.EgressOnlyInternetGateway": {}, + "VpcEncryptionControl.ResourceExclusions.EgressOnlyInternetGateway.State": {}, + "VpcEncryptionControl.ResourceExclusions.EgressOnlyInternetGateway.StateMessage": {}, + "VpcEncryptionControl.ResourceExclusions.ElasticFileSystem": {}, + "VpcEncryptionControl.ResourceExclusions.ElasticFileSystem.State": {}, + "VpcEncryptionControl.ResourceExclusions.ElasticFileSystem.StateMessage": {}, + "VpcEncryptionControl.ResourceExclusions.InternetGateway": {}, + "VpcEncryptionControl.ResourceExclusions.InternetGateway.State": {}, + "VpcEncryptionControl.ResourceExclusions.InternetGateway.StateMessage": {}, + "VpcEncryptionControl.ResourceExclusions.Lambda": {}, + "VpcEncryptionControl.ResourceExclusions.Lambda.State": {}, + "VpcEncryptionControl.ResourceExclusions.Lambda.StateMessage": {}, + "VpcEncryptionControl.ResourceExclusions.NatGateway": {}, + "VpcEncryptionControl.ResourceExclusions.NatGateway.State": {}, + "VpcEncryptionControl.ResourceExclusions.NatGateway.StateMessage": {}, + "VpcEncryptionControl.ResourceExclusions.VirtualPrivateGateway": {}, + "VpcEncryptionControl.ResourceExclusions.VirtualPrivateGateway.State": {}, + "VpcEncryptionControl.ResourceExclusions.VirtualPrivateGateway.StateMessage": {}, + "VpcEncryptionControl.ResourceExclusions.VpcLattice": {}, + "VpcEncryptionControl.ResourceExclusions.VpcLattice.State": {}, + "VpcEncryptionControl.ResourceExclusions.VpcLattice.StateMessage": {}, + "VpcEncryptionControl.ResourceExclusions.VpcPeering": {}, + "VpcEncryptionControl.ResourceExclusions.VpcPeering.State": {}, + "VpcEncryptionControl.ResourceExclusions.VpcPeering.StateMessage": {}, + "VpcEncryptionControl.State": {}, + "VpcEncryptionControl.StateMessage": {}, + "VpcEncryptionControl.VpcEncryptionControlId": {}, + "VpcEncryptionControl.VpcId": {}, + "VpcId": {} + } + }, + "AWS::EC2::VPCBlockPublicAccessExclusion": { + "Attributes": { + "ExclusionId": {} + } + }, + "AWS::EC2::VPCBlockPublicAccessOptions": { + "Attributes": { + "AccountId": {}, + "ExclusionsAllowed": {} + } + }, + "AWS::EC2::VPCCidrBlock": { + "Attributes": { + "Id": {}, + "IpSource": {}, + "Ipv6AddressAttribute": {} + } + }, + "AWS::EC2::VPCEncryptionControl": { + "Attributes": { + "ResourceExclusions": {}, + "ResourceExclusions.EgressOnlyInternetGateway": {}, + "ResourceExclusions.EgressOnlyInternetGateway.State": {}, + "ResourceExclusions.EgressOnlyInternetGateway.StateMessage": {}, + "ResourceExclusions.ElasticFileSystem": {}, + "ResourceExclusions.ElasticFileSystem.State": {}, + "ResourceExclusions.ElasticFileSystem.StateMessage": {}, + "ResourceExclusions.InternetGateway": {}, + "ResourceExclusions.InternetGateway.State": {}, + "ResourceExclusions.InternetGateway.StateMessage": {}, + "ResourceExclusions.Lambda": {}, + "ResourceExclusions.Lambda.State": {}, + "ResourceExclusions.Lambda.StateMessage": {}, + "ResourceExclusions.NatGateway": {}, + "ResourceExclusions.NatGateway.State": {}, + "ResourceExclusions.NatGateway.StateMessage": {}, + "ResourceExclusions.VirtualPrivateGateway": {}, + "ResourceExclusions.VirtualPrivateGateway.State": {}, + "ResourceExclusions.VirtualPrivateGateway.StateMessage": {}, + "ResourceExclusions.VpcLattice": {}, + "ResourceExclusions.VpcLattice.State": {}, + "ResourceExclusions.VpcLattice.StateMessage": {}, + "ResourceExclusions.VpcPeering": {}, + "ResourceExclusions.VpcPeering.State": {}, + "ResourceExclusions.VpcPeering.StateMessage": {}, + "State": {}, + "StateMessage": {}, + "VpcEncryptionControlId": {} + } + }, + "AWS::EC2::VPCEndpoint": { + "Attributes": { + "CreationTimestamp": {}, + "DnsEntries": {}, + "Id": {}, + "NetworkInterfaceIds": {} + } + }, + "AWS::EC2::VPCEndpointConnectionNotification": { + "Attributes": { + "VPCEndpointConnectionNotificationId": {} + } + }, + "AWS::EC2::VPCEndpointService": { + "Attributes": { + "PrivateDnsNameConfiguration.Name": {}, + "PrivateDnsNameConfiguration.State": {}, + "PrivateDnsNameConfiguration.Type": {}, + "PrivateDnsNameConfiguration.Value": {}, + "ServiceId": {} + } + }, + "AWS::EC2::VPCGatewayAttachment": { + "Attributes": { + "AttachmentType": {} + } + }, + "AWS::EC2::VPCPeeringConnection": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::VPNConcentrator": { + "Attributes": { + "TransitGatewayAttachmentId": {}, + "VpnConcentratorId": {} + } + }, + "AWS::EC2::VPNConnection": { + "Attributes": { + "VpnConnectionId": {} + } + }, + "AWS::EC2::VPNGateway": { + "Attributes": { + "VPNGatewayId": {} + } + }, + "AWS::EC2::VPNGatewayRoutePropagation": { + "Attributes": { + "Id": {} + } + }, + "AWS::EC2::VerifiedAccessEndpoint": { + "Attributes": { + "CreationTime": {}, + "DeviceValidationDomain": {}, + "EndpointDomain": {}, + "LastUpdatedTime": {}, + "Status": {}, + "VerifiedAccessEndpointId": {}, + "VerifiedAccessInstanceId": {} + } + }, + "AWS::EC2::VerifiedAccessGroup": { + "Attributes": { + "CreationTime": {}, + "LastUpdatedTime": {}, + "Owner": {}, + "VerifiedAccessGroupArn": {}, + "VerifiedAccessGroupId": {} + } + }, + "AWS::EC2::VerifiedAccessInstance": { + "Attributes": { + "CidrEndpointsCustomSubDomainNameServers": {}, + "CreationTime": {}, + "LastUpdatedTime": {}, + "VerifiedAccessInstanceId": {} + } + }, + "AWS::EC2::VerifiedAccessTrustProvider": { + "Attributes": { + "CreationTime": {}, + "LastUpdatedTime": {}, + "VerifiedAccessTrustProviderId": {} + } + }, + "AWS::EC2::Volume": { + "Attributes": { + "VolumeId": {} + } + }, + "AWS::EC2::VpnConnectionDeviceType": { + "Attributes": { + "Arn": {}, + "Platform": {}, + "Software": {}, + "Vendor": {}, + "VpnConnectionDeviceTypeId": {} + } + }, + "AWS::ECR::PublicRepository": { + "Attributes": { + "Arn": {} + } + }, + "AWS::ECR::RegistryPolicy": { + "Attributes": { + "RegistryId": {} + } + }, + "AWS::ECR::RegistryScanningConfiguration": { + "Attributes": { + "RegistryId": {} + } + }, + "AWS::ECR::ReplicationConfiguration": { + "Attributes": { + "RegistryId": {} + } + }, + "AWS::ECR::Repository": { + "Attributes": { + "Arn": {}, + "RepositoryUri": {} + } + }, + "AWS::ECR::RepositoryCreationTemplate": { + "Attributes": { + "CreatedAt": {}, + "UpdatedAt": {} + } + }, + "AWS::ECR::SigningConfiguration": { + "Attributes": { + "RegistryId": {} + } + }, + "AWS::ECRPublic::Registry": { + "Attributes": { + "Aliases": {}, + "DisplayName": {}, + "RegistryArn": {}, + "RegistryId": {}, + "RegistryUri": {}, + "Verified": {} + } + }, + "AWS::ECS::Cluster": { + "Attributes": { + "Arn": {} + } + }, + "AWS::ECS::ContainerInstance": { + "Attributes": { + "AgentConnected": {}, + "Attributes": {}, + "Cluster": {}, + "ContainerInstanceArn": {}, + "ContainerInstanceId": {}, + "Ec2InstanceId": {}, + "PendingTasksCount": {}, + "RegisteredAt": {}, + "RegisteredResources": {}, + "RemainingResources": {}, + "RunningTasksCount": {}, + "Status": {}, + "Tags": {}, + "Version": {}, + "VersionInfo": {}, + "VersionInfo.AgentHash": {}, + "VersionInfo.AgentVersion": {}, + "VersionInfo.DockerVersion": {} + } + }, + "AWS::ECS::Daemon": { + "Attributes": { + "CreatedAt": {}, + "DaemonArn": {}, + "DaemonStatus": {}, + "DeploymentArn": {}, + "UpdatedAt": {} + } + }, + "AWS::ECS::DaemonDeployment": { + "Attributes": { + "Alarms": {}, + "Alarms.AlarmNames": {}, + "Alarms.Status": {}, + "Alarms.TriggeredAlarmNames": {}, + "CircuitBreaker": {}, + "CircuitBreaker.FailureCount": {}, + "CircuitBreaker.Status": {}, + "CircuitBreaker.Threshold": {}, + "ClusterArn": {}, + "ClusterName": {}, + "CreatedAt": {}, + "DaemonDeploymentArn": {}, + "DaemonDeploymentId": {}, + "DaemonName": {}, + "DeploymentConfiguration": {}, + "DeploymentConfiguration.Alarms": {}, + "DeploymentConfiguration.Alarms.AlarmNames": {}, + "DeploymentConfiguration.Alarms.Enable": {}, + "DeploymentConfiguration.BakeTimeInMinutes": {}, + "DeploymentConfiguration.DrainPercent": {}, + "FinishedAt": {}, + "SourceDaemonRevisions": {}, + "StartedAt": {}, + "Status": {}, + "TargetDaemonRevision": {}, + "TargetDaemonRevision.Arn": {}, + "TargetDaemonRevision.CapacityProviders": {}, + "TargetDaemonRevision.TotalDrainingInstanceCount": {}, + "TargetDaemonRevision.TotalRunningInstanceCount": {} + } + }, + "AWS::ECS::DaemonTaskDefinition": { + "Attributes": { + "DaemonTaskDefinitionArn": {} + } + }, + "AWS::ECS::ExpressGatewayService": { + "Attributes": { + "ActiveConfigurations": {}, + "CreatedAt": {}, + "ECSManagedResourceArns": {}, + "ECSManagedResourceArns.AutoScaling": {}, + "ECSManagedResourceArns.AutoScaling.ApplicationAutoScalingPolicies": {}, + "ECSManagedResourceArns.AutoScaling.ScalableTarget": {}, + "ECSManagedResourceArns.IngressPath": {}, + "ECSManagedResourceArns.IngressPath.CertificateArn": {}, + "ECSManagedResourceArns.IngressPath.ListenerArn": {}, + "ECSManagedResourceArns.IngressPath.ListenerRuleArn": {}, + "ECSManagedResourceArns.IngressPath.LoadBalancerArn": {}, + "ECSManagedResourceArns.IngressPath.LoadBalancerSecurityGroups": {}, + "ECSManagedResourceArns.IngressPath.TargetGroupArns": {}, + "ECSManagedResourceArns.LogGroups": {}, + "ECSManagedResourceArns.MetricAlarms": {}, + "ECSManagedResourceArns.ServiceSecurityGroups": {}, + "Endpoint": {}, + "ServiceArn": {}, + "Status": {}, + "Status.StatusCode": {}, + "UpdatedAt": {} + } + }, + "AWS::ECS::Service": { + "Attributes": { + "Name": {}, + "ServiceArn": {} + } + }, + "AWS::ECS::ServiceDeployment": { + "Attributes": { + "Cluster": {}, + "ClusterArn": {}, + "CreatedAt": {}, + "Service": {}, + "ServiceArn": {}, + "ServiceDeploymentArn": {}, + "ServiceDeploymentId": {}, + "Status": {}, + "TargetServiceRevision": {}, + "TargetServiceRevision.Arn": {}, + "TargetServiceRevision.PendingTaskCount": {}, + "TargetServiceRevision.RequestedTaskCount": {}, + "TargetServiceRevision.RunningTaskCount": {}, + "UpdatedAt": {} + } + }, + "AWS::ECS::ServiceRevision": { + "Attributes": { + "CapacityProviderStrategy": {}, + "Cluster": {}, + "ClusterArn": {}, + "ContainerImages": {}, + "CreatedAt": {}, + "EcsManagedResources": {}, + "EcsManagedResources.AutoScaling": {}, + "EcsManagedResources.AutoScaling.ApplicationAutoScalingPolicies": {}, + "EcsManagedResources.AutoScaling.ScalableTarget": {}, + "EcsManagedResources.AutoScaling.ScalableTarget.Arn": {}, + "EcsManagedResources.AutoScaling.ScalableTarget.MaxCapacity": {}, + "EcsManagedResources.AutoScaling.ScalableTarget.MinCapacity": {}, + "EcsManagedResources.AutoScaling.ScalableTarget.Status": {}, + "EcsManagedResources.AutoScaling.ScalableTarget.StatusReason": {}, + "EcsManagedResources.AutoScaling.ScalableTarget.UpdatedAt": {}, + "EcsManagedResources.IngressPaths": {}, + "EcsManagedResources.LogGroups": {}, + "EcsManagedResources.MetricAlarms": {}, + "EcsManagedResources.ServiceSecurityGroups": {}, + "FargateEphemeralStorage": {}, + "FargateEphemeralStorage.KmsKeyId": {}, + "GuardDutyEnabled": {}, + "LaunchType": {}, + "LoadBalancers": {}, + "Monitoring": {}, + "Monitoring.MetricConfigurations": {}, + "NetworkConfiguration": {}, + "NetworkConfiguration.AwsvpcConfiguration": {}, + "NetworkConfiguration.AwsvpcConfiguration.AssignPublicIp": {}, + "NetworkConfiguration.AwsvpcConfiguration.SecurityGroups": {}, + "NetworkConfiguration.AwsvpcConfiguration.Subnets": {}, + "PlatformFamily": {}, + "PlatformVersion": {}, + "ResolvedConfiguration": {}, + "ResolvedConfiguration.LoadBalancers": {}, + "Service": {}, + "ServiceArn": {}, + "ServiceConnectConfiguration": {}, + "ServiceConnectConfiguration.AccessLogConfiguration": {}, + "ServiceConnectConfiguration.AccessLogConfiguration.Format": {}, + "ServiceConnectConfiguration.AccessLogConfiguration.IncludeQueryParameters": {}, + "ServiceConnectConfiguration.Enabled": {}, + "ServiceConnectConfiguration.LogConfiguration": {}, + "ServiceConnectConfiguration.LogConfiguration.LogDriver": {}, + "ServiceConnectConfiguration.LogConfiguration.Options": {}, + "ServiceConnectConfiguration.LogConfiguration.SecretOptions": {}, + "ServiceConnectConfiguration.Namespace": {}, + "ServiceConnectConfiguration.Services": {}, + "ServiceRegistries": {}, + "ServiceRevisionArn": {}, + "ServiceRevisionId": {}, + "TaskDefinition": {}, + "VolumeConfigurations": {}, + "VpcLatticeConfigurations": {} + } + }, + "AWS::ECS::Task": { + "Attributes": { + "Cluster": {}, + "Cpu": {}, + "CreatedAt": {}, + "DesiredStatus": {}, + "Group": {}, + "LastStatus": {}, + "LaunchType": {}, + "Memory": {}, + "StartedBy": {}, + "Tags": {}, + "TaskArn": {}, + "TaskDefinition": {}, + "TaskId": {} + } + }, + "AWS::ECS::TaskDefinition": { + "Attributes": { + "TaskDefinitionArn": {} + } + }, + "AWS::ECS::TaskSet": { + "Attributes": { + "Id": {} + } + }, + "AWS::EFS::AccessPoint": { + "Attributes": { + "AccessPointId": {}, + "Arn": {} + } + }, + "AWS::EFS::FileSystem": { + "Attributes": { + "Arn": {}, + "FileSystemId": {} + } + }, + "AWS::EFS::MountTarget": { + "Attributes": { + "Id": {}, + "IpAddress": {} + } + }, + "AWS::EKS::AccessEntry": { + "Attributes": { + "AccessEntryArn": {} + } + }, + "AWS::EKS::Addon": { + "Attributes": { + "Arn": {} + } + }, + "AWS::EKS::Capability": { + "Attributes": { + "Arn": {}, + "Configuration.ArgoCd.AwsIdc.IdcManagedApplicationArn": {}, + "Configuration.ArgoCd.ServerUrl": {}, + "CreatedAt": {}, + "ModifiedAt": {}, + "Status": {}, + "Version": {} + } + }, + "AWS::EKS::CertificateAuthority": { + "Attributes": { + "ActivatedAt": {}, + "ActivatedBy": {}, + "CreatedAt": {}, + "CreatedBy": {}, + "Data": {}, + "DistributionStatus": {}, + "Id": {}, + "RollbackAvailable": {}, + "ScheduledEvents": {}, + "ScheduledEvents.FinalAutoActivation": {}, + "ScheduledEvents.FirstAutoActivation": {}, + "SigningStatus": {}, + "Validity": {}, + "Validity.NotAfter": {}, + "Validity.NotBefore": {} + } + }, + "AWS::EKS::Cluster": { + "Attributes": { + "Arn": {}, + "CertificateAuthority.Active.ActivatedBy": {}, + "CertificateAuthority.Active.Id": {}, + "CertificateAuthority.Data": {}, + "CertificateAuthorityData": {}, + "ClusterSecurityGroupId": {}, + "EncryptionConfigKeyArn": {}, + "Endpoint": {}, + "Id": {}, + "KubernetesNetworkConfig.ServiceIpv6Cidr": {}, + "OpenIdConnectIssuerUrl": {} + } + }, + "AWS::EKS::FargateProfile": { + "Attributes": { + "Arn": {} + } + }, + "AWS::EKS::IdentityProviderConfig": { + "Attributes": { + "IdentityProviderConfigArn": {} + } + }, + "AWS::EKS::Nodegroup": { + "Attributes": { + "Arn": {}, + "ClusterName": {}, + "Id": {}, + "NodegroupName": {} + } + }, + "AWS::EKS::PodIdentityAssociation": { + "Attributes": { + "AssociationArn": {}, + "AssociationId": {}, + "ExternalId": {} + } + }, + "AWS::EMR::Cluster": { + "Attributes": { + "MasterPublicDNS": {} + } + }, + "AWS::EMR::NotebookExecution": { + "Attributes": { + "Arn": {}, + "NotebookExecutionId": {}, + "StartTime": {}, + "Status": {} + } + }, + "AWS::EMR::Step": { + "Attributes": { + "Id": {} + } + }, + "AWS::EMR::Studio": { + "Attributes": { + "Arn": {}, + "StudioId": {}, + "Url": {} + } + }, + "AWS::EMRContainers::Endpoint": { + "Attributes": { + "Arn": {}, + "AuthProxyUrl": {}, + "CertificateAuthority": {}, + "CertificateAuthority.CertificateArn": {}, + "CertificateAuthority.CertificateData": {}, + "CreatedAt": {}, + "FailureReason": {}, + "Id": {}, + "SecurityGroup": {}, + "ServerUrl": {}, + "State": {}, + "StateDetails": {} + } + }, + "AWS::EMRContainers::JobRun": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "CreatedBy": {}, + "FailureReason": {}, + "FinishedAt": {}, + "Id": {}, + "RetryPolicyExecution": {}, + "RetryPolicyExecution.CurrentAttemptCount": {}, + "State": {} + } + }, + "AWS::EMRContainers::SecurityConfiguration": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::EMRContainers::VirtualCluster": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::EMRServerless::Application": { + "Attributes": { + "ApplicationId": {}, + "Arn": {} + } + }, + "AWS::EMRServerless::JobRun": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "CreatedBy": {}, + "JobRunId": {}, + "ReleaseLabel": {}, + "State": {}, + "StateDetails": {}, + "UpdatedAt": {} + } + }, + "AWS::EMRServerless::Session": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "CreatedBy": {}, + "IdleSince": {}, + "NetworkConfiguration": {}, + "NetworkConfiguration.SecurityGroupIds": {}, + "NetworkConfiguration.SubnetIds": {}, + "ReleaseLabel": {}, + "SessionId": {}, + "StartedAt": {}, + "State": {}, + "StateDetails": {}, + "UpdatedAt": {} + } + }, + "AWS::EVS::Environment": { + "Attributes": { + "Checks": {}, + "CreatedAt": {}, + "Credentials": {}, + "EnvironmentArn": {}, + "EnvironmentId": {}, + "EnvironmentState": {}, + "ModifiedAt": {}, + "StateDetails": {} + } + }, + "AWS::ElastiCache::CacheCluster": { + "Attributes": { + "ConfigurationEndpoint": {}, + "ConfigurationEndpoint.Address": {}, + "ConfigurationEndpoint.Port": {}, + "RedisEndpoint": {}, + "RedisEndpoint.Address": {}, + "RedisEndpoint.Port": {} + } + }, + "AWS::ElastiCache::GlobalReplicationGroup": { + "Attributes": { + "GlobalReplicationGroupId": {}, + "Status": {} + } + }, + "AWS::ElastiCache::ParameterGroup": { + "Attributes": { + "CacheParameterGroupName": {} + } + }, + "AWS::ElastiCache::ReplicationGroup": { + "Attributes": { + "ConfigurationEndPoint": {}, + "ConfigurationEndPoint.Address": {}, + "ConfigurationEndPoint.Port": {}, + "EffectiveDurability": {}, + "PrimaryEndPoint": {}, + "PrimaryEndPoint.Address": {}, + "PrimaryEndPoint.Port": {}, + "ReadEndPoint": {}, + "ReadEndPoint.Addresses": {}, + "ReadEndPoint.AddressesList": {}, + "ReadEndPoint.Ports": {}, + "ReadEndPoint.PortsList": {}, + "ReaderEndPoint": {}, + "ReaderEndPoint.Address": {}, + "ReaderEndPoint.Port": {} + } + }, + "AWS::ElastiCache::ReservedCacheNode": { + "Attributes": { + "CacheNodeCount": {}, + "CacheNodeType": {}, + "Duration": {}, + "FixedPrice": {}, + "OfferingType": {}, + "ProductDescription": {}, + "RecurringCharges": {}, + "ReservationARN": {}, + "ReservedCacheNodeId": {}, + "ReservedCacheNodesOfferingId": {}, + "StartTime": {}, + "State": {}, + "Tags": {}, + "UsagePrice": {} + } + }, + "AWS::ElastiCache::ServerlessCache": { + "Attributes": { + "ARN": {}, + "CreateTime": {}, + "Endpoint.Address": {}, + "Endpoint.Port": {}, + "FullEngineVersion": {}, + "ReaderEndpoint.Address": {}, + "ReaderEndpoint.Port": {}, + "Status": {} + } + }, + "AWS::ElastiCache::ServerlessCacheSnapshot": { + "Attributes": { + "ARN": {}, + "BytesUsedForCache": {}, + "CreateTime": {}, + "ServerlessCacheConfiguration": {}, + "ServerlessCacheConfiguration.Engine": {}, + "ServerlessCacheConfiguration.MajorEngineVersion": {}, + "ServerlessCacheConfiguration.ServerlessCacheName": {}, + "SnapshotType": {}, + "Status": {} + } + }, + "AWS::ElastiCache::User": { + "Attributes": { + "Arn": {}, + "Status": {} + } + }, + "AWS::ElastiCache::UserGroup": { + "Attributes": { + "Arn": {}, + "Status": {} + } + }, + "AWS::ElasticBeanstalk::ApplicationVersion": { + "Attributes": { + "Id": {} + } + }, + "AWS::ElasticBeanstalk::ConfigurationTemplate": { + "Attributes": { + "TemplateName": {} + } + }, + "AWS::ElasticBeanstalk::Environment": { + "Attributes": { + "EndpointURL": {} + } + }, + "AWS::ElasticLoadBalancing::LoadBalancer": { + "Attributes": { + "CanonicalHostedZoneName": {}, + "CanonicalHostedZoneNameID": {}, + "DNSName": {}, + "SourceSecurityGroup": {}, + "SourceSecurityGroup.GroupName": {}, + "SourceSecurityGroup.OwnerAlias": {} + } + }, + "AWS::ElasticLoadBalancingV2::Listener": { + "Attributes": { + "ListenerArn": {} + } + }, + "AWS::ElasticLoadBalancingV2::ListenerRule": { + "Attributes": { + "IsDefault": {}, + "RuleArn": {} + } + }, + "AWS::ElasticLoadBalancingV2::LoadBalancer": { + "Attributes": { + "CanonicalHostedZoneID": {}, + "DNSName": {}, + "LoadBalancerArn": {}, + "LoadBalancerFullName": {}, + "LoadBalancerName": {}, + "SecurityGroups": {} + } + }, + "AWS::ElasticLoadBalancingV2::TargetGroup": { + "Attributes": { + "LoadBalancerArns": {}, + "TargetGroupArn": {}, + "TargetGroupFullName": {}, + "TargetGroupName": {} + } + }, + "AWS::ElasticLoadBalancingV2::TrustStore": { + "Attributes": { + "NumberOfCaCertificates": {}, + "Status": {}, + "TrustStoreArn": {} + } + }, + "AWS::ElasticLoadBalancingV2::TrustStoreRevocation": { + "Attributes": { + "RevocationId": {}, + "TrustStoreRevocations": {} + } + }, + "AWS::Elasticsearch::Domain": { + "Attributes": { + "Arn": {}, + "DomainArn": {}, + "DomainEndpoint": {} + } + }, + "AWS::ElementalInference::Dictionary": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::ElementalInference::Feed": { + "Attributes": { + "Arn": {}, + "DataEndpoints": {}, + "Id": {} + } + }, + "AWS::EntityResolution::IdMappingWorkflow": { + "Attributes": { + "CreatedAt": {}, + "UpdatedAt": {}, + "WorkflowArn": {} + } + }, + "AWS::EntityResolution::IdNamespace": { + "Attributes": { + "CreatedAt": {}, + "IdNamespaceArn": {}, + "UpdatedAt": {} + } + }, + "AWS::EntityResolution::MatchingWorkflow": { + "Attributes": { + "CreatedAt": {}, + "UpdatedAt": {}, + "WorkflowArn": {} + } + }, + "AWS::EntityResolution::SchemaMapping": { + "Attributes": { + "CreatedAt": {}, + "HasWorkflows": {}, + "SchemaArn": {}, + "UpdatedAt": {} + } + }, + "AWS::EventSchemas::Discoverer": { + "Attributes": { + "DiscovererArn": {}, + "DiscovererId": {}, + "State": {} + } + }, + "AWS::EventSchemas::Registry": { + "Attributes": { + "RegistryArn": {}, + "RegistryName": {} + } + }, + "AWS::EventSchemas::RegistryPolicy": { + "Attributes": { + "Id": {} + } + }, + "AWS::EventSchemas::Schema": { + "Attributes": { + "LastModified": {}, + "SchemaArn": {}, + "SchemaName": {}, + "SchemaVersion": {}, + "VersionCreatedDate": {} + } + }, + "AWS::Events::ApiDestination": { + "Attributes": { + "Arn": {}, + "ArnForPolicy": {} + } + }, + "AWS::Events::Archive": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Events::Connection": { + "Attributes": { + "Arn": {}, + "ArnForPolicy": {}, + "AuthParameters.ConnectivityParameters.ResourceParameters.ResourceAssociationArn": {}, + "InvocationConnectivityParameters.ResourceParameters.ResourceAssociationArn": {}, + "SecretArn": {} + } + }, + "AWS::Events::Endpoint": { + "Attributes": { + "Arn": {}, + "EndpointId": {}, + "EndpointUrl": {}, + "State": {}, + "StateReason": {} + } + }, + "AWS::Events::EventBus": { + "Attributes": { + "Arn": {}, + "Name": {} + } + }, + "AWS::Events::Replay": { + "Attributes": { + "ReplayArn": {}, + "ReplayStartTime": {}, + "State": {} + } + }, + "AWS::Events::Rule": { + "Attributes": { + "Arn": {}, + "RuleName": {} + } + }, + "AWS::Evidently::Experiment": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Evidently::Feature": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Evidently::Launch": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Evidently::Project": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Evidently::Segment": { + "Attributes": { + "Arn": {} + } + }, + "AWS::FIS::Action": { + "Attributes": { + "Arn": {}, + "Description": {}, + "Id": {}, + "Tags": {} + } + }, + "AWS::FIS::Experiment": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "EndTime": {}, + "Id": {}, + "RoleArn": {}, + "StartTime": {}, + "Tags": {} + } + }, + "AWS::FIS::ExperimentTemplate": { + "Attributes": { + "Id": {} + } + }, + "AWS::FIS::SafetyLever": { + "Attributes": { + "Arn": {}, + "Reason": {}, + "Status": {} + } + }, + "AWS::FMS::ApplicationsList": { + "Attributes": { + "Arn": {}, + "CreateTime": {}, + "LastUpdateTime": {}, + "ListId": {} + } + }, + "AWS::FMS::Policy": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::FMS::ProtocolsList": { + "Attributes": { + "Arn": {}, + "CreateTime": {}, + "LastUpdateTime": {}, + "ListId": {} + } + }, + "AWS::FMS::ResourceSet": { + "Attributes": { + "Id": {} + } + }, + "AWS::FSx::Backup": { + "Attributes": { + "BackupId": {}, + "CreationTime": {}, + "Lifecycle": {}, + "ResourceARN": {}, + "Type": {} + } + }, + "AWS::FSx::DataRepositoryAssociation": { + "Attributes": { + "AssociationId": {}, + "ResourceARN": {} + } + }, + "AWS::FSx::FileCache": { + "Attributes": { + "CreationTime": {}, + "DNSName": {}, + "DataRepositoryAssociationIds": {}, + "FileCacheId": {}, + "Lifecycle": {}, + "LustreConfiguration.LogConfiguration": {}, + "LustreConfiguration.LogConfiguration.Destination": {}, + "LustreConfiguration.LogConfiguration.Level": {}, + "LustreConfiguration.MountName": {}, + "NetworkInterfaceIds": {}, + "OwnerId": {}, + "ResourceARN": {}, + "VpcId": {} + } + }, + "AWS::FSx::FileSystem": { + "Attributes": { + "DNSName": {}, + "LustreMountName": {}, + "ResourceARN": {}, + "RootVolumeId": {} + } + }, + "AWS::FSx::S3AccessPointAttachment": { + "Attributes": { + "Lifecycle": {}, + "S3AccessPoint.Alias": {}, + "S3AccessPoint.ResourceARN": {} + } + }, + "AWS::FSx::Snapshot": { + "Attributes": { + "ResourceARN": {} + } + }, + "AWS::FSx::StorageVirtualMachine": { + "Attributes": { + "ResourceARN": {}, + "StorageVirtualMachineId": {}, + "UUID": {} + } + }, + "AWS::FSx::Volume": { + "Attributes": { + "ResourceARN": {}, + "UUID": {}, + "VolumeId": {} + } + }, + "AWS::FinSpace::Environment": { + "Attributes": { + "AwsAccountId": {}, + "DedicatedServiceAccountId": {}, + "EnvironmentArn": {}, + "EnvironmentId": {}, + "EnvironmentUrl": {}, + "SageMakerStudioDomainUrl": {}, + "Status": {} + } + }, + "AWS::Forecast::Dataset": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Forecast::DatasetGroup": { + "Attributes": { + "DatasetGroupArn": {} + } + }, + "AWS::FraudDetector::Detector": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "DetectorVersionId": {}, + "EventType.Arn": {}, + "EventType.CreatedTime": {}, + "EventType.LastUpdatedTime": {}, + "LastUpdatedTime": {} + } + }, + "AWS::FraudDetector::EntityType": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastUpdatedTime": {} + } + }, + "AWS::FraudDetector::EventType": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastUpdatedTime": {} + } + }, + "AWS::FraudDetector::Label": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastUpdatedTime": {} + } + }, + "AWS::FraudDetector::List": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastUpdatedTime": {} + } + }, + "AWS::FraudDetector::Outcome": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastUpdatedTime": {} + } + }, + "AWS::FraudDetector::Variable": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastUpdatedTime": {} + } + }, + "AWS::GameLift::Alias": { + "Attributes": { + "AliasArn": {}, + "AliasId": {} + } + }, + "AWS::GameLift::Build": { + "Attributes": { + "BuildArn": {}, + "BuildId": {} + } + }, + "AWS::GameLift::ContainerFleet": { + "Attributes": { + "CreationTime": {}, + "DeploymentDetails": {}, + "DeploymentDetails.LatestDeploymentId": {}, + "FleetArn": {}, + "FleetId": {}, + "GameServerContainerGroupDefinitionArn": {}, + "MaximumGameServerContainerGroupsPerInstance": {}, + "PerInstanceContainerGroupDefinitionArn": {}, + "Status": {} + } + }, + "AWS::GameLift::ContainerGroupDefinition": { + "Attributes": { + "ContainerGroupDefinitionArn": {}, + "CreationTime": {}, + "Status": {}, + "StatusReason": {}, + "VersionNumber": {} + } + }, + "AWS::GameLift::Fleet": { + "Attributes": { + "FleetArn": {}, + "FleetId": {} + } + }, + "AWS::GameLift::GameServerGroup": { + "Attributes": { + "AutoScalingGroupArn": {}, + "GameServerGroupArn": {} + } + }, + "AWS::GameLift::GameSessionQueue": { + "Attributes": { + "Arn": {}, + "Name": {} + } + }, + "AWS::GameLift::Location": { + "Attributes": { + "LocationArn": {} + } + }, + "AWS::GameLift::MatchmakingConfiguration": { + "Attributes": { + "Arn": {}, + "Name": {} + } + }, + "AWS::GameLift::MatchmakingRuleSet": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "Name": {} + } + }, + "AWS::GameLift::Script": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "Id": {}, + "SizeOnDisk": {} + } + }, + "AWS::GlobalAccelerator::Accelerator": { + "Attributes": { + "AcceleratorArn": {}, + "DnsName": {}, + "DualStackDnsName": {}, + "Ipv4Addresses": {}, + "Ipv6Addresses": {} + } + }, + "AWS::GlobalAccelerator::CrossAccountAttachment": { + "Attributes": { + "AttachmentArn": {} + } + }, + "AWS::GlobalAccelerator::EndpointGroup": { + "Attributes": { + "EndpointGroupArn": {} + } + }, + "AWS::GlobalAccelerator::Listener": { + "Attributes": { + "ListenerArn": {} + } + }, + "AWS::Glue::Blueprint": { + "Attributes": { + "Arn": {}, + "CreatedOn": {}, + "LastModifiedOn": {}, + "ParameterSpec": {}, + "Status": {} + } + }, + "AWS::Glue::Catalog": { + "Attributes": { + "CatalogId": {}, + "CatalogProperties.CustomProperties": {}, + "CatalogProperties.DataLakeAccessProperties.ManagedWorkgroupName": {}, + "CatalogProperties.DataLakeAccessProperties.ManagedWorkgroupStatus": {}, + "CatalogProperties.DataLakeAccessProperties.RedshiftDatabaseName": {}, + "CreateTime": {}, + "ResourceArn": {}, + "UpdateTime": {} + } + }, + "AWS::Glue::Classifier": { + "Attributes": { + "Name": {} + } + }, + "AWS::Glue::Connection": { + "Attributes": { + "Name": {} + } + }, + "AWS::Glue::ConnectionType": { + "Attributes": { + "ConnectionTypeArn": {} + } + }, + "AWS::Glue::IdentityCenterConfiguration": { + "Attributes": { + "AccountId": {}, + "ApplicationArn": {} + } + }, + "AWS::Glue::Integration": { + "Attributes": { + "CreateTime": {}, + "IntegrationArn": {}, + "Status": {} + } + }, + "AWS::Glue::IntegrationResourceProperty": { + "Attributes": { + "ResourcePropertyArn": {} + } + }, + "AWS::Glue::MLTransform": { + "Attributes": { + "TransformId": {} + } + }, + "AWS::Glue::Registry": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Glue::Schema": { + "Attributes": { + "Arn": {}, + "InitialSchemaVersionId": {} + } + }, + "AWS::Glue::SchemaVersion": { + "Attributes": { + "VersionId": {} + } + }, + "AWS::Glue::Session": { + "Attributes": { + "Arn": {}, + "CreatedOn": {}, + "Progress": {}, + "Status": {} + } + }, + "AWS::Glue::TableVersion": { + "Attributes": { + "Arn": {}, + "VersionId": {} + } + }, + "AWS::Glue::UsageProfile": { + "Attributes": { + "CreatedOn": {} + } + }, + "AWS::Glue::UserDefinedFunction": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Grafana::Workspace": { + "Attributes": { + "CreationTimestamp": {}, + "Endpoint": {}, + "GrafanaVersion": {}, + "Id": {}, + "ModificationTimestamp": {}, + "SamlConfigurationStatus": {}, + "SsoClientId": {}, + "Status": {} + } + }, + "AWS::Greengrass::ConnectorDefinition": { + "Attributes": { + "Arn": {}, + "Id": {}, + "LatestVersionArn": {}, + "Name": {} + } + }, + "AWS::Greengrass::CoreDefinition": { + "Attributes": { + "Arn": {}, + "Id": {}, + "LatestVersionArn": {}, + "Name": {} + } + }, + "AWS::Greengrass::DeviceDefinition": { + "Attributes": { + "Arn": {}, + "Id": {}, + "LatestVersionArn": {}, + "Name": {} + } + }, + "AWS::Greengrass::FunctionDefinition": { + "Attributes": { + "Arn": {}, + "Id": {}, + "LatestVersionArn": {}, + "Name": {} + } + }, + "AWS::Greengrass::Group": { + "Attributes": { + "Arn": {}, + "Id": {}, + "LatestVersionArn": {}, + "Name": {}, + "RoleArn": {}, + "RoleAttachedAt": {} + } + }, + "AWS::Greengrass::LoggerDefinition": { + "Attributes": { + "Arn": {}, + "Id": {}, + "LatestVersionArn": {}, + "Name": {} + } + }, + "AWS::Greengrass::ResourceDefinition": { + "Attributes": { + "Arn": {}, + "Id": {}, + "LatestVersionArn": {}, + "Name": {} + } + }, + "AWS::Greengrass::SubscriptionDefinition": { + "Attributes": { + "Arn": {}, + "Id": {}, + "LatestVersionArn": {}, + "Name": {} + } + }, + "AWS::GreengrassV2::Component": { + "Attributes": { + "Arn": {} + } + }, + "AWS::GreengrassV2::ComponentVersion": { + "Attributes": { + "Arn": {}, + "ComponentName": {}, + "ComponentVersion": {} + } + }, + "AWS::GreengrassV2::CoreDevice": { + "Attributes": { + "Architecture": {}, + "Arn": {}, + "CoreVersion": {}, + "LastStatusUpdateTimestamp": {}, + "Platform": {}, + "Runtime": {}, + "Status": {}, + "Tags": {} + } + }, + "AWS::GreengrassV2::Deployment": { + "Attributes": { + "DeploymentId": {} + } + }, + "AWS::GroundStation::Config": { + "Attributes": { + "Arn": {}, + "Id": {}, + "Type": {} + } + }, + "AWS::GroundStation::DataflowEndpointGroup": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::GroundStation::DataflowEndpointGroupV2": { + "Attributes": { + "Arn": {}, + "EndpointDetails": {}, + "Id": {} + } + }, + "AWS::GroundStation::MissionProfile": { + "Attributes": { + "Arn": {}, + "Id": {}, + "Region": {} + } + }, + "AWS::GroundStation::Satellite": { + "Attributes": { + "Arn": {}, + "CurrentEphemeris": {}, + "CurrentEphemeris.Epoch": {}, + "CurrentEphemeris.Source": {}, + "GroundStations": {}, + "NoradSatelliteID": {}, + "SatelliteId": {}, + "Tags": {} + } + }, + "AWS::GuardDuty::CustomDetectionRuleAssociation": { + "Attributes": { + "AccountId": {}, + "Arn": {}, + "AssociationId": {}, + "CreatedAt": {}, + "UpdatedAt": {} + } + }, + "AWS::GuardDuty::Detector": { + "Attributes": { + "Id": {} + } + }, + "AWS::GuardDuty::IPSet": { + "Attributes": { + "Id": {} + } + }, + "AWS::GuardDuty::MalwareProtectionPlan": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "MalwareProtectionPlanId": {}, + "Status": {}, + "StatusReasons": {} + } + }, + "AWS::GuardDuty::PublishingDestination": { + "Attributes": { + "Id": {}, + "PublishingFailureStartTimestamp": {}, + "Status": {} + } + }, + "AWS::GuardDuty::ThreatEntitySet": { + "Attributes": { + "CreatedAt": {}, + "ErrorDetails": {}, + "Id": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::GuardDuty::ThreatIntelSet": { + "Attributes": { + "Id": {} + } + }, + "AWS::GuardDuty::TrustedEntitySet": { + "Attributes": { + "CreatedAt": {}, + "ErrorDetails": {}, + "Id": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::HealthAgent::Domain": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "DomainId": {}, + "EncryptionContext": {}, + "EncryptionContext.EncryptionType": {}, + "Status": {} + } + }, + "AWS::HealthAgent::Subscription": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "LastUpdatedAt": {}, + "Status": {}, + "SubscriptionId": {} + } + }, + "AWS::HealthImaging::Datastore": { + "Attributes": { + "CreatedAt": {}, + "DatastoreArn": {}, + "DatastoreId": {}, + "DatastoreStatus": {}, + "UpdatedAt": {} + } + }, + "AWS::HealthLake::DataTransformationProfile": { + "Attributes": { + "Arn": {}, + "ProfileId": {}, + "TargetFormat": {} + } + }, + "AWS::HealthLake::FHIRDatastore": { + "Attributes": { + "CreatedAt": {}, + "CreatedAt.Nanos": {}, + "CreatedAt.Seconds": {}, + "DatastoreArn": {}, + "DatastoreEndpoint": {}, + "DatastoreId": {}, + "DatastoreStatus": {} + } + }, + "AWS::IAM::AccessKey": { + "Attributes": { + "SecretAccessKey": {} + } + }, + "AWS::IAM::Group": { + "Attributes": { + "Arn": {} + } + }, + "AWS::IAM::InstanceProfile": { + "Attributes": { + "Arn": {} + } + }, + "AWS::IAM::ManagedPolicy": { + "Attributes": { + "AttachmentCount": {}, + "CreateDate": {}, + "DefaultVersionId": {}, + "IsAttachable": {}, + "PermissionsBoundaryUsageCount": {}, + "PolicyArn": {}, + "PolicyId": {}, + "UpdateDate": {} + } + }, + "AWS::IAM::OIDCProvider": { + "Attributes": { + "Arn": {} + } + }, + "AWS::IAM::Policy": { + "Attributes": { + "Id": {} + } + }, + "AWS::IAM::Role": { + "Attributes": { + "Arn": {}, + "RoleId": {} + } + }, + "AWS::IAM::SAMLProvider": { + "Attributes": { + "Arn": {}, + "SamlProviderUUID": {} + } + }, + "AWS::IAM::ServerCertificate": { + "Attributes": { + "Arn": {} + } + }, + "AWS::IAM::ServiceLinkedRole": { + "Attributes": { + "RoleName": {} + } + }, + "AWS::IAM::User": { + "Attributes": { + "Arn": {} + } + }, + "AWS::IAM::VirtualMFADevice": { + "Attributes": { + "SerialNumber": {} + } + }, + "AWS::IVS::Channel": { + "Attributes": { + "Arn": {}, + "IngestEndpoint": {}, + "PlaybackUrl": {} + } + }, + "AWS::IVS::Composition": { + "Attributes": { + "Arn": {}, + "StartTime": {}, + "State": {} + } + }, + "AWS::IVS::EncoderConfiguration": { + "Attributes": { + "Arn": {} + } + }, + "AWS::IVS::IngestConfiguration": { + "Attributes": { + "Arn": {}, + "ParticipantId": {}, + "State": {}, + "StreamKey": {} + } + }, + "AWS::IVS::PlaybackKeyPair": { + "Attributes": { + "Arn": {}, + "Fingerprint": {} + } + }, + "AWS::IVS::PlaybackRestrictionPolicy": { + "Attributes": { + "Arn": {} + } + }, + "AWS::IVS::PublicKey": { + "Attributes": { + "Arn": {}, + "Fingerprint": {} + } + }, + "AWS::IVS::RecordingConfiguration": { + "Attributes": { + "Arn": {}, + "State": {} + } + }, + "AWS::IVS::Stage": { + "Attributes": { + "ActiveSessionId": {}, + "Arn": {} + } + }, + "AWS::IVS::StorageConfiguration": { + "Attributes": { + "Arn": {} + } + }, + "AWS::IVS::StreamKey": { + "Attributes": { + "Arn": {}, + "Value": {} + } + }, + "AWS::IVSChat::LoggingConfiguration": { + "Attributes": { + "Arn": {}, + "Id": {}, + "State": {} + } + }, + "AWS::IVSChat::Room": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::IdentityStore::AllGroupMemberships": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "CreatedBy": {}, + "MembershipId": {}, + "UpdatedAt": {}, + "UpdatedBy": {} + } + }, + "AWS::IdentityStore::Group": { + "Attributes": { + "GroupId": {} + } + }, + "AWS::IdentityStore::GroupMembership": { + "Attributes": { + "MembershipId": {} + } + }, + "AWS::IdentityStore::User": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "CreatedBy": {}, + "UpdatedAt": {}, + "UpdatedBy": {}, + "UserId": {}, + "UserStatus": {} + } + }, + "AWS::ImageBuilder::AllImageBuildVersions": { + "Attributes": { + "Arn": {}, + "BuildNumber": {}, + "BuildType": {}, + "DateCreated": {}, + "EnhancedImageMetadataEnabled": {}, + "ExecutionRole": {}, + "ImageTestsConfiguration": {}, + "ImageTestsConfiguration.ImageTestsEnabled": {}, + "ImageTestsConfiguration.TimeoutMinutes": {}, + "ImageVersionArn": {}, + "Name": {}, + "OsVersion": {}, + "Platform": {}, + "State": {}, + "State.Status": {}, + "Tags": {}, + "Type": {}, + "Version": {} + } + }, + "AWS::ImageBuilder::AllWorkflowBuildVersions": { + "Attributes": { + "Arn": {}, + "BuildNumber": {}, + "ChangeDescription": {}, + "Data": {}, + "DateCreated": {}, + "Description": {}, + "Name": {}, + "Owner": {}, + "Parameters": {}, + "Tags": {}, + "Version": {}, + "WorkflowType": {}, + "WorkflowVersionArn": {} + } + }, + "AWS::ImageBuilder::Component": { + "Attributes": { + "Arn": {}, + "Encrypted": {}, + "LatestVersion": {}, + "LatestVersion.Arn": {}, + "LatestVersion.Major": {}, + "LatestVersion.Minor": {}, + "LatestVersion.Patch": {}, + "Name": {}, + "Type": {} + } + }, + "AWS::ImageBuilder::ContainerRecipe": { + "Attributes": { + "Arn": {}, + "LatestVersion": {}, + "LatestVersion.Arn": {}, + "LatestVersion.Major": {}, + "LatestVersion.Minor": {}, + "LatestVersion.Patch": {}, + "Name": {} + } + }, + "AWS::ImageBuilder::DistributionConfiguration": { + "Attributes": { + "Arn": {}, + "Name": {} + } + }, + "AWS::ImageBuilder::Image": { + "Attributes": { + "Arn": {}, + "ImageId": {}, + "ImageUri": {}, + "LatestVersion": {}, + "LatestVersion.Arn": {}, + "LatestVersion.Major": {}, + "LatestVersion.Minor": {}, + "LatestVersion.Patch": {}, + "Name": {} + } + }, + "AWS::ImageBuilder::ImagePipeline": { + "Attributes": { + "Arn": {}, + "DeploymentId": {}, + "Name": {} + } + }, + "AWS::ImageBuilder::ImageRecipe": { + "Attributes": { + "Arn": {}, + "LatestVersion": {}, + "LatestVersion.Arn": {}, + "LatestVersion.Major": {}, + "LatestVersion.Minor": {}, + "LatestVersion.Patch": {}, + "Name": {} + } + }, + "AWS::ImageBuilder::InfrastructureConfiguration": { + "Attributes": { + "Arn": {}, + "Name": {} + } + }, + "AWS::ImageBuilder::LifecycleExecution": { + "Attributes": { + "Arn": {}, + "EndTime": {}, + "LifecycleExecutionId": {}, + "ResourcesImpactedSummary": {}, + "ResourcesImpactedSummary.HasImpactedResources": {}, + "StartTime": {}, + "State": {}, + "State.Status": {} + } + }, + "AWS::ImageBuilder::LifecyclePolicy": { + "Attributes": { + "Arn": {} + } + }, + "AWS::ImageBuilder::Workflow": { + "Attributes": { + "Arn": {}, + "LatestVersion": {}, + "LatestVersion.Arn": {}, + "LatestVersion.Major": {}, + "LatestVersion.Minor": {}, + "LatestVersion.Patch": {} + } + }, + "AWS::ImageBuilder::WorkflowExecution": { + "Attributes": { + "Arn": {}, + "EndTime": {}, + "StartTime": {}, + "Status": {}, + "TotalStepCount": {}, + "TotalStepsFailed": {}, + "TotalStepsSkipped": {}, + "TotalStepsSucceeded": {}, + "Type": {}, + "WorkflowBuildVersionArn": {}, + "WorkflowExecutionId": {} + } + }, + "AWS::ImageBuilder::WorkflowStepExecution": { + "Attributes": { + "Action": {}, + "EndTime": {}, + "ImageBuildVersionArn": {}, + "Inputs": {}, + "Name": {}, + "OnFailure": {}, + "Outputs": {}, + "StartTime": {}, + "Status": {}, + "StepExecutionId": {}, + "TimeoutSeconds": {}, + "WorkflowBuildVersionArn": {}, + "WorkflowExecutionId": {} + } + }, + "AWS::Inspector::AssessmentTarget": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Inspector::AssessmentTemplate": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Inspector::ResourceGroup": { + "Attributes": { + "Arn": {} + } + }, + "AWS::InspectorV2::CisScanConfiguration": { + "Attributes": { + "Arn": {} + } + }, + "AWS::InspectorV2::CodeSecurityIntegration": { + "Attributes": { + "Arn": {}, + "AuthorizationUrl": {}, + "CreatedAt": {}, + "LastUpdatedAt": {}, + "Status": {}, + "StatusReason": {} + } + }, + "AWS::InspectorV2::CodeSecurityScanConfiguration": { + "Attributes": { + "Arn": {} + } + }, + "AWS::InspectorV2::Connector": { + "Attributes": { + "ConnectorArn": {}, + "CreatedAt": {}, + "EnablementStatus": {}, + "EnablementStatusReason": {}, + "Health": {}, + "Health.ConnectorStatus": {}, + "Health.LastCheckedAt": {}, + "Health.Message": {}, + "LastUpdatedAt": {}, + "ProviderConfiguration.Azure.ScopeConfiguration.ContainerImageScanning.State": {}, + "ProviderConfiguration.Azure.ScopeConfiguration.ContainerImageScanning.StateReason": {}, + "ProviderConfiguration.Azure.ScopeConfiguration.ServerlessScanning.State": {}, + "ProviderConfiguration.Azure.ScopeConfiguration.ServerlessScanning.StateReason": {}, + "ProviderConfiguration.Azure.ScopeConfiguration.VmScanning.State": {}, + "ProviderConfiguration.Azure.ScopeConfiguration.VmScanning.StateReason": {} + } + }, + "AWS::InspectorV2::Filter": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Interconnect::Connection": { + "Attributes": { + "Arn": {}, + "BillingTier": {}, + "ConnectionId": {}, + "OwnerAccount": {}, + "Provider": {}, + "Provider.CloudServiceProvider": {}, + "Provider.LastMileProvider": {}, + "SharedId": {}, + "State": {}, + "Type": {} + } + }, + "AWS::InternetMonitor::InternetEvent": { + "Attributes": { + "ClientLocation": {}, + "ClientLocation.ASName": {}, + "ClientLocation.ASNumber": {}, + "ClientLocation.City": {}, + "ClientLocation.Country": {}, + "ClientLocation.Latitude": {}, + "ClientLocation.Longitude": {}, + "ClientLocation.Metro": {}, + "ClientLocation.Subdivision": {}, + "EndedAt": {}, + "EventArn": {}, + "EventId": {}, + "EventStatus": {}, + "EventType": {}, + "StartedAt": {} + } + }, + "AWS::InternetMonitor::Monitor": { + "Attributes": { + "CreatedAt": {}, + "ModifiedAt": {}, + "MonitorArn": {}, + "ProcessingStatus": {}, + "ProcessingStatusInfo": {} + } + }, + "AWS::Invoicing::InvoiceUnit": { + "Attributes": { + "InvoiceUnitArn": {}, + "LastModified": {} + } + }, + "AWS::Invoicing::ProcurementPortalPreference": { + "Attributes": { + "AwsAccountId": {}, + "CreateDate": {}, + "EinvoiceDeliveryPreferenceStatus": {}, + "LastUpdateDate": {}, + "ProcurementPortalPreferenceArn": {}, + "PurchaseOrderRetrievalEndpoint": {}, + "PurchaseOrderRetrievalPreferenceStatus": {}, + "Version": {} + } + }, + "AWS::IoT::Authorizer": { + "Attributes": { + "Arn": {} + } + }, + "AWS::IoT::BillingGroup": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::IoT::CACertificate": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::IoT::Certificate": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::IoT::CertificateProvider": { + "Attributes": { + "Arn": {} + } + }, + "AWS::IoT::Command": { + "Attributes": { + "CommandArn": {} + } + }, + "AWS::IoT::CustomMetric": { + "Attributes": { + "MetricArn": {} + } + }, + "AWS::IoT::Dimension": { + "Attributes": { + "Arn": {} + } + }, + "AWS::IoT::DomainConfiguration": { + "Attributes": { + "Arn": {}, + "DomainType": {}, + "ServerCertificates": {} + } + }, + "AWS::IoT::EncryptionConfiguration": { + "Attributes": { + "AccountId": {}, + "ConfigurationDetails": {}, + "ConfigurationDetails.ConfigurationStatus": {}, + "ConfigurationDetails.ErrorCode": {}, + "ConfigurationDetails.ErrorMessage": {}, + "LastModifiedDate": {} + } + }, + "AWS::IoT::FleetMetric": { + "Attributes": { + "CreationDate": {}, + "LastModifiedDate": {}, + "MetricArn": {}, + "Version": {} + } + }, + "AWS::IoT::Index": { + "Attributes": { + "Arn": {}, + "IndexStatus": {}, + "Schema": {} + } + }, + "AWS::IoT::Job": { + "Attributes": { + "Arn": {}, + "CreatedAt": {} + } + }, + "AWS::IoT::JobTemplate": { + "Attributes": { + "Arn": {} + } + }, + "AWS::IoT::MitigationAction": { + "Attributes": { + "MitigationActionArn": {}, + "MitigationActionId": {} + } + }, + "AWS::IoT::Policy": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::IoT::ProvisioningTemplate": { + "Attributes": { + "TemplateArn": {} + } + }, + "AWS::IoT::ResourceSpecificLogging": { + "Attributes": { + "TargetId": {} + } + }, + "AWS::IoT::RoleAlias": { + "Attributes": { + "RoleAliasArn": {} + } + }, + "AWS::IoT::ScheduledAudit": { + "Attributes": { + "ScheduledAuditArn": {} + } + }, + "AWS::IoT::SecurityProfile": { + "Attributes": { + "SecurityProfileArn": {} + } + }, + "AWS::IoT::SoftwarePackage": { + "Attributes": { + "PackageArn": {} + } + }, + "AWS::IoT::SoftwarePackageVersion": { + "Attributes": { + "ErrorReason": {}, + "PackageVersionArn": {}, + "SbomValidationStatus": {}, + "Status": {} + } + }, + "AWS::IoT::Stream": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "LastUpdatedAt": {}, + "StreamVersion": {} + } + }, + "AWS::IoT::Thing": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::IoT::ThingGroup": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::IoT::ThingType": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::IoT::TopicRule": { + "Attributes": { + "Arn": {} + } + }, + "AWS::IoT::TopicRuleDestination": { + "Attributes": { + "Arn": {}, + "StatusReason": {} + } + }, + "AWS::IoTAnalytics::Channel": { + "Attributes": { + "Id": {} + } + }, + "AWS::IoTAnalytics::Dataset": { + "Attributes": { + "Id": {} + } + }, + "AWS::IoTAnalytics::Datastore": { + "Attributes": { + "Id": {} + } + }, + "AWS::IoTAnalytics::Pipeline": { + "Attributes": { + "Id": {} + } + }, + "AWS::IoTCoreDeviceAdvisor::SuiteDefinition": { + "Attributes": { + "SuiteDefinitionArn": {}, + "SuiteDefinitionId": {}, + "SuiteDefinitionVersion": {} + } + }, + "AWS::IoTDeviceAdvisor::SuiteRun": { + "Attributes": { + "StartTime": {}, + "Status": {}, + "SuiteRunArn": {}, + "SuiteRunId": {} + } + }, + "AWS::IoTFleetWise::Campaign": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "LastModificationTime": {}, + "Status": {} + } + }, + "AWS::IoTFleetWise::DecoderManifest": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "LastModificationTime": {} + } + }, + "AWS::IoTFleetWise::Fleet": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "LastModificationTime": {} + } + }, + "AWS::IoTFleetWise::ModelManifest": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "LastModificationTime": {} + } + }, + "AWS::IoTFleetWise::SignalCatalog": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "LastModificationTime": {}, + "NodeCounts.TotalActuators": {}, + "NodeCounts.TotalAttributes": {}, + "NodeCounts.TotalBranches": {}, + "NodeCounts.TotalNodes": {}, + "NodeCounts.TotalSensors": {} + } + }, + "AWS::IoTFleetWise::StateTemplate": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "Id": {}, + "LastModificationTime": {} + } + }, + "AWS::IoTFleetWise::Vehicle": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "LastModificationTime": {} + } + }, + "AWS::IoTSecureTunneling::Tunnel": { + "Attributes": { + "Status": {}, + "TunnelArn": {}, + "TunnelId": {} + } + }, + "AWS::IoTSiteWise::AccessPolicy": { + "Attributes": { + "AccessPolicyArn": {}, + "AccessPolicyId": {} + } + }, + "AWS::IoTSiteWise::Asset": { + "Attributes": { + "AssetArn": {}, + "AssetId": {} + } + }, + "AWS::IoTSiteWise::AssetModel": { + "Attributes": { + "AssetModelArn": {}, + "AssetModelId": {} + } + }, + "AWS::IoTSiteWise::ComputationModel": { + "Attributes": { + "ComputationModelArn": {}, + "ComputationModelId": {} + } + }, + "AWS::IoTSiteWise::Dashboard": { + "Attributes": { + "DashboardArn": {}, + "DashboardId": {} + } + }, + "AWS::IoTSiteWise::Dataset": { + "Attributes": { + "DatasetArn": {}, + "DatasetId": {} + } + }, + "AWS::IoTSiteWise::Gateway": { + "Attributes": { + "GatewayId": {} + } + }, + "AWS::IoTSiteWise::Pipeline": { + "Attributes": { + "PipelineArn": {}, + "Status": {} + } + }, + "AWS::IoTSiteWise::Portal": { + "Attributes": { + "PortalArn": {}, + "PortalClientId": {}, + "PortalId": {}, + "PortalStartUrl": {} + } + }, + "AWS::IoTSiteWise::Project": { + "Attributes": { + "ProjectArn": {}, + "ProjectId": {} + } + }, + "AWS::IoTSiteWise::Task": { + "Attributes": { + "Status": {}, + "TaskArn": {} + } + }, + "AWS::IoTSiteWise::Workspace": { + "Attributes": { + "CreatedAt": {}, + "Status": {}, + "UpdatedAt": {}, + "WorkspaceArn": {} + } + }, + "AWS::IoTTwinMaker::ComponentType": { + "Attributes": { + "Arn": {}, + "CreationDateTime": {}, + "IsAbstract": {}, + "IsSchemaInitialized": {}, + "Status": {}, + "Status.Error": {}, + "Status.Error.Code": {}, + "Status.Error.Message": {}, + "Status.State": {}, + "UpdateDateTime": {} + } + }, + "AWS::IoTTwinMaker::Entity": { + "Attributes": { + "Arn": {}, + "CreationDateTime": {}, + "HasChildEntities": {}, + "Status": {}, + "Status.Error": {}, + "Status.State": {}, + "UpdateDateTime": {} + } + }, + "AWS::IoTTwinMaker::MetadataTransferJob": { + "Attributes": { + "Arn": {}, + "CreationDateTime": {}, + "Status": {}, + "Status.State": {}, + "UpdateDateTime": {} + } + }, + "AWS::IoTTwinMaker::Scene": { + "Attributes": { + "Arn": {}, + "CreationDateTime": {}, + "GeneratedSceneMetadata": {}, + "UpdateDateTime": {} + } + }, + "AWS::IoTTwinMaker::SyncJob": { + "Attributes": { + "Arn": {}, + "CreationDateTime": {}, + "State": {}, + "UpdateDateTime": {} + } + }, + "AWS::IoTTwinMaker::Workspace": { + "Attributes": { + "Arn": {}, + "CreationDateTime": {}, + "UpdateDateTime": {} + } + }, + "AWS::IoTWireless::Destination": { + "Attributes": { + "Arn": {} + } + }, + "AWS::IoTWireless::DeviceProfile": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::IoTWireless::FuotaTask": { + "Attributes": { + "Arn": {}, + "FuotaTaskStatus": {}, + "Id": {}, + "LoRaWAN.StartTime": {} + } + }, + "AWS::IoTWireless::MulticastGroup": { + "Attributes": { + "Arn": {}, + "Id": {}, + "LoRaWAN.NumberOfDevicesInGroup": {}, + "LoRaWAN.NumberOfDevicesRequested": {}, + "Status": {} + } + }, + "AWS::IoTWireless::NetworkAnalyzerConfiguration": { + "Attributes": { + "Arn": {} + } + }, + "AWS::IoTWireless::PartnerAccount": { + "Attributes": { + "Arn": {}, + "Fingerprint": {} + } + }, + "AWS::IoTWireless::ServiceProfile": { + "Attributes": { + "Arn": {}, + "Id": {}, + "LoRaWAN.ChannelMask": {}, + "LoRaWAN.DevStatusReqFreq": {}, + "LoRaWAN.DlBucketSize": {}, + "LoRaWAN.DlRate": {}, + "LoRaWAN.DlRatePolicy": {}, + "LoRaWAN.DrMax": {}, + "LoRaWAN.DrMin": {}, + "LoRaWAN.HrAllowed": {}, + "LoRaWAN.MinGwDiversity": {}, + "LoRaWAN.NwkGeoLoc": {}, + "LoRaWAN.ReportDevStatusBattery": {}, + "LoRaWAN.ReportDevStatusMargin": {}, + "LoRaWAN.TargetPer": {}, + "LoRaWAN.UlBucketSize": {}, + "LoRaWAN.UlRate": {}, + "LoRaWAN.UlRatePolicy": {} + } + }, + "AWS::IoTWireless::TaskDefinition": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::IoTWireless::WirelessDevice": { + "Attributes": { + "Arn": {}, + "Id": {}, + "ThingName": {} + } + }, + "AWS::IoTWireless::WirelessDeviceImportTask": { + "Attributes": { + "Arn": {}, + "CreationDate": {}, + "FailedImportedDevicesCount": {}, + "Id": {}, + "InitializedImportedDevicesCount": {}, + "OnboardedImportedDevicesCount": {}, + "PendingImportedDevicesCount": {}, + "Sidewalk.DeviceCreationFileList": {}, + "Status": {}, + "StatusReason": {} + } + }, + "AWS::IoTWireless::WirelessGateway": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::KMS::Key": { + "Attributes": { + "Arn": {}, + "KeyId": {} + } + }, + "AWS::KMS::ReplicaKey": { + "Attributes": { + "Arn": {}, + "KeyId": {} + } + }, + "AWS::KafkaConnect::Connector": { + "Attributes": { + "ConnectorArn": {} + } + }, + "AWS::KafkaConnect::ConnectorOperation": { + "Attributes": { + "ConnectorArn": {}, + "ConnectorOperationArn": {}, + "ConnectorOperationState": {}, + "ConnectorOperationType": {}, + "CreationTime": {}, + "EndTime": {}, + "OperationSteps": {}, + "OriginWorkerSetting": {}, + "OriginWorkerSetting.Capacity": {}, + "OriginWorkerSetting.Capacity.ProvisionedCapacity": {}, + "OriginWorkerSetting.Capacity.ProvisionedCapacity.McuCount": {}, + "OriginWorkerSetting.Capacity.ProvisionedCapacity.WorkerCount": {}, + "TargetWorkerSetting": {}, + "TargetWorkerSetting.Capacity": {} + } + }, + "AWS::KafkaConnect::CustomPlugin": { + "Attributes": { + "CustomPluginArn": {}, + "FileDescription": {}, + "FileDescription.FileMd5": {}, + "FileDescription.FileSize": {}, + "Revision": {} + } + }, + "AWS::KafkaConnect::WorkerConfiguration": { + "Attributes": { + "Revision": {}, + "WorkerConfigurationArn": {} + } + }, + "AWS::Kendra::DataSource": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::Kendra::Faq": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::Kendra::FeaturedResultsSet": { + "Attributes": { + "Arn": {}, + "FeaturedResultsSetId": {} + } + }, + "AWS::Kendra::Index": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::Kendra::QuerySuggestionsBlockList": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::Kendra::Thesaurus": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::KendraRanking::ExecutionPlan": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::Kinesis::Channel": { + "Attributes": { + "ChannelARN": {}, + "ChannelCreationTimestamp": {}, + "ChannelId": {}, + "ChannelStatus": {} + } + }, + "AWS::Kinesis::Stream": { + "Attributes": { + "Arn": {}, + "WarmThroughputObject": {}, + "WarmThroughputObject.CurrentMiBps": {}, + "WarmThroughputObject.TargetMiBps": {} + } + }, + "AWS::Kinesis::StreamConsumer": { + "Attributes": { + "ConsumerARN": {}, + "ConsumerCreationTimestamp": {}, + "ConsumerName": {}, + "ConsumerStatus": {}, + "StreamARN": {} + } + }, + "AWS::KinesisFirehose::DeliveryStream": { + "Attributes": { + "Arn": {} + } + }, + "AWS::KinesisVideo::SignalingChannel": { + "Attributes": { + "Arn": {} + } + }, + "AWS::KinesisVideo::Stream": { + "Attributes": { + "Arn": {} + } + }, + "AWS::LakeFormation::PrincipalPermissions": { + "Attributes": { + "PrincipalIdentifier": {}, + "ResourceIdentifier": {} + } + }, + "AWS::LakeFormation::TagAssociation": { + "Attributes": { + "ResourceIdentifier": {}, + "TagsIdentifier": {} + } + }, + "AWS::Lambda::Alias": { + "Attributes": { + "AliasArn": {} + } + }, + "AWS::Lambda::CapacityProvider": { + "Attributes": { + "Arn": {}, + "State": {} + } + }, + "AWS::Lambda::CodeSigningConfig": { + "Attributes": { + "CodeSigningConfigArn": {}, + "CodeSigningConfigId": {} + } + }, + "AWS::Lambda::DurableExecution": { + "Attributes": { + "DurableExecutionArn": {}, + "DurableExecutionName": {}, + "EndTimestamp": {}, + "FunctionArn": {}, + "StartTimestamp": {}, + "Status": {}, + "Version": {} + } + }, + "AWS::Lambda::EventSourceMapping": { + "Attributes": { + "EventSourceMappingArn": {}, + "Id": {} + } + }, + "AWS::Lambda::Function": { + "Attributes": { + "Arn": {}, + "SnapStartResponse": {}, + "SnapStartResponse.ApplyOn": {}, + "SnapStartResponse.OptimizationStatus": {} + } + }, + "AWS::Lambda::LayerVersion": { + "Attributes": { + "LayerVersionArn": {} + } + }, + "AWS::Lambda::LayerVersionPermission": { + "Attributes": { + "Id": {} + } + }, + "AWS::Lambda::MicrovmImage": { + "Attributes": { + "CreatedAt": {}, + "ImageArn": {}, + "LatestActiveImageVersion": {}, + "LatestFailedImageVersion": {}, + "State": {}, + "UpdatedAt": {} + } + }, + "AWS::Lambda::NetworkConnector": { + "Attributes": { + "Arn": {}, + "State": {} + } + }, + "AWS::Lambda::Permission": { + "Attributes": { + "Id": {} + } + }, + "AWS::Lambda::Url": { + "Attributes": { + "FunctionArn": {}, + "FunctionUrl": {} + } + }, + "AWS::Lambda::Version": { + "Attributes": { + "FunctionArn": {}, + "Version": {} + } + }, + "AWS::LaunchWizard::Deployment": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "DeletedAt": {}, + "DeploymentId": {}, + "ResourceGroup": {}, + "Status": {} + } + }, + "AWS::Lex::Bot": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::Lex::BotAlias": { + "Attributes": { + "Arn": {}, + "BotAliasId": {}, + "BotAliasStatus": {} + } + }, + "AWS::Lex::BotVersion": { + "Attributes": { + "BotVersion": {} + } + }, + "AWS::Lex::ResourcePolicy": { + "Attributes": { + "Id": {}, + "RevisionId": {} + } + }, + "AWS::LicenseManager::Grant": { + "Attributes": { + "GrantArn": {}, + "Version": {} + } + }, + "AWS::LicenseManager::License": { + "Attributes": { + "LicenseArn": {}, + "Version": {} + } + }, + "AWS::LicenseManager::LicenseAssetGroup": { + "Attributes": { + "LicenseAssetGroupArn": {} + } + }, + "AWS::LicenseManager::LicenseAssetRuleSet": { + "Attributes": { + "LicenseAssetRulesetArn": {} + } + }, + "AWS::Lightsail::Alarm": { + "Attributes": { + "AlarmArn": {}, + "State": {} + } + }, + "AWS::Lightsail::Bucket": { + "Attributes": { + "AbleToUpdateBundle": {}, + "BucketArn": {}, + "Url": {} + } + }, + "AWS::Lightsail::Certificate": { + "Attributes": { + "CertificateArn": {}, + "Status": {} + } + }, + "AWS::Lightsail::ContactMethod": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Name": {}, + "ResourceType": {}, + "Status": {}, + "SupportCode": {} + } + }, + "AWS::Lightsail::Container": { + "Attributes": { + "ContainerArn": {}, + "PrincipalArn": {}, + "PrivateRegistryAccess.EcrImagePullerRole.PrincipalArn": {}, + "Url": {} + } + }, + "AWS::Lightsail::Database": { + "Attributes": { + "DatabaseArn": {} + } + }, + "AWS::Lightsail::DatabaseSnapshot": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Engine": {}, + "EngineVersion": {}, + "FromRelationalDatabaseArn": {}, + "FromRelationalDatabaseBlueprintId": {}, + "FromRelationalDatabaseBundleId": {}, + "FromRelationalDatabaseName": {}, + "Location": {}, + "Location.AvailabilityZone": {}, + "Location.RegionName": {}, + "Name": {}, + "ResourceType": {}, + "SizeInGb": {}, + "State": {}, + "SupportCode": {} + } + }, + "AWS::Lightsail::Disk": { + "Attributes": { + "AttachedTo": {}, + "AttachmentState": {}, + "DiskArn": {}, + "Iops": {}, + "IsAttached": {}, + "Location.AvailabilityZone": {}, + "Location.RegionName": {}, + "Path": {}, + "ResourceType": {}, + "State": {}, + "SupportCode": {} + } + }, + "AWS::Lightsail::DiskSnapshot": { + "Attributes": { + "CreatedAt": {}, + "DiskSnapshotArn": {}, + "FromDiskName": {}, + "IsFromAutoSnapshot": {}, + "Location": {}, + "Location.AvailabilityZone": {}, + "Location.RegionName": {}, + "Progress": {}, + "ResourceType": {}, + "SizeInGb": {}, + "State": {}, + "SupportCode": {} + } + }, + "AWS::Lightsail::Distribution": { + "Attributes": { + "AbleToUpdateBundle": {}, + "DistributionArn": {}, + "Status": {} + } + }, + "AWS::Lightsail::Domain": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Location": {}, + "Location.AvailabilityZone": {}, + "Location.RegionName": {}, + "ResourceType": {}, + "SupportCode": {} + } + }, + "AWS::Lightsail::ExportSnapshotRecord": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "DestinationInfo": {}, + "DestinationInfo.Id": {}, + "DestinationInfo.Service": {}, + "Location": {}, + "Location.AvailabilityZone": {}, + "Location.RegionName": {}, + "Name": {}, + "RecordId": {}, + "ResourceType": {}, + "SourceInfo": {}, + "SourceInfo.Arn": {}, + "SourceInfo.CreatedAt": {}, + "SourceInfo.FromResourceArn": {}, + "SourceInfo.FromResourceName": {}, + "SourceInfo.InstanceSnapshotInfo": {}, + "SourceInfo.InstanceSnapshotInfo.FromBlueprintId": {}, + "SourceInfo.InstanceSnapshotInfo.FromBundleId": {}, + "SourceInfo.InstanceSnapshotInfo.FromDiskInfo": {}, + "SourceInfo.Name": {}, + "SourceInfo.ResourceType": {}, + "State": {} + } + }, + "AWS::Lightsail::Instance": { + "Attributes": { + "Hardware.CpuCount": {}, + "Hardware.RamSizeInGb": {}, + "InstanceArn": {}, + "Ipv6Addresses": {}, + "IsStaticIp": {}, + "Location.AvailabilityZone": {}, + "Location.RegionName": {}, + "Networking.MonthlyTransfer.GbPerMonthAllocated": {}, + "PrivateIpAddress": {}, + "PublicIpAddress": {}, + "ResourceType": {}, + "SshKeyName": {}, + "State.Code": {}, + "State.Name": {}, + "SupportCode": {}, + "UserName": {} + } + }, + "AWS::Lightsail::InstanceSnapshot": { + "Attributes": { + "Arn": {}, + "FromInstanceArn": {}, + "FromInstanceName": {}, + "IsFromAutoSnapshot": {}, + "Location": {}, + "Location.AvailabilityZone": {}, + "Location.RegionName": {}, + "ResourceType": {}, + "SizeInGb": {}, + "State": {}, + "SupportCode": {} + } + }, + "AWS::Lightsail::KeyPair": { + "Attributes": { + "CreatedAt": {}, + "Fingerprint": {}, + "KeyPairArn": {}, + "Location": {}, + "Location.AvailabilityZone": {}, + "Location.RegionName": {}, + "ResourceType": {}, + "SupportCode": {} + } + }, + "AWS::Lightsail::LoadBalancer": { + "Attributes": { + "LoadBalancerArn": {} + } + }, + "AWS::Lightsail::LoadBalancerTlsCertificate": { + "Attributes": { + "LoadBalancerTlsCertificateArn": {}, + "Status": {} + } + }, + "AWS::Lightsail::StaticIp": { + "Attributes": { + "IpAddress": {}, + "IsAttached": {}, + "StaticIpArn": {} + } + }, + "AWS::Location::APIKey": { + "Attributes": { + "Arn": {}, + "CreateTime": {}, + "KeyArn": {}, + "UpdateTime": {} + } + }, + "AWS::Location::GeofenceCollection": { + "Attributes": { + "Arn": {}, + "CollectionArn": {}, + "CreateTime": {}, + "UpdateTime": {} + } + }, + "AWS::Location::Job": { + "Attributes": { + "CreatedAt": {}, + "JobArn": {}, + "JobId": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::Location::Map": { + "Attributes": { + "Arn": {}, + "CreateTime": {}, + "MapArn": {}, + "UpdateTime": {} + } + }, + "AWS::Location::PlaceIndex": { + "Attributes": { + "Arn": {}, + "CreateTime": {}, + "IndexArn": {}, + "UpdateTime": {} + } + }, + "AWS::Location::RouteCalculator": { + "Attributes": { + "Arn": {}, + "CalculatorArn": {}, + "CreateTime": {}, + "UpdateTime": {} + } + }, + "AWS::Location::Tracker": { + "Attributes": { + "Arn": {}, + "CreateTime": {}, + "TrackerArn": {}, + "UpdateTime": {} + } + }, + "AWS::Logs::AccountPolicy": { + "Attributes": { + "AccountId": {} + } + }, + "AWS::Logs::Delivery": { + "Attributes": { + "Arn": {}, + "DeliveryDestinationType": {}, + "DeliveryId": {} + } + }, + "AWS::Logs::DeliveryDestination": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Logs::DeliverySource": { + "Attributes": { + "Arn": {}, + "ResourceArns": {}, + "Service": {}, + "Status": {}, + "StatusReason": {} + } + }, + "AWS::Logs::Destination": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Logs::Integration": { + "Attributes": { + "IntegrationStatus": {} + } + }, + "AWS::Logs::LogAnomalyDetector": { + "Attributes": { + "AnomalyDetectorArn": {}, + "AnomalyDetectorStatus": {}, + "CreationTimeStamp": {}, + "LastModifiedTimeStamp": {} + } + }, + "AWS::Logs::LogGroup": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Logs::QueryDefinition": { + "Attributes": { + "QueryDefinitionId": {} + } + }, + "AWS::Logs::ScheduledQuery": { + "Attributes": { + "CreationTime": {}, + "LastExecutionStatus": {}, + "LastTriggeredTime": {}, + "LastUpdatedTime": {}, + "ScheduledQueryArn": {} + } + }, + "AWS::Logs::StorageTierPolicy": { + "Attributes": { + "AccountId": {}, + "LastUpdatedTime": {} + } + }, + "AWS::LookoutEquipment::InferenceScheduler": { + "Attributes": { + "InferenceSchedulerArn": {} + } + }, + "AWS::LookoutVision::Project": { + "Attributes": { + "Arn": {} + } + }, + "AWS::M2::Application": { + "Attributes": { + "ApplicationArn": {}, + "ApplicationId": {} + } + }, + "AWS::M2::Deployment": { + "Attributes": { + "DeploymentId": {}, + "Status": {} + } + }, + "AWS::M2::Environment": { + "Attributes": { + "EnvironmentArn": {}, + "EnvironmentId": {} + } + }, + "AWS::MGN::NetworkMigrationDefinition": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "NetworkMigrationDefinitionID": {}, + "UpdatedAt": {} + } + }, + "AWS::MPA::ApprovalTeam": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "LastUpdateTime": {}, + "NumberOfApprovers": {}, + "Status": {}, + "StatusCode": {}, + "StatusMessage": {}, + "UpdateSessionArn": {}, + "VersionId": {} + } + }, + "AWS::MPA::IdentitySource": { + "Attributes": { + "CreationTime": {}, + "IdentitySourceArn": {}, + "IdentitySourceParameters.IamIdentityCenter.ApprovalPortalUrl": {}, + "IdentitySourceType": {}, + "Status": {}, + "StatusCode": {}, + "StatusMessage": {} + } + }, + "AWS::MSK::Channel": { + "Attributes": { + "ChannelArn": {}, + "StateInfo": {}, + "StateInfo.Code": {}, + "StateInfo.Message": {}, + "Status": {} + } + }, + "AWS::MSK::Cluster": { + "Attributes": { + "Arn": {}, + "CurrentVersion": {} + } + }, + "AWS::MSK::ClusterPolicy": { + "Attributes": { + "CurrentVersion": {} + } + }, + "AWS::MSK::Configuration": { + "Attributes": { + "Arn": {}, + "LatestRevision.CreationTime": {}, + "LatestRevision.Description": {}, + "LatestRevision.Revision": {} + } + }, + "AWS::MSK::Replicator": { + "Attributes": { + "CurrentVersion": {}, + "ReplicatorArn": {} + } + }, + "AWS::MSK::ServerlessCluster": { + "Attributes": { + "Arn": {} + } + }, + "AWS::MSK::Topic": { + "Attributes": { + "TopicArn": {} + } + }, + "AWS::MSK::VpcConnection": { + "Attributes": { + "Arn": {} + } + }, + "AWS::MWAA::Environment": { + "Attributes": { + "Arn": {}, + "CeleryExecutorQueue": {}, + "DatabaseVpcEndpointService": {}, + "LoggingConfiguration.DagProcessingLogs.CloudWatchLogGroupArn": {}, + "LoggingConfiguration.SchedulerLogs.CloudWatchLogGroupArn": {}, + "LoggingConfiguration.TaskLogs.CloudWatchLogGroupArn": {}, + "LoggingConfiguration.WebserverLogs.CloudWatchLogGroupArn": {}, + "LoggingConfiguration.WorkerLogs.CloudWatchLogGroupArn": {}, + "WebserverUrl": {}, + "WebserverVpcEndpointService": {} + } + }, + "AWS::MWAAServerless::Workflow": { + "Attributes": { + "CodeSnapshottedAt": {}, + "CreatedAt": {}, + "ModifiedAt": {}, + "ScheduleConfiguration": {}, + "ScheduleConfiguration.CronExpression": {}, + "WorkflowArn": {}, + "WorkflowStatus": {}, + "WorkflowVersion": {} + } + }, + "AWS::Macie2::ClassificationJob": { + "Attributes": { + "CreatedAt": {}, + "JobArn": {}, + "JobId": {}, + "JobStatus": {}, + "S3JobDefinition.Scoping": {} + } + }, + "AWS::Macie::AllowList": { + "Attributes": { + "Arn": {}, + "Id": {}, + "Status": {} + } + }, + "AWS::Macie::CustomDataIdentifier": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::Macie::FindingsFilter": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::Macie::Member": { + "Attributes": { + "AdministratorAccountId": {}, + "Arn": {}, + "RelationshipStatus": {}, + "UpdatedAt": {} + } + }, + "AWS::Macie::Session": { + "Attributes": { + "AutomatedDiscoveryStatus": {}, + "AwsAccountId": {}, + "ServiceRole": {} + } + }, + "AWS::ManagedBlockchain::Accessor": { + "Attributes": { + "Arn": {}, + "BillingToken": {}, + "CreationDate": {}, + "Id": {}, + "Status": {} + } + }, + "AWS::ManagedBlockchain::Member": { + "Attributes": { + "MemberId": {}, + "NetworkId": {} + } + }, + "AWS::ManagedBlockchain::Node": { + "Attributes": { + "Arn": {}, + "MemberId": {}, + "NetworkId": {}, + "NodeId": {} + } + }, + "AWS::MediaConnect::Bridge": { + "Attributes": { + "BridgeArn": {}, + "BridgeState": {} + } + }, + "AWS::MediaConnect::Flow": { + "Attributes": { + "EgressIp": {}, + "FlowArn": {}, + "FlowAvailabilityZone": {}, + "FlowNdiMachineName": {}, + "Source.IngestIp": {}, + "Source.SourceArn": {}, + "Source.SourceIngestPort": {} + } + }, + "AWS::MediaConnect::FlowEntitlement": { + "Attributes": { + "EntitlementArn": {} + } + }, + "AWS::MediaConnect::FlowOutput": { + "Attributes": { + "OutputArn": {} + } + }, + "AWS::MediaConnect::FlowSource": { + "Attributes": { + "IngestIp": {}, + "SourceArn": {}, + "SourceIngestPort": {} + } + }, + "AWS::MediaConnect::FlowVpcInterface": { + "Attributes": { + "NetworkInterfaceIds": {} + } + }, + "AWS::MediaConnect::Gateway": { + "Attributes": { + "GatewayArn": {}, + "GatewayState": {} + } + }, + "AWS::MediaConnect::Offering": { + "Attributes": { + "CurrencyCode": {}, + "Duration": {}, + "DurationUnits": {}, + "OfferingArn": {}, + "OfferingDescription": {}, + "PricePerUnit": {}, + "PriceUnits": {}, + "ResourceSpecification": {}, + "ResourceSpecification.ReservedBitrate": {}, + "ResourceSpecification.ResourceType": {} + } + }, + "AWS::MediaConnect::Reservation": { + "Attributes": { + "CurrencyCode": {}, + "Duration": {}, + "DurationUnits": {}, + "End": {}, + "OfferingArn": {}, + "OfferingDescription": {}, + "PricePerUnit": {}, + "PriceUnits": {}, + "ReservationArn": {}, + "ReservationName": {}, + "ReservationState": {}, + "ResourceSpecification": {}, + "ResourceSpecification.ReservedBitrate": {}, + "ResourceSpecification.ResourceType": {}, + "Start": {} + } + }, + "AWS::MediaConnect::RouterInput": { + "Attributes": { + "Arn": {}, + "ContentQualityAnalysisType": {}, + "CreatedAt": {}, + "Id": {}, + "InputType": {}, + "IpAddress": {}, + "MaintenanceType": {}, + "RoutedOutputs": {}, + "State": {}, + "UpdatedAt": {} + } + }, + "AWS::MediaConnect::RouterNetworkInterface": { + "Attributes": { + "Arn": {}, + "AssociatedInputCount": {}, + "AssociatedOutputCount": {}, + "CreatedAt": {}, + "Id": {}, + "NetworkInterfaceType": {}, + "State": {}, + "UpdatedAt": {} + } + }, + "AWS::MediaConnect::RouterOutput": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "IpAddress": {}, + "MaintenanceType": {}, + "OutputType": {}, + "RoutedState": {}, + "State": {}, + "UpdatedAt": {} + } + }, + "AWS::MediaConvert::Job": { + "Attributes": { + "AccelerationStatus": {}, + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "Messages": {}, + "Messages.Info": {}, + "Messages.Warning": {}, + "OutputGroupDetails": {}, + "ShareStatus": {}, + "Status": {}, + "Timing": {}, + "Timing.FinishTime": {}, + "Timing.StartTime": {}, + "Timing.SubmitTime": {} + } + }, + "AWS::MediaConvert::JobTemplate": { + "Attributes": { + "Arn": {}, + "Name": {} + } + }, + "AWS::MediaConvert::Preset": { + "Attributes": { + "Arn": {}, + "Name": {} + } + }, + "AWS::MediaConvert::Queue": { + "Attributes": { + "Arn": {}, + "Name": {} + } + }, + "AWS::MediaLive::Channel": { + "Attributes": { + "Arn": {}, + "Inputs": {} + } + }, + "AWS::MediaLive::ChannelPlacementGroup": { + "Attributes": { + "Arn": {}, + "Channels": {}, + "Id": {}, + "State": {} + } + }, + "AWS::MediaLive::CloudWatchAlarmTemplate": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "GroupId": {}, + "Id": {}, + "Identifier": {}, + "ModifiedAt": {} + } + }, + "AWS::MediaLive::CloudWatchAlarmTemplateGroup": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "Identifier": {}, + "ModifiedAt": {} + } + }, + "AWS::MediaLive::Cluster": { + "Attributes": { + "Arn": {}, + "ChannelIds": {}, + "Id": {}, + "State": {} + } + }, + "AWS::MediaLive::EventBridgeRuleTemplate": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "GroupId": {}, + "Id": {}, + "Identifier": {}, + "ModifiedAt": {} + } + }, + "AWS::MediaLive::EventBridgeRuleTemplateGroup": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "Identifier": {}, + "ModifiedAt": {} + } + }, + "AWS::MediaLive::Input": { + "Attributes": { + "Arn": {}, + "Destinations": {}, + "Sources": {} + } + }, + "AWS::MediaLive::InputSecurityGroup": { + "Attributes": { + "Arn": {} + } + }, + "AWS::MediaLive::Multiplex": { + "Attributes": { + "Arn": {}, + "Id": {}, + "PipelinesRunningCount": {}, + "ProgramCount": {}, + "State": {} + } + }, + "AWS::MediaLive::Multiplexprogram": { + "Attributes": { + "ChannelId": {} + } + }, + "AWS::MediaLive::Network": { + "Attributes": { + "Arn": {}, + "AssociatedClusterIds": {}, + "Id": {}, + "State": {} + } + }, + "AWS::MediaLive::Node": { + "Attributes": { + "Arn": {}, + "ChannelPlacementGroups": {}, + "ConnectionState": {}, + "Id": {}, + "InstanceArn": {}, + "State": {} + } + }, + "AWS::MediaLive::Offering": { + "Attributes": { + "Arn": {}, + "CurrencyCode": {}, + "Duration": {}, + "DurationUnits": {}, + "FixedPrice": {}, + "OfferingDescription": {}, + "OfferingId": {}, + "OfferingType": {}, + "Region": {}, + "ResourceSpecification": {}, + "ResourceSpecification.ChannelClass": {}, + "ResourceSpecification.Codec": {}, + "ResourceSpecification.MaximumBitrate": {}, + "ResourceSpecification.MaximumFramerate": {}, + "ResourceSpecification.Resolution": {}, + "ResourceSpecification.ResourceType": {}, + "ResourceSpecification.SpecialFeature": {}, + "ResourceSpecification.VideoQuality": {}, + "UsagePrice": {} + } + }, + "AWS::MediaLive::SdiSource": { + "Attributes": { + "Arn": {}, + "Id": {}, + "Inputs": {}, + "State": {} + } + }, + "AWS::MediaLive::SignalMap": { + "Attributes": { + "Arn": {}, + "CloudWatchAlarmTemplateGroupIds": {}, + "CreatedAt": {}, + "ErrorMessage": {}, + "EventBridgeRuleTemplateGroupIds": {}, + "FailedMediaResourceMap": {}, + "Id": {}, + "Identifier": {}, + "LastDiscoveredAt": {}, + "LastSuccessfulMonitorDeployment": {}, + "LastSuccessfulMonitorDeployment.DetailsUri": {}, + "LastSuccessfulMonitorDeployment.Status": {}, + "MediaResourceMap": {}, + "ModifiedAt": {}, + "MonitorChangesPendingDeployment": {}, + "MonitorDeployment": {}, + "MonitorDeployment.DetailsUri": {}, + "MonitorDeployment.ErrorMessage": {}, + "MonitorDeployment.Status": {}, + "Status": {} + } + }, + "AWS::MediaPackage::Asset": { + "Attributes": { + "Arn": {}, + "CreatedAt": {} + } + }, + "AWS::MediaPackage::Channel": { + "Attributes": { + "Arn": {} + } + }, + "AWS::MediaPackage::HarvestJob": { + "Attributes": { + "Arn": {}, + "ChannelId": {}, + "CreatedAt": {}, + "Status": {} + } + }, + "AWS::MediaPackage::OriginEndpoint": { + "Attributes": { + "Arn": {}, + "Url": {} + } + }, + "AWS::MediaPackage::PackagingConfiguration": { + "Attributes": { + "Arn": {} + } + }, + "AWS::MediaPackage::PackagingGroup": { + "Attributes": { + "Arn": {}, + "DomainName": {} + } + }, + "AWS::MediaPackageV2::Channel": { + "Attributes": { + "Arn": {}, + "AttachedMultiviewChannels": {}, + "CreatedAt": {}, + "IngestEndpointUrls": {}, + "IngestEndpoints": {}, + "ModifiedAt": {} + } + }, + "AWS::MediaPackageV2::ChannelGroup": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "EgressDomain": {}, + "ModifiedAt": {} + } + }, + "AWS::MediaPackageV2::HarvestJob": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "ModifiedAt": {}, + "Status": {} + } + }, + "AWS::MediaPackageV2::OriginEndpoint": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "DashManifestUrls": {}, + "HlsManifestUrls": {}, + "LowLatencyHlsManifestUrls": {}, + "ModifiedAt": {}, + "MssManifestUrls": {} + } + }, + "AWS::MediaStore::Container": { + "Attributes": { + "Endpoint": {} + } + }, + "AWS::MediaTailor::Channel": { + "Attributes": { + "Arn": {} + } + }, + "AWS::MediaTailor::Function": { + "Attributes": { + "Arn": {} + } + }, + "AWS::MediaTailor::LiveSource": { + "Attributes": { + "Arn": {} + } + }, + "AWS::MediaTailor::PlaybackConfiguration": { + "Attributes": { + "DashConfiguration.ManifestEndpointPrefix": {}, + "HlsConfiguration.ManifestEndpointPrefix": {}, + "PlaybackConfigurationArn": {}, + "PlaybackEndpointPrefix": {}, + "SessionInitializationEndpointPrefix": {} + } + }, + "AWS::MediaTailor::PrefetchSchedule": { + "Attributes": { + "Arn": {} + } + }, + "AWS::MediaTailor::SourceLocation": { + "Attributes": { + "Arn": {} + } + }, + "AWS::MediaTailor::VodSource": { + "Attributes": { + "Arn": {} + } + }, + "AWS::MedicalImaging::ImageSet": { + "Attributes": { + "CreatedAt": {}, + "ImageSetArn": {}, + "ImageSetId": {}, + "ImageSetState": {}, + "ImageSetWorkflowStatus": {}, + "IsPrimary": {}, + "LastAccessedAt": {}, + "StorageTier": {}, + "Tags": {}, + "UpdatedAt": {}, + "VersionId": {} + } + }, + "AWS::MemoryDB::ACL": { + "Attributes": { + "Arn": {}, + "Status": {} + } + }, + "AWS::MemoryDB::Cluster": { + "Attributes": { + "ARN": {}, + "ClusterEndpoint.Address": {}, + "ClusterEndpoint.Port": {}, + "ParameterGroupStatus": {}, + "Status": {} + } + }, + "AWS::MemoryDB::MultiRegionCluster": { + "Attributes": { + "ARN": {}, + "MultiRegionClusterName": {}, + "Status": {} + } + }, + "AWS::MemoryDB::MultiRegionParameterGroup": { + "Attributes": { + "Arn": {}, + "Description": {}, + "Family": {}, + "MultiRegionParameterGroupName": {} + } + }, + "AWS::MemoryDB::ParameterGroup": { + "Attributes": { + "ARN": {} + } + }, + "AWS::MemoryDB::ReservedNode": { + "Attributes": { + "Arn": {}, + "Duration": {}, + "FixedPrice": {}, + "NodeCount": {}, + "NodeType": {}, + "OfferingType": {}, + "RecurringCharges": {}, + "ReservationId": {}, + "ReservedNodesOfferingId": {}, + "StartTime": {}, + "State": {}, + "Tags": {} + } + }, + "AWS::MemoryDB::Snapshot": { + "Attributes": { + "Arn": {}, + "ClusterConfiguration": {}, + "ClusterConfiguration.Description": {}, + "ClusterConfiguration.Engine": {}, + "ClusterConfiguration.EngineVersion": {}, + "ClusterConfiguration.MaintenanceWindow": {}, + "ClusterConfiguration.Name": {}, + "ClusterConfiguration.NodeType": {}, + "ClusterConfiguration.NumShards": {}, + "ClusterConfiguration.ParameterGroupName": {}, + "ClusterConfiguration.Port": {}, + "ClusterConfiguration.SnapshotRetentionLimit": {}, + "ClusterConfiguration.SnapshotWindow": {}, + "ClusterConfiguration.SubnetGroupName": {}, + "ClusterConfiguration.TopicArn": {}, + "ClusterConfiguration.VpcId": {}, + "DataTiering": {}, + "Source": {}, + "Status": {} + } + }, + "AWS::MemoryDB::SubnetGroup": { + "Attributes": { + "ARN": {}, + "SupportedNetworkTypes": {} + } + }, + "AWS::MemoryDB::User": { + "Attributes": { + "Arn": {}, + "Status": {} + } + }, + "AWS::Neptune::DBCluster": { + "Attributes": { + "ClusterResourceId": {}, + "Endpoint": {}, + "Port": {}, + "ReadEndpoint": {} + } + }, + "AWS::Neptune::DBInstance": { + "Attributes": { + "Endpoint": {}, + "Port": {} + } + }, + "AWS::NeptuneGraph::ExportTask": { + "Attributes": { + "Arn": {}, + "Status": {}, + "TaskId": {} + } + }, + "AWS::NeptuneGraph::Graph": { + "Attributes": { + "Endpoint": {}, + "GraphArn": {}, + "GraphId": {} + } + }, + "AWS::NeptuneGraph::GraphSnapshot": { + "Attributes": { + "Arn": {}, + "Id": {}, + "KmsKeyIdentifier": {}, + "SnapshotCreateTime": {}, + "Status": {} + } + }, + "AWS::NeptuneGraph::PrivateGraphEndpoint": { + "Attributes": { + "PrivateGraphEndpointIdentifier": {}, + "VpcEndpointId": {} + } + }, + "AWS::NetworkFirewall::ContainerAssociation": { + "Attributes": { + "ContainerAssociationArn": {}, + "ResolvedCidrCount": {}, + "Status": {} + } + }, + "AWS::NetworkFirewall::Firewall": { + "Attributes": { + "EndpointIds": {}, + "FirewallArn": {}, + "FirewallId": {}, + "TransitGatewayAttachmentId": {} + } + }, + "AWS::NetworkFirewall::FirewallPolicy": { + "Attributes": { + "FirewallPolicyArn": {}, + "FirewallPolicyId": {} + } + }, + "AWS::NetworkFirewall::RuleGroup": { + "Attributes": { + "RuleGroupArn": {}, + "RuleGroupId": {} + } + }, + "AWS::NetworkFirewall::TLSInspectionConfiguration": { + "Attributes": { + "TLSInspectionConfigurationArn": {}, + "TLSInspectionConfigurationId": {} + } + }, + "AWS::NetworkFirewall::VpcEndpointAssociation": { + "Attributes": { + "EndpointId": {}, + "VpcEndpointAssociationArn": {}, + "VpcEndpointAssociationId": {} + } + }, + "AWS::NetworkFlowMonitor::Monitor": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "ModifiedAt": {}, + "MonitorStatus": {} + } + }, + "AWS::NetworkManager::ConnectAttachment": { + "Attributes": { + "AttachmentId": {}, + "AttachmentPolicyRuleNumber": {}, + "AttachmentType": {}, + "CoreNetworkArn": {}, + "CreatedAt": {}, + "LastModificationErrors": {}, + "OwnerAccountId": {}, + "ProposedNetworkFunctionGroupChange": {}, + "ProposedNetworkFunctionGroupChange.AttachmentPolicyRuleNumber": {}, + "ProposedNetworkFunctionGroupChange.NetworkFunctionGroupName": {}, + "ProposedNetworkFunctionGroupChange.Tags": {}, + "ProposedSegmentChange": {}, + "ProposedSegmentChange.AttachmentPolicyRuleNumber": {}, + "ProposedSegmentChange.SegmentName": {}, + "ProposedSegmentChange.Tags": {}, + "ResourceArn": {}, + "SegmentName": {}, + "State": {}, + "UpdatedAt": {} + } + }, + "AWS::NetworkManager::ConnectPeer": { + "Attributes": { + "Configuration": {}, + "Configuration.BgpConfigurations": {}, + "Configuration.CoreNetworkAddress": {}, + "Configuration.InsideCidrBlocks": {}, + "Configuration.PeerAddress": {}, + "Configuration.Protocol": {}, + "ConnectPeerId": {}, + "CoreNetworkId": {}, + "CreatedAt": {}, + "EdgeLocation": {}, + "LastModificationErrors": {}, + "State": {} + } + }, + "AWS::NetworkManager::CoreNetwork": { + "Attributes": { + "CoreNetworkArn": {}, + "CoreNetworkId": {}, + "CreatedAt": {}, + "Edges": {}, + "NetworkFunctionGroups": {}, + "OwnerAccount": {}, + "Segments": {}, + "State": {} + } + }, + "AWS::NetworkManager::Device": { + "Attributes": { + "CreatedAt": {}, + "DeviceArn": {}, + "DeviceId": {}, + "State": {} + } + }, + "AWS::NetworkManager::DirectConnectGatewayAttachment": { + "Attributes": { + "AttachmentId": {}, + "AttachmentPolicyRuleNumber": {}, + "AttachmentType": {}, + "CoreNetworkArn": {}, + "CreatedAt": {}, + "LastModificationErrors": {}, + "NetworkFunctionGroupName": {}, + "OwnerAccountId": {}, + "ProposedNetworkFunctionGroupChange": {}, + "ProposedNetworkFunctionGroupChange.AttachmentPolicyRuleNumber": {}, + "ProposedNetworkFunctionGroupChange.NetworkFunctionGroupName": {}, + "ProposedNetworkFunctionGroupChange.Tags": {}, + "ProposedSegmentChange": {}, + "ProposedSegmentChange.AttachmentPolicyRuleNumber": {}, + "ProposedSegmentChange.SegmentName": {}, + "ProposedSegmentChange.Tags": {}, + "ResourceArn": {}, + "SegmentName": {}, + "State": {}, + "UpdatedAt": {} + } + }, + "AWS::NetworkManager::GlobalNetwork": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::NetworkManager::Link": { + "Attributes": { + "CreatedAt": {}, + "LinkArn": {}, + "LinkId": {}, + "State": {} + } + }, + "AWS::NetworkManager::Site": { + "Attributes": { + "CreatedAt": {}, + "SiteArn": {}, + "SiteId": {}, + "State": {} + } + }, + "AWS::NetworkManager::SiteToSiteVpnAttachment": { + "Attributes": { + "AttachmentId": {}, + "AttachmentPolicyRuleNumber": {}, + "AttachmentType": {}, + "CoreNetworkArn": {}, + "CreatedAt": {}, + "EdgeLocation": {}, + "LastModificationErrors": {}, + "OwnerAccountId": {}, + "ProposedNetworkFunctionGroupChange": {}, + "ProposedNetworkFunctionGroupChange.AttachmentPolicyRuleNumber": {}, + "ProposedNetworkFunctionGroupChange.NetworkFunctionGroupName": {}, + "ProposedNetworkFunctionGroupChange.Tags": {}, + "ProposedSegmentChange": {}, + "ProposedSegmentChange.AttachmentPolicyRuleNumber": {}, + "ProposedSegmentChange.SegmentName": {}, + "ProposedSegmentChange.Tags": {}, + "ResourceArn": {}, + "SegmentName": {}, + "State": {}, + "UpdatedAt": {} + } + }, + "AWS::NetworkManager::TransitGatewayPeering": { + "Attributes": { + "CoreNetworkArn": {}, + "CreatedAt": {}, + "EdgeLocation": {}, + "LastModificationErrors": {}, + "OwnerAccountId": {}, + "PeeringId": {}, + "PeeringType": {}, + "ResourceArn": {}, + "State": {}, + "TransitGatewayPeeringAttachmentId": {} + } + }, + "AWS::NetworkManager::TransitGatewayRouteTableAttachment": { + "Attributes": { + "AttachmentId": {}, + "AttachmentPolicyRuleNumber": {}, + "AttachmentType": {}, + "CoreNetworkArn": {}, + "CoreNetworkId": {}, + "CreatedAt": {}, + "EdgeLocation": {}, + "LastModificationErrors": {}, + "OwnerAccountId": {}, + "ProposedNetworkFunctionGroupChange": {}, + "ProposedNetworkFunctionGroupChange.AttachmentPolicyRuleNumber": {}, + "ProposedNetworkFunctionGroupChange.NetworkFunctionGroupName": {}, + "ProposedNetworkFunctionGroupChange.Tags": {}, + "ProposedSegmentChange": {}, + "ProposedSegmentChange.AttachmentPolicyRuleNumber": {}, + "ProposedSegmentChange.SegmentName": {}, + "ProposedSegmentChange.Tags": {}, + "ResourceArn": {}, + "SegmentName": {}, + "State": {}, + "UpdatedAt": {} + } + }, + "AWS::NetworkManager::VpcAttachment": { + "Attributes": { + "AttachmentId": {}, + "AttachmentPolicyRuleNumber": {}, + "AttachmentType": {}, + "CoreNetworkArn": {}, + "CreatedAt": {}, + "EdgeLocation": {}, + "LastModificationErrors": {}, + "NetworkFunctionGroupName": {}, + "OwnerAccountId": {}, + "ProposedNetworkFunctionGroupChange": {}, + "ProposedNetworkFunctionGroupChange.AttachmentPolicyRuleNumber": {}, + "ProposedNetworkFunctionGroupChange.NetworkFunctionGroupName": {}, + "ProposedNetworkFunctionGroupChange.Tags": {}, + "ProposedSegmentChange": {}, + "ProposedSegmentChange.AttachmentPolicyRuleNumber": {}, + "ProposedSegmentChange.SegmentName": {}, + "ProposedSegmentChange.Tags": {}, + "ResourceArn": {}, + "SegmentName": {}, + "State": {}, + "UpdatedAt": {} + } + }, + "AWS::Notifications::EventRule": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "ManagedRules": {}, + "StatusSummaryByRegion": {} + } + }, + "AWS::Notifications::NotificationConfiguration": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "Status": {} + } + }, + "AWS::Notifications::NotificationHub": { + "Attributes": { + "CreationTime": {}, + "NotificationHubStatusSummary": {}, + "NotificationHubStatusSummary.NotificationHubStatus": {}, + "NotificationHubStatusSummary.NotificationHubStatusReason": {} + } + }, + "AWS::NotificationsContacts::EmailContact": { + "Attributes": { + "Arn": {}, + "EmailContact": {}, + "EmailContact.Address": {}, + "EmailContact.Arn": {}, + "EmailContact.CreationTime": {}, + "EmailContact.Name": {}, + "EmailContact.Status": {}, + "EmailContact.UpdateTime": {} + } + }, + "AWS::NovaAct::WorkflowDefinition": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Status": {} + } + }, + "AWS::NovaAct::WorkflowRun": { + "Attributes": { + "StartedAt": {}, + "Status": {}, + "WorkflowRunArn": {}, + "WorkflowRunId": {} + } + }, + "AWS::ODB::CloudAutonomousVmCluster": { + "Attributes": { + "AutonomousDataStoragePercentage": {}, + "AvailableAutonomousDataStorageSizeInTBs": {}, + "AvailableContainerDatabases": {}, + "AvailableCpus": {}, + "CloudAutonomousVmClusterArn": {}, + "CloudAutonomousVmClusterId": {}, + "ComputeModel": {}, + "CpuCoreCount": {}, + "CpuPercentage": {}, + "DataStorageSizeInGBs": {}, + "DataStorageSizeInTBs": {}, + "DbNodeStorageSizeInGBs": {}, + "Domain": {}, + "ExadataStorageInTBsLowestScaledValue": {}, + "Hostname": {}, + "MaxAcdsLowestScaledValue": {}, + "MemorySizeInGBs": {}, + "NodeCount": {}, + "NonProvisionableAutonomousContainerDatabases": {}, + "OciResourceAnchorName": {}, + "OciUrl": {}, + "Ocid": {}, + "ProvisionableAutonomousContainerDatabases": {}, + "ProvisionedAutonomousContainerDatabases": {}, + "ProvisionedCpus": {}, + "ReclaimableCpus": {}, + "ReservedCpus": {}, + "Shape": {} + } + }, + "AWS::ODB::CloudExadataInfrastructure": { + "Attributes": { + "ActivatedStorageCount": {}, + "AdditionalStorageCount": {}, + "AvailableStorageSizeInGBs": {}, + "CloudExadataInfrastructureArn": {}, + "CloudExadataInfrastructureId": {}, + "ComputeModel": {}, + "CpuCount": {}, + "DataStorageSizeInTBs": {}, + "DbNodeStorageSizeInGBs": {}, + "DbServerIds": {}, + "DbServerVersion": {}, + "MaxCpuCount": {}, + "MaxDataStorageInTBs": {}, + "MaxDbNodeStorageSizeInGBs": {}, + "MaxMemoryInGBs": {}, + "MemorySizeInGBs": {}, + "OciResourceAnchorName": {}, + "OciUrl": {}, + "Ocid": {}, + "StorageServerVersion": {}, + "TotalStorageSizeInGBs": {} + } + }, + "AWS::ODB::CloudVmCluster": { + "Attributes": { + "CloudVmClusterArn": {}, + "CloudVmClusterId": {}, + "ComputeModel": {}, + "DiskRedundancy": {}, + "Domain": {}, + "ListenerPort": {}, + "NodeCount": {}, + "OciResourceAnchorName": {}, + "OciUrl": {}, + "Ocid": {}, + "ScanDnsName": {}, + "ScanIpIds": {}, + "Shape": {}, + "StorageSizeInGBs": {}, + "VipIds": {} + } + }, + "AWS::ODB::OdbNetwork": { + "Attributes": { + "Ec2PlacementGroupIds": {}, + "ManagedServices": {}, + "ManagedServices.CrossRegionS3RestoreSourcesAccess": {}, + "ManagedServices.KmsAccess": {}, + "ManagedServices.KmsAccess.DomainName": {}, + "ManagedServices.KmsAccess.Ipv4Addresses": {}, + "ManagedServices.KmsAccess.KmsPolicyDocument": {}, + "ManagedServices.KmsAccess.Status": {}, + "ManagedServices.ManagedS3BackupAccess": {}, + "ManagedServices.ManagedS3BackupAccess.Ipv4Addresses": {}, + "ManagedServices.ManagedS3BackupAccess.Status": {}, + "ManagedServices.ManagedServicesIpv4Cidrs": {}, + "ManagedServices.ResourceGatewayArn": {}, + "ManagedServices.S3Access": {}, + "ManagedServices.S3Access.DomainName": {}, + "ManagedServices.S3Access.Ipv4Addresses": {}, + "ManagedServices.S3Access.S3PolicyDocument": {}, + "ManagedServices.S3Access.Status": {}, + "ManagedServices.ServiceNetworkArn": {}, + "ManagedServices.ServiceNetworkEndpoint": {}, + "ManagedServices.ServiceNetworkEndpoint.VpcEndpointId": {}, + "ManagedServices.ServiceNetworkEndpoint.VpcEndpointType": {}, + "ManagedServices.StsAccess": {}, + "ManagedServices.StsAccess.DomainName": {}, + "ManagedServices.StsAccess.Ipv4Addresses": {}, + "ManagedServices.StsAccess.Status": {}, + "ManagedServices.StsAccess.StsPolicyDocument": {}, + "ManagedServices.ZeroEtlAccess": {}, + "ManagedServices.ZeroEtlAccess.Cidr": {}, + "ManagedServices.ZeroEtlAccess.Status": {}, + "OciNetworkAnchorId": {}, + "OciResourceAnchorName": {}, + "OciVcnUrl": {}, + "OdbNetworkArn": {}, + "OdbNetworkId": {} + } + }, + "AWS::ODB::OdbPeeringConnection": { + "Attributes": { + "OdbNetworkArn": {}, + "OdbPeeringConnectionArn": {}, + "OdbPeeringConnectionId": {}, + "PeerNetworkArn": {}, + "PeerNetworkCidrs": {} + } + }, + "AWS::OSIS::Pipeline": { + "Attributes": { + "IngestEndpointUrls": {}, + "PipelineArn": {}, + "VpcEndpointService": {}, + "VpcEndpoints": {} + } + }, + "AWS::OSIS::PipelineBlueprint": { + "Attributes": { + "Arn": {}, + "DisplayDescription": {}, + "DisplayName": {}, + "PipelineConfigurationBody": {}, + "Service": {}, + "UseCase": {} + } + }, + "AWS::Oam::Link": { + "Attributes": { + "Arn": {}, + "Label": {} + } + }, + "AWS::Oam::Sink": { + "Attributes": { + "Arn": {} + } + }, + "AWS::ObservabilityAdmin::OrganizationCentralizationRule": { + "Attributes": { + "RuleArn": {} + } + }, + "AWS::ObservabilityAdmin::OrganizationTelemetryRule": { + "Attributes": { + "RegionStatuses": {}, + "RuleArn": {} + } + }, + "AWS::ObservabilityAdmin::S3TableIntegration": { + "Attributes": { + "Arn": {} + } + }, + "AWS::ObservabilityAdmin::TelemetryEnrichment": { + "Attributes": { + "Status": {} + } + }, + "AWS::ObservabilityAdmin::TelemetryPipelines": { + "Attributes": { + "Arn": {}, + "Pipeline": {}, + "Pipeline.Arn": {}, + "Pipeline.Configuration": {}, + "Pipeline.Configuration.Body": {}, + "Pipeline.CreatedTimeStamp": {}, + "Pipeline.LastUpdateTimeStamp": {}, + "Pipeline.Name": {}, + "Pipeline.Status": {}, + "Pipeline.StatusReason": {}, + "Pipeline.StatusReason.Description": {}, + "Pipeline.Tags": {}, + "PipelineIdentifier": {}, + "Status": {}, + "StatusReason": {}, + "StatusReason.Description": {} + } + }, + "AWS::ObservabilityAdmin::TelemetryRule": { + "Attributes": { + "RegionStatuses": {}, + "RuleArn": {} + } + }, + "AWS::Omics::AnnotationStore": { + "Attributes": { + "CreationTime": {}, + "Id": {}, + "Status": {}, + "StatusMessage": {}, + "StoreArn": {}, + "StoreSizeBytes": {}, + "UpdateTime": {} + } + }, + "AWS::Omics::Configuration": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "Status": {}, + "Uuid": {} + } + }, + "AWS::Omics::ReadSet": { + "Attributes": { + "Arn": {}, + "CreationJobId": {}, + "CreationTime": {}, + "CreationType": {}, + "Etag": {}, + "Etag.Algorithm": {}, + "Etag.Source1": {}, + "Etag.Source2": {}, + "Files": {}, + "Files.Index": {}, + "Files.Index.ContentLength": {}, + "Files.Index.PartSize": {}, + "Files.Index.S3Access": {}, + "Files.Index.TotalParts": {}, + "Files.Source1": {}, + "Files.Source1.ContentLength": {}, + "Files.Source1.PartSize": {}, + "Files.Source1.S3Access": {}, + "Files.Source1.TotalParts": {}, + "Files.Source2": {}, + "Files.Source2.ContentLength": {}, + "Files.Source2.PartSize": {}, + "Files.Source2.S3Access": {}, + "Files.Source2.S3Access.S3Uri": {}, + "Files.Source2.TotalParts": {}, + "ReadSetId": {}, + "SequenceInformation": {}, + "SequenceInformation.Alignment": {}, + "SequenceInformation.GeneratedFrom": {}, + "SequenceInformation.TotalBaseCount": {}, + "SequenceInformation.TotalReadCount": {}, + "Status": {}, + "Tags": {} + } + }, + "AWS::Omics::Reference": { + "Attributes": { + "Arn": {}, + "CreationJobId": {}, + "CreationTime": {}, + "CreationType": {}, + "Id": {}, + "Md5": {}, + "Status": {}, + "UpdateTime": {} + } + }, + "AWS::Omics::ReferenceStore": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "ReferenceStoreId": {} + } + }, + "AWS::Omics::Run": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "Id": {}, + "RunOutputUri": {}, + "StartTime": {}, + "StartedBy": {}, + "Status": {}, + "Uuid": {} + } + }, + "AWS::Omics::RunBatch": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "Id": {}, + "RunSummary": {}, + "RunSummary.CancelledRunCount": {}, + "RunSummary.CompletedRunCount": {}, + "RunSummary.DeletedRunCount": {}, + "RunSummary.FailedRunCount": {}, + "RunSummary.PendingRunCount": {}, + "RunSummary.RunningRunCount": {}, + "RunSummary.StartingRunCount": {}, + "RunSummary.StoppingRunCount": {}, + "Status": {}, + "SubmissionSummary": {}, + "SubmissionSummary.FailedCancelSubmissionCount": {}, + "SubmissionSummary.FailedDeleteSubmissionCount": {}, + "SubmissionSummary.FailedStartSubmissionCount": {}, + "SubmissionSummary.PendingStartSubmissionCount": {}, + "SubmissionSummary.SuccessfulCancelSubmissionCount": {}, + "SubmissionSummary.SuccessfulDeleteSubmissionCount": {}, + "SubmissionSummary.SuccessfulStartSubmissionCount": {}, + "SubmittedTime": {}, + "TotalRuns": {}, + "Uuid": {} + } + }, + "AWS::Omics::RunCache": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "Id": {}, + "Status": {} + } + }, + "AWS::Omics::RunGroup": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "Id": {} + } + }, + "AWS::Omics::SequenceStore": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "S3AccessPointArn": {}, + "S3Uri": {}, + "SequenceStoreId": {}, + "Status": {}, + "StatusMessage": {}, + "UpdateTime": {} + } + }, + "AWS::Omics::Task": { + "Attributes": { + "Arn": {}, + "Cpus": {}, + "CreationTime": {}, + "Gpus": {}, + "InstanceType": {}, + "LogStream": {}, + "Memory": {}, + "Name": {}, + "Status": {}, + "TaskId": {} + } + }, + "AWS::Omics::VariantStore": { + "Attributes": { + "CreationTime": {}, + "Id": {}, + "Status": {}, + "StatusMessage": {}, + "StoreArn": {}, + "StoreSizeBytes": {}, + "UpdateTime": {} + } + }, + "AWS::Omics::Workflow": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "Id": {}, + "Status": {}, + "Type": {}, + "Uuid": {} + } + }, + "AWS::Omics::WorkflowVersion": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "Status": {}, + "Type": {}, + "Uuid": {} + } + }, + "AWS::OpenSearch::DataSource": { + "Attributes": { + "Arn": {}, + "Status": {} + } + }, + "AWS::OpenSearchServerless::Collection": { + "Attributes": { + "Arn": {}, + "CollectionEndpoint": {}, + "DashboardEndpoint": {}, + "FipsEndpoints": {}, + "FipsEndpoints.CollectionEndpoint": {}, + "FipsEndpoints.DashboardEndpoint": {}, + "Id": {}, + "KmsKeyArn": {} + } + }, + "AWS::OpenSearchServerless::CollectionGroup": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::OpenSearchServerless::Index": { + "Attributes": { + "Uuid": {} + } + }, + "AWS::OpenSearchServerless::SecurityConfig": { + "Attributes": { + "IamIdentityCenterOptions.ApplicationArn": {}, + "IamIdentityCenterOptions.ApplicationDescription": {}, + "IamIdentityCenterOptions.ApplicationName": {}, + "Id": {} + } + }, + "AWS::OpenSearchServerless::VpcEndpoint": { + "Attributes": { + "Id": {} + } + }, + "AWS::OpenSearchService::Application": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::OpenSearchService::Domain": { + "Attributes": { + "AdvancedSecurityOptions.AnonymousAuthDisableDate": {}, + "Arn": {}, + "DomainArn": {}, + "DomainEndpoint": {}, + "DomainEndpointV2": {}, + "DomainEndpoints": {}, + "Id": {}, + "IdentityCenterOptions.IdentityCenterApplicationARN": {}, + "IdentityCenterOptions.IdentityStoreId": {}, + "ServiceSoftwareOptions": {}, + "ServiceSoftwareOptions.AutomatedUpdateDate": {}, + "ServiceSoftwareOptions.Cancellable": {}, + "ServiceSoftwareOptions.CurrentVersion": {}, + "ServiceSoftwareOptions.Description": {}, + "ServiceSoftwareOptions.NewVersion": {}, + "ServiceSoftwareOptions.OptionalDeployment": {}, + "ServiceSoftwareOptions.UpdateAvailable": {}, + "ServiceSoftwareOptions.UpdateStatus": {} + } + }, + "AWS::OpsWorks::Instance": { + "Attributes": { + "AvailabilityZone": {}, + "PrivateDnsName": {}, + "PrivateIp": {}, + "PublicDnsName": {}, + "PublicIp": {} + } + }, + "AWS::OpsWorks::UserProfile": { + "Attributes": { + "SshUsername": {} + } + }, + "AWS::Organizations::Account": { + "Attributes": { + "AccountId": {}, + "Arn": {}, + "JoinedMethod": {}, + "JoinedTimestamp": {}, + "Paths": {}, + "State": {}, + "Status": {} + } + }, + "AWS::Organizations::Organization": { + "Attributes": { + "Arn": {}, + "Id": {}, + "ManagementAccountArn": {}, + "ManagementAccountEmail": {}, + "ManagementAccountId": {}, + "RootId": {} + } + }, + "AWS::Organizations::OrganizationalUnit": { + "Attributes": { + "Arn": {}, + "Id": {}, + "Path": {} + } + }, + "AWS::Organizations::Policy": { + "Attributes": { + "Arn": {}, + "AwsManaged": {}, + "Id": {} + } + }, + "AWS::Organizations::ResourcePolicy": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::Organizations::Root": { + "Attributes": { + "Arn": {}, + "Id": {}, + "Name": {}, + "OrganizationId": {}, + "PolicyTypes": {}, + "Tags": {} + } + }, + "AWS::Outposts::Outpost": { + "Attributes": { + "LifeCycleStatus": {}, + "OutpostArn": {}, + "OutpostId": {}, + "OwnerId": {}, + "SiteArn": {} + } + }, + "AWS::Outposts::Site": { + "Attributes": { + "SiteArn": {}, + "SiteId": {} + } + }, + "AWS::PCAConnectorAD::Connector": { + "Attributes": { + "ConnectorArn": {} + } + }, + "AWS::PCAConnectorAD::DirectoryRegistration": { + "Attributes": { + "DirectoryRegistrationArn": {} + } + }, + "AWS::PCAConnectorAD::Template": { + "Attributes": { + "TemplateArn": {} + } + }, + "AWS::PCAConnectorSCEP::Challenge": { + "Attributes": { + "ChallengeArn": {} + } + }, + "AWS::PCAConnectorSCEP::Connector": { + "Attributes": { + "ConnectorArn": {}, + "Endpoint": {}, + "OpenIdConfiguration": {}, + "OpenIdConfiguration.Audience": {}, + "OpenIdConfiguration.Issuer": {}, + "OpenIdConfiguration.Subject": {}, + "Type": {} + } + }, + "AWS::PCS::Cluster": { + "Attributes": { + "Arn": {}, + "Endpoints": {}, + "ErrorInfo": {}, + "Id": {}, + "Status": {} + } + }, + "AWS::PCS::ComputeNodeGroup": { + "Attributes": { + "Arn": {}, + "ErrorInfo": {}, + "Id": {}, + "Status": {} + } + }, + "AWS::PCS::Queue": { + "Attributes": { + "Arn": {}, + "ErrorInfo": {}, + "Id": {}, + "Status": {} + } + }, + "AWS::Panorama::ApplicationInstance": { + "Attributes": { + "ApplicationInstanceId": {}, + "Arn": {}, + "CreatedTime": {}, + "DefaultRuntimeContextDeviceName": {}, + "HealthStatus": {}, + "LastUpdatedTime": {}, + "Status": {}, + "StatusDescription": {} + } + }, + "AWS::Panorama::Package": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "PackageId": {}, + "StorageLocation.BinaryPrefixLocation": {}, + "StorageLocation.Bucket": {}, + "StorageLocation.GeneratedPrefixLocation": {}, + "StorageLocation.ManifestPrefixLocation": {}, + "StorageLocation.RepoPrefixLocation": {} + } + }, + "AWS::Panorama::PackageVersion": { + "Attributes": { + "IsLatestPatch": {}, + "PackageArn": {}, + "PackageName": {}, + "RegisteredTime": {}, + "Status": {}, + "StatusDescription": {} + } + }, + "AWS::PartnerCentral::ConnectionPreferences": { + "Attributes": { + "AccessType": {}, + "Arn": {}, + "ExcludedParticipantIds": {}, + "Revision": {}, + "UpdatedAt": {} + } + }, + "AWS::PartnerCentral::Partner": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "LegalName": {}, + "PrimarySolutionType": {} + } + }, + "AWS::PaymentCryptography::Key": { + "Attributes": { + "KeyIdentifier": {}, + "KeyOrigin": {}, + "KeyState": {}, + "ReplicationStatus": {} + } + }, + "AWS::Personalize::BatchInferenceJob": { + "Attributes": { + "BatchInferenceJobArn": {}, + "CreationDateTime": {}, + "LastUpdatedDateTime": {}, + "Status": {} + } + }, + "AWS::Personalize::BatchSegmentJob": { + "Attributes": { + "BatchSegmentJobArn": {}, + "CreationDateTime": {}, + "LastUpdatedDateTime": {}, + "Status": {} + } + }, + "AWS::Personalize::DataDeletionJob": { + "Attributes": { + "CreationDateTime": {}, + "DataDeletionJobArn": {}, + "LastUpdatedDateTime": {}, + "Status": {} + } + }, + "AWS::Personalize::Dataset": { + "Attributes": { + "DatasetArn": {} + } + }, + "AWS::Personalize::DatasetExportJob": { + "Attributes": { + "CreationDateTime": {}, + "DatasetExportJobArn": {}, + "LastUpdatedDateTime": {}, + "Status": {} + } + }, + "AWS::Personalize::DatasetGroup": { + "Attributes": { + "DatasetGroupArn": {} + } + }, + "AWS::Personalize::DatasetImportJob": { + "Attributes": { + "CreationDateTime": {}, + "DatasetImportJobArn": {}, + "LastUpdatedDateTime": {}, + "Status": {} + } + }, + "AWS::Personalize::EventTracker": { + "Attributes": { + "EventTrackerArn": {}, + "TrackingId": {} + } + }, + "AWS::Personalize::MetricAttribution": { + "Attributes": { + "MetricAttributionArn": {}, + "Status": {} + } + }, + "AWS::Personalize::Recipe": { + "Attributes": { + "AlgorithmArn": {}, + "CreationDateTime": {}, + "Description": {}, + "FeatureTransformationArn": {}, + "LastUpdatedDateTime": {}, + "RecipeArn": {}, + "RecipeType": {}, + "Status": {} + } + }, + "AWS::Personalize::Schema": { + "Attributes": { + "SchemaArn": {} + } + }, + "AWS::Personalize::Solution": { + "Attributes": { + "SolutionArn": {} + } + }, + "AWS::Pinpoint::App": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Pinpoint::Campaign": { + "Attributes": { + "Arn": {}, + "CampaignId": {} + } + }, + "AWS::Pinpoint::EmailTemplate": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Pinpoint::InAppTemplate": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Pinpoint::PushTemplate": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Pinpoint::Segment": { + "Attributes": { + "Arn": {}, + "SegmentId": {} + } + }, + "AWS::Pinpoint::SmsTemplate": { + "Attributes": { + "Arn": {} + } + }, + "AWS::PinpointEmail::Identity": { + "Attributes": { + "IdentityDNSRecordName1": {}, + "IdentityDNSRecordName2": {}, + "IdentityDNSRecordName3": {}, + "IdentityDNSRecordValue1": {}, + "IdentityDNSRecordValue2": {}, + "IdentityDNSRecordValue3": {} + } + }, + "AWS::Pipes::Pipe": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "CurrentState": {}, + "LastModifiedTime": {}, + "StateReason": {} + } + }, + "AWS::PricingPlanManager::Subscription": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "CurrentPlanTier": {}, + "Status": {}, + "StatusReason": {}, + "UpdatedAt": {} + } + }, + "AWS::Proton::EnvironmentAccountConnection": { + "Attributes": { + "Arn": {}, + "Id": {}, + "Status": {} + } + }, + "AWS::Proton::EnvironmentTemplate": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Proton::ServiceTemplate": { + "Attributes": { + "Arn": {} + } + }, + "AWS::QBusiness::Application": { + "Attributes": { + "ApplicationArn": {}, + "ApplicationId": {}, + "CreatedAt": {}, + "IdentityCenterApplicationArn": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::QBusiness::DataAccessor": { + "Attributes": { + "CreatedAt": {}, + "DataAccessorArn": {}, + "DataAccessorId": {}, + "IdcApplicationArn": {}, + "UpdatedAt": {} + } + }, + "AWS::QBusiness::DataSource": { + "Attributes": { + "CreatedAt": {}, + "DataSourceArn": {}, + "DataSourceId": {}, + "Status": {}, + "Type": {}, + "UpdatedAt": {} + } + }, + "AWS::QBusiness::Index": { + "Attributes": { + "CreatedAt": {}, + "IndexArn": {}, + "IndexId": {}, + "IndexStatistics": {}, + "IndexStatistics.TextDocumentStatistics": {}, + "IndexStatistics.TextDocumentStatistics.IndexedTextBytes": {}, + "IndexStatistics.TextDocumentStatistics.IndexedTextDocumentCount": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::QBusiness::Plugin": { + "Attributes": { + "BuildStatus": {}, + "CreatedAt": {}, + "PluginArn": {}, + "PluginId": {}, + "UpdatedAt": {} + } + }, + "AWS::QBusiness::Retriever": { + "Attributes": { + "CreatedAt": {}, + "RetrieverArn": {}, + "RetrieverId": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::QBusiness::WebExperience": { + "Attributes": { + "CreatedAt": {}, + "DefaultEndpoint": {}, + "Status": {}, + "UpdatedAt": {}, + "WebExperienceArn": {}, + "WebExperienceId": {} + } + }, + "AWS::QLDB::Stream": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::QuickSight::ActionConnector": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "EnabledActions": {}, + "LastUpdatedTime": {}, + "Status": {} + } + }, + "AWS::QuickSight::Agent": { + "Attributes": { + "AgentStatus": {}, + "Arn": {}, + "CreatedAt": {}, + "Creator": {}, + "CustomPromptInterface": {}, + "CustomPromptInterface.CustomInstructions": {}, + "CustomPromptInterface.Identity": {}, + "CustomPromptInterface.ModelProfileId": {}, + "CustomPromptInterface.OutputStyle": {}, + "CustomPromptInterface.PromptSummary": {}, + "CustomPromptInterface.QbsAwsAccountId": {}, + "CustomPromptInterface.ResponseLength": {}, + "CustomPromptInterface.SubscriptionId": {}, + "CustomPromptInterface.Tone": {}, + "ErrorMessage": {}, + "UpdatedAt": {} + } + }, + "AWS::QuickSight::Analysis": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "DataSetArns": {}, + "LastUpdatedTime": {} + } + }, + "AWS::QuickSight::ApprovalPolicy": { + "Attributes": { + "CreatedAt": {}, + "PolicyArn": {}, + "UpdatedAt": {} + } + }, + "AWS::QuickSight::AssetBundleExportJob": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "JobStatus": {} + } + }, + "AWS::QuickSight::AssetBundleImportJob": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "JobStatus": {} + } + }, + "AWS::QuickSight::CustomPermissions": { + "Attributes": { + "Arn": {} + } + }, + "AWS::QuickSight::Customization": { + "Attributes": { + "Arn": {}, + "AwsAccountId": {} + } + }, + "AWS::QuickSight::DLPSetting": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::QuickSight::Dashboard": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastPublishedTime": {}, + "LastUpdatedTime": {}, + "Version": {}, + "Version.Arn": {}, + "Version.CreatedTime": {}, + "Version.DataSetArns": {}, + "Version.Description": {}, + "Version.Errors": {}, + "Version.Sheets": {}, + "Version.SourceEntityArn": {}, + "Version.Status": {}, + "Version.ThemeArn": {}, + "Version.VersionNumber": {} + } + }, + "AWS::QuickSight::DataSet": { + "Attributes": { + "Arn": {}, + "ConsumedSpiceCapacityInBytes": {}, + "CreatedTime": {}, + "LastUpdatedTime": {}, + "OutputColumns": {} + } + }, + "AWS::QuickSight::DataSource": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastUpdatedTime": {}, + "Status": {} + } + }, + "AWS::QuickSight::Flow": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "FlowId": {}, + "LastUpdatedTime": {}, + "PublishState": {}, + "StepAliases": {} + } + }, + "AWS::QuickSight::Folder": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastUpdatedTime": {} + } + }, + "AWS::QuickSight::KnowledgeBase": { + "Attributes": { + "CreatedAt": {}, + "DocumentCount": {}, + "KnowledgeBaseArn": {}, + "KnowledgeBaseSizeBytes": {}, + "PrimaryOwnerUsername": {}, + "Status": {}, + "Type": {}, + "UpdatedAt": {} + } + }, + "AWS::QuickSight::LimitsProfile": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "ProfileId": {}, + "UpdatedAt": {} + } + }, + "AWS::QuickSight::OAuthClientApplication": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastUpdatedTime": {} + } + }, + "AWS::QuickSight::RefreshSchedule": { + "Attributes": { + "Arn": {} + } + }, + "AWS::QuickSight::Space": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "CreatedBy": {}, + "UpdatedAt": {} + } + }, + "AWS::QuickSight::Template": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastUpdatedTime": {}, + "Version": {}, + "Version.CreatedTime": {}, + "Version.DataSetConfigurations": {}, + "Version.Description": {}, + "Version.Errors": {}, + "Version.Sheets": {}, + "Version.SourceEntityArn": {}, + "Version.Status": {}, + "Version.ThemeArn": {}, + "Version.VersionNumber": {} + } + }, + "AWS::QuickSight::Theme": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastUpdatedTime": {}, + "Type": {}, + "Version": {}, + "Version.Arn": {}, + "Version.BaseThemeId": {}, + "Version.Configuration": {}, + "Version.Configuration.DataColorPalette": {}, + "Version.Configuration.Sheet": {}, + "Version.Configuration.Typography": {}, + "Version.Configuration.UIColorPalette": {}, + "Version.CreatedTime": {}, + "Version.Description": {}, + "Version.Errors": {}, + "Version.Status": {}, + "Version.VersionNumber": {} + } + }, + "AWS::QuickSight::Topic": { + "Attributes": { + "Arn": {} + } + }, + "AWS::QuickSight::TopicV2": { + "Attributes": { + "Arn": {} + } + }, + "AWS::QuickSight::VPCConnection": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastUpdatedTime": {}, + "NetworkInterfaces": {}, + "Status": {}, + "VPCId": {} + } + }, + "AWS::RAM::Permission": { + "Attributes": { + "Arn": {}, + "IsResourceTypeDefault": {}, + "PermissionType": {}, + "Version": {} + } + }, + "AWS::RAM::ResourceShare": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "FeatureSet": {}, + "LastUpdatedTime": {}, + "OwningAccountId": {}, + "Status": {} + } + }, + "AWS::RDS::ClusterSnapshot": { + "Attributes": { + "AllocatedStorage": {}, + "AvailabilityZones": {}, + "ClusterCreateTime": {}, + "DBClusterSnapshotArn": {}, + "DbClusterResourceId": {}, + "Engine": {}, + "EngineMode": {}, + "EngineVersion": {}, + "IAMDatabaseAuthenticationEnabled": {}, + "KmsKeyId": {}, + "LicenseModel": {}, + "MasterUsername": {}, + "Port": {}, + "SnapshotCreateTime": {}, + "SnapshotType": {}, + "Status": {}, + "StorageEncrypted": {}, + "VpcId": {} + } + }, + "AWS::RDS::CustomDBEngineVersion": { + "Attributes": { + "DBEngineVersionArn": {} + } + }, + "AWS::RDS::DBCluster": { + "Attributes": { + "DBClusterArn": {}, + "DBClusterResourceId": {}, + "Endpoint": {}, + "Endpoint.Address": {}, + "Endpoint.Port": {}, + "MasterUserSecret.SecretArn": {}, + "ReadEndpoint": {}, + "ReadEndpoint.Address": {}, + "StorageEncryptionType": {}, + "StorageThroughput": {} + } + }, + "AWS::RDS::DBClusterAutomatedBackup": { + "Attributes": { + "AllocatedStorage": {}, + "AvailabilityZones": {}, + "ClusterCreateTime": {}, + "DBClusterArn": {}, + "DBClusterAutomatedBackupsArn": {}, + "DbClusterResourceId": {}, + "KmsKeyId": {}, + "Region": {}, + "RestoreWindow": {}, + "RestoreWindow.EarliestTime": {}, + "RestoreWindow.LatestTime": {}, + "Status": {}, + "Tags": {} + } + }, + "AWS::RDS::DBInstance": { + "Attributes": { + "AutomaticRestartTime": {}, + "CertificateDetails": {}, + "CertificateDetails.CAIdentifier": {}, + "CertificateDetails.ValidTill": {}, + "DBInstanceArn": {}, + "DBInstanceStatus": {}, + "DBSystemId": {}, + "DbiResourceId": {}, + "Endpoint": {}, + "Endpoint.Address": {}, + "Endpoint.HostedZoneId": {}, + "Endpoint.Port": {}, + "InstanceCreateTime": {}, + "IsStorageConfigUpgradeAvailable": {}, + "LatestRestorableTime": {}, + "ListenerEndpoint": {}, + "ListenerEndpoint.Address": {}, + "ListenerEndpoint.HostedZoneId": {}, + "ListenerEndpoint.Port": {}, + "MasterUserSecret.SecretArn": {}, + "PercentProgress": {}, + "ReadReplicaDBClusterIdentifiers": {}, + "ReadReplicaDBInstanceIdentifiers": {}, + "ResumeFullAutomationModeTime": {}, + "SecondaryAvailabilityZone": {}, + "StatusInfos": {}, + "StorageOperationPercentProgress": {}, + "StorageOperationStatus": {} + } + }, + "AWS::RDS::DBInstanceAutomatedBackup": { + "Attributes": { + "AvailabilityZone": {}, + "BackupTarget": {}, + "DBInstanceArn": {}, + "DBInstanceAutomatedBackupsArn": {}, + "DbiResourceId": {}, + "EngineVersion": {}, + "IAMDatabaseAuthenticationEnabled": {}, + "InstanceCreateTime": {}, + "Iops": {}, + "KmsKeyId": {}, + "LicenseModel": {}, + "OptionGroupName": {}, + "Region": {}, + "RestoreWindow": {}, + "RestoreWindow.EarliestTime": {}, + "RestoreWindow.LatestTime": {}, + "Status": {}, + "StorageThroughput": {}, + "Tags": {}, + "VpcId": {} + } + }, + "AWS::RDS::DBParameterGroup": { + "Attributes": { + "DBParameterGroupArn": {}, + "DBParameterGroupName": {} + } + }, + "AWS::RDS::DBProxy": { + "Attributes": { + "DBProxyArn": {}, + "Endpoint": {}, + "VpcId": {} + } + }, + "AWS::RDS::DBProxyEndpoint": { + "Attributes": { + "DBProxyEndpointArn": {}, + "Endpoint": {}, + "IsDefault": {}, + "VpcId": {} + } + }, + "AWS::RDS::DBProxyTargetGroup": { + "Attributes": { + "TargetGroupArn": {} + } + }, + "AWS::RDS::DBShardGroup": { + "Attributes": { + "DBShardGroupResourceId": {}, + "Endpoint": {} + } + }, + "AWS::RDS::DBSnapshot": { + "Attributes": { + "AllocatedStorage": {}, + "AvailabilityZone": {}, + "DBSnapshotArn": {}, + "DbiResourceId": {}, + "Encrypted": {}, + "Engine": {}, + "EngineVersion": {}, + "IAMDatabaseAuthenticationEnabled": {}, + "InstanceCreateTime": {}, + "Iops": {}, + "KmsKeyId": {}, + "LicenseModel": {}, + "MasterUsername": {}, + "OptionGroupName": {}, + "OriginalSnapshotCreateTime": {}, + "Port": {}, + "SnapshotCreateTime": {}, + "SnapshotType": {}, + "Status": {}, + "StorageThroughput": {}, + "StorageType": {}, + "VpcId": {} + } + }, + "AWS::RDS::DBSubnetGroup": { + "Attributes": { + "DBSubnetGroupArn": {} + } + }, + "AWS::RDS::GlobalCluster": { + "Attributes": { + "GlobalEndpoint": {}, + "GlobalEndpoint.Address": {} + } + }, + "AWS::RDS::Integration": { + "Attributes": { + "CreateTime": {}, + "IntegrationArn": {} + } + }, + "AWS::RDS::ReservedDBInstance": { + "Attributes": { + "CurrencyCode": {}, + "DBInstanceClass": {}, + "Duration": {}, + "FixedPrice": {}, + "MultiAZ": {}, + "OfferingType": {}, + "ProductDescription": {}, + "RecurringCharges": {}, + "ReservedDBInstanceArn": {}, + "ReservedDBInstanceId": {}, + "ReservedDBInstancesOfferingId": {}, + "StartTime": {}, + "State": {}, + "Tags": {}, + "UsagePrice": {} + } + }, + "AWS::RTBFabric::InboundExternalLink": { + "Attributes": { + "Arn": {}, + "CreatedTimestamp": {}, + "DomainName": {}, + "LinkId": {}, + "LinkStatus": {}, + "UpdatedTimestamp": {} + } + }, + "AWS::RTBFabric::Link": { + "Attributes": { + "Arn": {}, + "CreatedTimestamp": {}, + "LinkDirection": {}, + "LinkId": {}, + "LinkStatus": {}, + "UpdatedTimestamp": {} + } + }, + "AWS::RTBFabric::LinkRoutingRule": { + "Attributes": { + "Arn": {}, + "CreatedTimestamp": {}, + "RuleId": {}, + "Status": {}, + "UpdatedTimestamp": {} + } + }, + "AWS::RTBFabric::OutboundExternalLink": { + "Attributes": { + "Arn": {}, + "CreatedTimestamp": {}, + "LinkId": {}, + "LinkStatus": {}, + "UpdatedTimestamp": {} + } + }, + "AWS::RTBFabric::RequesterGateway": { + "Attributes": { + "ActiveLinksCount": {}, + "Arn": {}, + "CreatedTimestamp": {}, + "DomainName": {}, + "GatewayId": {}, + "RequesterGatewayStatus": {}, + "TotalLinksCount": {}, + "UpdatedTimestamp": {} + } + }, + "AWS::RTBFabric::ResponderGateway": { + "Attributes": { + "Arn": {}, + "CertificateAssociationStatus": {}, + "CreatedTimestamp": {}, + "ExternalInboundEndpoint": {}, + "GatewayId": {}, + "ResponderGatewayStatus": {}, + "UpdatedTimestamp": {} + } + }, + "AWS::RUM::AppMonitor": { + "Attributes": { + "Id": {} + } + }, + "AWS::Rbin::Rule": { + "Attributes": { + "Arn": {}, + "Identifier": {}, + "LockState": {} + } + }, + "AWS::Redshift::Cluster": { + "Attributes": { + "ClusterNamespaceArn": {}, + "DeferMaintenanceIdentifier": {}, + "Endpoint.Address": {}, + "Endpoint.Port": {}, + "MasterPasswordSecretArn": {} + } + }, + "AWS::Redshift::ClusterSubnetGroup": { + "Attributes": { + "ClusterSubnetGroupName": {} + } + }, + "AWS::Redshift::DataShare": { + "Attributes": { + "AllowPubliclyAccessibleConsumers": {}, + "DataShareArn": {}, + "DataShareAssociations": {}, + "ProducerArn": {} + } + }, + "AWS::Redshift::EndpointAccess": { + "Attributes": { + "Address": {}, + "EndpointCreateTime": {}, + "EndpointStatus": {}, + "Port": {}, + "VpcEndpoint": {}, + "VpcEndpoint.NetworkInterfaces": {}, + "VpcEndpoint.VpcEndpointId": {}, + "VpcEndpoint.VpcId": {}, + "VpcSecurityGroups": {} + } + }, + "AWS::Redshift::EndpointAuthorization": { + "Attributes": { + "AllowedAllVPCs": {}, + "AllowedVPCs": {}, + "AuthorizeTime": {}, + "ClusterStatus": {}, + "EndpointCount": {}, + "Grantee": {}, + "Grantor": {}, + "Status": {} + } + }, + "AWS::Redshift::EventSubscription": { + "Attributes": { + "CustSubscriptionId": {}, + "CustomerAwsId": {}, + "EventCategoriesList": {}, + "SourceIdsList": {}, + "Status": {}, + "SubscriptionCreationTime": {} + } + }, + "AWS::Redshift::Integration": { + "Attributes": { + "CreateTime": {}, + "IntegrationArn": {} + } + }, + "AWS::Redshift::QEV2IdcApplication": { + "Attributes": { + "IdcManagedApplicationArn": {}, + "IdcOnboardStatus": {}, + "Qev2IdcApplicationArn": {} + } + }, + "AWS::Redshift::ScheduledAction": { + "Attributes": { + "NextInvocations": {}, + "State": {} + } + }, + "AWS::Redshift::Snapshot": { + "Attributes": { + "AvailabilityZone": {}, + "ClusterCreateTime": {}, + "ClusterVersion": {}, + "DBName": {}, + "Encrypted": {}, + "EncryptedWithHSM": {}, + "EngineFullVersion": {}, + "EnhancedVpcRouting": {}, + "KmsKeyId": {}, + "MaintenanceTrackName": {}, + "MasterUsername": {}, + "NodeType": {}, + "NumberOfNodes": {}, + "OwnerAccount": {}, + "Port": {}, + "SnapshotArn": {}, + "SnapshotCreateTime": {}, + "SnapshotType": {}, + "Status": {}, + "VpcId": {} + } + }, + "AWS::Redshift::SnapshotCopyGrant": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Redshift::SnapshotSchedule": { + "Attributes": { + "Arn": {}, + "AssociatedClusterCount": {} + } + }, + "AWS::Redshift::UsageLimit": { + "Attributes": { + "Arn": {}, + "UsageLimitId": {} + } + }, + "AWS::RedshiftServerless::Namespace": { + "Attributes": { + "Namespace": {}, + "Namespace.AdminPasswordSecretArn": {}, + "Namespace.AdminPasswordSecretKmsKeyId": {}, + "Namespace.AdminUsername": {}, + "Namespace.CreationDate": {}, + "Namespace.DbName": {}, + "Namespace.DefaultIamRoleArn": {}, + "Namespace.IamRoles": {}, + "Namespace.KmsKeyId": {}, + "Namespace.LogExports": {}, + "Namespace.NamespaceArn": {}, + "Namespace.NamespaceId": {}, + "Namespace.NamespaceName": {}, + "Namespace.Status": {} + } + }, + "AWS::RedshiftServerless::RecoveryPoint": { + "Attributes": { + "Arn": {}, + "NamespaceArn": {}, + "RecoveryPointCreateTime": {}, + "RecoveryPointId": {}, + "Tags": {}, + "TotalSizeInMegaBytes": {} + } + }, + "AWS::RedshiftServerless::Snapshot": { + "Attributes": { + "OwnerAccount": {}, + "Snapshot": {}, + "Snapshot.AdminUsername": {}, + "Snapshot.KmsKeyId": {}, + "Snapshot.NamespaceArn": {}, + "Snapshot.NamespaceName": {}, + "Snapshot.OwnerAccount": {}, + "Snapshot.RetentionPeriod": {}, + "Snapshot.SnapshotArn": {}, + "Snapshot.SnapshotCreateTime": {}, + "Snapshot.SnapshotName": {}, + "Snapshot.Status": {} + } + }, + "AWS::RedshiftServerless::Workgroup": { + "Attributes": { + "Workgroup.BaseCapacity": {}, + "Workgroup.ConfigParameters": {}, + "Workgroup.CreationDate": {}, + "Workgroup.Endpoint.Address": {}, + "Workgroup.Endpoint.Port": {}, + "Workgroup.EnhancedVpcRouting": {}, + "Workgroup.MaxCapacity": {}, + "Workgroup.NamespaceName": {}, + "Workgroup.PubliclyAccessible": {}, + "Workgroup.SecurityGroupIds": {}, + "Workgroup.Status": {}, + "Workgroup.SubnetIds": {}, + "Workgroup.TrackName": {}, + "Workgroup.WorkgroupArn": {}, + "Workgroup.WorkgroupId": {}, + "Workgroup.WorkgroupName": {} + } + }, + "AWS::RefactorSpaces::Application": { + "Attributes": { + "ApiGatewayId": {}, + "ApplicationIdentifier": {}, + "Arn": {}, + "NlbArn": {}, + "NlbName": {}, + "ProxyUrl": {}, + "StageName": {}, + "VpcLinkId": {} + } + }, + "AWS::RefactorSpaces::Environment": { + "Attributes": { + "Arn": {}, + "EnvironmentIdentifier": {}, + "TransitGatewayId": {} + } + }, + "AWS::RefactorSpaces::Route": { + "Attributes": { + "Arn": {}, + "PathResourceToId": {}, + "RouteIdentifier": {} + } + }, + "AWS::RefactorSpaces::Service": { + "Attributes": { + "Arn": {}, + "ServiceIdentifier": {} + } + }, + "AWS::Rekognition::Collection": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Rekognition::Dataset": { + "Attributes": { + "DatasetArn": {} + } + }, + "AWS::Rekognition::Project": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Rekognition::StreamProcessor": { + "Attributes": { + "Arn": {}, + "Status": {}, + "StatusMessage": {} + } + }, + "AWS::ResilienceHub::App": { + "Attributes": { + "AppArn": {}, + "DriftStatus": {} + } + }, + "AWS::ResilienceHub::AppAssessment": { + "Attributes": { + "AppVersion": {}, + "AssessmentArn": {}, + "AssessmentStatus": {}, + "Compliance": {}, + "ComplianceStatus": {}, + "Cost": {}, + "Cost.Amount": {}, + "Cost.Currency": {}, + "Cost.Frequency": {}, + "DriftStatus": {}, + "EndTime": {}, + "Invoker": {}, + "Policy": {}, + "Policy.DataLocationConstraint": {}, + "Policy.Policy": {}, + "Policy.PolicyArn": {}, + "Policy.PolicyName": {}, + "ResiliencyScore": {}, + "ResiliencyScore.ComponentScore": {}, + "ResiliencyScore.DisruptionScore": {}, + "ResiliencyScore.Score": {}, + "StartTime": {}, + "VersionName": {} + } + }, + "AWS::ResilienceHub::RecommendationTemplate": { + "Attributes": { + "AppArn": {}, + "RecommendationTemplateArn": {}, + "Status": {}, + "TemplatesLocation": {}, + "TemplatesLocation.Bucket": {}, + "TemplatesLocation.Prefix": {} + } + }, + "AWS::ResilienceHub::ResiliencyPolicy": { + "Attributes": { + "PolicyArn": {} + } + }, + "AWS::ResilienceHubV2::Policy": { + "Attributes": { + "AssociatedServiceCount": {}, + "CreatedAt": {}, + "PolicyArn": {}, + "UpdatedAt": {} + } + }, + "AWS::ResilienceHubV2::Service": { + "Attributes": { + "CreatedAt": {}, + "EffectivePolicyValues": {}, + "EffectivePolicyValues.AvailabilitySlo": {}, + "EffectivePolicyValues.AvailabilitySlo.PolicyName": {}, + "EffectivePolicyValues.AvailabilitySlo.Value": {}, + "EffectivePolicyValues.MultiAzDrApproach": {}, + "EffectivePolicyValues.MultiAzDrApproach.PolicyName": {}, + "EffectivePolicyValues.MultiAzDrApproach.Value": {}, + "EffectivePolicyValues.MultiAzRpo": {}, + "EffectivePolicyValues.MultiAzRpo.PolicyName": {}, + "EffectivePolicyValues.MultiAzRpo.Value": {}, + "EffectivePolicyValues.MultiAzRto": {}, + "EffectivePolicyValues.MultiAzRto.PolicyName": {}, + "EffectivePolicyValues.MultiAzRto.Value": {}, + "EffectivePolicyValues.MultiRegionDrApproach": {}, + "EffectivePolicyValues.MultiRegionDrApproach.PolicyName": {}, + "EffectivePolicyValues.MultiRegionDrApproach.Value": {}, + "EffectivePolicyValues.MultiRegionRpo": {}, + "EffectivePolicyValues.MultiRegionRpo.PolicyName": {}, + "EffectivePolicyValues.MultiRegionRpo.Value": {}, + "EffectivePolicyValues.MultiRegionRto": {}, + "EffectivePolicyValues.MultiRegionRto.PolicyName": {}, + "EffectivePolicyValues.MultiRegionRto.Value": {}, + "ServiceArn": {}, + "UpdatedAt": {} + } + }, + "AWS::ResilienceHubV2::ServiceFunction": { + "Attributes": { + "CreatedAt": {}, + "ResourceCount": {}, + "ServiceFunctionId": {}, + "Source": {}, + "UpdatedAt": {} + } + }, + "AWS::ResilienceHubV2::System": { + "Attributes": { + "CreatedAt": {}, + "SystemArn": {}, + "SystemId": {}, + "UpdatedAt": {} + } + }, + "AWS::ResilienceHubV2::UserJourney": { + "Attributes": { + "CreatedAt": {}, + "UpdatedAt": {}, + "UserJourneyId": {} + } + }, + "AWS::ResourceExplorer2::DefaultViewAssociation": { + "Attributes": { + "AssociatedAwsPrincipal": {} + } + }, + "AWS::ResourceExplorer2::Index": { + "Attributes": { + "Arn": {}, + "IndexState": {} + } + }, + "AWS::ResourceExplorer2::View": { + "Attributes": { + "ViewArn": {} + } + }, + "AWS::ResourceGroups::Group": { + "Attributes": { + "Arn": {} + } + }, + "AWS::ResourceGroups::TagSyncTask": { + "Attributes": { + "GroupArn": {}, + "GroupName": {}, + "Status": {}, + "TaskArn": {} + } + }, + "AWS::RoboMaker::Fleet": { + "Attributes": { + "Arn": {} + } + }, + "AWS::RoboMaker::Robot": { + "Attributes": { + "Arn": {} + } + }, + "AWS::RoboMaker::RobotApplication": { + "Attributes": { + "Arn": {}, + "CurrentRevisionId": {} + } + }, + "AWS::RoboMaker::RobotApplicationVersion": { + "Attributes": { + "ApplicationVersion": {}, + "Arn": {} + } + }, + "AWS::RoboMaker::SimulationApplication": { + "Attributes": { + "Arn": {}, + "CurrentRevisionId": {} + } + }, + "AWS::RoboMaker::SimulationApplicationVersion": { + "Attributes": { + "ApplicationVersion": {}, + "Arn": {} + } + }, + "AWS::RolesAnywhere::CRL": { + "Attributes": { + "CrlId": {} + } + }, + "AWS::RolesAnywhere::Profile": { + "Attributes": { + "ProfileArn": {}, + "ProfileId": {} + } + }, + "AWS::RolesAnywhere::TrustAnchor": { + "Attributes": { + "TrustAnchorArn": {}, + "TrustAnchorId": {} + } + }, + "AWS::Route53::CidrCollection": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::Route53::HealthCheck": { + "Attributes": { + "HealthCheckId": {} + } + }, + "AWS::Route53::HostedZone": { + "Attributes": { + "Id": {}, + "NameServers": {} + } + }, + "AWS::Route53::TrafficPolicy": { + "Attributes": { + "Arn": {}, + "Id": {}, + "Type": {}, + "Version": {} + } + }, + "AWS::Route53::TrafficPolicyInstance": { + "Attributes": { + "Arn": {}, + "Id": {}, + "State": {}, + "TrafficPolicyType": {} + } + }, + "AWS::Route53GlobalResolver::AccessSource": { + "Attributes": { + "AccessSourceId": {}, + "Arn": {}, + "CreatedAt": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::Route53GlobalResolver::AccessToken": { + "Attributes": { + "AccessTokenId": {}, + "Arn": {}, + "CreatedAt": {}, + "GlobalResolverId": {}, + "Status": {}, + "UpdatedAt": {}, + "Value": {} + } + }, + "AWS::Route53GlobalResolver::DnsView": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "DnsViewId": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::Route53GlobalResolver::FirewallDomainList": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "DomainCount": {}, + "FirewallDomainListId": {}, + "Status": {}, + "StatusMessage": {}, + "UpdatedAt": {} + } + }, + "AWS::Route53GlobalResolver::FirewallRule": { + "Attributes": { + "CreatedAt": {}, + "FirewallRuleId": {}, + "QueryType": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::Route53GlobalResolver::GlobalResolver": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "DnsName": {}, + "GlobalResolverId": {}, + "IPv4Addresses": {}, + "IPv6Addresses": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::Route53GlobalResolver::HostedZoneAssociation": { + "Attributes": { + "CreatedAt": {}, + "HostedZoneAssociationId": {}, + "HostedZoneName": {}, + "Status": {}, + "UpdatedAt": {} + } + }, + "AWS::Route53Profiles::Profile": { + "Attributes": { + "Arn": {}, + "ClientToken": {}, + "Id": {}, + "ShareStatus": {} + } + }, + "AWS::Route53Profiles::ProfileAssociation": { + "Attributes": { + "Id": {} + } + }, + "AWS::Route53Profiles::ProfileResourceAssociation": { + "Attributes": { + "Id": {}, + "ResourceType": {} + } + }, + "AWS::Route53RecoveryControl::Cluster": { + "Attributes": { + "ClusterArn": {}, + "ClusterEndpoints": {}, + "Status": {} + } + }, + "AWS::Route53RecoveryControl::ControlPanel": { + "Attributes": { + "ControlPanelArn": {}, + "DefaultControlPanel": {}, + "RoutingControlCount": {}, + "Status": {} + } + }, + "AWS::Route53RecoveryControl::RoutingControl": { + "Attributes": { + "RoutingControlArn": {}, + "Status": {} + } + }, + "AWS::Route53RecoveryControl::SafetyRule": { + "Attributes": { + "SafetyRuleArn": {}, + "Status": {} + } + }, + "AWS::Route53RecoveryReadiness::Cell": { + "Attributes": { + "CellArn": {}, + "ParentReadinessScopes": {} + } + }, + "AWS::Route53RecoveryReadiness::ReadinessCheck": { + "Attributes": { + "ReadinessCheckArn": {} + } + }, + "AWS::Route53RecoveryReadiness::RecoveryGroup": { + "Attributes": { + "RecoveryGroupArn": {} + } + }, + "AWS::Route53RecoveryReadiness::ResourceSet": { + "Attributes": { + "ResourceSetArn": {} + } + }, + "AWS::Route53Resolver::FirewallConfig": { + "Attributes": { + "Arn": {}, + "Id": {}, + "OwnerId": {} + } + }, + "AWS::Route53Resolver::FirewallDomainList": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "CreatorRequestId": {}, + "DomainCount": {}, + "Id": {}, + "ManagedOwnerName": {}, + "ModificationTime": {}, + "Status": {}, + "StatusMessage": {} + } + }, + "AWS::Route53Resolver::FirewallRuleGroup": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "CreatorRequestId": {}, + "Id": {}, + "ModificationTime": {}, + "OwnerId": {}, + "RuleCount": {}, + "ShareStatus": {}, + "Status": {}, + "StatusMessage": {} + } + }, + "AWS::Route53Resolver::FirewallRuleGroupAssociation": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "CreatorRequestId": {}, + "Id": {}, + "ManagedOwnerName": {}, + "ModificationTime": {}, + "Status": {}, + "StatusMessage": {} + } + }, + "AWS::Route53Resolver::OutpostResolver": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "CreatorRequestId": {}, + "Id": {}, + "ModificationTime": {}, + "Status": {}, + "StatusMessage": {} + } + }, + "AWS::Route53Resolver::ResolverConfig": { + "Attributes": { + "AutodefinedReverse": {}, + "Id": {}, + "OwnerId": {} + } + }, + "AWS::Route53Resolver::ResolverDNSSECConfig": { + "Attributes": { + "Id": {}, + "OwnerId": {}, + "ValidationStatus": {} + } + }, + "AWS::Route53Resolver::ResolverEndpoint": { + "Attributes": { + "Arn": {}, + "Direction": {}, + "HostVPCId": {}, + "IpAddressCount": {}, + "Name": {}, + "ResolverEndpointId": {} + } + }, + "AWS::Route53Resolver::ResolverQueryLoggingConfig": { + "Attributes": { + "Arn": {}, + "AssociationCount": {}, + "CreationTime": {}, + "CreatorRequestId": {}, + "Id": {}, + "OwnerId": {}, + "ShareStatus": {}, + "Status": {} + } + }, + "AWS::Route53Resolver::ResolverQueryLoggingConfigAssociation": { + "Attributes": { + "CreationTime": {}, + "Error": {}, + "ErrorMessage": {}, + "Id": {}, + "Status": {} + } + }, + "AWS::Route53Resolver::ResolverRule": { + "Attributes": { + "Arn": {}, + "DomainName": {}, + "Name": {}, + "ResolverEndpointId": {}, + "ResolverRuleId": {}, + "TargetIps": {} + } + }, + "AWS::Route53Resolver::ResolverRuleAssociation": { + "Attributes": { + "Name": {}, + "ResolverRuleAssociationId": {}, + "ResolverRuleId": {}, + "VPCId": {} + } + }, + "AWS::S3::AccessGrant": { + "Attributes": { + "AccessGrantArn": {}, + "AccessGrantId": {}, + "GrantScope": {} + } + }, + "AWS::S3::AccessGrantsInstance": { + "Attributes": { + "AccessGrantsInstanceArn": {}, + "AccessGrantsInstanceId": {} + } + }, + "AWS::S3::AccessGrantsLocation": { + "Attributes": { + "AccessGrantsLocationArn": {}, + "AccessGrantsLocationId": {} + } + }, + "AWS::S3::AccessPoint": { + "Attributes": { + "Alias": {}, + "Arn": {}, + "Name": {}, + "NetworkOrigin": {} + } + }, + "AWS::S3::Bucket": { + "Attributes": { + "Arn": {}, + "DomainName": {}, + "DualStackDomainName": {}, + "MetadataConfiguration.AnnotationTableConfiguration.TableArn": {}, + "MetadataConfiguration.AnnotationTableConfiguration.TableName": {}, + "MetadataConfiguration.Destination": {}, + "MetadataConfiguration.Destination.TableBucketArn": {}, + "MetadataConfiguration.Destination.TableBucketType": {}, + "MetadataConfiguration.Destination.TableNamespace": {}, + "MetadataConfiguration.InventoryTableConfiguration.TableArn": {}, + "MetadataConfiguration.InventoryTableConfiguration.TableName": {}, + "MetadataConfiguration.JournalTableConfiguration.TableArn": {}, + "MetadataConfiguration.JournalTableConfiguration.TableName": {}, + "MetadataTableConfiguration.S3TablesDestination.TableArn": {}, + "MetadataTableConfiguration.S3TablesDestination.TableNamespace": {}, + "RegionalDomainName": {}, + "WebsiteURL": {} + } + }, + "AWS::S3::MultiRegionAccessPoint": { + "Attributes": { + "Alias": {}, + "CreatedAt": {} + } + }, + "AWS::S3::MultiRegionAccessPointPolicy": { + "Attributes": { + "PolicyStatus": {}, + "PolicyStatus.IsPublic": {} + } + }, + "AWS::S3::StorageLens": { + "Attributes": { + "StorageLensConfiguration.StorageLensArn": {} + } + }, + "AWS::S3::StorageLensGroup": { + "Attributes": { + "StorageLensGroupArn": {} + } + }, + "AWS::S3Express::AccessPoint": { + "Attributes": { + "Arn": {}, + "NetworkOrigin": {} + } + }, + "AWS::S3Express::DirectoryBucket": { + "Attributes": { + "Arn": {}, + "AvailabilityZoneName": {} + } + }, + "AWS::S3Files::AccessPoint": { + "Attributes": { + "AccessPointArn": {}, + "AccessPointId": {}, + "OwnerId": {}, + "Status": {} + } + }, + "AWS::S3Files::FileSystem": { + "Attributes": { + "CreationTime": {}, + "FileSystemArn": {}, + "FileSystemId": {}, + "OwnerId": {}, + "Status": {}, + "StatusMessage": {}, + "SynchronizationConfiguration.LatestVersionNumber": {} + } + }, + "AWS::S3Files::MountTarget": { + "Attributes": { + "AvailabilityZoneId": {}, + "MountTargetId": {}, + "NetworkInterfaceId": {}, + "OwnerId": {}, + "Status": {}, + "StatusMessage": {}, + "VpcId": {} + } + }, + "AWS::S3ObjectLambda::AccessPoint": { + "Attributes": { + "Alias": {}, + "Alias.Status": {}, + "Alias.Value": {}, + "Arn": {}, + "CreationDate": {}, + "PublicAccessBlockConfiguration": {}, + "PublicAccessBlockConfiguration.BlockPublicAcls": {}, + "PublicAccessBlockConfiguration.BlockPublicPolicy": {}, + "PublicAccessBlockConfiguration.IgnorePublicAcls": {}, + "PublicAccessBlockConfiguration.RestrictPublicBuckets": {} + } + }, + "AWS::S3Outposts::AccessPoint": { + "Attributes": { + "Arn": {} + } + }, + "AWS::S3Outposts::Bucket": { + "Attributes": { + "Arn": {} + } + }, + "AWS::S3Outposts::Endpoint": { + "Attributes": { + "Arn": {}, + "CidrBlock": {}, + "CreationTime": {}, + "Id": {}, + "NetworkInterfaces": {}, + "Status": {} + } + }, + "AWS::S3Tables::Table": { + "Attributes": { + "TableARN": {}, + "VersionToken": {}, + "WarehouseLocation": {} + } + }, + "AWS::S3Tables::TableBucket": { + "Attributes": { + "TableBucketARN": {} + } + }, + "AWS::S3Tables::TablePolicy": { + "Attributes": { + "Namespace": {}, + "TableBucketARN": {}, + "TableName": {} + } + }, + "AWS::S3Vectors::Index": { + "Attributes": { + "CreationTime": {}, + "IndexArn": {} + } + }, + "AWS::S3Vectors::VectorBucket": { + "Attributes": { + "CreationTime": {}, + "VectorBucketArn": {} + } + }, + "AWS::SCN::Dataset": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastModifiedTime": {} + } + }, + "AWS::SCN::Namespace": { + "Attributes": { + "Arn": {}, + "CreatedTime": {}, + "LastModifiedTime": {} + } + }, + "AWS::SDB::Domain": { + "Attributes": { + "Id": {} + } + }, + "AWS::SES::ConfigurationSetEventDestination": { + "Attributes": { + "Id": {} + } + }, + "AWS::SES::EmailIdentity": { + "Attributes": { + "DkimDNSTokenName1": {}, + "DkimDNSTokenName2": {}, + "DkimDNSTokenName3": {}, + "DkimDNSTokenValue1": {}, + "DkimDNSTokenValue2": {}, + "DkimDNSTokenValue3": {} + } + }, + "AWS::SES::MailManagerAddonInstance": { + "Attributes": { + "AddonInstanceArn": {}, + "AddonInstanceId": {}, + "AddonName": {} + } + }, + "AWS::SES::MailManagerAddonSubscription": { + "Attributes": { + "AddonSubscriptionArn": {}, + "AddonSubscriptionId": {} + } + }, + "AWS::SES::MailManagerAddressList": { + "Attributes": { + "AddressListArn": {}, + "AddressListId": {} + } + }, + "AWS::SES::MailManagerArchive": { + "Attributes": { + "ArchiveArn": {}, + "ArchiveId": {}, + "ArchiveState": {} + } + }, + "AWS::SES::MailManagerIngressPoint": { + "Attributes": { + "ARecord": {}, + "IngressPointArn": {}, + "IngressPointId": {}, + "Status": {} + } + }, + "AWS::SES::MailManagerRelay": { + "Attributes": { + "RelayArn": {}, + "RelayId": {} + } + }, + "AWS::SES::MailManagerRuleSet": { + "Attributes": { + "RuleSetArn": {}, + "RuleSetId": {} + } + }, + "AWS::SES::MailManagerTrafficPolicy": { + "Attributes": { + "TrafficPolicyArn": {}, + "TrafficPolicyId": {} + } + }, + "AWS::SES::ReceiptFilter": { + "Attributes": { + "Id": {} + } + }, + "AWS::SES::ReceiptRule": { + "Attributes": { + "RuleName": {} + } + }, + "AWS::SES::Template": { + "Attributes": { + "Id": {} + } + }, + "AWS::SES::Tenant": { + "Attributes": { + "Arn": {} + } + }, + "AWS::SES::VdmAttributes": { + "Attributes": { + "VdmAttributesResourceId": {} + } + }, + "AWS::SMSVOICE::ConfigurationSet": { + "Attributes": { + "Arn": {} + } + }, + "AWS::SMSVOICE::OptOutList": { + "Attributes": { + "Arn": {} + } + }, + "AWS::SMSVOICE::PhoneNumber": { + "Attributes": { + "Arn": {}, + "PhoneNumber": {}, + "PhoneNumberId": {} + } + }, + "AWS::SMSVOICE::Pool": { + "Attributes": { + "Arn": {}, + "PoolId": {} + } + }, + "AWS::SMSVOICE::ProtectConfiguration": { + "Attributes": { + "Arn": {}, + "ProtectConfigurationId": {} + } + }, + "AWS::SMSVOICE::Registration": { + "Attributes": { + "CreatedTimestamp": {}, + "CurrentVersionNumber": {}, + "RegistrationArn": {}, + "RegistrationId": {}, + "RegistrationStatus": {} + } + }, + "AWS::SMSVOICE::SenderId": { + "Attributes": { + "Arn": {} + } + }, + "AWS::SNS::Subscription": { + "Attributes": { + "Arn": {} + } + }, + "AWS::SNS::Topic": { + "Attributes": { + "TopicArn": {}, + "TopicName": {} + } + }, + "AWS::SNS::TopicPolicy": { + "Attributes": { + "Id": {} + } + }, + "AWS::SQS::Queue": { + "Attributes": { + "Arn": {}, + "QueueName": {}, + "QueueUrl": {} + } + }, + "AWS::SQS::QueuePolicy": { + "Attributes": { + "Id": {} + } + }, + "AWS::SSM::Association": { + "Attributes": { + "AssociationId": {} + } + }, + "AWS::SSM::AutomationExecution": { + "Attributes": { + "Arn": {}, + "AutomationExecutionId": {}, + "AutomationExecutionStatus": {}, + "DocumentVersion": {}, + "ExecutedBy": {}, + "ExecutionStartTime": {}, + "Mode": {} + } + }, + "AWS::SSM::CloudConnector": { + "Attributes": { + "CloudConnectorArn": {}, + "CloudConnectorId": {}, + "CreatedAt": {}, + "UpdatedAt": {} + } + }, + "AWS::SSM::MaintenanceWindow": { + "Attributes": { + "WindowId": {} + } + }, + "AWS::SSM::MaintenanceWindowTarget": { + "Attributes": { + "WindowTargetId": {} + } + }, + "AWS::SSM::MaintenanceWindowTask": { + "Attributes": { + "WindowTaskId": {} + } + }, + "AWS::SSM::ManagedInstance": { + "Attributes": { + "AgentVersion": {}, + "Arn": {}, + "ComputerName": {}, + "IPAddress": {}, + "InstanceId": {}, + "IsLatestVersion": {}, + "PingStatus": {}, + "PlatformName": {}, + "PlatformType": {}, + "PlatformVersion": {}, + "ResourceType": {} + } + }, + "AWS::SSM::OpsItem": { + "Attributes": { + "CreatedBy": {}, + "CreatedTime": {}, + "LastModifiedBy": {}, + "LastModifiedTime": {}, + "OpsItemArn": {}, + "OpsItemId": {}, + "OpsItemType": {}, + "Status": {}, + "Version": {} + } + }, + "AWS::SSM::Parameter": { + "Attributes": { + "Arn": {}, + "Type": {}, + "Value": {} + } + }, + "AWS::SSM::PatchBaseline": { + "Attributes": { + "Id": {} + } + }, + "AWS::SSM::ResourcePolicy": { + "Attributes": { + "PolicyHash": {}, + "PolicyId": {} + } + }, + "AWS::SSM::ServiceSetting": { + "Attributes": { + "Arn": {}, + "LastModifiedDate": {}, + "LastModifiedUser": {}, + "Status": {} + } + }, + "AWS::SSM::Session": { + "Attributes": { + "AccessType": {}, + "Arn": {}, + "Owner": {}, + "SessionId": {}, + "StartDate": {}, + "Status": {} + } + }, + "AWS::SSMContacts::Contact": { + "Attributes": { + "Arn": {} + } + }, + "AWS::SSMContacts::ContactChannel": { + "Attributes": { + "Arn": {} + } + }, + "AWS::SSMContacts::Plan": { + "Attributes": { + "Arn": {} + } + }, + "AWS::SSMContacts::Rotation": { + "Attributes": { + "Arn": {} + } + }, + "AWS::SSMGuiConnect::Preferences": { + "Attributes": { + "AccountId": {} + } + }, + "AWS::SSMIncidents::ReplicationSet": { + "Attributes": { + "Arn": {} + } + }, + "AWS::SSMIncidents::ResponsePlan": { + "Attributes": { + "Arn": {} + } + }, + "AWS::SSMQuickSetup::ConfigurationManager": { + "Attributes": { + "CreatedAt": {}, + "LastModifiedAt": {}, + "ManagerArn": {}, + "StatusSummaries": {} + } + }, + "AWS::SSMQuickSetup::LifecycleAutomation": { + "Attributes": { + "AssociationId": {} + } + }, + "AWS::SSO::Application": { + "Attributes": { + "ApplicationArn": {}, + "IdentityStoreArn": {} + } + }, + "AWS::SSO::ApplicationProvider": { + "Attributes": { + "ApplicationProviderArn": {}, + "DisplayData": {}, + "DisplayData.Description": {}, + "DisplayData.DisplayName": {}, + "DisplayData.IconUrl": {}, + "FederationProtocol": {}, + "ResourceServerConfig": {}, + "ResourceServerConfig.Scopes": {} + } + }, + "AWS::SSO::Instance": { + "Attributes": { + "IdentityStoreId": {}, + "InstanceArn": {}, + "OwnerAccountId": {}, + "Status": {} + } + }, + "AWS::SSO::PermissionSet": { + "Attributes": { + "PermissionSetArn": {} + } + }, + "AWS::SWF::Domain": { + "Attributes": { + "Arn": {}, + "Description": {}, + "Name": {}, + "Tags": {}, + "WorkflowExecutionRetentionPeriodInDays": {} + } + }, + "AWS::SageMaker::AIBenchmarkJob": { + "Attributes": { + "AIBenchmarkJobArn": {}, + "AIBenchmarkJobStatus": {}, + "CreationTime": {}, + "EndTime": {} + } + }, + "AWS::SageMaker::AIWorkloadConfig": { + "Attributes": { + "AIWorkloadConfigArn": {}, + "CreationTime": {} + } + }, + "AWS::SageMaker::Action": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "LastModifiedTime": {} + } + }, + "AWS::SageMaker::Algorithm": { + "Attributes": { + "AlgorithmArn": {}, + "CreationTime": {} + } + }, + "AWS::SageMaker::App": { + "Attributes": { + "AppArn": {}, + "BuiltInLifecycleConfigArn": {} + } + }, + "AWS::SageMaker::AppImageConfig": { + "Attributes": { + "AppImageConfigArn": {} + } + }, + "AWS::SageMaker::Artifact": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "LastModifiedTime": {} + } + }, + "AWS::SageMaker::AutoMLJob": { + "Attributes": { + "AutoMLJobArn": {}, + "AutoMLJobName": {}, + "AutoMLJobSecondaryStatus": {}, + "AutoMLJobStatus": {}, + "AutoMLProblemTypeConfigName": {}, + "CreationTime": {}, + "EndTime": {}, + "LastModifiedTime": {} + } + }, + "AWS::SageMaker::Cluster": { + "Attributes": { + "ClusterArn": {}, + "ClusterStatus": {}, + "CreationTime": {}, + "FailureMessage": {} + } + }, + "AWS::SageMaker::CodeRepository": { + "Attributes": { + "CodeRepositoryArn": {}, + "CodeRepositoryName": {} + } + }, + "AWS::SageMaker::Context": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "LastModifiedTime": {} + } + }, + "AWS::SageMaker::DataQualityJobDefinition": { + "Attributes": { + "CreationTime": {}, + "JobDefinitionArn": {} + } + }, + "AWS::SageMaker::Domain": { + "Attributes": { + "DomainArn": {}, + "DomainId": {}, + "HomeEfsFileSystemId": {}, + "SecurityGroupIdForDomainBoundary": {}, + "SingleSignOnApplicationArn": {}, + "SingleSignOnManagedApplicationInstanceId": {}, + "Url": {} + } + }, + "AWS::SageMaker::Endpoint": { + "Attributes": { + "EndpointArn": {}, + "EndpointName": {} + } + }, + "AWS::SageMaker::EndpointConfig": { + "Attributes": { + "EndpointConfigArn": {}, + "EndpointConfigName": {} + } + }, + "AWS::SageMaker::Experiment": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "LastModifiedTime": {} + } + }, + "AWS::SageMaker::FeatureGroup": { + "Attributes": { + "CreationTime": {}, + "FeatureGroupStatus": {} + } + }, + "AWS::SageMaker::Hub": { + "Attributes": { + "CreationTime": {}, + "HubArn": {}, + "HubStatus": {}, + "LastModifiedTime": {} + } + }, + "AWS::SageMaker::HubContentVersion": { + "Attributes": { + "CreationTime": {}, + "DocumentSchemaVersion": {}, + "HubArn": {}, + "HubContentArn": {}, + "HubContentDescription": {}, + "HubContentDisplayName": {}, + "HubContentStatus": {}, + "HubContentVersion": {}, + "LastModifiedTime": {}, + "ReferenceMinVersion": {}, + "ResourceArn": {}, + "SageMakerPublicHubContentArn": {}, + "SupportStatus": {} + } + }, + "AWS::SageMaker::HumanTaskUi": { + "Attributes": { + "CreationTime": {}, + "HumanTaskUiArn": {} + } + }, + "AWS::SageMaker::HyperParameterTuningJob": { + "Attributes": { + "CreationTime": {}, + "HyperParameterTuningJobArn": {}, + "HyperParameterTuningJobStatus": {}, + "ObjectiveStatusCounters": {}, + "ObjectiveStatusCounters.Failed": {}, + "ObjectiveStatusCounters.Pending": {}, + "ObjectiveStatusCounters.Succeeded": {}, + "TrainingJobStatusCounters": {}, + "TrainingJobStatusCounters.Completed": {}, + "TrainingJobStatusCounters.InProgress": {}, + "TrainingJobStatusCounters.NonRetryableError": {}, + "TrainingJobStatusCounters.RetryableError": {}, + "TrainingJobStatusCounters.Stopped": {} + } + }, + "AWS::SageMaker::Image": { + "Attributes": { + "ImageArn": {} + } + }, + "AWS::SageMaker::ImageVersion": { + "Attributes": { + "ContainerImage": {}, + "ImageArn": {}, + "ImageVersionArn": {}, + "Version": {} + } + }, + "AWS::SageMaker::InferenceComponent": { + "Attributes": { + "CreationTime": {}, + "FailureReason": {}, + "InferenceComponentArn": {}, + "InferenceComponentStatus": {}, + "LastModifiedTime": {}, + "RuntimeConfig.CurrentCopyCount": {}, + "RuntimeConfig.DesiredCopyCount": {}, + "RuntimeConfig.PlacementStatus": {}, + "Specification.Container.DeployedImage": {}, + "Specification.Container.DeployedImage.ResolutionTime": {}, + "Specification.Container.DeployedImage.ResolvedImage": {}, + "Specification.Container.DeployedImage.SpecifiedImage": {}, + "Specification.CurrentDataCacheConfig": {}, + "Specification.CurrentDataCacheConfig.EnableCaching": {} + } + }, + "AWS::SageMaker::InferenceExperiment": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "EndpointMetadata": {}, + "EndpointMetadata.EndpointConfigName": {}, + "EndpointMetadata.EndpointName": {}, + "EndpointMetadata.EndpointStatus": {}, + "LastModifiedTime": {}, + "Status": {} + } + }, + "AWS::SageMaker::MlflowApp": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "LastModifiedTime": {}, + "MlflowAppId": {}, + "MlflowVersion": {}, + "Status": {} + } + }, + "AWS::SageMaker::MlflowTrackingServer": { + "Attributes": { + "TrackingServerArn": {} + } + }, + "AWS::SageMaker::Model": { + "Attributes": { + "ModelArn": {}, + "ModelName": {} + } + }, + "AWS::SageMaker::ModelBiasJobDefinition": { + "Attributes": { + "CreationTime": {}, + "JobDefinitionArn": {} + } + }, + "AWS::SageMaker::ModelCard": { + "Attributes": { + "CreatedBy.DomainId": {}, + "CreatedBy.UserProfileArn": {}, + "CreatedBy.UserProfileName": {}, + "CreationTime": {}, + "LastModifiedBy.DomainId": {}, + "LastModifiedBy.UserProfileArn": {}, + "LastModifiedBy.UserProfileName": {}, + "LastModifiedTime": {}, + "ModelCardArn": {}, + "ModelCardProcessingStatus": {}, + "ModelCardVersion": {} + } + }, + "AWS::SageMaker::ModelCardExportJob": { + "Attributes": { + "CreatedAt": {}, + "ExportArtifacts": {}, + "ExportArtifacts.S3ExportArtifacts": {}, + "LastModifiedAt": {}, + "ModelCardExportJobArn": {}, + "Status": {} + } + }, + "AWS::SageMaker::ModelExplainabilityJobDefinition": { + "Attributes": { + "CreationTime": {}, + "JobDefinitionArn": {} + } + }, + "AWS::SageMaker::ModelPackage": { + "Attributes": { + "CreationTime": {}, + "LastModifiedTime": {}, + "ModelPackageArn": {}, + "ModelPackageStatus": {} + } + }, + "AWS::SageMaker::ModelPackageGroup": { + "Attributes": { + "CreationTime": {}, + "ModelPackageGroupArn": {}, + "ModelPackageGroupStatus": {} + } + }, + "AWS::SageMaker::ModelQualityJobDefinition": { + "Attributes": { + "CreationTime": {}, + "JobDefinitionArn": {} + } + }, + "AWS::SageMaker::MonitoringSchedule": { + "Attributes": { + "CreationTime": {}, + "LastModifiedTime": {}, + "MonitoringScheduleArn": {} + } + }, + "AWS::SageMaker::MonitoringScheduleAlert": { + "Attributes": { + "Actions": {}, + "Actions.ModelDashboardIndicator": {}, + "Actions.ModelDashboardIndicator.Enabled": {}, + "AlertStatus": {}, + "Arn": {}, + "CreationTime": {}, + "LastModifiedTime": {} + } + }, + "AWS::SageMaker::NotebookInstance": { + "Attributes": { + "NotebookInstanceArn": {}, + "NotebookInstanceName": {} + } + }, + "AWS::SageMaker::NotebookInstanceLifecycleConfig": { + "Attributes": { + "NotebookInstanceLifecycleConfigName": {} + } + }, + "AWS::SageMaker::OptimizationJob": { + "Attributes": { + "CreationTime": {}, + "LastModifiedTime": {}, + "OptimizationJobArn": {}, + "OptimizationJobStatus": {} + } + }, + "AWS::SageMaker::PartnerApp": { + "Attributes": { + "Arn": {}, + "BaseUrl": {}, + "CurrentVersionEolDate": {} + } + }, + "AWS::SageMaker::PipelineExecution": { + "Attributes": { + "CreatedBy": {}, + "CreatedBy.IamIdentity": {}, + "CreationTime": {}, + "LastModifiedBy": {}, + "LastModifiedBy.IamIdentity": {}, + "LastModifiedBy.IamIdentity.Arn": {}, + "LastModifiedBy.IamIdentity.PrincipalId": {}, + "LastModifiedBy.IamIdentity.SourceIdentity": {}, + "LastModifiedTime": {}, + "ParallelismConfiguration": {}, + "ParallelismConfiguration.MaxParallelExecutionSteps": {}, + "PipelineArn": {}, + "PipelineExecutionArn": {}, + "PipelineExecutionDescription": {}, + "PipelineExecutionDisplayName": {}, + "PipelineExecutionId": {}, + "PipelineExecutionStatus": {}, + "PipelineName": {}, + "PipelineVersionId": {}, + "Tags": {} + } + }, + "AWS::SageMaker::ProcessingJob": { + "Attributes": { + "AutoMLJobArn": {}, + "CreationTime": {}, + "ExitMessage": {}, + "FailureReason": {}, + "LastModifiedTime": {}, + "MonitoringScheduleArn": {}, + "ProcessingEndTime": {}, + "ProcessingJobArn": {}, + "ProcessingJobStatus": {}, + "ProcessingStartTime": {}, + "TrainingJobArn": {} + } + }, + "AWS::SageMaker::Project": { + "Attributes": { + "CreationTime": {}, + "ProjectArn": {}, + "ProjectId": {}, + "ProjectStatus": {} + } + }, + "AWS::SageMaker::Space": { + "Attributes": { + "SpaceArn": {}, + "Url": {} + } + }, + "AWS::SageMaker::StudioLifecycleConfig": { + "Attributes": { + "StudioLifecycleConfigArn": {} + } + }, + "AWS::SageMaker::TrainingJob": { + "Attributes": { + "BillableTimeInSeconds": {}, + "CreationTime": {}, + "LastModifiedTime": {}, + "ProfilingStatus": {}, + "SecondaryStatus": {}, + "SecondaryStatusTransitions": {}, + "TrainingJobArn": {}, + "TrainingJobStatus": {}, + "TrainingTimeInSeconds": {} + } + }, + "AWS::SageMaker::TransformJob": { + "Attributes": { + "CreationTime": {}, + "TransformEndTime": {}, + "TransformJobArn": {}, + "TransformJobName": {}, + "TransformJobStatus": {}, + "TransformStartTime": {} + } + }, + "AWS::SageMaker::TrialComponent": { + "Attributes": { + "CreationTime": {}, + "LastModifiedTime": {}, + "LineageGroupArn": {}, + "TrialComponentArn": {} + } + }, + "AWS::SageMaker::UserProfile": { + "Attributes": { + "UserProfileArn": {} + } + }, + "AWS::SageMaker::Workforce": { + "Attributes": { + "SubDomain": {}, + "WorkforceArn": {} + } + }, + "AWS::SageMaker::Workteam": { + "Attributes": { + "WorkteamName": {} + } + }, + "AWS::SavingsPlans::SavingsPlan": { + "Attributes": { + "Commitment": {}, + "Currency": {}, + "Description": {}, + "End": {}, + "PaymentOption": {}, + "ProductTypes": {}, + "RecurringPaymentAmount": {}, + "SavingsPlanArn": {}, + "SavingsPlanId": {}, + "SavingsPlanOfferingId": {}, + "SavingsPlanType": {}, + "Start": {}, + "State": {}, + "Tags": {}, + "TermDurationInSeconds": {}, + "UpfrontPaymentAmount": {} + } + }, + "AWS::Scheduler::Schedule": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Scheduler::ScheduleGroup": { + "Attributes": { + "Arn": {}, + "CreationDate": {}, + "LastModificationDate": {}, + "State": {} + } + }, + "AWS::SecretsManager::ResourcePolicy": { + "Attributes": { + "Id": {} + } + }, + "AWS::SecretsManager::RotationSchedule": { + "Attributes": { + "Id": {} + } + }, + "AWS::SecretsManager::Secret": { + "Attributes": { + "Id": {} + } + }, + "AWS::SecretsManager::SecretTargetAttachment": { + "Attributes": { + "Id": {} + } + }, + "AWS::SecurityAgent::AgentSpace": { + "Attributes": { + "AgentSpaceId": {}, + "CreatedAt": {}, + "UpdatedAt": {} + } + }, + "AWS::SecurityAgent::Application": { + "Attributes": { + "ApplicationId": {}, + "ApplicationName": {}, + "Domain": {}, + "IdCConfiguration.IdCApplicationArn": {} + } + }, + "AWS::SecurityAgent::Artifact": { + "Attributes": { + "Arn": {}, + "ArtifactId": {}, + "UpdatedAt": {} + } + }, + "AWS::SecurityAgent::Pentest": { + "Attributes": { + "CreatedAt": {}, + "PentestId": {}, + "UpdatedAt": {} + } + }, + "AWS::SecurityAgent::PentestTask": { + "Attributes": { + "Arn": {}, + "Categories": {}, + "CreatedAt": {}, + "Description": {}, + "ExecutionStatus": {}, + "PentestId": {}, + "PentestJobId": {}, + "TargetEndpoint": {}, + "TargetEndpoint.Uri": {}, + "TaskHours": {}, + "TaskId": {}, + "Title": {}, + "UpdatedAt": {} + } + }, + "AWS::SecurityAgent::SecurityRequirementPack": { + "Attributes": { + "PackId": {} + } + }, + "AWS::SecurityAgent::TargetDomain": { + "Attributes": { + "CreatedAt": {}, + "TargetDomainId": {}, + "VerificationDetails": {}, + "VerificationDetails.DnsTxt": {}, + "VerificationDetails.DnsTxt.DnsRecordName": {}, + "VerificationDetails.DnsTxt.DnsRecordType": {}, + "VerificationDetails.DnsTxt.Token": {}, + "VerificationDetails.HttpRoute": {}, + "VerificationDetails.HttpRoute.RoutePath": {}, + "VerificationDetails.HttpRoute.Token": {}, + "VerificationDetails.Method": {}, + "VerificationStatus": {}, + "VerificationStatusReason": {}, + "VerifiedAt": {} + } + }, + "AWS::SecurityHub::AggregatorV2": { + "Attributes": { + "AggregationRegion": {}, + "AggregatorV2Arn": {} + } + }, + "AWS::SecurityHub::AutomationRule": { + "Attributes": { + "CreatedAt": {}, + "CreatedBy": {}, + "RuleArn": {}, + "UpdatedAt": {} + } + }, + "AWS::SecurityHub::AutomationRuleV2": { + "Attributes": { + "CreatedAt": {}, + "RuleArn": {}, + "RuleId": {}, + "UpdatedAt": {} + } + }, + "AWS::SecurityHub::ConfigurationPolicy": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "ServiceEnabled": {}, + "UpdatedAt": {} + } + }, + "AWS::SecurityHub::Connector": { + "Attributes": { + "ConnectorArn": {}, + "ConnectorId": {}, + "ConnectorStatus": {}, + "CreatedAt": {}, + "CreatedBy": {}, + "EnablementStatus": {}, + "Issues": {}, + "LastCheckedAt": {}, + "LastUpdatedAt": {}, + "Message": {} + } + }, + "AWS::SecurityHub::ConnectorV2": { + "Attributes": { + "ConnectorArn": {}, + "ConnectorId": {}, + "ConnectorStatus": {}, + "CreatedAt": {}, + "EnablementStatus": {}, + "EnablementStatusReason": {}, + "Issues": {}, + "LastCheckedAt": {}, + "LastUpdatedAt": {}, + "Message": {} + } + }, + "AWS::SecurityHub::DelegatedAdmin": { + "Attributes": { + "DelegatedAdminIdentifier": {}, + "Status": {} + } + }, + "AWS::SecurityHub::FindingAggregator": { + "Attributes": { + "FindingAggregationRegion": {}, + "FindingAggregatorArn": {} + } + }, + "AWS::SecurityHub::Hub": { + "Attributes": { + "ARN": {}, + "SubscribedAt": {} + } + }, + "AWS::SecurityHub::HubV2": { + "Attributes": { + "HubV2Arn": {}, + "SubscribedAt": {} + } + }, + "AWS::SecurityHub::Insight": { + "Attributes": { + "InsightArn": {} + } + }, + "AWS::SecurityHub::OrganizationConfiguration": { + "Attributes": { + "MemberAccountLimitReached": {}, + "OrganizationConfigurationIdentifier": {}, + "Status": {}, + "StatusMessage": {} + } + }, + "AWS::SecurityHub::PolicyAssociation": { + "Attributes": { + "AssociationIdentifier": {}, + "AssociationStatus": {}, + "AssociationStatusMessage": {}, + "AssociationType": {}, + "UpdatedAt": {} + } + }, + "AWS::SecurityHub::ProductSubscription": { + "Attributes": { + "ProductSubscriptionArn": {} + } + }, + "AWS::SecurityHub::Standard": { + "Attributes": { + "StandardsSubscriptionArn": {} + } + }, + "AWS::SecurityLake::DataLake": { + "Attributes": { + "Arn": {}, + "S3BucketArn": {} + } + }, + "AWS::SecurityLake::Subscriber": { + "Attributes": { + "ResourceShareArn": {}, + "ResourceShareName": {}, + "S3BucketArn": {}, + "SubscriberArn": {}, + "SubscriberRoleArn": {} + } + }, + "AWS::SecurityLake::SubscriberNotification": { + "Attributes": { + "SubscriberEndpoint": {} + } + }, + "AWS::ServerlessRepo::Application": { + "Attributes": { + "ApplicationId": {}, + "CreationTime": {}, + "IsVerifiedAuthor": {} + } + }, + "AWS::ServiceCatalog::CloudFormationProduct": { + "Attributes": { + "Id": {}, + "ProductName": {}, + "ProvisioningArtifactIds": {}, + "ProvisioningArtifactNames": {} + } + }, + "AWS::ServiceCatalog::CloudFormationProvisionedProduct": { + "Attributes": { + "CloudformationStackArn": {}, + "Outputs": {}, + "ProvisionedProductId": {}, + "RecordId": {} + } + }, + "AWS::ServiceCatalog::LaunchNotificationConstraint": { + "Attributes": { + "Id": {} + } + }, + "AWS::ServiceCatalog::LaunchRoleConstraint": { + "Attributes": { + "Id": {} + } + }, + "AWS::ServiceCatalog::LaunchTemplateConstraint": { + "Attributes": { + "Id": {} + } + }, + "AWS::ServiceCatalog::Portfolio": { + "Attributes": { + "Id": {}, + "PortfolioName": {} + } + }, + "AWS::ServiceCatalog::ResourceUpdateConstraint": { + "Attributes": { + "Id": {} + } + }, + "AWS::ServiceCatalog::ServiceAction": { + "Attributes": { + "Id": {} + } + }, + "AWS::ServiceCatalog::StackSetConstraint": { + "Attributes": { + "Id": {} + } + }, + "AWS::ServiceCatalog::TagOption": { + "Attributes": { + "Id": {} + } + }, + "AWS::ServiceCatalogAppRegistry::Application": { + "Attributes": { + "ApplicationName": {}, + "ApplicationTagKey": {}, + "ApplicationTagValue": {}, + "Arn": {}, + "Id": {} + } + }, + "AWS::ServiceCatalogAppRegistry::AttributeGroup": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::ServiceCatalogAppRegistry::AttributeGroupAssociation": { + "Attributes": { + "ApplicationArn": {}, + "AttributeGroupArn": {} + } + }, + "AWS::ServiceCatalogAppRegistry::ResourceAssociation": { + "Attributes": { + "ApplicationArn": {}, + "ResourceArn": {} + } + }, + "AWS::ServiceDiscovery::HttpNamespace": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::ServiceDiscovery::PrivateDnsNamespace": { + "Attributes": { + "Arn": {}, + "HostedZoneId": {}, + "Id": {} + } + }, + "AWS::ServiceDiscovery::PublicDnsNamespace": { + "Attributes": { + "Arn": {}, + "HostedZoneId": {}, + "Id": {} + } + }, + "AWS::ServiceDiscovery::Service": { + "Attributes": { + "Arn": {}, + "Id": {}, + "Name": {} + } + }, + "AWS::ServiceQuotas::Quota": { + "Attributes": { + "Adjustable": {}, + "Description": {}, + "GlobalQuota": {}, + "Period": {}, + "Period.PeriodUnit": {}, + "Period.PeriodValue": {}, + "QuotaAppliedAtLevel": {}, + "QuotaArn": {}, + "QuotaContext": {}, + "QuotaContext.ContextId": {}, + "QuotaContext.ContextScope": {}, + "QuotaContext.ContextScopeType": {}, + "QuotaName": {}, + "ServiceName": {}, + "Tags": {}, + "Unit": {}, + "UsageMetric": {}, + "UsageMetric.MetricDimensions": {}, + "UsageMetric.MetricName": {}, + "UsageMetric.MetricNamespace": {}, + "UsageMetric.MetricStatisticRecommendation": {}, + "Value": {} + } + }, + "AWS::Shield::DRTAccess": { + "Attributes": { + "AccountId": {} + } + }, + "AWS::Shield::ProactiveEngagement": { + "Attributes": { + "AccountId": {} + } + }, + "AWS::Shield::Protection": { + "Attributes": { + "ProtectionArn": {}, + "ProtectionId": {} + } + }, + "AWS::Shield::ProtectionGroup": { + "Attributes": { + "ProtectionGroupArn": {} + } + }, + "AWS::Signer::SigningJob": { + "Attributes": { + "Arn": {}, + "CompletedAt": {}, + "CreatedAt": {}, + "JobId": {}, + "JobInvoker": {}, + "JobOwner": {}, + "PlatformDisplayName": {}, + "PlatformId": {}, + "ProfileVersion": {}, + "RequestedBy": {}, + "SignatureExpiresAt": {}, + "SignedObject": {}, + "SignedObject.S3": {}, + "SignedObject.S3.BucketName": {}, + "SignedObject.S3.Key": {}, + "Source": {}, + "Source.S3": {}, + "Source.S3.BucketName": {}, + "Source.S3.Key": {}, + "Source.S3.Version": {}, + "Status": {} + } + }, + "AWS::Signer::SigningProfile": { + "Attributes": { + "Arn": {}, + "ProfileName": {}, + "ProfileVersion": {}, + "ProfileVersionArn": {} + } + }, + "AWS::SimSpaceWeaver::Simulation": { + "Attributes": { + "DescribePayload": {} + } + }, + "AWS::States::Execution": { + "Attributes": { + "ExecutionArn": {}, + "RedriveCount": {}, + "RedriveStatus": {}, + "StartDate": {}, + "StateMachineName": {}, + "Status": {} + } + }, + "AWS::StepFunctions::Activity": { + "Attributes": { + "Arn": {}, + "Name": {} + } + }, + "AWS::StepFunctions::MapRun": { + "Attributes": { + "ExecutionCounts": {}, + "ExecutionCounts.Aborted": {}, + "ExecutionCounts.Failed": {}, + "ExecutionCounts.FailuresNotRedrivable": {}, + "ExecutionCounts.Pending": {}, + "ExecutionCounts.PendingRedrive": {}, + "ExecutionCounts.ResultsWritten": {}, + "ExecutionCounts.Running": {}, + "ExecutionCounts.Succeeded": {}, + "ExecutionCounts.TimedOut": {}, + "ExecutionCounts.Total": {}, + "ItemCounts": {}, + "ItemCounts.Aborted": {}, + "ItemCounts.Failed": {}, + "ItemCounts.FailuresNotRedrivable": {}, + "ItemCounts.Pending": {}, + "ItemCounts.PendingRedrive": {}, + "ItemCounts.ResultsWritten": {}, + "ItemCounts.Running": {}, + "ItemCounts.Succeeded": {}, + "ItemCounts.TimedOut": {}, + "ItemCounts.Total": {}, + "MapRunArn": {}, + "MaxConcurrency": {}, + "StartDate": {}, + "Status": {}, + "StopDate": {}, + "ToleratedFailureCount": {}, + "ToleratedFailurePercentage": {} + } + }, + "AWS::StepFunctions::StateMachine": { + "Attributes": { + "Arn": {}, + "Name": {}, + "StateMachineRevisionId": {} + } + }, + "AWS::StepFunctions::StateMachineAlias": { + "Attributes": { + "Arn": {} + } + }, + "AWS::StepFunctions::StateMachineVersion": { + "Attributes": { + "Arn": {} + } + }, + "AWS::StorageGateway::CacheReport": { + "Attributes": { + "CacheReportARN": {}, + "CacheReportStatus": {}, + "EndTime": {}, + "ReportCompletionPercent": {}, + "ReportName": {}, + "StartTime": {} + } + }, + "AWS::StorageGateway::Device": { + "Attributes": { + "DeviceiSCSIAttributes": {}, + "DeviceiSCSIAttributes.ChapEnabled": {}, + "DeviceiSCSIAttributes.NetworkInterfaceId": {}, + "DeviceiSCSIAttributes.NetworkInterfacePort": {}, + "DeviceiSCSIAttributes.TargetARN": {}, + "GatewayARN": {}, + "GatewayId": {}, + "VTLDeviceARN": {}, + "VTLDeviceName": {}, + "VTLDeviceProductIdentifier": {}, + "VTLDeviceType": {}, + "VTLDeviceVendor": {} + } + }, + "AWS::StorageGateway::Gateway": { + "Attributes": { + "Ec2InstanceId": {}, + "Ec2InstanceRegion": {}, + "EndpointType": {}, + "GatewayARN": {}, + "GatewayId": {}, + "GatewayNetworkInterfaces": {}, + "GatewayState": {}, + "HostEnvironment": {}, + "Tags": {} + } + }, + "AWS::StorageGateway::Tape": { + "Attributes": { + "TapeARN": {}, + "TapeCreatedDate": {}, + "TapeStatus": {}, + "TapeUsedInBytes": {} + } + }, + "AWS::StorageGateway::TapePool": { + "Attributes": { + "PoolARN": {}, + "PoolId": {} + } + }, + "AWS::SupportApp::AccountAlias": { + "Attributes": { + "AccountAliasResourceId": {} + } + }, + "AWS::SupportAuthZ::SupportPermit": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "PermitId": {}, + "Status": {} + } + }, + "AWS::Synthetics::Canary": { + "Attributes": { + "Code.SourceLocationArn": {}, + "Id": {}, + "State": {} + } + }, + "AWS::Synthetics::Group": { + "Attributes": { + "Id": {} + } + }, + "AWS::SystemsManagerSAP::Application": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Textract::Adapter": { + "Attributes": { + "AdapterId": {}, + "Arn": {}, + "CreationTime": {} + } + }, + "AWS::ThinClient::SoftwareSet": { + "Attributes": { + "Arn": {}, + "Id": {}, + "ReleasedAt": {}, + "Software": {}, + "ValidationStatus": {}, + "Version": {} + } + }, + "AWS::Timestream::Database": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Timestream::InfluxDBCluster": { + "Attributes": { + "Arn": {}, + "Endpoint": {}, + "EngineType": {}, + "Id": {}, + "InfluxAuthParametersSecretArn": {}, + "NextMaintenanceTime": {}, + "ReaderEndpoint": {}, + "Status": {} + } + }, + "AWS::Timestream::InfluxDBInstance": { + "Attributes": { + "Arn": {}, + "AvailabilityZone": {}, + "Endpoint": {}, + "Id": {}, + "InfluxAuthParametersSecretArn": {}, + "NextMaintenanceTime": {}, + "SecondaryAvailabilityZone": {}, + "Status": {} + } + }, + "AWS::Timestream::InfluxDBParameterGroup": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::Timestream::ScheduledQuery": { + "Attributes": { + "Arn": {}, + "SQErrorReportConfiguration": {}, + "SQKmsKeyId": {}, + "SQName": {}, + "SQNotificationConfiguration": {}, + "SQQueryString": {}, + "SQScheduleConfiguration": {}, + "SQScheduledQueryExecutionRoleArn": {}, + "SQTargetConfiguration": {} + } + }, + "AWS::Timestream::Table": { + "Attributes": { + "Arn": {}, + "Name": {} + } + }, + "AWS::Transcribe::CallAnalyticsCategory": { + "Attributes": { + "Arn": {}, + "CreateTime": {}, + "LastUpdateTime": {} + } + }, + "AWS::Transcribe::CallAnalyticsJob": { + "Attributes": { + "Arn": {}, + "CallAnalyticsJobStatus": {}, + "CreationTime": {}, + "LanguageCode": {}, + "MediaFormat": {}, + "MediaSampleRateHertz": {}, + "Transcript": {}, + "Transcript.TranscriptFileUri": {} + } + }, + "AWS::Transcribe::MedicalScribeJob": { + "Attributes": { + "Arn": {}, + "CompletionTime": {}, + "CreationTime": {}, + "LanguageCode": {}, + "MedicalScribeContextProvided": {}, + "MedicalScribeJobStatus": {}, + "MedicalScribeOutput": {}, + "MedicalScribeOutput.ClinicalDocumentUri": {}, + "MedicalScribeOutput.TranscriptFileUri": {}, + "StartTime": {} + } + }, + "AWS::Transcribe::MedicalTranscriptionJob": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "Transcript": {}, + "Transcript.TranscriptFileUri": {}, + "TranscriptionJobStatus": {} + } + }, + "AWS::Transcribe::TranscriptionJob": { + "Attributes": { + "Arn": {}, + "CreationTime": {}, + "Transcript": {}, + "Transcript.RedactedTranscriptFileUri": {}, + "Transcript.TranscriptFileUri": {}, + "TranscriptionJobStatus": {} + } + }, + "AWS::Transcribe::Vocabulary": { + "Attributes": { + "Arn": {}, + "LastModifiedTime": {}, + "VocabularyState": {} + } + }, + "AWS::Transcribe::VocabularyFilter": { + "Attributes": { + "Arn": {} + } + }, + "AWS::Transfer::Agreement": { + "Attributes": { + "AgreementId": {}, + "Arn": {} + } + }, + "AWS::Transfer::Certificate": { + "Attributes": { + "Arn": {}, + "CertificateId": {}, + "NotAfterDate": {}, + "NotBeforeDate": {}, + "Serial": {}, + "Status": {}, + "Type": {} + } + }, + "AWS::Transfer::Connector": { + "Attributes": { + "Arn": {}, + "ConnectorId": {}, + "ErrorMessage": {}, + "ServiceManagedEgressIpAddresses": {}, + "Status": {} + } + }, + "AWS::Transfer::HostKey": { + "Attributes": { + "Arn": {}, + "DateImported": {}, + "HostKeyFingerprint": {}, + "HostKeyId": {}, + "Type": {} + } + }, + "AWS::Transfer::Profile": { + "Attributes": { + "Arn": {}, + "ProfileId": {} + } + }, + "AWS::Transfer::Server": { + "Attributes": { + "Arn": {}, + "As2ServiceManagedEgressIpAddresses": {}, + "ServerId": {}, + "State": {} + } + }, + "AWS::Transfer::User": { + "Attributes": { + "Arn": {}, + "ServerId": {}, + "UserName": {} + } + }, + "AWS::Transfer::WebApp": { + "Attributes": { + "Arn": {}, + "IdentityProviderDetails.ApplicationArn": {}, + "VpcEndpointId": {}, + "WebAppId": {} + } + }, + "AWS::Transfer::Workflow": { + "Attributes": { + "Arn": {}, + "WorkflowId": {} + } + }, + "AWS::Translate::ParallelData": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "FailedRecordCount": {}, + "ImportedDataSize": {}, + "ImportedRecordCount": {}, + "LastUpdatedAt": {}, + "SkippedRecordCount": {}, + "SourceLanguageCode": {}, + "Status": {}, + "TargetLanguageCodes": {} + } + }, + "AWS::UXC::AccountCustomization": { + "Attributes": { + "AccountId": {} + } + }, + "AWS::UserNotifications::ManagedNotificationConfiguration": { + "Attributes": { + "Arn": {}, + "Description": {}, + "Name": {} + } + }, + "AWS::UserNotifications::NotificationEvent": { + "Attributes": { + "AggregationEventType": {}, + "Arn": {}, + "CreationTime": {}, + "EventStatus": {}, + "Id": {}, + "MessageComponents": {}, + "MessageComponents.Dimensions": {}, + "MessageComponents.Headline": {}, + "MessageComponents.ParagraphSummary": {}, + "NotificationConfigurationArn": {}, + "NotificationType": {}, + "SchemaVersion": {}, + "SourceEventDetailUrl": {}, + "SourceEventMetadata": {}, + "SourceEventMetadata.EventOccurrenceTime": {}, + "SourceEventMetadata.EventOriginRegion": {}, + "SourceEventMetadata.EventType": {}, + "SourceEventMetadata.EventTypeVersion": {}, + "SourceEventMetadata.RelatedAccount": {}, + "SourceEventMetadata.RelatedResources": {}, + "SourceEventMetadata.Source": {}, + "SourceEventMetadata.SourceEventId": {} + } + }, + "AWS::VerifiedPermissions::IdentitySource": { + "Attributes": { + "IdentitySourceId": {} + } + }, + "AWS::VerifiedPermissions::Policy": { + "Attributes": { + "PolicyId": {}, + "PolicyType": {} + } + }, + "AWS::VerifiedPermissions::PolicyStore": { + "Attributes": { + "Arn": {}, + "EncryptionState": {}, + "EncryptionState.Default": {}, + "EncryptionState.KmsEncryptionState": {}, + "EncryptionState.KmsEncryptionState.EncryptionContext": {}, + "EncryptionState.KmsEncryptionState.Key": {}, + "PolicyStoreId": {} + } + }, + "AWS::VerifiedPermissions::PolicyTemplate": { + "Attributes": { + "PolicyTemplateId": {} + } + }, + "AWS::VoiceID::Domain": { + "Attributes": { + "DomainId": {} + } + }, + "AWS::VpcLattice::AccessLogSubscription": { + "Attributes": { + "Arn": {}, + "Id": {}, + "ResourceArn": {}, + "ResourceId": {} + } + }, + "AWS::VpcLattice::AuthPolicy": { + "Attributes": { + "State": {} + } + }, + "AWS::VpcLattice::DomainVerification": { + "Attributes": { + "Arn": {}, + "Id": {}, + "Status": {}, + "TxtMethodConfig": {}, + "TxtMethodConfig.name": {}, + "TxtMethodConfig.value": {} + } + }, + "AWS::VpcLattice::Listener": { + "Attributes": { + "Arn": {}, + "Id": {}, + "ServiceArn": {}, + "ServiceId": {} + } + }, + "AWS::VpcLattice::ResourceConfiguration": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::VpcLattice::ResourceEndpointAssociation": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "ResourceConfigurationArn": {}, + "ResourceConfigurationId": {}, + "VpcEndpointId": {}, + "VpcEndpointOwner": {} + } + }, + "AWS::VpcLattice::ResourceGateway": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::VpcLattice::Rule": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::VpcLattice::Service": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "DnsEntry.DomainName": {}, + "DnsEntry.HostedZoneId": {}, + "Id": {}, + "LastUpdatedAt": {}, + "Status": {} + } + }, + "AWS::VpcLattice::ServiceNetwork": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "LastUpdatedAt": {} + } + }, + "AWS::VpcLattice::ServiceNetworkResourceAssociation": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::VpcLattice::ServiceNetworkServiceAssociation": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "DnsEntry.DomainName": {}, + "DnsEntry.HostedZoneId": {}, + "Id": {}, + "ServiceArn": {}, + "ServiceId": {}, + "ServiceName": {}, + "ServiceNetworkArn": {}, + "ServiceNetworkId": {}, + "ServiceNetworkName": {}, + "Status": {} + } + }, + "AWS::VpcLattice::ServiceNetworkVpcAssociation": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "ServiceNetworkArn": {}, + "ServiceNetworkId": {}, + "ServiceNetworkName": {}, + "Status": {}, + "VpcId": {} + } + }, + "AWS::VpcLattice::TargetGroup": { + "Attributes": { + "Arn": {}, + "CreatedAt": {}, + "Id": {}, + "LastUpdatedAt": {}, + "Status": {} + } + }, + "AWS::WAFv2::IPSet": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::WAFv2::LoggingConfiguration": { + "Attributes": { + "ManagedByFirewallManager": {} + } + }, + "AWS::WAFv2::RegexPatternSet": { + "Attributes": { + "Arn": {}, + "Id": {} + } + }, + "AWS::WAFv2::RuleGroup": { + "Attributes": { + "Arn": {}, + "Id": {}, + "LabelNamespace": {} + } + }, + "AWS::WAFv2::WebACL": { + "Attributes": { + "Arn": {}, + "Capacity": {}, + "Id": {}, + "LabelNamespace": {} + } + }, + "AWS::WellArchitected::Lens": { + "Attributes": { + "Description": {}, + "LensArn": {}, + "LensId": {}, + "Name": {}, + "Owner": {} + } + }, + "AWS::WellArchitected::Profile": { + "Attributes": { + "CreatedAt": {}, + "Owner": {}, + "ProfileArn": {}, + "ProfileVersion": {}, + "UpdatedAt": {} + } + }, + "AWS::WellArchitected::ReviewTemplate": { + "Attributes": { + "Owner": {}, + "TemplateArn": {}, + "UpdateStatus": {}, + "UpdatedAt": {} + } + }, + "AWS::WellArchitected::Workload": { + "Attributes": { + "ImprovementStatus": {}, + "WorkloadArn": {}, + "WorkloadId": {} + } + }, + "AWS::Wickr::Network": { + "Attributes": { + "AwsAccountId": {}, + "MigrationState": {}, + "NetworkArn": {}, + "NetworkId": {}, + "Standing": {} + } + }, + "AWS::Wisdom::AIAgent": { + "Attributes": { + "AIAgentArn": {}, + "AIAgentId": {}, + "AssistantArn": {}, + "ModifiedTimeSeconds": {} + } + }, + "AWS::Wisdom::AIAgentVersion": { + "Attributes": { + "AIAgentArn": {}, + "AIAgentVersionId": {}, + "AssistantArn": {}, + "VersionNumber": {} + } + }, + "AWS::Wisdom::AIGuardrail": { + "Attributes": { + "AIGuardrailArn": {}, + "AIGuardrailId": {}, + "AssistantArn": {}, + "ModifiedTimeSeconds": {} + } + }, + "AWS::Wisdom::AIGuardrailVersion": { + "Attributes": { + "AIGuardrailArn": {}, + "AIGuardrailVersionId": {}, + "AssistantArn": {}, + "VersionNumber": {} + } + }, + "AWS::Wisdom::AIPrompt": { + "Attributes": { + "AIPromptArn": {}, + "AIPromptId": {}, + "AssistantArn": {}, + "ModifiedTimeSeconds": {} + } + }, + "AWS::Wisdom::AIPromptVersion": { + "Attributes": { + "AIPromptArn": {}, + "AIPromptVersionId": {}, + "AssistantArn": {}, + "VersionNumber": {} + } + }, + "AWS::Wisdom::Assistant": { + "Attributes": { + "AssistantArn": {}, + "AssistantId": {} + } + }, + "AWS::Wisdom::AssistantAssociation": { + "Attributes": { + "AssistantArn": {}, + "AssistantAssociationArn": {}, + "AssistantAssociationId": {} + } + }, + "AWS::Wisdom::KnowledgeBase": { + "Attributes": { + "KnowledgeBaseArn": {}, + "KnowledgeBaseId": {} + } + }, + "AWS::Wisdom::MessageTemplate": { + "Attributes": { + "MessageTemplateArn": {}, + "MessageTemplateContentSha256": {}, + "MessageTemplateId": {} + } + }, + "AWS::Wisdom::MessageTemplateVersion": { + "Attributes": { + "MessageTemplateVersionArn": {}, + "MessageTemplateVersionNumber": {} + } + }, + "AWS::Wisdom::QuickResponse": { + "Attributes": { + "Contents": {}, + "Contents.Markdown": {}, + "Contents.Markdown.Content": {}, + "Contents.PlainText": {}, + "Contents.PlainText.Content": {}, + "QuickResponseArn": {}, + "QuickResponseId": {}, + "Status": {} + } + }, + "AWS::Wisdom::Session": { + "Attributes": { + "SessionArn": {}, + "SessionId": {} + } + }, + "AWS::WorkSpaces::ConnectionAlias": { + "Attributes": { + "AliasId": {}, + "Associations": {}, + "ConnectionAliasState": {} + } + }, + "AWS::WorkSpaces::WorkSpaceApplication": { + "Attributes": { + "ApplicationId": {}, + "Arn": {}, + "Created": {}, + "Description": {}, + "LicenseType": {}, + "Name": {}, + "Owner": {}, + "State": {}, + "SupportedComputeTypeNames": {}, + "SupportedOperatingSystemNames": {} + } + }, + "AWS::WorkSpaces::Workspace": { + "Attributes": { + "Id": {}, + "WorkspaceId": {} + } + }, + "AWS::WorkSpaces::WorkspaceIpGroup": { + "Attributes": { + "Arn": {}, + "GroupId": {} + } + }, + "AWS::WorkSpaces::WorkspacesPool": { + "Attributes": { + "CreatedAt": {}, + "PoolArn": {}, + "PoolId": {} + } + }, + "AWS::WorkSpacesThinClient::Environment": { + "Attributes": { + "ActivationCode": {}, + "Arn": {}, + "CreatedAt": {}, + "DesktopType": {}, + "Id": {}, + "PendingSoftwareSetId": {}, + "PendingSoftwareSetVersion": {}, + "RegisteredDevicesCount": {}, + "SoftwareSetComplianceStatus": {}, + "UpdatedAt": {} + } + }, + "AWS::WorkSpacesWeb::BrowserSettings": { + "Attributes": { + "AssociatedPortalArns": {}, + "BrowserSettingsArn": {} + } + }, + "AWS::WorkSpacesWeb::DataProtectionSettings": { + "Attributes": { + "AssociatedPortalArns": {}, + "CreationDate": {}, + "DataProtectionSettingsArn": {} + } + }, + "AWS::WorkSpacesWeb::IdentityProvider": { + "Attributes": { + "IdentityProviderArn": {} + } + }, + "AWS::WorkSpacesWeb::IpAccessSettings": { + "Attributes": { + "AssociatedPortalArns": {}, + "CreationDate": {}, + "IpAccessSettingsArn": {} + } + }, + "AWS::WorkSpacesWeb::NetworkSettings": { + "Attributes": { + "AssociatedPortalArns": {}, + "NetworkSettingsArn": {} + } + }, + "AWS::WorkSpacesWeb::Portal": { + "Attributes": { + "BrowserType": {}, + "CreationDate": {}, + "PortalArn": {}, + "PortalEndpoint": {}, + "PortalStatus": {}, + "RendererType": {}, + "ServiceProviderSamlMetadata": {}, + "StatusReason": {} + } + }, + "AWS::WorkSpacesWeb::SessionLogger": { + "Attributes": { + "AssociatedPortalArns": {}, + "CreationDate": {}, + "SessionLoggerArn": {} + } + }, + "AWS::WorkSpacesWeb::TrustStore": { + "Attributes": { + "AssociatedPortalArns": {}, + "TrustStoreArn": {} + } + }, + "AWS::WorkSpacesWeb::UserAccessLoggingSettings": { + "Attributes": { + "AssociatedPortalArns": {}, + "UserAccessLoggingSettingsArn": {} + } + }, + "AWS::WorkSpacesWeb::UserSettings": { + "Attributes": { + "AssociatedPortalArns": {}, + "BrandingConfiguration.FaviconMetadata": {}, + "BrandingConfiguration.FaviconMetadata.FileExtension": {}, + "BrandingConfiguration.FaviconMetadata.LastUploadTimestamp": {}, + "BrandingConfiguration.FaviconMetadata.MimeType": {}, + "BrandingConfiguration.LogoMetadata": {}, + "BrandingConfiguration.LogoMetadata.FileExtension": {}, + "BrandingConfiguration.LogoMetadata.LastUploadTimestamp": {}, + "BrandingConfiguration.LogoMetadata.MimeType": {}, + "BrandingConfiguration.WallpaperMetadata": {}, + "BrandingConfiguration.WallpaperMetadata.FileExtension": {}, + "BrandingConfiguration.WallpaperMetadata.LastUploadTimestamp": {}, + "BrandingConfiguration.WallpaperMetadata.MimeType": {}, + "UserSettingsArn": {} + } + }, + "AWS::WorkspacesInstances::Volume": { + "Attributes": { + "VolumeId": {} + } + }, + "AWS::WorkspacesInstances::WorkspaceInstance": { + "Attributes": { + "EC2ManagedInstance": {}, + "EC2ManagedInstance.InstanceId": {}, + "ProvisionState": {}, + "WorkspaceInstanceId": {} + } + }, + "AWS::XRay::Group": { + "Attributes": { + "GroupARN": {} + } + }, + "AWS::XRay::SamplingRule": { + "Attributes": { + "RuleARN": {} + } + }, + "AWS::XRay::TransactionSearchConfig": { + "Attributes": { + "AccountId": {} + } + } + } +} diff --git a/cwlogs_subscription_delivery_test.go b/cwlogs_subscription_delivery_test.go index 07d974ec37..19fea7749c 100644 --- a/cwlogs_subscription_delivery_test.go +++ b/cwlogs_subscription_delivery_test.go @@ -3,6 +3,7 @@ package main import ( "context" "testing" + "time" kinesisbackend "github.com/blackbirdworks/gopherstack/services/kinesis" lambdabackend "github.com/blackbirdworks/gopherstack/services/lambda" @@ -17,13 +18,15 @@ func TestCWLogsSubscriptionDeliverer_Routing(t *testing.T) { t.Run("kinesis destination receives the payload", func(t *testing.T) { t.Parallel() - kb := kinesisbackend.NewInMemoryBackend() + clock := newKinesisFakeClock(time.Now()) + kb := kinesisbackend.NewInMemoryBackend().WithClock(clock.Now) if err := kb.CreateStream( context.Background(), &kinesisbackend.CreateStreamInput{StreamName: "logs", ShardCount: 1}, ); err != nil { t.Fatalf("CreateStream: %v", err) } + clock.Advance(kinesisStreamSettleWait) d := &cwlogsSubscriptionDeliverer{kinesis: kb} arn := "arn:aws:kinesis:us-east-1:000000000000:stream/logs" diff --git a/go.mod b/go.mod index 37a257ff56..3b1ffb538e 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/alicebob/miniredis/v2 v2.39.0 github.com/aws/aws-dax-go v1.2.15 github.com/aws/aws-sdk-go v1.55.8 - github.com/aws/aws-sdk-go-v2 v1.46.0 + github.com/aws/aws-sdk-go-v2 v1.47.1 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 github.com/aws/aws-sdk-go-v2/config v1.33.3 github.com/aws/aws-sdk-go-v2/credentials v1.20.3 @@ -68,10 +68,12 @@ require ( github.com/aws/aws-sdk-go-v2/service/directoryservice v1.41.4 github.com/aws/aws-sdk-go-v2/service/dlm v1.39.4 github.com/aws/aws-sdk-go-v2/service/docdb v1.51.4 + github.com/aws/aws-sdk-go-v2/service/dsql v1.22.1 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.67.0 github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.40.0 github.com/aws/aws-sdk-go-v2/service/ec2 v1.329.0 github.com/aws/aws-sdk-go-v2/service/ecr v1.64.0 + github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.47.1 github.com/aws/aws-sdk-go-v2/service/ecs v1.96.0 github.com/aws/aws-sdk-go-v2/service/efs v1.48.0 github.com/aws/aws-sdk-go-v2/service/eks v1.98.0 @@ -99,9 +101,11 @@ require ( github.com/aws/aws-sdk-go-v2/service/iotdataplane v1.35.4 github.com/aws/aws-sdk-go-v2/service/iotwireless v1.59.4 github.com/aws/aws-sdk-go-v2/service/kafka v1.57.2 + github.com/aws/aws-sdk-go-v2/service/kafkaconnect v1.39.1 github.com/aws/aws-sdk-go-v2/service/kinesis v1.53.0 github.com/aws/aws-sdk-go-v2/service/kinesisanalytics v1.33.4 github.com/aws/aws-sdk-go-v2/service/kinesisanalyticsv2 v1.41.4 + github.com/aws/aws-sdk-go-v2/service/kinesisvideo v1.41.1 github.com/aws/aws-sdk-go-v2/service/kms v1.59.0 github.com/aws/aws-sdk-go-v2/service/lakeformation v1.50.4 github.com/aws/aws-sdk-go-v2/service/lambda v1.107.0 @@ -219,8 +223,8 @@ require ( github.com/agnivade/levenshtein v1.2.1 // indirect github.com/antlr/antlr4 v0.0.0-20181218183524-be58ebffde8e // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 // indirect diff --git a/go.sum b/go.sum index 1378117100..49cca66b6b 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ github.com/aws/aws-dax-go v1.2.15 h1:30rH3+QgjpjemrVg0NGIG5FnB1izJZ7jUZuBb1Fy8ak github.com/aws/aws-dax-go v1.2.15/go.mod h1:4f/qGLBQlPYd+fmAfG4n4oSvN19JdKNYYmsr90/MPso= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= -github.com/aws/aws-sdk-go-v2 v1.46.0 h1:1kt7m/EKcEHt5mlyyxx9cSlMddRPIKbjb6DIQsu4HPk= -github.com/aws/aws-sdk-go-v2 v1.46.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= +github.com/aws/aws-sdk-go-v2 v1.47.1 h1:uOIZnp4PK3ZhKI0dNrJrhTEsLxbpXHTAJlwoS1pvAtw= +github.com/aws/aws-sdk-go-v2 v1.47.1/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= github.com/aws/aws-sdk-go-v2/config v1.33.3 h1:h090b3O5S17bF87/0ysHZuIT/7DCb4EBRFQX2PMVPCw= @@ -54,10 +54,10 @@ github.com/aws/aws-sdk-go-v2/credentials v1.20.3 h1:tToOYM/LXev4NpfWlIYGDvBvjHmJ github.com/aws/aws-sdk-go-v2/credentials v1.20.3/go.mod h1:wfGneWyncO7p67wqXV2IQhPk14JqIc25woKlaArT3WI= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.2 h1:Ldv7RPHs7qwwTscRjAl3YBud32f3BvdAGRmSvAx5L38= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.2/go.mod h1:XyK6UV8xbo66ysVqLd2783C09pBYHOm8aKTRV5DVJ30= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.2 h1:q/PSLGuRWCChWg+dLnb9dWOnrCxJtnboXbBtFoqqRrI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.2/go.mod h1:TD1jvU2LvXkJexct5vBqcd8QlNXh5EmRUeL/Z32p0n4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.2 h1:6fl86IPqKEXoySqiOWdfgbEp9OVbn44zTfEICNEBDhY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.2/go.mod h1:63HDfhFkdzBpI8WGXTSKUHPKS6mqldj4u3LJW7RZtSU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4 h1:CLq4+8UHCI+ZZYl/EuJxXovaIVN2xeeT8JV+dsApQ5E= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.4/go.mod h1:Wv4q5sAM04xAMkoOedxLx2inVf6K5FdxYp+A61L+q/0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4 h1:dD4MR81I7YkpEBRk6UP9rocC2QnT3qVuXwzlYTtfGEs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.4/go.mod h1:EcXV1kAFd5XwSkDHlj94gnF3q5CkJyYiIJfH8N0VmrE= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.2 h1:XMgIRS+uW9F3yFKnXGRrI9pkHi99CXTmoz2kz2/TGBA= github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.2/go.mod h1:vorxDzK+n3jiv9a5ST/LG0Eu9cSv1CRdKTpG6pDMs+M= github.com/aws/aws-sdk-go-v2/service/accessanalyzer v1.51.4 h1:DD5SFDxWC2jxmzOTM4b3fWeDSwlYtF52EFbCwyrxLy0= @@ -162,6 +162,8 @@ github.com/aws/aws-sdk-go-v2/service/dlm v1.39.4 h1:8iumILxX2NecA6Y9e+2AwsK/mQkY github.com/aws/aws-sdk-go-v2/service/dlm v1.39.4/go.mod h1:WsfwRJMXmG3vC98Sg27QDj2j+PZ7sTc+tfC1poL3XW8= github.com/aws/aws-sdk-go-v2/service/docdb v1.51.4 h1:v9APiIk1o8rcc1UibTO+b4wULni1u4uARTOfsMFlZ9w= github.com/aws/aws-sdk-go-v2/service/docdb v1.51.4/go.mod h1:DixlM7ytFFg7slVDNA/RkJbU+f81SLQKZU9RJOkvE/s= +github.com/aws/aws-sdk-go-v2/service/dsql v1.22.1 h1:Jnr7wkgRjhwkatsBA9XpqzPpotEF1AKE56mG6Ys1Btk= +github.com/aws/aws-sdk-go-v2/service/dsql v1.22.1/go.mod h1:23eSdtdlcwDGNyGsrlpWdUoDOEKOSALjPKRxm7Z9qNc= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.67.0 h1:Qs5KY7LC+/lMasuJOujgDtl3In6MHxnceULr2+V/Ldc= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.67.0/go.mod h1:xZ68tXuET4F9jrz4kQCXN6JIex933g0Izh6FM+YtFsg= github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.40.0 h1:gyRsDiymu1UGrBPu4/viOrY08ifTSdUcrAwpYHXyXd4= @@ -170,6 +172,8 @@ github.com/aws/aws-sdk-go-v2/service/ec2 v1.329.0 h1:diNdfHw/832G8QYVQ6uz5kwOC0p github.com/aws/aws-sdk-go-v2/service/ec2 v1.329.0/go.mod h1:mILEu29lo7lpbpkBIy4BtVFoHPLyM/ngOKYCZHEqciM= github.com/aws/aws-sdk-go-v2/service/ecr v1.64.0 h1:iOYGE9bHGhMQYtbjEcgDJEobWIhKoUvE71m+Jm0vZgU= github.com/aws/aws-sdk-go-v2/service/ecr v1.64.0/go.mod h1:5ccNgipT/aF9MWzTrKkyGJaCozPt+D6LOD4RFIdP22k= +github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.47.1 h1:v5YoVRgpKjrRoE0dMLg+uHPanOO9g9oUWg5bl2Vv7lU= +github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.47.1/go.mod h1:jDq6WJurFrXwl6HkEIzh6Kq7+uzvriNder8irvx11dY= github.com/aws/aws-sdk-go-v2/service/ecs v1.96.0 h1:kAOWRGbOwkvD+IQLOklSNSVY6pF2nGCmemBHG4iRLNk= github.com/aws/aws-sdk-go-v2/service/ecs v1.96.0/go.mod h1:djHxMBR2H6gax8sZIXbRic+c5O6NQKm/r7NU2kezbZk= github.com/aws/aws-sdk-go-v2/service/efs v1.48.0 h1:Bo8wTE02aOyvJeuiwA7nEHZDEMMOOUh1igWVItxGu1k= @@ -234,12 +238,16 @@ github.com/aws/aws-sdk-go-v2/service/iotwireless v1.59.4 h1:/3tu+ozqXFjYQDcnlO0S github.com/aws/aws-sdk-go-v2/service/iotwireless v1.59.4/go.mod h1:73vaf69mm3qGN3x2OvKaHkbn2xNDyfTA2MQHTelKip0= github.com/aws/aws-sdk-go-v2/service/kafka v1.57.2 h1:nGGNUc4pBJgAD5H7/et7lKAZhD0i0JRyTHXpjB5iym8= github.com/aws/aws-sdk-go-v2/service/kafka v1.57.2/go.mod h1:v4Wwc/lfF7eoE/dezUXD46lpQ/B1B4Cv5w+XPOGfHx0= +github.com/aws/aws-sdk-go-v2/service/kafkaconnect v1.39.1 h1:Ne7KMO3cPD1jq09kGjo0deOPzAxqyGe8rPikc8ooM+E= +github.com/aws/aws-sdk-go-v2/service/kafkaconnect v1.39.1/go.mod h1:b2vR4U6pl0srcPy4qo35ze6yZSI+ECmQsVEHDSjyVFE= github.com/aws/aws-sdk-go-v2/service/kinesis v1.53.0 h1:jFGpuCKudK8UUYAN8gokcq9NbavYnzX4DM0G5YYG/rM= github.com/aws/aws-sdk-go-v2/service/kinesis v1.53.0/go.mod h1:2h8s6B+Dk/9p2qrajjQHh3/OvlieQTuxRucQ+1l8VUc= github.com/aws/aws-sdk-go-v2/service/kinesisanalytics v1.33.4 h1:XbC82YaaogjUXeciT8I86BOXmdsCUgHsrPkk5sW2unA= github.com/aws/aws-sdk-go-v2/service/kinesisanalytics v1.33.4/go.mod h1:0rM3wUqWSiBPMlw0pvWhILHQICKVOvm1aMjPo8Idzuc= github.com/aws/aws-sdk-go-v2/service/kinesisanalyticsv2 v1.41.4 h1:DkAPWjRHgTQtGfeGDFWfLO6vkT7puNauJHEBf2uJeO8= github.com/aws/aws-sdk-go-v2/service/kinesisanalyticsv2 v1.41.4/go.mod h1:dLC1r0GeGKy1WL6nzDX3uH0AHsfcq99WyxW6a5zJZ2k= +github.com/aws/aws-sdk-go-v2/service/kinesisvideo v1.41.1 h1:zA/ZvubYiYUxb+BVgvVe/0kpxqYXtnF020Xx7EOL2xA= +github.com/aws/aws-sdk-go-v2/service/kinesisvideo v1.41.1/go.mod h1:IlHmk+bal4iFxUsZ4UJwW6S6yBgz/AZvQDUhI/1aOhY= github.com/aws/aws-sdk-go-v2/service/kms v1.59.0 h1:qvCvEQxFqL4LLCFLuvrNovy2iLYSU74gedrahGvT6vI= github.com/aws/aws-sdk-go-v2/service/kms v1.59.0/go.mod h1:p1tptb9enFZSeQxQjODPDwcDAuFrKKXumkHUi+R5C8A= github.com/aws/aws-sdk-go-v2/service/lakeformation v1.50.4 h1:X/dDCuk20MDnquyeA9oHxgr4KPIjLAQut9QUq27JJzA= diff --git a/kinesis_faketime_test.go b/kinesis_faketime_test.go new file mode 100644 index 0000000000..8bb8de240b --- /dev/null +++ b/kinesis_faketime_test.go @@ -0,0 +1,41 @@ +package main + +import ( + "sync/atomic" + "time" +) + +// kinesisStreamSettleWait safely exceeds Kinesis's internal transient-state +// transition delay (streamTransitionDelay, 250ms in services/kinesis/models.go) so a +// single kinesisFakeClock.Advance call is guaranteed to move a CREATING/UPDATING +// stream past its ReadyAt deadline. +const kinesisStreamSettleWait = time.Second + +// kinesisFakeClock is a goroutine-safe, manually-advanced clock for driving +// Kinesis's lazy stream-transition deadlines deterministically in tests that +// exercise it in-process (via InMemoryBackend.WithClock), including through +// another service's fire-and-forget delivery goroutine (e.g. DynamoDB's +// KinesisEmitter, EventBridge's target retry loop). Mirrors +// services/kinesis/faketime_test.go's fakeClock. +type kinesisFakeClock struct { + now atomic.Pointer[time.Time] +} + +// newKinesisFakeClock creates a kinesisFakeClock starting at start. +func newKinesisFakeClock(start time.Time) *kinesisFakeClock { + c := &kinesisFakeClock{} + c.now.Store(&start) + + return c +} + +// Now returns the clock's current time. Suitable as InMemoryBackend.WithClock's argument. +func (c *kinesisFakeClock) Now() time.Time { + return *c.now.Load() +} + +// Advance moves the clock forward by d. +func (c *kinesisFakeClock) Advance(d time.Duration) { + next := c.Now().Add(d) + c.now.Store(&next) +} diff --git a/pkgs/condeval/condeval.go b/pkgs/condeval/condeval.go new file mode 100644 index 0000000000..438b6d082b --- /dev/null +++ b/pkgs/condeval/condeval.go @@ -0,0 +1,103 @@ +// Package condeval holds IAM policy / STS trust-policy condition-operator +// logic shared between services/iam and services/sts: ARN segment-wise +// matching (ArnEquals/ArnLike/ArnNotEquals/ArnNotLike) and Date operand +// parsing/comparison (DateEquals/.../DateGreaterThanEquals). Each caller +// keeps its own wildcard matcher and its own operator-dispatch/IfExists/ +// set-qualifier plumbing; only the AWS-documented matching rules that were +// starting to be copy-pasted verbatim between the two packages live here. +package condeval + +import ( + "strconv" + "strings" + "time" +) + +// ArnSegmentCount is the number of colon-delimited components in an ARN +// (arn:partition:service:region:account-id:resource). +const ArnSegmentCount = 6 + +// ArnMatch implements ArnEquals/ArnLike, which AWS documents as behaving +// identically: case-sensitive, with each of the six colon-delimited ARN +// components wildcard-matched separately via match, rather than one glob +// over the whole string (which would let a wildcard span a segment +// boundary). +// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html#Conditions_ARN +func ArnMatch(pattern, value string, match func(pattern, value string) bool) bool { + pParts := strings.SplitN(pattern, ":", ArnSegmentCount) + vParts := strings.SplitN(value, ":", ArnSegmentCount) + + if len(pParts) != len(vParts) { + return false + } + + for i, p := range pParts { + if !match(p, vParts[i]) { + return false + } + } + + return true +} + +// AnyArnMatch reports whether value matches any ARN pattern in patterns. +func AnyArnMatch(patterns []string, value string, match func(pattern, value string) bool) bool { + for _, p := range patterns { + if ArnMatch(p, value, match) { + return true + } + } + + return false +} + +// isoDateLayouts are the W3C ISO 8601 profiles AWS documents for Date +// condition values, tried in order before falling back to epoch seconds. +// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html#Conditions_Date +var isoDateLayouts = []string{ //nolint:gochecknoglobals // read-only lookup table + time.RFC3339Nano, + time.RFC3339, + "2006-01-02T15:04:05", + "2006-01-02", +} + +// ParseDate parses a Date condition operand, accepting both ISO 8601 and +// epoch (UNIX) seconds, as AWS documents for aws:CurrentTime/aws:EpochTime. +func ParseDate(v string) (time.Time, bool) { + for _, layout := range isoDateLayouts { + if t, err := time.Parse(layout, v); err == nil { + return t, true + } + } + + if secs, err := strconv.ParseFloat(v, 64); err == nil { + whole := int64(secs) + nanos := int64((secs - float64(whole)) * float64(time.Second)) + + return time.Unix(whole, nanos).UTC(), true + } + + return time.Time{}, false +} + +// CompareDate evaluates one of the six Date condition operators (already +// lower-cased and IfExists-stripped) for a single actual/candidate pair. +// Unrecognized operators return false. +func CompareDate(op string, actual, candidate time.Time) bool { + switch op { + case "dateequals": + return actual.Equal(candidate) + case "datenotequals": + return !actual.Equal(candidate) + case "datelessthan": + return actual.Before(candidate) + case "datelessthanequals": + return !actual.After(candidate) + case "dategreaterthan": + return actual.After(candidate) + case "dategreaterthanequals": + return !actual.Before(candidate) + default: + return false + } +} diff --git a/pkgs/condeval/condeval_test.go b/pkgs/condeval/condeval_test.go new file mode 100644 index 0000000000..1a5f5e8970 --- /dev/null +++ b/pkgs/condeval/condeval_test.go @@ -0,0 +1,145 @@ +package condeval_test + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/condeval" +) + +// glob is a minimal '*'/'?' matcher, standing in for each caller's own +// wildcardMatch (services/iam and services/sts each keep their own). +func glob(pattern, value string) bool { + if pattern == "*" { + return true + } + + prefix, suffix, ok := strings.Cut(pattern, "*") + if !ok { + return pattern == value + } + + return strings.HasPrefix(value, prefix) && strings.HasSuffix(value, suffix) +} + +func TestArnMatch(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + pattern string + value string + want bool + }{ + { + name: "wildcard_confined_to_last_segment", + pattern: "arn:aws:sqs:us-east-1:123456789012:my-*", + value: "arn:aws:sqs:us-east-1:123456789012:my-queue", + want: true, + }, + { + name: "wildcard_does_not_span_segments", + pattern: "arn:aws:s3:*:mybucket", + value: "arn:aws:s3:us-east-1:123456789012:mybucket", + want: false, + }, + { + name: "case_sensitive_no_match", + pattern: "arn:aws:iam::123456789012:role/prod", + value: "arn:aws:iam::123456789012:role/Prod", + want: false, + }, + { + name: "differing_segment_count_no_match", + pattern: "arn:aws:iam::123456789012:role/prod", + value: "not-an-arn-at-all", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, condeval.ArnMatch(tt.pattern, tt.value, glob)) + }) + } +} + +func TestAnyArnMatch(t *testing.T) { + t.Parallel() + + patterns := []string{"arn:aws:iam::111111111111:role/other", "arn:aws:iam::222222222222:role/*"} + + assert.True(t, condeval.AnyArnMatch(patterns, "arn:aws:iam::222222222222:role/deploy", glob)) + assert.False(t, condeval.AnyArnMatch(patterns, "arn:aws:iam::333333333333:role/deploy", glob)) +} + +func TestParseDate(t *testing.T) { + t.Parallel() + + tests := []struct { + want time.Time + name string + in string + }{ + {name: "rfc3339", in: "2023-06-15T12:00:00Z", want: time.Date(2023, 6, 15, 12, 0, 0, 0, time.UTC)}, + {name: "date_only", in: "2023-06-15", want: time.Date(2023, 6, 15, 0, 0, 0, 0, time.UTC)}, + {name: "epoch_seconds", in: "1686830400", want: time.Date(2023, 6, 15, 12, 0, 0, 0, time.UTC)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, ok := condeval.ParseDate(tt.in) + require.True(t, ok) + assert.True(t, tt.want.Equal(got), "got %v, want %v", got, tt.want) + }) + } + + t.Run("invalid", func(t *testing.T) { + t.Parallel() + + _, ok := condeval.ParseDate("not-a-date") + assert.False(t, ok) + }) +} + +func TestCompareDate(t *testing.T) { + t.Parallel() + + earlier := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + later := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + + tests := []struct { + actual, candidate time.Time + name string + op string + want bool + }{ + {name: "equals_true", op: "dateequals", actual: earlier, candidate: earlier, want: true}, + {name: "equals_false", op: "dateequals", actual: earlier, candidate: later, want: false}, + {name: "not_equals", op: "datenotequals", actual: earlier, candidate: later, want: true}, + {name: "less_than", op: "datelessthan", actual: earlier, candidate: later, want: true}, + {name: "less_than_equal_at_boundary", op: "datelessthanequals", actual: later, candidate: later, want: true}, + {name: "greater_than", op: "dategreaterthan", actual: later, candidate: earlier, want: true}, + { + name: "greater_than_equal_at_boundary", op: "dategreaterthanequals", + actual: earlier, candidate: earlier, want: true, + }, + {name: "unrecognized_op", op: "bogus", actual: earlier, candidate: later, want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, condeval.CompareDate(tt.op, tt.actual, tt.candidate)) + }) + } +} diff --git a/pkgs/dynamoattr/attr.go b/pkgs/dynamoattr/attr.go index 1b54e056e0..5d567ad6a8 100644 --- a/pkgs/dynamoattr/attr.go +++ b/pkgs/dynamoattr/attr.go @@ -8,19 +8,49 @@ import ( ) // UnwrapAttributeValue converts a DynamoDB wire attribute map into a bare value when possible. +// +// Direct lookups instead of ranging: a wire map has exactly one type key, and a +// map iterator costs more than checking the (overwhelmingly common) S/N keys directly. func UnwrapAttributeValue(v any) any { - m, ok := v.(map[string]any) - if !ok || len(m) == 0 { + m, isMap := v.(map[string]any) + if !isMap || len(m) == 0 { return v } - for k, val := range m { - switch k { - case "S", "N", "B", "BOOL", "M", "L", "SS", "NS", "BS": - return val - case "NULL": - return nil - } + if val, ok := m["S"]; ok { + return val + } + if val, ok := m["N"]; ok { + return val + } + + return unwrapAttributeValueRare(m, v) +} + +func unwrapAttributeValueRare(m map[string]any, v any) any { + if val, ok := m["B"]; ok { + return val + } + if val, ok := m["BOOL"]; ok { + return val + } + if _, ok := m["NULL"]; ok { + return nil + } + if val, ok := m["M"]; ok { + return val + } + if val, ok := m["L"]; ok { + return val + } + if val, ok := m["SS"]; ok { + return val + } + if val, ok := m["NS"]; ok { + return val + } + if val, ok := m["BS"]; ok { + return val } return v diff --git a/pkgs/httputils/pool.go b/pkgs/httputils/pool.go index 11bdb112f1..4f57db1202 100644 --- a/pkgs/httputils/pool.go +++ b/pkgs/httputils/pool.go @@ -42,6 +42,12 @@ func PutBuffer(buf *bytes.Buffer) { bufferPool.Put(buf) } +// WillPool reports whether PutBuffer would retain buf; if so, clone buf.Bytes() +// before handing it out. +func WillPool(buf *bytes.Buffer) bool { + return buf != nil && buf.Cap() <= maxPooledBufferSize +} + var crc32Pool = sync.Pool{ //nolint:gochecknoglobals // sync.Pool requires package-level allocation New: func() any { return crc32.NewIEEE() diff --git a/pkgs/lockmetrics/bench_lock_test.go b/pkgs/lockmetrics/bench_lock_test.go new file mode 100644 index 0000000000..f9b12d7314 --- /dev/null +++ b/pkgs/lockmetrics/bench_lock_test.go @@ -0,0 +1,32 @@ +package lockmetrics_test + +import ( + "testing" + + "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" +) + +// BenchmarkRWMutex_LockUnlock_SameOp measures the common case of a backend +// repeatedly locking the same operation name. +func BenchmarkRWMutex_LockUnlock_SameOp(b *testing.B) { + m := lockmetrics.New("bench.lock." + b.Name()) + + b.ReportAllocs() + + for b.Loop() { + m.Lock("PutItem") + m.Unlock() + } +} + +// BenchmarkRWMutex_RLockRUnlock_SameOp is RLock/RUnlock's analogue. +func BenchmarkRWMutex_RLockRUnlock_SameOp(b *testing.B) { + m := lockmetrics.New("bench.rlock." + b.Name()) + + b.ReportAllocs() + + for b.Loop() { + m.RLock("GetItem") + m.RUnlock() + } +} diff --git a/pkgs/lockmetrics/close_series_test.go b/pkgs/lockmetrics/close_series_test.go new file mode 100644 index 0000000000..ba4bd74971 --- /dev/null +++ b/pkgs/lockmetrics/close_series_test.go @@ -0,0 +1,104 @@ +package lockmetrics_test + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" +) + +// TestRWMutex_CloseInvalidatesCachedHandles pins that a post-Close +// observation lands on a live series, not a handle Close already deleted. +func TestRWMutex_CloseInvalidatesCachedHandles(t *testing.T) { + t.Parallel() + + tests := []struct { + build func(name string) *lockmetrics.RWMutex + name string + }{ + { + name: "use_after_close_same_instance", + build: func(name string) *lockmetrics.RWMutex { + m := lockmetrics.New(name) + m.Lock("first") + m.Unlock() + m.Close() + + return m + }, + }, + { + name: "recreate_same_name_after_close", + build: func(name string) *lockmetrics.RWMutex { + old := lockmetrics.New(name) + old.Lock("first") + old.Unlock() + old.Close() + + return lockmetrics.New(name) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + name := "close.invalidate." + t.Name() + m := tt.build(name) + t.Cleanup(m.Close) + + m.Lock("second") + m.Unlock() + m.RLock("second") + m.RUnlock() + + mfs, err := prometheus.DefaultGatherer.Gather() + require.NoError(t, err) + + held := seriesFor(mfs, "gopherstack_lock_hold_seconds", name) + require.Len(t, held, 1, "post-Close write observation must land on a live series") + assert.EqualValues(t, 1, held[0].GetHistogram().GetSampleCount()) + assert.Equal(t, "second", labelValue(held[0], "operation")) + + waited := seriesFor(mfs, "gopherstack_lock_wait_seconds", name) + assert.Len(t, waited, 2, "post-Close read and write wait observations must land on live series") + + active := seriesFor(mfs, "gopherstack_lock_active_readers", name) + require.Len(t, active, 1, "post-Close RLock/RUnlock must re-curry the series, not write the deleted one") + assert.InDelta(t, 0, active[0].GetGauge().GetValue(), 0, "a paired RLock/RUnlock after Close nets to zero") + }) + } +} + +// TestRWMutex_CloseDuringHeldRLockTolerated pins the accepted race: Close between RLock +// and RUnlock may leave the gauge negative, but the mutex keeps working. +func TestRWMutex_CloseDuringHeldRLockTolerated(t *testing.T) { + t.Parallel() + + name := "close.during-hold." + t.Name() + m := lockmetrics.New(name) + + m.RLock("held") + m.Close() + m.RUnlock() + + // The mutex must remain usable: a fresh RLock/RUnlock re-curries cleanly. + m2 := lockmetrics.New(name) + t.Cleanup(m2.Close) + + m2.RLock("after") + m2.RUnlock() + + mfs, err := prometheus.DefaultGatherer.Gather() + require.NoError(t, err) + + active := seriesFor(mfs, "gopherstack_lock_active_readers", name) + require.Len(t, active, 1) + // Documents the tolerated race: RUnlock's post-Close re-curry can't see + // the Inc that landed on the deleted series, so it nets negative here. + assert.InDelta(t, -1, active[0].GetGauge().GetValue(), 0) +} diff --git a/pkgs/lockmetrics/lockmetrics.go b/pkgs/lockmetrics/lockmetrics.go index 90ed45c11a..5c8b8ad093 100644 --- a/pkgs/lockmetrics/lockmetrics.go +++ b/pkgs/lockmetrics/lockmetrics.go @@ -204,30 +204,37 @@ func registerOrReuse[T prometheus.Collector](c T) T { return c } +// writeOpMetrics caches one write operation's curried WithLabelValues +// handles, so repeated Lock/Unlock calls for the same op skip the lookup. +type writeOpMetrics struct { + wait prometheus.Observer + hold prometheus.Observer + active prometheus.Gauge +} + // RWMutex is a drop-in replacement for [sync.RWMutex] that records Prometheus // metrics on every Lock/RLock call. // // The zero value is not usable; always create via New. type RWMutex struct { - // *prometheus pointer fields first; they are 8 bytes each (pure pointer). - waitSeconds *prometheus.HistogramVec - holdSeconds *prometheus.HistogramVec - activeWriters *prometheus.GaugeVec - activeReaders *prometheus.GaugeVec - // activeReadersLock is a curried gauge pre-scoped to this lock name, - // eliminating the per-call label hash lookup on RLock/RUnlock. - activeReadersLock prometheus.Gauge - // writeOp and name follow; each contains a pointer so the GC scan extends - // through them, but their trailing non-pointer word (len/cap) falls outside - // the scan range. - writeOp atomic.Value // string — current write-lock operation name - name string - - // Non-pointer fields: GC scan stops above this line. - mu sync.RWMutex - - writeStart atomic.Int64 // unix nanoseconds; 0 when write lock is not held - + // activeReadersLock caches the curried active-readers gauge; Close clears it so + // later use re-curries instead of writing to the deleted series. + activeReadersLock atomic.Pointer[prometheus.Gauge] + writeOp atomic.Value // string — current write-lock operation name + // writeMetricsCur holds the current write-lock's metrics, set in Lock and + // read by the matching Unlock; safe since the write lock is exclusive. + writeMetricsCur atomic.Pointer[writeOpMetrics] + activeReaders *prometheus.GaugeVec + activeWriters *prometheus.GaugeVec + holdSeconds *prometheus.HistogramVec + waitSeconds *prometheus.HistogramVec + // writeOpCache/readOpCache memoize per-op WithLabelValues results + // (map[string]*writeOpMetrics / map[string]prometheus.Observer). + writeOpCache sync.Map + readOpCache sync.Map + name string + writeStart atomic.Int64 // unix nanoseconds; 0 when write lock is not held + mu sync.RWMutex // writeWaiters and readWaiters count goroutines currently blocked // waiting to acquire the respective lock. A non-zero count that stays // non-zero indefinitely indicates a deadlock or severe starvation. @@ -235,6 +242,57 @@ type RWMutex struct { readWaiters atomic.Int32 } +// writeMetricsFor returns the cached [writeOpMetrics] for op, creating and +// caching it on first use. +func (m *RWMutex) writeMetricsFor(op string) *writeOpMetrics { + if v, ok := m.writeOpCache.Load(op); ok { + wm, _ := v.(*writeOpMetrics) + + return wm + } + + wm := &writeOpMetrics{ + wait: m.waitSeconds.WithLabelValues(m.name, op, "write"), + hold: m.holdSeconds.WithLabelValues(m.name, op), + active: m.activeWriters.WithLabelValues(m.name, op), + } + + actual, _ := m.writeOpCache.LoadOrStore(op, wm) + wm, _ = actual.(*writeOpMetrics) + + return wm +} + +// activeReaderGauge returns the cached active-readers gauge, re-currying it +// if Close cleared the cache since the last call. +func (m *RWMutex) activeReaderGauge() prometheus.Gauge { + if p := m.activeReadersLock.Load(); p != nil { + return *p + } + + g := m.activeReaders.WithLabelValues(m.name) + m.activeReadersLock.Store(&g) + + return g +} + +// readWaitFor returns the cached read-wait [prometheus.Observer] for op, +// creating and caching it on first use. +func (m *RWMutex) readWaitFor(op string) prometheus.Observer { + if v, ok := m.readOpCache.Load(op); ok { + obs, _ := v.(prometheus.Observer) + + return obs + } + + obs := m.waitSeconds.WithLabelValues(m.name, op, "read") + + actual, _ := m.readOpCache.LoadOrStore(op, obs) + obs, _ = actual.(prometheus.Observer) + + return obs +} + // New creates a new [RWMutex]. The name appears as the labelLock label in all // emitted metrics and should be a stable, human-readable identifier // (e.g. "s3", "ddb.table.users"). @@ -254,14 +312,17 @@ func New(name string) *RWMutex { // Pre-curry the activeReaders gauge to this lock's name so RLock/RUnlock // avoid a label hash lookup on every call. - m.activeReadersLock = m.activeReaders.WithLabelValues(m.name) + g := m.activeReaders.WithLabelValues(m.name) + m.activeReadersLock.Store(&g) return m } // Close removes the [RWMutex] from the global metrics registry. // It must be called when the mutex is no longer needed (e.g. on table/bucket deletion) -// to prevent memory leaks and performance degradation. +// to prevent memory leaks and performance degradation, and only once no goroutine +// holds an active Lock/RLock: a Close racing an in-flight RLock/RUnlock pair can +// leave the recreated active-readers series transiently negative. func (m *RWMutex) Close() { if m == nil { return @@ -276,6 +337,13 @@ func (m *RWMutex) Close() { m.holdSeconds.DeletePartialMatch(prometheus.Labels{labelLock: m.name}) m.activeWriters.DeletePartialMatch(prometheus.Labels{labelLock: m.name}) m.activeReaders.DeleteLabelValues(m.name) + + // Drop cached handles: they point at series just deleted above. A stray + // call after Close recreates fresh series, matching pre-cache behavior. + m.writeOpCache.Clear() + m.readOpCache.Clear() + m.writeMetricsCur.Store(nil) + m.activeReadersLock.Store(nil) } // WriteWaiters returns the current number of goroutines blocked waiting for @@ -311,10 +379,11 @@ func (m *RWMutex) Lock(op string) { m.mu.Lock() m.writeWaiters.Add(-1) // acquired — no longer waiting - waited := time.Since(start).Seconds() - m.waitSeconds.WithLabelValues(m.name, op, "write").Observe(waited) - m.activeWriters.WithLabelValues(m.name, op).Inc() + wm := m.writeMetricsFor(op) + wm.wait.Observe(time.Since(start).Seconds()) + wm.active.Inc() m.writeOp.Store(op) + m.writeMetricsCur.Store(wm) m.writeStart.Store(time.Now().UnixNano()) } @@ -322,15 +391,19 @@ func (m *RWMutex) Lock(op string) { // during Lock is used to attribute the hold-duration histogram. func (m *RWMutex) Unlock() { ts := m.writeStart.Load() - op, _ := m.writeOp.Load().(string) + + wm := m.writeMetricsCur.Load() + if wm == nil { + wm = m.writeMetricsFor("") + } var held float64 if ts != 0 { held = time.Since(time.Unix(0, ts)).Seconds() } - m.holdSeconds.WithLabelValues(m.name, op).Observe(held) - m.activeWriters.WithLabelValues(m.name, op).Dec() + wm.hold.Observe(held) + wm.active.Dec() m.writeStart.Store(0) m.writeOp.Store("") m.mu.Unlock() @@ -344,12 +417,12 @@ func (m *RWMutex) RLock(op string) { m.mu.RLock() m.readWaiters.Add(-1) // acquired — no longer waiting - m.waitSeconds.WithLabelValues(m.name, op, "read").Observe(time.Since(start).Seconds()) - m.activeReadersLock.Inc() + m.readWaitFor(op).Observe(time.Since(start).Seconds()) + m.activeReaderGauge().Inc() } // RUnlock releases the shared read lock. func (m *RWMutex) RUnlock() { - m.activeReadersLock.Dec() + m.activeReaderGauge().Dec() m.mu.RUnlock() } diff --git a/pkgs/persistence/testdata/snapshot_inventory.json b/pkgs/persistence/testdata/snapshot_inventory.json index 14e683002d..5858e39f41 100644 --- a/pkgs/persistence/testdata/snapshot_inventory.json +++ b/pkgs/persistence/testdata/snapshot_inventory.json @@ -496,10 +496,17 @@ "CorsConfiguration.ExposeHeaders []string `json:\"exposeHeaders,omitempty\"`", "CorsConfiguration.MaxAge int `json:\"maxAge,omitempty\"`", "Deployment.APISummary map[string]map[string]MethodSnapshot `json:\"apiSummary,omitempty\"`", + "Deployment.Config *DeploymentConfig `json:\"-\"`", "Deployment.CreatedDate unixEpochTime `json:\"createdDate\"`", "Deployment.Description string `json:\"description,omitempty\"`", "Deployment.ID string `json:\"id\"`", "Deployment.RestAPIID string `json:\"-\"`", + "DeploymentConfig.Authorizers map[string]*Authorizer `json:\"authorizers,omitempty\"`", + "DeploymentConfig.GatewayResponses map[string]*GatewayResponse `json:\"gatewayResponses,omitempty\"`", + "DeploymentConfig.MinimumCompressionSize int `json:\"minimumCompressionSize,omitempty\"`", + "DeploymentConfig.Models map[string]*Model `json:\"models,omitempty\"`", + "DeploymentConfig.RequestValidators map[string]*RequestValidator `json:\"requestValidators,omitempty\"`", + "DeploymentConfig.Resources []Resource `json:\"resources,omitempty\"`", "DocumentationLocation.Method string `json:\"method,omitempty\"`", "DocumentationLocation.Name string `json:\"name,omitempty\"`", "DocumentationLocation.Path string `json:\"path,omitempty\"`", @@ -687,6 +694,7 @@ "backendSnapshot.Tables map[string]json.RawMessage `json:\"tables\"`", "backendSnapshot.UsageOverrides map[string]map[string]int64 `json:\"usageOverrides,omitempty\"`", "deploymentSnapshot.APISummary map[string]map[string]MethodSnapshot `json:\"apiSummary,omitempty\"`", + "deploymentSnapshot.Config *DeploymentConfig `json:\"config,omitempty\"`", "deploymentSnapshot.CreatedDate unixEpochTime `json:\"createdDate\"`", "deploymentSnapshot.Description string `json:\"description,omitempty\"`", "deploymentSnapshot.ID string `json:\"id\"`", @@ -8123,6 +8131,40 @@ ], "version": 1 }, + "dsql": { + "fields": [ + "Cluster.ARN string", + "Cluster.AccountID string", + "Cluster.CreationTime time.Time", + "Cluster.DeletionProtectionEnabled bool", + "Cluster.Endpoint string", + "Cluster.Identifier string", + "Cluster.KmsEncryptionKey string", + "Cluster.MultiRegion *MultiRegionProperties", + "Cluster.PendingUntil time.Time", + "Cluster.Policy *ClusterPolicy", + "Cluster.Region string", + "Cluster.Status string", + "Cluster.Tags map[string]string", + "ClusterPolicy.Policy string", + "MultiRegionProperties.Clusters []string", + "MultiRegionProperties.WitnessRegion string", + "Stream.ARN string", + "Stream.ClusterIdentifier string", + "Stream.CreationTime time.Time", + "Stream.Format string", + "Stream.Ordering string", + "Stream.PendingUntil time.Time", + "Stream.Status string", + "Stream.StreamIdentifier string", + "Stream.Tags map[string]string", + "Stream.Target *StreamTarget", + "StreamTarget.RoleArn string", + "StreamTarget.StreamArn string", + "backendSnapshot.Tables map[string]json.RawMessage `json:\"tables\"`" + ], + "version": 1 + }, "dynamodb": { "fields": [ "Backup.AttributeDefinitions []models.AttributeDefinition `json:\"AttributeDefinitions\"`", @@ -8190,6 +8232,7 @@ "Table.PITRSnapshots []pitrSnapshot `json:\"PITRSnapshots,omitempty\"`", "Table.ProvisionedThroughput models.ProvisionedThroughputDescription `json:\"ProvisionedThroughput\"`", "Table.RecoveryPeriodInDays int32 `json:\"RecoveryPeriodInDays,omitempty\"`", + "Table.ReplicaAutoScaling map[string]*replicaAutoScalingSettings `json:\"ReplicaAutoScaling,omitempty\"`", "Table.Replicas []models.ReplicaDescription `json:\"Replicas,omitempty\"`", "Table.ResourcePolicy string `json:\"ResourcePolicy,omitempty\"`", "Table.ResourcePolicyRevision string `json:\"ResourcePolicyRevision,omitempty\"`", @@ -8225,9 +8268,14 @@ "autoScalingSettings.GlobalSecondaryIndexes map[string]*autoScalingThroughput `json:\"GlobalSecondaryIndexes,omitempty\"`", "autoScalingSettings.Read *autoScalingThroughput `json:\"Read,omitempty\"`", "autoScalingSettings.Write *autoScalingThroughput `json:\"Write,omitempty\"`", + "autoScalingThroughput.DisableScaleIn *bool `json:\"DisableScaleIn,omitempty\"`", "autoScalingThroughput.Disabled bool `json:\"AutoScalingDisabled,omitempty\"`", "autoScalingThroughput.MaxCapacity *int64 `json:\"MaxCapacity,omitempty\"`", "autoScalingThroughput.MinCapacity *int64 `json:\"MinCapacity,omitempty\"`", + "autoScalingThroughput.PolicyName *string `json:\"PolicyName,omitempty\"`", + "autoScalingThroughput.RoleArn *string `json:\"AutoScalingRoleArn,omitempty\"`", + "autoScalingThroughput.ScaleInCooldown *int32 `json:\"ScaleInCooldown,omitempty\"`", + "autoScalingThroughput.ScaleOutCooldown *int32 `json:\"ScaleOutCooldown,omitempty\"`", "autoScalingThroughput.TargetUtilizPct *float64 `json:\"TargetUtilizationPct,omitempty\"`", "dbSnapshot.AccountID string `json:\"accountID\"`", "dbSnapshot.Backups []*Backup `json:\"backups,omitempty\"`", @@ -8242,6 +8290,8 @@ "globalTableSettingsSnapshot.writeCapacityUnits *int64", "pitrSnapshot.Items []map[string]any `json:\"Items\"`", "pitrSnapshot.Taken time.Time `json:\"Taken\"`", + "replicaAutoScalingSettings.GlobalSecondaryIndexes map[string]*autoScalingThroughput `json:\"GlobalSecondaryIndexes,omitempty\"`", + "replicaAutoScalingSettings.Read *autoScalingThroughput `json:\"Read,omitempty\"`", "secondaryIndex.pkOnly map[string]map[int]struct{}", "secondaryIndex.pksk map[string]map[string]map[int]struct{}", "storedExport.BilledSizeBytes int64", @@ -10132,6 +10182,42 @@ ], "version": 2 }, + "ecrpublic": { + "fields": [ + "CatalogData.AboutText string", + "CatalogData.Architectures []string", + "CatalogData.Description string", + "CatalogData.LogoImageBlob []byte", + "CatalogData.MarketplaceCertified bool", + "CatalogData.OperatingSystems []string", + "CatalogData.UsageText string", + "Image.ArtifactMediaType string", + "Image.ImageDigest string", + "Image.ImageManifest string", + "Image.ImageManifestMediaType string", + "Image.ImagePushedAt time.Time", + "Image.ImageSizeInBytes int64", + "Image.RegistryID string", + "Image.RepositoryName string", + "RegistryCatalogData.DisplayName string", + "Repository.CatalogData CatalogData", + "Repository.CreatedAt time.Time", + "Repository.HasPolicy bool", + "Repository.PolicyText string", + "Repository.RegistryID string", + "Repository.RepositoryArn string", + "Repository.RepositoryName string", + "Repository.RepositoryURI string", + "Repository.Tags map[string]string", + "backendSnapshot.RegistryCatalogData RegistryCatalogData `json:\"registryCatalogData\"`", + "backendSnapshot.Tables map[string]json.RawMessage `json:\"tables\"`", + "backendSnapshot.TagIndex map[string]map[string]tagBinding `json:\"tagIndex,omitempty\"`", + "backendSnapshot.UploadedLayers map[string]map[string]int64 `json:\"uploadedLayers,omitempty\"`", + "tagBinding.CreatedAt time.Time", + "tagBinding.Digest string" + ], + "version": 1 + }, "ecs": { "fields": [ "AccountSetting.Name string `json:\"name\"`", @@ -15476,6 +15562,95 @@ ], "version": 1 }, + "kafkaconnect": { + "fields": [ + "ApacheKafkaCluster.BootstrapServers string", + "ApacheKafkaCluster.Vpc Vpc", + "AutoScaling.MaxAutoscalingTaskCount int32", + "AutoScaling.MaxWorkerCount int32", + "AutoScaling.McuCount int32", + "AutoScaling.MinWorkerCount int32", + "AutoScaling.ScaleInCPUPercent int32", + "AutoScaling.ScaleOutCPUPercent int32", + "Capacity.AutoScaling *AutoScaling", + "Capacity.Provisioned *ProvisionedCapacity", + "CloudWatchLogsDelivery.Enabled bool", + "CloudWatchLogsDelivery.LogGroup string", + "Connector.ARN string", + "Connector.ApacheKafkaCluster ApacheKafkaCluster", + "Connector.Capacity Capacity", + "Connector.ConnectorConfiguration map[string]string", + "Connector.CreationTime time.Time", + "Connector.CurrentVersion string", + "Connector.Description string", + "Connector.KafkaClusterClientAuthentication string", + "Connector.KafkaClusterEncryptionInTransit string", + "Connector.KafkaConnectVersion string", + "Connector.Name string", + "Connector.NetworkType string", + "Connector.Plugins []PluginRef", + "Connector.ServiceExecutionRoleArn string", + "Connector.State string", + "Connector.Tags map[string]string", + "Connector.WorkerConfiguration *WorkerConfigRef", + "Connector.WorkerLogDelivery *WorkerLogDelivery", + "ConnectorOperation.ARN string", + "ConnectorOperation.ConnectorArn string", + "ConnectorOperation.CreationTime time.Time", + "ConnectorOperation.EndTime time.Time", + "ConnectorOperation.OriginCapacity *Capacity", + "ConnectorOperation.OriginConnectorConfiguration map[string]string", + "ConnectorOperation.State string", + "ConnectorOperation.Steps []ConnectorOperationStep", + "ConnectorOperation.TargetCapacity *Capacity", + "ConnectorOperation.TargetConnectorConfiguration map[string]string", + "ConnectorOperation.Type string", + "ConnectorOperationStep.StepState string", + "ConnectorOperationStep.StepType string", + "CustomPlugin.ARN string", + "CustomPlugin.BucketArn string", + "CustomPlugin.ContentType string", + "CustomPlugin.CreationTime time.Time", + "CustomPlugin.Description string", + "CustomPlugin.FileKey string", + "CustomPlugin.FileMD5 string", + "CustomPlugin.FileSizeBytes int64", + "CustomPlugin.Name string", + "CustomPlugin.ObjectVersion string", + "CustomPlugin.Revision int64", + "CustomPlugin.State string", + "CustomPlugin.Tags map[string]string", + "FirehoseDelivery.DeliveryStream string", + "FirehoseDelivery.Enabled bool", + "PluginRef.CustomPluginArn string", + "PluginRef.Revision int64", + "ProvisionedCapacity.McuCount int32", + "ProvisionedCapacity.WorkerCount int32", + "S3LogDelivery.Bucket string", + "S3LogDelivery.Enabled bool", + "S3LogDelivery.Prefix string", + "Vpc.SecurityGroups []string", + "Vpc.Subnets []string", + "WorkerConfigRef.Arn string", + "WorkerConfigRef.Revision int64", + "WorkerConfigRevision.CreationTime time.Time", + "WorkerConfigRevision.Description string", + "WorkerConfigRevision.PropertiesFileContent string", + "WorkerConfigRevision.Revision int64", + "WorkerConfiguration.ARN string", + "WorkerConfiguration.CreationTime time.Time", + "WorkerConfiguration.Description string", + "WorkerConfiguration.LatestRevision WorkerConfigRevision", + "WorkerConfiguration.Name string", + "WorkerConfiguration.State string", + "WorkerConfiguration.Tags map[string]string", + "WorkerLogDelivery.CloudWatchLogs *CloudWatchLogsDelivery", + "WorkerLogDelivery.Firehose *FirehoseDelivery", + "WorkerLogDelivery.S3 *S3LogDelivery", + "backendSnapshot.Tables map[string]json.RawMessage `json:\"tables\"`" + ], + "version": 1 + }, "kinesis": { "fields": [ "Channel.ChannelARN string `json:\"channelARN\"`", @@ -15558,6 +15733,7 @@ "Stream.KeyID string `json:\"keyId,omitempty\"`", "Stream.MaxRecordSizeBytes int `json:\"maxRecordSizeBytes,omitempty\"`", "Stream.Name string `json:\"name\"`", + "Stream.ReadyAt time.Time `json:\"readyAt\"`", "Stream.Region string `json:\"region,omitempty\"`", "Stream.RetentionPeriod int `json:\"retentionPeriod\"`", "Stream.Shards []*Shard `json:\"shards\"`", @@ -15847,6 +16023,42 @@ ], "version": 2 }, + "kinesisvideo": { + "fields": [ + "Channel.ARN string", + "Channel.CreationTime time.Time", + "Channel.MessageTTLSeconds int32", + "Channel.Name string", + "Channel.Status string", + "Channel.Tags map[string]string", + "Channel.Type string", + "ImageGenerationConfig.DestinationRegion string", + "ImageGenerationConfig.Format string", + "ImageGenerationConfig.FormatConfig map[string]string", + "ImageGenerationConfig.HeightPixels int32", + "ImageGenerationConfig.ImageSelectorType string", + "ImageGenerationConfig.SamplingInterval int32", + "ImageGenerationConfig.Status string", + "ImageGenerationConfig.URI string", + "ImageGenerationConfig.WidthPixels int32", + "NotificationConfig.DestinationURI string", + "NotificationConfig.Status string", + "Stream.ARN string", + "Stream.CreationTime time.Time", + "Stream.DataRetentionInHours int32", + "Stream.DefaultStorageTier string", + "Stream.DeviceName string", + "Stream.ImageGeneration *ImageGenerationConfig", + "Stream.KmsKeyID string", + "Stream.MediaType string", + "Stream.Name string", + "Stream.Notification *NotificationConfig", + "Stream.Status string", + "Stream.Tags map[string]string", + "backendSnapshot.Tables map[string]json.RawMessage `json:\"tables\"`" + ], + "version": 1 + }, "kms": { "fields": [ "Alias.AliasArn string `json:\"AliasArn\"`", @@ -23757,12 +23969,12 @@ "backendSnapshot.DefaultRegion string `json:\"defaultRegion\"`", "backendSnapshot.Tables map[string]json.RawMessage `json:\"tables\"`", "backendSnapshot.Tags map[string][]types.Tag `json:\"tags\"`", - "sseInfo.Algorithm string", - "sseInfo.EncryptionContext string", - "sseInfo.KMSKeyID string", - "sseInfo.SSECAlgorithm string", + "sseInfo.Algorithm string `json:\"Algorithm\"`", + "sseInfo.EncryptionContext string `json:\"EncryptionContext\"`", + "sseInfo.KMSKeyID string `json:\"KMSKeyID\"`", + "sseInfo.SSECAlgorithm string `json:\"SSECAlgorithm\"`", "sseInfo.SSECKeyB64 string `json:\"-\"`", - "sseInfo.SSECKeyMD5 string", + "sseInfo.SSECKeyMD5 string `json:\"SSECKeyMD5\"`", "versionSnapshot.checksumAlgorithm string", "versionSnapshot.deleted bool", "versionSnapshot.etag string", @@ -26253,12 +26465,12 @@ "Queue.Region string", "Queue.Tags *tags.Tags", "Queue.URL string", + "Queue.blockedGroupsScratch map[string]bool", "Queue.deduplicationMsgIDs map[string]string", "Queue.delayedCount int", "Queue.dlq *Queue", - "Queue.fifoSendTimes map[string][]time.Time", - "Queue.fifoSendTimesQueue []time.Time", "Queue.fifoSeqCounter uint64", + "Queue.fifoThroughput map[fifoThroughputKey]*fifoRateWindow", "Queue.hasActivity atomic.Bool", "Queue.inFlightByHandle map[string]*InFlightMessage", "Queue.inFlightMessages []*InFlightMessage", @@ -26276,6 +26488,10 @@ "backendSnapshot.Tables map[string]json.RawMessage `json:\"tables\"`", "encodedMessageAttribute.Bytes []byte", "encodedMessageAttribute.Name string", + "fifoRateWindow.calls []time.Time", + "fifoRateWindow.messages []time.Time", + "fifoThroughputKey.method fifoAPIMethod", + "fifoThroughputKey.scopeKey string", "moveTaskSnapshot.DestArn string `json:\"destArn\"`", "moveTaskSnapshot.FailureReason string `json:\"failureReason,omitempty\"`", "moveTaskSnapshot.MaxPerSec int32 `json:\"maxPerSec,omitempty\"`", diff --git a/pkgs/service/cloudtrail_capture.go b/pkgs/service/cloudtrail_capture.go index e259cf36a1..f23edfd11d 100644 --- a/pkgs/service/cloudtrail_capture.go +++ b/pkgs/service/cloudtrail_capture.go @@ -209,6 +209,8 @@ func (w *captureResponseWriter) WriteHeader(code int) { // a wrapped handler that writes an error body without ever setting // Content-Type would otherwise let net/http sniff the tee'd bytes -- which // can include request-derived text -- as text/html, enabling reflected XSS. +// The tee itself only runs for error statuses; extractErrorInfo ignores the +// body otherwise, so buffering every success response was wasted work. func (w *captureResponseWriter) Write(b []byte) (int, error) { if w.status == 0 { w.WriteHeader(http.StatusOK) @@ -218,7 +220,9 @@ func (w *captureResponseWriter) Write(b []byte) (int, error) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") } - w.body.Write(b) + if w.status >= httpErrorStatusThreshold { + w.body.Write(b) + } return w.ResponseWriter.Write(b) } diff --git a/pkgs/service/cloudtrail_capture_test.go b/pkgs/service/cloudtrail_capture_test.go index 8b1d89b04c..a70fd2e76e 100644 --- a/pkgs/service/cloudtrail_capture_test.go +++ b/pkgs/service/cloudtrail_capture_test.go @@ -219,3 +219,62 @@ func TestWrapCloudTrailCapture(t *testing.T) { }) } } + +// TestWrapCloudTrailCapture_ErrorExtraction pins that a failed response still +// yields ErrorCode/ErrorMessage now that the tee only runs for status >= 400. +func TestWrapCloudTrailCapture_ErrorExtraction(t *testing.T) { + t.Parallel() + + rec := &mockRecorder{} + svc := &dummyService{name: "S3", extractOperation: "PutObject", extractResource: "bucket"} + next := func(c *echo.Context) error { + return c.JSON(http.StatusBadRequest, map[string]string{ + "__type": "NoSuchBucket", + "message": "bucket does not exist", + }) + } + handler := wrapCloudTrailCapture(rec, svc, next) + + e := echo.New() + req := httptest.NewRequest(http.MethodPut, "/", nil) + c := e.NewContext(req, httptest.NewRecorder()) + + require.NoError(t, handler(c)) + require.Len(t, rec.events, 1) + assert.Equal(t, "NoSuchBucket", rec.events[0].ErrorCode) + assert.Equal(t, "bucket does not exist", rec.events[0].ErrorMessage) +} + +// BenchmarkCaptureResponseWriterWrite_Success proves the success path no +// longer tees the response body into captureResponseWriter.body. +func BenchmarkCaptureResponseWriterWrite_Success(b *testing.B) { + payload := []byte(`{"ok":true,"items":[1,2,3,4,5]}`) + + b.ReportAllocs() + + for b.Loop() { + w := &captureResponseWriter{ResponseWriter: httptest.NewRecorder()} + w.WriteHeader(http.StatusOK) + + if _, err := w.Write(payload); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkCaptureResponseWriterWrite_Error covers the still-buffered path so +// the A/B comparison shows the error path's cost is unchanged. +func BenchmarkCaptureResponseWriterWrite_Error(b *testing.B) { + payload := []byte(`{"__type":"SomeException","message":"bad"}`) + + b.ReportAllocs() + + for b.Loop() { + w := &captureResponseWriter{ResponseWriter: httptest.NewRecorder()} + w.WriteHeader(http.StatusBadRequest) + + if _, err := w.Write(payload); err != nil { + b.Fatal(err) + } + } +} diff --git a/pkgs/testleak/testleak.go b/pkgs/testleak/testleak.go index c30ba4e7e5..d577093bbd 100644 --- a/pkgs/testleak/testleak.go +++ b/pkgs/testleak/testleak.go @@ -37,3 +37,9 @@ func VerifyTestMain(m *testing.M, extra ...goleak.Option) { opts := append(defaultIgnores(), extra...) goleak.VerifyTestMain(m, opts...) } + +// DefaultIgnores exposes the shared ignore list for callers that verify +// goroutines directly (e.g. goleak.VerifyNone) instead of via TestMain. +func DefaultIgnores() []goleak.Option { + return defaultIgnores() +} diff --git a/services/_ROUTE_COLLISIONS.md b/services/_ROUTE_COLLISIONS.md index 629058bf4d..d60d48eede 100644 --- a/services/_ROUTE_COLLISIONS.md +++ b/services/_ROUTE_COLLISIONS.md @@ -935,3 +935,47 @@ overcorrect). See `services/detective/PARITY.md` and `services/guardduty/PARITY.md`'s matching 2026-09-12 entries for the full gate results. + +## kafkaconnect (new service, 2026-09-25): `/v1/` bare prefix (batch vs. kafkaconnect) + +Adding the MSK Connect service (`services/kafkaconnect`, paths +`/v1/connectors`, `/v1/custom-plugins`, `/v1/worker-configurations`, +`/v1/connectorOperations/{arn}`, all at `PriorityPathVersioned` = 85) +reproduced the same species of bug as the first pass's batch/kafka case +above, except this time it was live, not a false positive: `batch`'s +`RouteMatcher` falls through to an unconditional +`strings.HasPrefix(path, "/v1/")` after excluding only `/v1/tags/`, +`/v1/apis`, the CodeArtifact paths, and `/v1/clusters`/`/v1/configurations` +(Kafka MSK) — none of kafkaconnect's paths were excluded, and unlike MSK, +kafkaconnect does not bump its own `MatchPriority` above batch's, so at a +tied priority-85 the router's stable sort falls back to registration order, +and batch registers before kafkaconnect in `cli.go`. Batch's matcher would +therefore have won every kafkaconnect request. + +**Fix** (per this file's own rule and `.claude/memories`'s +route-matcher-prefix-collision entry: never fix by raising `MatchPriority`): +`services/batch/handler.go`'s +`RouteMatcher` gained a third exclusion block, `kafkaConnectConnectorPrefix += "/v1/connector"` (covers `/v1/connectors` and `/v1/connectorOperations/`), +`kafkaConnectPluginPrefix = "/v1/custom-plugins"`, and +`kafkaConnectWorkerPrefix = "/v1/worker-configurations"`, mirroring the +existing MSK exclusion shape exactly. + +`go run ./cmd/routecollisions` still lists `batch shadows kafkaconnect` and +`polly shadows kafkaconnect` after the fix — both `(guarded/guarded)`, the +same coarse-tool residue the batch/kafka and polly/appsync false positives +above already document: the tool flags any narrower same-or-lower-priority +`/v1/...` claim against batch's/polly's bare `/v1/` literal regardless of +what carve-outs precede it in the source, because `isExclusion` only +suppresses a literal from the *owning* service's own claim list, it does +not cross-reference another service's specific claims. Verified functionally +correct instead: `services/batch/handler_test.go`'s existing RouteMatcher +table plus `services/kafkaconnect`'s own routing tests +(`routes_whitebox_test.go`) pass, and `TestTerraform_MskConnect` +(`test/terraform/msk_connect_test.go`) — which exercises the real +`service.NewServiceRouter` with every service registered, including batch — +passes, proving kafkaconnect's requests reach `services/kafkaconnect`, not +batch. `polly` needed no change: its bare `/v1/` claim is already gated by +`parseRoute(...).operation != opUnknown`, which rejects any kafkaconnect +path (confirmed by reading `services/polly/handler.go`, not just the tool's +"guarded" bit). diff --git a/services/acm/certificate_lifecycle_test.go b/services/acm/certificate_lifecycle_test.go index 95883b5f9f..8f3ebdab26 100644 --- a/services/acm/certificate_lifecycle_test.go +++ b/services/acm/certificate_lifecycle_test.go @@ -3,6 +3,7 @@ package acm_test import ( "context" "testing" + "testing/synctest" "time" sdktypes "github.com/aws/aws-sdk-go-v2/service/acm/types" @@ -16,30 +17,24 @@ import ( func TestACMBackend_AutoValidation(t *testing.T) { t.Parallel() - b := acm.NewInMemoryBackend("000000000000", "us-east-1") - cert, err := b.RequestCertificate(context.Background(), "auto.example.com", "", "DNS", "", "", "", "", nil) - require.NoError(t, err) - assert.Equal(t, "PENDING_VALIDATION", cert.Status) + synctest.Test(t, func(t *testing.T) { + b := acm.NewInMemoryBackend("000000000000", "us-east-1") + cert, err := b.RequestCertificate(context.Background(), "auto.example.com", "", "DNS", "", "", "", "", nil) + require.NoError(t, err) + assert.Equal(t, "PENDING_VALIDATION", cert.Status) - // Wait for auto-validation (should happen within 500ms) - require.Eventually(t, func() bool { - c, descErr := b.DescribeCertificate(context.Background(), cert.ARN) - if descErr != nil { - return false - } + // autoValidateDelayMS is 100ms; cross it so auto-validation fires. + time.Sleep(150 * time.Millisecond) + synctest.Wait() - if c.Status != "ISSUED" { - return false - } + c, err := b.DescribeCertificate(context.Background(), cert.ARN) + require.NoError(t, err) + require.Equal(t, "ISSUED", c.Status) for _, dvo := range c.DomainValidationOptions { - if dvo.ValidationStatus != "SUCCESS" { - return false - } + assert.Equal(t, "SUCCESS", dvo.ValidationStatus) } - - return true - }, 2*time.Second, 50*time.Millisecond) + }) } // TestACMBackend_StatusLifecycle verifies the full certificate status lifecycle transitions. diff --git a/services/acm/certificates.go b/services/acm/certificates.go index a8e2a172ee..86d18ebec1 100644 --- a/services/acm/certificates.go +++ b/services/acm/certificates.go @@ -8,6 +8,8 @@ import ( "strings" "time" + "github.com/google/uuid" + "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/page" ) @@ -50,7 +52,7 @@ func (b *InMemoryBackend) RequestCertificate( return existing, nil } - id := fmt.Sprintf("%x", time.Now().UnixNano()) + id := uuid.NewString() certARN := arn.Build("acm", region, b.accountID, "certificate/"+id) if certType == "" { @@ -401,7 +403,7 @@ func (b *InMemoryBackend) ImportCertificate( return &cp, nil } - id := fmt.Sprintf("%x", time.Now().UnixNano()) + id := uuid.NewString() certARN := arn.Build("acm", region, b.accountID, "certificate/"+id) cert := &Certificate{ diff --git a/services/acm/certificates_test.go b/services/acm/certificates_test.go index d28d0c0713..db13d31c82 100644 --- a/services/acm/certificates_test.go +++ b/services/acm/certificates_test.go @@ -2,8 +2,10 @@ package acm_test import ( "context" + "regexp" "strings" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -12,6 +14,80 @@ import ( "github.com/blackbirdworks/gopherstack/services/acm" ) +// uuidCertArnPattern matches an ACM certificate ARN whose id is a UUID v4, +// e.g. arn:aws:acm:us-east-1:000000000000:certificate/12345678-1234-1234-1234-123456789012. +var uuidCertArnPattern = regexp.MustCompile( + `^arn:aws:acm:[\w-]+:\d+:certificate/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`, +) + +// TestACMBackend_CertificateIDs_Unique locks in the fix for gopherstack-k1b28: +// certificate IDs were derived from time.Now().UnixNano(), so two requests in +// the same nanosecond (guaranteed under synctest's fake clock) collided and +// the second silently overwrote the first. +func TestACMBackend_CertificateIDs_Unique(t *testing.T) { + t.Parallel() + + tests := []struct { + create func(t *testing.T, b *acm.InMemoryBackend) string + name string + }{ + { + name: "request_certificate", + create: func(t *testing.T, b *acm.InMemoryBackend) string { + t.Helper() + + cert, err := b.RequestCertificate( + context.Background(), "unique.example.com", "", "", "", "", "", "", nil, + ) + require.NoError(t, err) + + return cert.ARN + }, + }, + { + name: "import_certificate", + create: func(t *testing.T, b *acm.InMemoryBackend) string { + t.Helper() + + certPEM, keyPEM := generateTestCert(t) + cert, err := b.ImportCertificate(context.Background(), certPEM, keyPEM, "", "") + require.NoError(t, err) + + return cert.ARN + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + b := acm.NewInMemoryBackend("000000000000", "us-east-1") + + // No sleep: both calls land in the same synctest instant, the + // exact scenario that used to collide. + arn1 := tt.create(t, b) + arn2 := tt.create(t, b) + + assert.NotEqual(t, arn1, arn2, "two certificates created back-to-back must get distinct ARNs") + assert.Regexp(t, uuidCertArnPattern, arn1) + assert.Regexp(t, uuidCertArnPattern, arn2) + + p, err := b.ListCertificates(context.Background(), acm.ListCertificatesParams{}) + require.NoError(t, err) + + gotARNs := make([]string, 0, len(p.Data)) + for _, c := range p.Data { + gotARNs = append(gotARNs, c.ARN) + } + + assert.ElementsMatch(t, []string{arn1, arn2}, gotARNs, "both certificates must be listed") + }) + }) + } +} + func TestACMBackend_RequestCertificate(t *testing.T) { t.Parallel() @@ -65,41 +141,44 @@ func TestACMBackend_RequestCertificate(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := acm.NewInMemoryBackend("000000000000", "us-east-1") - cert, err := b.RequestCertificate( - context.Background(), - tt.domain, - "", - tt.validationMethod, - "", - "", - "", - "", - nil, - ) + synctest.Test(t, func(t *testing.T) { + b := acm.NewInMemoryBackend("000000000000", "us-east-1") + cert, err := b.RequestCertificate( + context.Background(), + tt.domain, + "", + tt.validationMethod, + "", + "", + "", + "", + nil, + ) - if tt.wantErr != nil { - require.Error(t, err) - assert.ErrorIs(t, err, tt.wantErr) + if tt.wantErr != nil { + require.Error(t, err) + assert.ErrorIs(t, err, tt.wantErr) - return - } + return + } - require.NoError(t, err) - assert.Contains(t, cert.ARN, "arn:aws:acm:") - assert.Equal(t, tt.wantDomain, cert.DomainName) - assert.Equal(t, tt.wantStatus, cert.Status) - assert.Equal(t, tt.wantType, cert.Type) - assert.NotEmpty(t, cert.CertificateBody, "CertificateBody should be set") + require.NoError(t, err) + assert.Contains(t, cert.ARN, "arn:aws:acm:") + assert.Equal(t, tt.wantDomain, cert.DomainName) + assert.Equal(t, tt.wantStatus, cert.Status) + assert.Equal(t, tt.wantType, cert.Type) + assert.NotEmpty(t, cert.CertificateBody, "CertificateBody should be set") - if tt.wantPendingFirst { - // Wait for auto-validation - require.Eventually(t, func() bool { - c, descErr := b.DescribeCertificate(context.Background(), cert.ARN) + if tt.wantPendingFirst { + // autoValidateDelayMS is 100ms; cross it so auto-validation fires. + time.Sleep(150 * time.Millisecond) + synctest.Wait() - return descErr == nil && c.Status == "ISSUED" - }, 2*time.Second, 50*time.Millisecond, "certificate should transition to ISSUED") - } + c, descErr := b.DescribeCertificate(context.Background(), cert.ARN) + require.NoError(t, descErr) + assert.Equal(t, "ISSUED", c.Status, "certificate should transition to ISSUED") + } + }) }) } } diff --git a/services/acm/handler_certificate_lifecycle_test.go b/services/acm/handler_certificate_lifecycle_test.go index 60511c7a0e..a3a67fee04 100644 --- a/services/acm/handler_certificate_lifecycle_test.go +++ b/services/acm/handler_certificate_lifecycle_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -17,54 +18,55 @@ import ( func TestACMHandler_DNSValidationWorkflow(t *testing.T) { t.Parallel() - h := newACMHandler() + synctest.Test(t, func(t *testing.T) { + h := newACMHandler() - // Request with DNS validation - reqRec := postACMJSON(t, h, "RequestCertificate", - `{"DomainName":"workflow.example.com","ValidationMethod":"DNS"}`) - require.Equal(t, http.StatusOK, reqRec.Code) + // Request with DNS validation + reqRec := postACMJSON(t, h, "RequestCertificate", + `{"DomainName":"workflow.example.com","ValidationMethod":"DNS"}`) + require.Equal(t, http.StatusOK, reqRec.Code) - var reqOut struct { - CertificateArn string `json:"CertificateArn"` - } - require.NoError(t, json.Unmarshal(reqRec.Body.Bytes(), &reqOut)) - require.NotEmpty(t, reqOut.CertificateArn) + var reqOut struct { + CertificateArn string `json:"CertificateArn"` + } + require.NoError(t, json.Unmarshal(reqRec.Body.Bytes(), &reqOut)) + require.NotEmpty(t, reqOut.CertificateArn) - // Describe should show PENDING_VALIDATION with CNAME records - descBody, _ := json.Marshal(map[string]string{"CertificateArn": reqOut.CertificateArn}) - descRec := postACMJSON(t, h, "DescribeCertificate", string(descBody)) - require.Equal(t, http.StatusOK, descRec.Code) + // Describe should show PENDING_VALIDATION with CNAME records + descBody, _ := json.Marshal(map[string]string{"CertificateArn": reqOut.CertificateArn}) + descRec := postACMJSON(t, h, "DescribeCertificate", string(descBody)) + require.Equal(t, http.StatusOK, descRec.Code) + + var descOut struct { + Certificate struct { + Status string `json:"Status"` + DomainValidationOptions []struct { + ResourceRecord *struct { + Type string `json:"Type"` + } `json:"ResourceRecord"` + ValidationStatus string `json:"ValidationStatus"` + } `json:"DomainValidationOptions"` + } `json:"Certificate"` + } + require.NoError(t, json.Unmarshal(descRec.Body.Bytes(), &descOut)) + require.Equal(t, "PENDING_VALIDATION", descOut.Certificate.Status) + require.NotEmpty(t, descOut.Certificate.DomainValidationOptions) + assert.NotNil(t, descOut.Certificate.DomainValidationOptions[0].ResourceRecord) + assert.Equal(t, "CNAME", descOut.Certificate.DomainValidationOptions[0].ResourceRecord.Type) + + // autoValidateDelayMS is 100ms; cross it so the cert transitions to ISSUED. + time.Sleep(150 * time.Millisecond) + synctest.Wait() - var descOut struct { - Certificate struct { - Status string `json:"Status"` - DomainValidationOptions []struct { - ResourceRecord *struct { - Type string `json:"Type"` - } `json:"ResourceRecord"` - ValidationStatus string `json:"ValidationStatus"` - } `json:"DomainValidationOptions"` - } `json:"Certificate"` - } - require.NoError(t, json.Unmarshal(descRec.Body.Bytes(), &descOut)) - // Initial describe may already show ISSUED (auto-validate is quick), so accept either. - assert.Contains(t, []string{"PENDING_VALIDATION", "ISSUED"}, descOut.Certificate.Status) - require.NotEmpty(t, descOut.Certificate.DomainValidationOptions) - assert.NotNil(t, descOut.Certificate.DomainValidationOptions[0].ResourceRecord) - assert.Equal(t, "CNAME", descOut.Certificate.DomainValidationOptions[0].ResourceRecord.Type) - - // Wait for auto-transition to ISSUED - require.Eventually(t, func() bool { rec := postACMJSON(t, h, "DescribeCertificate", string(descBody)) var out struct { Certificate struct { Status string `json:"Status"` } `json:"Certificate"` } - _ = json.Unmarshal(rec.Body.Bytes(), &out) - - return out.Certificate.Status == "ISSUED" - }, 2*time.Second, 50*time.Millisecond, "cert should transition to ISSUED") + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out)) + assert.Equal(t, "ISSUED", out.Certificate.Status, "cert should transition to ISSUED") + }) } func TestACMHandler_ResendValidationEmail(t *testing.T) { diff --git a/services/acm/handler_certificate_status_errors_test.go b/services/acm/handler_certificate_status_errors_test.go index d73ef8ad62..cce0dcd514 100644 --- a/services/acm/handler_certificate_status_errors_test.go +++ b/services/acm/handler_certificate_status_errors_test.go @@ -138,12 +138,8 @@ func TestACMHandler_ExportCertificate_AmazonIssued(t *testing.T) { } } -// requestAndAwaitIssued creates an AMAZON_ISSUED certificate (optionally with -// Options.Export set) and polls DescribeCertificate until it reaches ISSUED -// (auto-validation fires after acm.autoValidateDelayMS, matching the existing -// TestACMHandler_GetCertificate_Issued_Succeeds pattern in this file), so -// tests exercising post-issuance behavior aren't racing the auto-validate -// timer. +// requestAndAwaitIssued creates an AMAZON_ISSUED certificate and returns its ARN; +// without ValidationMethod it issues synchronously. func requestAndAwaitIssued(t *testing.T, h *acm.Handler, domainName, exportOption string) string { t.Helper() @@ -163,19 +159,15 @@ func requestAndAwaitIssued(t *testing.T, h *acm.Handler, domainName, exportOptio } require.NoError(t, json.Unmarshal(reqRec.Body.Bytes(), &reqOut)) - require.Eventually(t, func() bool { - rec := postACMJSON(t, h, "DescribeCertificate", - `{"CertificateArn":"`+reqOut.CertificateArn+`"}`) + descRec := postACMJSON(t, h, "DescribeCertificate", `{"CertificateArn":"`+reqOut.CertificateArn+`"}`) - var out struct { - Certificate struct { - Status string `json:"Status"` - } `json:"Certificate"` - } - _ = json.Unmarshal(rec.Body.Bytes(), &out) - - return out.Certificate.Status == "ISSUED" - }, 2*time.Second, 20*time.Millisecond) + var out struct { + Certificate struct { + Status string `json:"Status"` + } `json:"Certificate"` + } + require.NoError(t, json.Unmarshal(descRec.Body.Bytes(), &out)) + require.Equal(t, "ISSUED", out.Certificate.Status) return reqOut.CertificateArn } @@ -509,19 +501,16 @@ func TestACMHandler_GetCertificate_Issued_Succeeds(t *testing.T) { } require.NoError(t, json.Unmarshal(reqRec.Body.Bytes(), &reqOut)) - // Wait for ISSUED status (immediate for no-validation certs) - require.Eventually(t, func() bool { - rec := postACMJSON(t, h, "DescribeCertificate", - `{"CertificateArn":"`+reqOut.CertificateArn+`"}`) - var out struct { - Certificate struct { - Status string `json:"Status"` - } `json:"Certificate"` - } - _ = json.Unmarshal(rec.Body.Bytes(), &out) - - return out.Certificate.Status == "ISSUED" - }, 2*time.Second, 20*time.Millisecond) + // No ValidationMethod is set, so the cert issues synchronously. + descRec := postACMJSON(t, h, "DescribeCertificate", + `{"CertificateArn":"`+reqOut.CertificateArn+`"}`) + var descOut struct { + Certificate struct { + Status string `json:"Status"` + } `json:"Certificate"` + } + require.NoError(t, json.Unmarshal(descRec.Body.Bytes(), &descOut)) + require.Equal(t, "ISSUED", descOut.Certificate.Status) body, _ := json.Marshal(map[string]string{"CertificateArn": reqOut.CertificateArn}) rec := postACMJSON(t, h, "GetCertificate", string(body)) diff --git a/services/acm/handler_certificates_list_test.go b/services/acm/handler_certificates_list_test.go index f97ad6d7e1..53561b2a54 100644 --- a/services/acm/handler_certificates_list_test.go +++ b/services/acm/handler_certificates_list_test.go @@ -6,6 +6,7 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -18,36 +19,41 @@ import ( func TestACMHandler_ListCertificates_StatusFilter(t *testing.T) { t.Parallel() - h := newACMHandler() - - // Create one regular (ISSUED) cert - rec1 := postACMJSON(t, h, "RequestCertificate", `{"DomainName":"issued-filter.example.com"}`) - require.Equal(t, http.StatusOK, rec1.Code) - - // Create one cert that starts in PENDING_VALIDATION - rec2 := postACMJSON(t, h, "RequestCertificate", - `{"DomainName":"pending-filter.example.com","ValidationMethod":"DNS"}`) - require.Equal(t, http.StatusOK, rec2.Code) - - // Filter for ISSUED only (immediately-issued should show; wait for pending to not match) - time.Sleep(10 * time.Millisecond) // give autoValidate timer a head start - - filterRec := postACMJSON(t, h, "ListCertificates", - `{"CertificateStatuses":["ISSUED"]}`) - require.Equal(t, http.StatusOK, filterRec.Code) - - var out struct { - CertificateSummaryList []struct { - DomainName string `json:"DomainName"` - Status string `json:"Status"` - } `json:"CertificateSummaryList"` - } - require.NoError(t, json.Unmarshal(filterRec.Body.Bytes(), &out)) - - for _, s := range out.CertificateSummaryList { - assert.Equal(t, "ISSUED", s.Status, - "filtered list should only contain ISSUED certs; got %s for %s", s.Status, s.DomainName) - } + synctest.Test(t, func(t *testing.T) { + h := newACMHandler() + + // Create one regular (ISSUED) cert + rec1 := postACMJSON(t, h, "RequestCertificate", `{"DomainName":"issued-filter.example.com"}`) + require.Equal(t, http.StatusOK, rec1.Code) + + // Create one cert that starts in PENDING_VALIDATION + rec2 := postACMJSON(t, h, "RequestCertificate", + `{"DomainName":"pending-filter.example.com","ValidationMethod":"DNS"}`) + require.Equal(t, http.StatusOK, rec2.Code) + + // autoValidateDelayMS is 100ms; stay well under it so the pending + // cert's timer cannot have fired yet. + time.Sleep(10 * time.Millisecond) + synctest.Wait() + + filterRec := postACMJSON(t, h, "ListCertificates", + `{"CertificateStatuses":["ISSUED"]}`) + require.Equal(t, http.StatusOK, filterRec.Code) + + var out struct { + CertificateSummaryList []struct { + DomainName string `json:"DomainName"` + Status string `json:"Status"` + } `json:"CertificateSummaryList"` + } + require.NoError(t, json.Unmarshal(filterRec.Body.Bytes(), &out)) + require.Len(t, out.CertificateSummaryList, 1, "only the immediately-issued cert should match") + + for _, s := range out.CertificateSummaryList { + assert.Equal(t, "ISSUED", s.Status, + "filtered list should only contain ISSUED certs; got %s for %s", s.Status, s.DomainName) + } + }) } // TestACMHandler_ListCertificates_EnrichedSummary verifies that summary includes Status and KeyAlgorithm. @@ -607,30 +613,32 @@ func TestACMHandler_SearchCertificates(t *testing.T) { run: func(t *testing.T, h *acm.Handler) { t.Helper() - postACMJSON(t, h, "RequestCertificate", `{"DomainName":"sort-first.example.com"}`) - time.Sleep(2 * time.Millisecond) - postACMJSON(t, h, "RequestCertificate", `{"DomainName":"sort-second.example.com"}`) - - body := `{"SortBy":"CREATED_AT","SortOrder":"DESCENDING"}` - rec := postACMJSON(t, h, "SearchCertificates", body) - require.Equal(t, http.StatusOK, rec.Code) - - var out struct { - Results []struct { - CertificateMetadata struct { - AcmCertificateMetadata struct { - CreatedAt int64 `json:"CreatedAt"` - } `json:"AcmCertificateMetadata"` - } `json:"CertificateMetadata"` - } `json:"Results"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out)) - require.Len(t, out.Results, 2) - assert.GreaterOrEqual(t, - out.Results[0].CertificateMetadata.AcmCertificateMetadata.CreatedAt, - out.Results[1].CertificateMetadata.AcmCertificateMetadata.CreatedAt, - "DESCENDING sort must put the newer cert first", - ) + synctest.Test(t, func(t *testing.T) { + postACMJSON(t, h, "RequestCertificate", `{"DomainName":"sort-first.example.com"}`) + time.Sleep(2 * time.Millisecond) + postACMJSON(t, h, "RequestCertificate", `{"DomainName":"sort-second.example.com"}`) + + body := `{"SortBy":"CREATED_AT","SortOrder":"DESCENDING"}` + rec := postACMJSON(t, h, "SearchCertificates", body) + require.Equal(t, http.StatusOK, rec.Code) + + var out struct { + Results []struct { + CertificateMetadata struct { + AcmCertificateMetadata struct { + CreatedAt int64 `json:"CreatedAt"` + } `json:"AcmCertificateMetadata"` + } `json:"CertificateMetadata"` + } `json:"Results"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out)) + require.Len(t, out.Results, 2) + assert.GreaterOrEqual(t, + out.Results[0].CertificateMetadata.AcmCertificateMetadata.CreatedAt, + out.Results[1].CertificateMetadata.AcmCertificateMetadata.CreatedAt, + "DESCENDING sort must put the newer cert first", + ) + }) }, }, } diff --git a/services/acm/janitor_test.go b/services/acm/janitor_test.go index 6deda53e3c..6eb4c10917 100644 --- a/services/acm/janitor_test.go +++ b/services/acm/janitor_test.go @@ -3,6 +3,7 @@ package acm_test import ( "context" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -45,24 +46,28 @@ func TestJanitor_TimesOutAbandonedPendingValidation(t *testing.T) { func TestJanitor_ExpiresPastNotAfter(t *testing.T) { t.Parallel() - b := acm.NewInMemoryBackend("000000000000", "us-east-1") + synctest.Test(t, func(t *testing.T) { + b := acm.NewInMemoryBackend("000000000000", "us-east-1") - cert, err := b.RequestCertificate(context.Background(), "expiring.example.com", "", "DNS", "", "", "", "", nil) - require.NoError(t, err) + cert, err := b.RequestCertificate(context.Background(), "expiring.example.com", "", "DNS", "", "", "", "", nil) + require.NoError(t, err) - require.Eventually(t, func() bool { - c, descErr := b.DescribeCertificate(context.Background(), cert.ARN) + // autoValidateDelayMS is 100ms; cross it so the cert auto-validates. + time.Sleep(150 * time.Millisecond) + synctest.Wait() - return descErr == nil && c.Status == "ISSUED" - }, 2*time.Second, 50*time.Millisecond, "certificate must auto-validate to ISSUED") + c, err := b.DescribeCertificate(context.Background(), cert.ARN) + require.NoError(t, err) + require.Equal(t, "ISSUED", c.Status, "certificate must auto-validate to ISSUED") - b.BackdateCertForTest("us-east-1", cert.ARN, cert.CreatedAt, time.Now().UTC().Add(-time.Hour)) + b.BackdateCertForTest("us-east-1", cert.ARN, cert.CreatedAt, time.Now().UTC().Add(-time.Hour)) - b.SweepJanitorOnceForTest() + b.SweepJanitorOnceForTest() - got, err := b.DescribeCertificate(context.Background(), cert.ARN) - require.NoError(t, err) - require.Equal(t, "EXPIRED", got.Status) + got, err := b.DescribeCertificate(context.Background(), cert.ARN) + require.NoError(t, err) + require.Equal(t, "EXPIRED", got.Status) + }) } // TestJanitor_TimeoutDoesNotSetFailureReason is a regression test: the diff --git a/services/acm/leak_test.go b/services/acm/leak_test.go index 61110cef85..f3e596113a 100644 --- a/services/acm/leak_test.go +++ b/services/acm/leak_test.go @@ -2,6 +2,7 @@ package acm_test import ( "testing" + "testing/synctest" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -105,39 +106,43 @@ func TestDeleteCertificate_TimersDoNotAccumulateAcrossCreateDelete(t *testing.T) func TestDeleteCertificate_StopsRenewalTimer(t *testing.T) { t.Parallel() - b := acm.NewInMemoryBackend("123456789012", "us-east-1") + synctest.Test(t, func(t *testing.T) { + b := acm.NewInMemoryBackend("123456789012", "us-east-1") - // Request a DNS-validated cert; it starts PENDING_VALIDATION with an auto-validate timer. - cert, err := b.RequestCertificate( - t.Context(), - "renew-leak.example.com", - "AMAZON_ISSUED", - "DNS", - "", - "", - "", - "", - nil, - ) - require.NoError(t, err) + // Request a DNS-validated cert; it starts PENDING_VALIDATION with an auto-validate timer. + cert, err := b.RequestCertificate( + t.Context(), + "renew-leak.example.com", + "AMAZON_ISSUED", + "DNS", + "", + "", + "", + "", + nil, + ) + require.NoError(t, err) - // Wait for auto-validate to fire and transition the cert to ISSUED (clears the timer). - require.Eventually(t, func() bool { - c, descErr := b.DescribeCertificate(t.Context(), cert.ARN) + // autoValidateDelayMS is 100ms; cross it so auto-validate fires and + // transitions the cert to ISSUED (clearing the timer). + time.Sleep(150 * time.Millisecond) + synctest.Wait() - return descErr == nil && c.Status == "ISSUED" - }, time.Second, 10*time.Millisecond, "cert must reach ISSUED before renewal") + c, err := b.DescribeCertificate(t.Context(), cert.ARN) + require.NoError(t, err) + require.Equal(t, "ISSUED", c.Status, "cert must reach ISSUED before renewal") - require.Equal(t, 0, b.TimerCountForTest(), "no timers expected after initial auto-validate") + require.Equal(t, 0, b.TimerCountForTest(), "no timers expected after initial auto-validate") - // Renew the cert — this schedules a new autoValidateRenewal timer. - require.NoError(t, b.RenewCertificate(t.Context(), cert.ARN)) - require.Equal(t, 1, b.TimerCountForTest(), "renewal must register one timer") + // Renew the cert — this schedules a new autoValidateRenewal timer. + require.NoError(t, b.RenewCertificate(t.Context(), cert.ARN)) + require.Equal(t, 1, b.TimerCountForTest(), "renewal must register one timer") - // Delete the cert — the renewal timer must be stopped and removed. - require.NoError(t, b.DeleteCertificate(t.Context(), cert.ARN)) - require.Equal(t, 0, b.TimerCountForTest(), - "renewal timer must be stopped and removed after DeleteCertificate") + // Delete the cert — the renewal timer must be stopped and removed. + require.NoError(t, b.DeleteCertificate(t.Context(), cert.ARN)) + require.Equal(t, 0, b.TimerCountForTest(), + "renewal timer must be stopped and removed after DeleteCertificate") + }) } // TestDeleteCertificate_StopsResendValidationEmailTimer verifies that deleting a diff --git a/services/apigateway/PARITY.md b/services/apigateway/PARITY.md index 10210a45d3..a61d46f979 100644 --- a/services/apigateway/PARITY.md +++ b/services/apigateway/PARITY.md @@ -137,6 +137,51 @@ overall: A # closed all 5 documented gaps + 3 deferred items from the # deliberately does not reject CreateDeployment for a method with no integration — # guessing a rejection rule is worse than not enforcing one (a wrong rejection breaks # working user code; a missing one only under-enforces). +# 2026-09-26 follow-up (bd: gopherstack-fum): FIXED the per-deployment snapshot gap the +# two notes above deferred as structural/out of scope ("Properly fixing this needs a +# real per-deployment snapshot plus stage-to-deployment pinning in the data plane"). +# CreateDeployment (deployment_snapshot.go) now deep-copies the API's resources (with +# their nested methods, integrations, and method/integration responses), models, +# request validators, authorizers, gateway responses, and minimumCompressionSize onto +# the new Deployment's Config field (internal-only, json:"-", not on the real +# GetDeploymentOutput wire shape -- apiSummary remains the only wire-visible summary, +# still display-only). The data plane now resolves every stage request against the +# stage's pinned deployment Config instead of live backend state: +# handleProxyRequest/routingTrie (proxy.go/proxy_routing.go) resolve resources/methods/ +# integrations from it; runRequestValidator/requestModelSchema (proxy_validation.go) +# resolve request validators and models from it; runAuthorizer (proxy_authorizer.go) +# resolves authorizers from it; minCompressSize (proxy_integrations.go) resolves +# minimumCompressionSize from it. A resource/method/integration edit made after +# CreateDeployment is now correctly invisible to an already-deployed stage until the +# next CreateDeployment (proven by TestDeploymentSnapshot_ServesCapturedConfig); a stage +# repointed to an older deployment via UpdateStage's "/deploymentId" replace correctly +# rolls back to that deployment's snapshot (same test). Stage variables remain resolved +# live from the stage (unaffected -- they're stage state, not deployment state, +# confirmed unchanged by TestDeploymentSnapshot_StageVariables). A stage with +# DeploymentID == "" (unreachable via the public API today -- CreateStage and +# CreateDeployment's inline stage creation both require a deploymentId -- but defended +# against for a restored-from-an-older-snapshot or future-regression stage) 403s with +# "Missing Authentication Token", the same response real API Gateway returns for an +# undeployed stage/API ("Why did I receive a 403 Missing Authentication Token error from +# an API Gateway API endpoint?" lists "you didn't deploy the API" as a cause) -- +# TestHandleProxyRequest_StageWithNoDeployment / TestDeploymentSnapshot_StageWithNoDeployment. +# The routing-trie cache (h.trieCache) is now keyed by deploymentID instead of RestApi +# ID -- a deployment's snapshot is immutable once created, so no version-based +# invalidation is needed, only eviction when the deployment (or its owning RestApi) is +# deleted (deleteDeploymentAction / deleteRestAPIAction), freeing the snapshot's memory; +# TestDeleteRestAPI_EvictsTrieCache updated for the new key scheme, still passing. +# Persistence: Deployment.Config is a new, additive (omitempty) field on the existing +# deploymentSnapshot DTO -- no snapshot version bump, verified via +# TestInMemoryBackend_SnapshotRestore_DeploymentConfig (a restored backend can still +# serve real stage traffic from the restored Config, and a live edit made before the +# snapshot but after CreateDeployment does not leak into it, proving the deep copy is +# real). GatewayResponses are captured in the snapshot for completeness/future-proofing +# even though no proxy code path reads them today (see gaps: DEFAULT_4XX/DEFAULT_5XX +# etc. are not yet wired into the data plane's error responses at all, unchanged by this +# pass). apigatewayv2 (HTTP APIs) already had the equivalent fix (gopherstack-cfr1, +# 2026-09-06) including autoDeploy=true handling; WebSocket APIs there remain a +# separate, disclosed gap (apigatewayv2/PARITY.md), out of this pass's scope (v1 has no +# WebSocket support at all). ops: UpdateStage: {wire: ok, errors: fixed, state: fixed, persist: ok, note: "prior sweep: PATCH semantics rewritten (/variables/{name}, canary-promotion copy op, /canarySettings/*, /accessLogSettings/*, per-route method settings, cacheCluster* fields). Prior sweep 2: documentationVersion field + PATCH added; /canarySettings/stageVariableOverrides whole-map-replace PATCH added; caching/dataEncrypted + caching/unauthorizedCacheControlHeaderStrategy per-route PATCH properties added. 2026-09-08 (gopherstack-9ard): FIXED — deploymentId was never validated against real Deployment state (unlike CreateStage's existing guard); now rejects a nonexistent deploymentId with NotFoundException. See the dated note above for detail; TestUpdateStage_RejectsNonexistentDeploymentID."} UpdateRestApi: {wire: ok, errors: ok, state: ok, persist: ok, note: "prior sweep: PATCH /binaryMediaTypes/{escaped} add/remove merge, minimumCompressionSize coercion. This sweep: ApiStatus/ApiStatusMessage/DisableExecuteApiEndpoint/EndpointAccessMode fields added (Create + Update + PATCH replace); Description switched to *string so PATCH remove on /description actually clears it (was a silent no-op) — see Notes"} @@ -180,7 +225,7 @@ ops: PutIntegrationResponse: {wire: ok, errors: ok, state: ok, persist: ok} GetIntegrationResponse: {wire: ok, errors: ok, state: ok, persist: ok} DeleteIntegrationResponse: {wire: ok, errors: ok, state: ok, persist: ok} - CreateDeployment: {wire: ok, errors: ok, state: ok, persist: ok, note: "inline stage create/update via stageName param. 2026-09-08 (gopherstack-9ard): the 'real snapshot of resources/methods/integrations at deploy time' claim previously on this line was INACCURATE — corrected, see the dated note above and gopherstack-fum's gaps entry below: apiSummary is a display-only metadata summary, NOT something the data plane routes against. Investigated whether a method with no integration should reject CreateDeployment (BadRequestException) — found no authoritative evidence (neither the pinned Go SDK module nor botocore's wire model documents this precondition; only third-party tooling claimed it), so deliberately left unenforced rather than guessed at. Deploying an API with zero resources/methods at all remains allowed, matching real AWS (TestBackend_DeploymentAndStage/create_deployment_and_stage)."} + CreateDeployment: {wire: ok, errors: ok, state: fixed, persist: fixed, note: "inline stage create/update via stageName param. 2026-09-08 (gopherstack-9ard): the 'real snapshot of resources/methods/integrations at deploy time' claim previously on this line was INACCURATE at the time — apiSummary is a display-only metadata summary, NOT something the data plane routes against, and no other snapshot existed yet. FIXED 2026-09-26 (gopherstack-fum): CreateDeployment now also captures a real Deployment.Config (deployment_snapshot.go, internal-only -- json:\"-\", not part of the real GetDeploymentOutput wire shape) deep-copying the API's resources (with their nested methods, integrations, and method/integration responses), models, request validators, authorizers, gateway responses, and minimumCompressionSize; the data plane (proxy.go/proxy_routing.go/proxy_validation.go/proxy_authorizer.go/proxy_integrations.go) now resolves every stage request against the stage's pinned deployment's Config instead of live state -- see the dated note below and Notes. apiSummary itself remains display-only, unchanged. Investigated whether a method with no integration should reject CreateDeployment (BadRequestException) — found no authoritative evidence (neither the pinned Go SDK module nor botocore's wire model documents this precondition; only third-party tooling claimed it), so deliberately left unenforced rather than guessed at. Deploying an API with zero resources/methods at all remains allowed, matching real AWS (TestBackend_DeploymentAndStage/create_deployment_and_stage)."} GetDeployment: {wire: ok, errors: ok, state: ok, persist: ok} GetDeployments: {wire: fixed, errors: ok, state: ok, persist: ok, note: "2026-08-29 wrapper-key sweep: REQUEST direction verified against apigateway@v1.42.4 serializers.go (prior grading was response-only). limit/position were never read at all -- every call returned the full unpaginated list regardless of Limit; now paginated via paginatePageByKey. Also found and fixed a service-wide bug in injectJSONFieldAPIGW: query-string limit was always JSON-quoted, so a real client's numeric Limit 500'd on json.Unmarshal into every Limit-typed handler struct (affected every list op with pagination, not just this one) -- limit is now injected as a bare JSON number."} DeleteDeployment: {wire: ok, errors: ok, state: ok, persist: ok, note: "2026-09-08 (gopherstack-9ard): audited the 'delete a deployment a stage still references' precondition — ALREADY CORRECT (rejects with BadRequestException, matching api_op_DeleteDeployment.go's doc comment), pinned by pre-existing TestDeleteDeployment_StageProtection. No change needed."} @@ -269,7 +314,6 @@ gaps: [] items_still_open: - "UpdateAuthorizer's PATCH table documents \"/authType\" (types.Authorizer.AuthType, distinct from the existing \"Type\"/authorizerType) and UpdateRestApi's documents \"/securityPolicy\" (only DomainName has SecurityPolicy today) -- both real, doc-documented PATCH paths with no backing model field anywhere in this backend. Unmodeled, not a casing or plumbing bug; not fabricated. (gopherstack-6q5h)" - "'AWS' (non-proxy) integration target: sqs path-style and sns action-style dispatch for real (gopherstack-is2a); every other target (DynamoDB, Step Functions, S3, ...) is still accepted at PutIntegration with no validation and unconditionally invoked as Lambda at request time. Fixing the rest needs per-service invoker interfaces or a real VTL + AWS query-protocol encoder -- out of a targeted pass's scope. (gopherstack-fum)" - - "CreateDeployment does not freeze a routable snapshot: the data plane always matches the RestApi's LIVE resource/method/integration state, not the state at deploy time (Deployment.ApiSummary is display-only metadata). Reproduced by deleting a resource post-deploy with no redeploy -- the already-deployed stage 403s immediately. Fixing this needs a real per-deployment snapshot plus stage-to-deployment pinning in the data plane, a substantial redesign; deliberately not attempted in a targeted pass. (gopherstack-fum, gopherstack-9ard)" deferred: - "Method.AuthorizationScopes is not modeled (not on Method, not on PutMethodInput/CreateAuthorizerInput's COGNITO_USER_POOLS flow) even though UpdateMethod's \"/authorizationScopes\" is documented add/remove-supported; UpdateMethod explicitly REJECTS this path (BadRequestException) rather than silently no-opping. Needs PutMethod/PutMethodInput plumbing too, a larger change than a PATCH-focused pass. (gopherstack-oius)" leaks: {status: fixed, note: "no new goroutines/tickers/persistent state introduced this sweep — all new code (StageKeyInput resolution, patch.go's new resolvers/stagedValue helper) is request-scoped and synchronous under the existing coarse b.mu; UpdateUsagePlan's missing defensive copy (return p instead of a copy, found while extending it for per-route throttle) was also fixed, closing a latent aliasing hole where a caller mutating the returned *UsagePlan would have corrupted backend state directly. 2026-09-04 (bd: gopherstack-fum): FIXED -- h.trieCache (the compiled per-API routing-trie cache, a sync.Map keyed by RestApi ID) was never evicted on DeleteRestApi; since IDs are fresh-random per CreateRestApi a deleted API's cached trie could never be overwritten by a later Store and stayed in process memory for the server's remaining lifetime. Fixed in handler_rest_apis.go's deleteRestAPIAction (h.trieCache.Delete after a successful backend delete); TestDeleteRestAPI_EvictsTrieCache confirmed failing pre-fix, passing post-fix."} diff --git a/services/apigateway/README.md b/services/apigateway/README.md index 03974d70bf..66441f1b84 100644 --- a/services/apigateway/README.md +++ b/services/apigateway/README.md @@ -9,7 +9,7 @@ | --- | --- | | PARITY entries audited | 123 (123 ok) | | Feature families | 3 (3 ok) | -| Known gaps | 3 | +| Known gaps | 2 | | Deferred items | 1 | | Resource leaks | fixed | @@ -17,7 +17,6 @@ - UpdateAuthorizer's PATCH table documents "/authType" (types.Authorizer.AuthType, distinct from the existing "Type"/authorizerType) and UpdateRestApi's documents "/securityPolicy" (only DomainName has SecurityPolicy today) -- both real, doc-documented PATCH paths with no backing model field anywhere in this backend. Unmodeled, not a casing or plumbing bug; not fabricated. (gopherstack-6q5h) - 'AWS' (non-proxy) integration target: sqs path-style and sns action-style dispatch for real (gopherstack-is2a); every other target (DynamoDB, Step Functions, S3, ...) is still accepted at PutIntegration with no validation and unconditionally invoked as Lambda at request time. Fixing the rest needs per-service invoker interfaces or a real VTL + AWS query-protocol encoder -- out of a targeted pass's scope. (gopherstack-fum) -- CreateDeployment does not freeze a routable snapshot: the data plane always matches the RestApi's LIVE resource/method/integration state, not the state at deploy time (Deployment.ApiSummary is display-only metadata). Reproduced by deleting a resource post-deploy with no redeploy -- the already-deployed stage 403s immediately. Fixing this needs a real per-deployment snapshot plus stage-to-deployment pinning in the data plane, a substantial redesign; deliberately not attempted in a targeted pass. (gopherstack-fum, gopherstack-9ard) ### Deferred diff --git a/services/apigateway/deployment_snapshot.go b/services/apigateway/deployment_snapshot.go new file mode 100644 index 0000000000..6a053d5fa2 --- /dev/null +++ b/services/apigateway/deployment_snapshot.go @@ -0,0 +1,174 @@ +package apigateway + +import ( + "fmt" + "maps" + "slices" +) + +// DeploymentConfig is the immutable snapshot of a REST API's invocable +// configuration captured by CreateDeployment. Real API Gateway serves a +// stage's traffic from the snapshot taken when it was deployed, not from the +// live (possibly since-edited) resources/methods/integrations -- +// api-gateway-basic-concept.html: "deploying an API...creates a snapshot of +// the API and makes it callable"; edits after that point are invisible to +// the stage until the next CreateDeployment. The data-plane proxy (proxy*.go) +// resolves every request against a stage's deployment Config instead of +// InMemoryBackend's live tables. +type DeploymentConfig struct { + Models map[string]*Model `json:"models,omitempty"` + RequestValidators map[string]*RequestValidator `json:"requestValidators,omitempty"` + Authorizers map[string]*Authorizer `json:"authorizers,omitempty"` + GatewayResponses map[string]*GatewayResponse `json:"gatewayResponses,omitempty"` + Resources []Resource `json:"resources,omitempty"` + MinimumCompressionSize int `json:"minimumCompressionSize,omitempty"` +} + +// snapshotDeploymentConfig deep-copies restAPIID's current resources (with +// their nested methods, integrations, and method/integration responses), +// models, request validators, authorizers, gateway responses, and the +// RestApi-level settings the invoke path reads (minimumCompressionSize), for +// CreateDeployment to attach to the new Deployment. A deep copy is required, +// not a slice/map copy: Resource/Method/Integration are stored as pointers +// mutated in place by PutMethod/PutIntegration/etc, so anything less would +// let a later edit leak into deployments that already captured this state. +// Caller must hold b.mu (CreateDeployment does). +func (b *InMemoryBackend) snapshotDeploymentConfig(restAPIID string) *DeploymentConfig { + live := b.resourcesByAPI.Get(restAPIID) + resources := make([]Resource, 0, len(live)) + + for _, r := range live { + resources = append(resources, deepCopyResource(r)) + } + + cfg := &DeploymentConfig{ + Resources: resources, + Models: make(map[string]*Model), + RequestValidators: make(map[string]*RequestValidator), + Authorizers: make(map[string]*Authorizer), + GatewayResponses: make(map[string]*GatewayResponse), + } + + for _, m := range b.modelsByAPI.Get(restAPIID) { + cp := *m + cfg.Models[cp.Name] = &cp + } + + for _, v := range b.requestValidatorsByAPI.Get(restAPIID) { + cp := *v + cfg.RequestValidators[cp.ID] = &cp + } + + for _, a := range b.authorizersByAPI.Get(restAPIID) { + cfg.Authorizers[a.ID] = deepCopyAuthorizer(a) + } + + for _, rt := range gatewayResponseTypes { + if gr, ok := b.gatewayResponses.Get(gatewayResponseKey(restAPIID, rt)); ok { + cfg.GatewayResponses[rt] = deepCopyGatewayResponse(gr) + } + } + + if api, ok := b.restApis.Get(restAPIID); ok { + cfg.MinimumCompressionSize = api.MinimumCompressionSize + } + + return cfg +} + +// deepCopyResource copies r and everything the invoke path reads through it: +// its per-method map (methods, their integration, and method/integration +// responses) and its CORS configuration. +func deepCopyResource(r *Resource) Resource { + cp := *r + + if r.ResourceMethods != nil { + cp.ResourceMethods = make(map[string]*Method, len(r.ResourceMethods)) + for httpMethod, m := range r.ResourceMethods { + mc := deepCopyMethod(m) + cp.ResourceMethods[httpMethod] = &mc + } + } + + if r.CorsConfiguration != nil { + corsCopy := *r.CorsConfiguration + cp.CorsConfiguration = &corsCopy + } + + return cp +} + +func deepCopyMethod(m *Method) Method { + cp := *m + cp.RequestParameters = maps.Clone(m.RequestParameters) + cp.RequestModels = maps.Clone(m.RequestModels) + + if m.MethodIntegration != nil { + ic := deepCopyIntegration(m.MethodIntegration) + cp.MethodIntegration = &ic + } + + if m.MethodResponses != nil { + cp.MethodResponses = make(map[string]*MethodResponse, len(m.MethodResponses)) + for status, resp := range m.MethodResponses { + rc := *resp + rc.ResponseModels = maps.Clone(resp.ResponseModels) + rc.ResponseParameters = maps.Clone(resp.ResponseParameters) + cp.MethodResponses[status] = &rc + } + } + + return cp +} + +func deepCopyIntegration(i *Integration) Integration { + cp := *i + cp.RequestTemplates = maps.Clone(i.RequestTemplates) + cp.RequestParameters = maps.Clone(i.RequestParameters) + cp.CacheKeyParameters = slices.Clone(i.CacheKeyParameters) + + if i.IntegrationResponses != nil { + cp.IntegrationResponses = make(map[string]*IntegrationResponse, len(i.IntegrationResponses)) + for status, resp := range i.IntegrationResponses { + rc := *resp + rc.ResponseTemplates = maps.Clone(resp.ResponseTemplates) + rc.ResponseParameters = maps.Clone(resp.ResponseParameters) + cp.IntegrationResponses[status] = &rc + } + } + + return cp +} + +func deepCopyAuthorizer(a *Authorizer) *Authorizer { + cp := *a + cp.ProviderARNs = slices.Clone(a.ProviderARNs) + + return &cp +} + +func deepCopyGatewayResponse(gr *GatewayResponse) *GatewayResponse { + cp := *gr + cp.ResponseParameters = maps.Clone(gr.ResponseParameters) + cp.ResponseTemplates = maps.Clone(gr.ResponseTemplates) + + return &cp +} + +// DeploymentConfig returns the deployment snapshot deploymentID captured, for +// the data-plane proxy to resolve a request against instead of live state. +func (b *InMemoryBackend) DeploymentConfig(restAPIID, deploymentID string) (*DeploymentConfig, error) { + b.mu.RLock("DeploymentConfig") + defer b.mu.RUnlock() + + depl, ok := b.deployments.Get(deploymentKey(restAPIID, deploymentID)) + if !ok { + return nil, fmt.Errorf("%w: deployment %s not found", ErrDeploymentNotFound, deploymentID) + } + + if depl.Config == nil { + return nil, fmt.Errorf("%w: deployment %s has no snapshot", ErrDeploymentNotFound, deploymentID) + } + + return depl.Config, nil +} diff --git a/services/apigateway/deployment_snapshot_internal_test.go b/services/apigateway/deployment_snapshot_internal_test.go new file mode 100644 index 0000000000..6c2da3a5dd --- /dev/null +++ b/services/apigateway/deployment_snapshot_internal_test.go @@ -0,0 +1,43 @@ +package apigateway + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestHandleProxyRequest_StageWithNoDeployment defends the stage.DeploymentID +// == "" branch in handleProxyRequest. Every client-reachable way to create a +// stage (CreateStage, CreateDeployment's inline stage) requires a +// deploymentId, so this constructs the state directly to prove the fallback +// still returns AWS's real 403 rather than a panic or 500 if that invariant +// is ever broken by a future change (e.g. a restored snapshot from an older, +// buggier version). +func TestHandleProxyRequest_StageWithNoDeployment(t *testing.T) { + t.Parallel() + + backend := NewInMemoryBackend() + h := NewHandler(backend) + e := echo.New() + + api, err := backend.CreateRestAPI(CreateRestAPIInput{Name: "orphan-stage-api"}) + require.NoError(t, err) + + backend.stages.Put(&Stage{ + RestAPIID: api.ID, + StageName: "orphan", + Variables: map[string]string{}, + }) + + req := httptest.NewRequest(http.MethodGet, "/restapis/"+api.ID+"/orphan/_user_request_/", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + + assert.Equal(t, http.StatusForbidden, rec.Code) + assert.Contains(t, rec.Body.String(), "Missing Authentication Token") +} diff --git a/services/apigateway/deployment_snapshot_test.go b/services/apigateway/deployment_snapshot_test.go new file mode 100644 index 0000000000..9a22566ffc --- /dev/null +++ b/services/apigateway/deployment_snapshot_test.go @@ -0,0 +1,304 @@ +package apigateway_test + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awscfg "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + apigwsdk "github.com/aws/aws-sdk-go-v2/service/apigateway" + apigwtypes "github.com/aws/aws-sdk-go-v2/service/apigateway/types" + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/apigateway" +) + +// newDeploymentInvokeServer stands up the real aws-sdk-go-v2 apigateway client +// (for control-plane calls) alongside the raw invoke base URL (for data-plane +// requests against a deployed stage -- there is no typed SDK for invoking an +// arbitrary deployed REST API, only plain HTTP). +func newDeploymentInvokeServer(t *testing.T) (*apigwsdk.Client, string) { + t.Helper() + + h := apigateway.NewHandler(apigateway.NewInMemoryBackend()) + e := echo.New() + registry := service.NewRegistry() + require.NoError(t, registry.Register(h)) + e.Use(service.NewServiceRouter(registry).RouteHandler()) + + srv := httptest.NewServer(e) + t.Cleanup(srv.Close) + + cfg, err := awscfg.LoadDefaultConfig( + t.Context(), + awscfg.WithRegion("us-east-1"), + awscfg.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", "")), + ) + require.NoError(t, err) + + client := apigwsdk.NewFromConfig(cfg, func(o *apigwsdk.Options) { + o.BaseEndpoint = aws.String(srv.URL) + }) + + return client, srv.URL +} + +// invokeStage issues a real HTTP GET against the "prod" stage's invoke URL +// and returns its status code and body. +func invokeStage(t *testing.T, baseURL, apiID, path string) (int, string) { + t.Helper() + + req, err := http.NewRequestWithContext( + t.Context(), http.MethodGet, baseURL+"/restapis/"+apiID+"/prod/_user_request_"+path, nil, + ) + require.NoError(t, err) + + resp, err := http.DefaultClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + return resp.StatusCode, string(body) +} + +// putMockResponse (re)configures resourceID's GET MOCK integration to return +// body verbatim, so tests can tell "which deployment served this request" +// apart by the body they got back. +func putMockResponse(t *testing.T, client *apigwsdk.Client, apiID, resourceID, body string) { + t.Helper() + ctx := t.Context() + + _, err := client.PutIntegration(ctx, &apigwsdk.PutIntegrationInput{ + RestApiId: aws.String(apiID), + ResourceId: aws.String(resourceID), + HttpMethod: aws.String(http.MethodGet), + Type: apigwtypes.IntegrationTypeMock, + RequestTemplates: map[string]string{ + "application/json": `{"statusCode": 200}`, + }, + }) + require.NoError(t, err) + + _, err = client.PutIntegrationResponse(ctx, &apigwsdk.PutIntegrationResponseInput{ + RestApiId: aws.String(apiID), + ResourceId: aws.String(resourceID), + HttpMethod: aws.String(http.MethodGet), + StatusCode: aws.String("200"), + ResponseTemplates: map[string]string{ + "application/json": body, + }, + }) + require.NoError(t, err) +} + +// setupMockAPI creates a REST API with a single GET /widgets resource wired +// to a MOCK integration returning initialBody, returning the API and +// resource IDs. +func setupMockAPI(t *testing.T, client *apigwsdk.Client, initialBody string) (string, string) { + t.Helper() + ctx := t.Context() + + api, err := client.CreateRestApi(ctx, &apigwsdk.CreateRestApiInput{Name: aws.String("deploy-snapshot-api")}) + require.NoError(t, err) + apiID := aws.ToString(api.Id) + + resources, err := client.GetResources(ctx, &apigwsdk.GetResourcesInput{RestApiId: api.Id}) + require.NoError(t, err) + rootID := resources.Items[0].Id + + resource, err := client.CreateResource(ctx, &apigwsdk.CreateResourceInput{ + RestApiId: api.Id, + ParentId: rootID, + PathPart: aws.String("widgets"), + }) + require.NoError(t, err) + resourceID := aws.ToString(resource.Id) + + _, err = client.PutMethod(ctx, &apigwsdk.PutMethodInput{ + RestApiId: api.Id, + ResourceId: resource.Id, + HttpMethod: aws.String(http.MethodGet), + AuthorizationType: aws.String("NONE"), + }) + require.NoError(t, err) + + putMockResponse(t, client, apiID, resourceID, initialBody) + + return apiID, resourceID +} + +// TestDeploymentSnapshot_ServesCapturedConfig covers the core gap: the invoke +// path must read the resource/method/integration snapshot CreateDeployment +// captured, not the live configuration, and must pick up a new snapshot only +// after a fresh CreateDeployment. +func TestDeploymentSnapshot_ServesCapturedConfig(t *testing.T) { + t.Parallel() + + client, baseURL := newDeploymentInvokeServer(t) + ctx := t.Context() + + apiID, resourceID := setupMockAPI(t, client, "v1") + + depl1, err := client.CreateDeployment(ctx, &apigwsdk.CreateDeploymentInput{ + RestApiId: aws.String(apiID), + StageName: aws.String("prod"), + }) + require.NoError(t, err) + + status, body := invokeStage(t, baseURL, apiID, "/widgets") + require.Equal(t, http.StatusOK, status) + assert.Equal(t, "v1", body) + + // Change the integration response live, without redeploying: the stage + // must keep serving the deployed snapshot's old behaviour. + putMockResponse(t, client, apiID, resourceID, "v2") + + status, body = invokeStage(t, baseURL, apiID, "/widgets") + require.Equal(t, http.StatusOK, status) + assert.Equal(t, "v1", body, "live edits must not be visible until redeployed") + + // Redeploy: the new behaviour is now served. + depl2, err := client.CreateDeployment(ctx, &apigwsdk.CreateDeploymentInput{ + RestApiId: aws.String(apiID), + StageName: aws.String("prod"), + }) + require.NoError(t, err) + + status, body = invokeStage(t, baseURL, apiID, "/widgets") + require.Equal(t, http.StatusOK, status) + assert.Equal(t, "v2", body, "redeploying must pick up the live edit") + + // Roll back to the first deployment via UpdateStage: old behaviour again. + _, err = client.UpdateStage(ctx, &apigwsdk.UpdateStageInput{ + RestApiId: aws.String(apiID), + StageName: aws.String("prod"), + PatchOperations: []apigwtypes.PatchOperation{ + {Op: apigwtypes.OpReplace, Path: aws.String("/deploymentId"), Value: depl1.Id}, + }, + }) + require.NoError(t, err) + + status, body = invokeStage(t, baseURL, apiID, "/widgets") + require.Equal(t, http.StatusOK, status) + assert.Equal(t, "v1", body, "UpdateStage deploymentId must roll back to the old snapshot") + + // Rolling forward again reaches the second deployment's snapshot. + _, err = client.UpdateStage(ctx, &apigwsdk.UpdateStageInput{ + RestApiId: aws.String(apiID), + StageName: aws.String("prod"), + PatchOperations: []apigwtypes.PatchOperation{ + {Op: apigwtypes.OpReplace, Path: aws.String("/deploymentId"), Value: depl2.Id}, + }, + }) + require.NoError(t, err) + + status, body = invokeStage(t, baseURL, apiID, "/widgets") + require.Equal(t, http.StatusOK, status) + assert.Equal(t, "v2", body) +} + +// TestDeploymentSnapshot_StageWithNoDeployment covers CreateStage without a +// deployment, which the real CreateStage API also always requires -- there is +// no client-reachable way to leave deploymentId off, so this exercises the +// same 403 documented for an undeployed API/stage +// ("Why did I receive a 403 Missing Authentication Token error from an API +// Gateway API endpoint?" lists "you didn't deploy the API" as a cause) by +// asserting a never-deployed stage name simply isn't invocable. +func TestDeploymentSnapshot_StageWithNoDeployment(t *testing.T) { + t.Parallel() + + client, baseURL := newDeploymentInvokeServer(t) + + apiID, _ := setupMockAPI(t, client, "v1") + + status, _ := invokeStage(t, baseURL, apiID, "/widgets") + assert.Equal(t, http.StatusForbidden, status) +} + +// TestDeploymentSnapshot_StageVariables covers stage-variable interpolation +// in an integration URI, which is resolved from the stage's live Variables +// (not the deployment snapshot -- stage variables are stage state, not part +// of what CreateDeployment captures) at invoke time. +func TestDeploymentSnapshot_StageVariables(t *testing.T) { + t.Parallel() + + client, baseURL := newDeploymentInvokeServer(t) + ctx := t.Context() + + upstreamA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("upstream-a")) + })) + t.Cleanup(upstreamA.Close) + + upstreamB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("upstream-b")) + })) + t.Cleanup(upstreamB.Close) + + api, err := client.CreateRestApi(ctx, &apigwsdk.CreateRestApiInput{Name: aws.String("stagevar-api")}) + require.NoError(t, err) + apiID := aws.ToString(api.Id) + + resources, err := client.GetResources(ctx, &apigwsdk.GetResourcesInput{RestApiId: api.Id}) + require.NoError(t, err) + rootID := resources.Items[0].Id + + resource, err := client.CreateResource(ctx, &apigwsdk.CreateResourceInput{ + RestApiId: api.Id, + ParentId: rootID, + PathPart: aws.String("proxy"), + }) + require.NoError(t, err) + + _, err = client.PutMethod(ctx, &apigwsdk.PutMethodInput{ + RestApiId: api.Id, + ResourceId: resource.Id, + HttpMethod: aws.String(http.MethodGet), + AuthorizationType: aws.String("NONE"), + }) + require.NoError(t, err) + + _, err = client.PutIntegration(ctx, &apigwsdk.PutIntegrationInput{ + RestApiId: api.Id, + ResourceId: resource.Id, + HttpMethod: aws.String(http.MethodGet), + Type: apigwtypes.IntegrationTypeHttpProxy, + IntegrationHttpMethod: aws.String(http.MethodGet), + Uri: aws.String("${stageVariables.upstream}"), + }) + require.NoError(t, err) + + _, err = client.CreateDeployment(ctx, &apigwsdk.CreateDeploymentInput{ + RestApiId: aws.String(apiID), + StageName: aws.String("prod"), + Variables: map[string]string{"upstream": upstreamA.URL}, + }) + require.NoError(t, err) + + status, body := invokeStage(t, baseURL, apiID, "/proxy") + require.Equal(t, http.StatusOK, status) + assert.Equal(t, "upstream-a", body) + + // Changing the stage variable takes effect immediately -- no redeploy + // needed, since stage variables are stage state, not deployment state. + _, err = client.UpdateStage(ctx, &apigwsdk.UpdateStageInput{ + RestApiId: aws.String(apiID), + StageName: aws.String("prod"), + PatchOperations: []apigwtypes.PatchOperation{ + {Op: apigwtypes.OpReplace, Path: aws.String("/variables/upstream"), Value: aws.String(upstreamB.URL)}, + }, + }) + require.NoError(t, err) + + status, body = invokeStage(t, baseURL, apiID, "/proxy") + require.Equal(t, http.StatusOK, status) + assert.Equal(t, "upstream-b", body) +} diff --git a/services/apigateway/deployments.go b/services/apigateway/deployments.go index b2dca135fc..481fc8ae08 100644 --- a/services/apigateway/deployments.go +++ b/services/apigateway/deployments.go @@ -25,6 +25,7 @@ func (b *InMemoryBackend) CreateDeployment(restAPIID, stageName, description str Description: description, CreatedDate: now, APISummary: b.apiSummary(restAPIID), + Config: b.snapshotDeploymentConfig(restAPIID), } b.deployments.Put(depl) diff --git a/services/apigateway/gateway_responses.go b/services/apigateway/gateway_responses.go index 7344d36a18..0392caf6e4 100644 --- a/services/apigateway/gateway_responses.go +++ b/services/apigateway/gateway_responses.go @@ -55,6 +55,19 @@ func gatewayResponseDefaultStatus(responseType string) string { } } +// gatewayResponseTypes lists every AWS gateway response type, used both to +// list a REST API's responses (defaulting the ones never PUT) and to build a +// deployment's gateway-response snapshot (see deployment_snapshot.go). +// +//nolint:gochecknoglobals // fixed lookup table, mirrors dirtyTableNames elsewhere +var gatewayResponseTypes = []string{ + "UNAUTHORIZED", "ACCESS_DENIED", "RESOURCE_NOT_FOUND", + "THROTTLED", "QUOTA_EXCEEDED", "BAD_REQUEST_BODY", + "BAD_REQUEST_PARAMETERS", "REQUEST_TOO_LARGE", + "AUTHORIZER_FAILURE", "AUTHORIZER_CONFIGURATION_ERROR", + "DEFAULT_4XX", "DEFAULT_5XX", +} + // GetGatewayResponses retrieves all gateway responses for a REST API. func (b *InMemoryBackend) GetGatewayResponses(restAPIID string) ([]GatewayResponse, error) { b.mu.RLock("GetGatewayResponses") @@ -64,17 +77,9 @@ func (b *InMemoryBackend) GetGatewayResponses(restAPIID string) ([]GatewayRespon return nil, fmt.Errorf("%w: REST API %s not found", ErrRestAPINotFound, restAPIID) } - defaultTypes := []string{ - "UNAUTHORIZED", "ACCESS_DENIED", "RESOURCE_NOT_FOUND", - "THROTTLED", "QUOTA_EXCEEDED", "BAD_REQUEST_BODY", - "BAD_REQUEST_PARAMETERS", "REQUEST_TOO_LARGE", - "AUTHORIZER_FAILURE", "AUTHORIZER_CONFIGURATION_ERROR", - "DEFAULT_4XX", "DEFAULT_5XX", - } - - result := make([]GatewayResponse, 0, len(defaultTypes)) + result := make([]GatewayResponse, 0, len(gatewayResponseTypes)) - for _, rt := range defaultTypes { + for _, rt := range gatewayResponseTypes { key := gatewayResponseKey(restAPIID, rt) if gr, ok := b.gatewayResponses.Get(key); ok { result = append(result, *gr) diff --git a/services/apigateway/handler.go b/services/apigateway/handler.go index 52312800ef..1ba364f29c 100644 --- a/services/apigateway/handler.go +++ b/services/apigateway/handler.go @@ -57,8 +57,10 @@ type Handler struct { // dispatchCache is the op→handler table, built exactly once (see dispatchOnce) // instead of per request. dispatchCache map[string]actionFn - // trieCache holds the per-API routing trie (map[apiID]*trieCacheEntry). It is - // rebuilt only when the API's resource-set version changes. + // trieCache holds the per-deployment routing trie (map[deploymentID]*resourcePathTrie). + // A deployment's snapshot is immutable once created, so entries are built once and + // evicted only when their deployment is deleted (see deleteDeploymentAction / + // deleteRestAPIAction). trieCache sync.Map // dispatchOnce guards the one-time build of dispatchCache. dispatchOnce sync.Once diff --git a/services/apigateway/handler_deployments.go b/services/apigateway/handler_deployments.go index 0011a27841..b76fc98b35 100644 --- a/services/apigateway/handler_deployments.go +++ b/services/apigateway/handler_deployments.go @@ -133,6 +133,11 @@ func (h *Handler) deleteDeploymentAction(b []byte) (int, any, error) { return 0, nil, err } + // Free the deployment's cached routing trie along with its snapshot -- + // otherwise a long-lived API that churns through many deployments leaks + // one trie per deleted deployment for the life of the process. + h.trieCache.Delete(input.DeploymentID) + return http.StatusNoContent, map[string]any{}, nil } diff --git a/services/apigateway/handler_rest_apis.go b/services/apigateway/handler_rest_apis.go index 8decbb5d0e..74e4c24e3f 100644 --- a/services/apigateway/handler_rest_apis.go +++ b/services/apigateway/handler_rest_apis.go @@ -53,14 +53,19 @@ func (h *Handler) deleteRestAPIAction(b []byte) (int, any, error) { if err := json.Unmarshal(b, &input); err != nil { return 0, nil, err } + // Deployment IDs are freshly random and never reused, so the trie cache + // (now keyed by deploymentID -- see routingTrie's doc) never overwrites a + // stale entry on its own; evict every deployment this API owned before + // they're gone from the backend and can no longer be listed. + depls, _ := h.Backend.GetDeployments(input.RestAPIID) + if err := h.Backend.DeleteRestAPI(input.RestAPIID); err != nil { return 0, nil, err } - // Evict the cached routing trie -- otherwise every RestApi ID ever routed to - // stays in h.trieCache forever, since fresh random IDs never reuse a deleted - // entry's key for the cache to overwrite. - h.trieCache.Delete(input.RestAPIID) + for _, d := range depls { + h.trieCache.Delete(d.ID) + } return http.StatusAccepted, map[string]any{}, nil } diff --git a/services/apigateway/handler_router_test.go b/services/apigateway/handler_router_test.go index c63224b533..41020cd1cf 100644 --- a/services/apigateway/handler_router_test.go +++ b/services/apigateway/handler_router_test.go @@ -683,6 +683,10 @@ func (n *noopBackend) GetDeployments(_ string) ([]apigateway.Deployment, error) return nil, nil } +func (n *noopBackend) DeploymentConfig(_ string, _ string) (*apigateway.DeploymentConfig, error) { + return nil, errNoopNotImplemented +} + func (n *noopBackend) DeleteDeployment(_ string, _ string) error { return nil } func (n *noopBackend) GetStages(_ string) ([]apigateway.Stage, error) { return nil, nil } diff --git a/services/apigateway/models.go b/services/apigateway/models.go index ea275cdc62..7140083b50 100644 --- a/services/apigateway/models.go +++ b/services/apigateway/models.go @@ -336,11 +336,15 @@ type Deployment struct { // APISummary is a snapshot, taken at deployment time, of every resource // path's methods (types.Deployment.ApiSummary in the SDK), keyed // resourcePath -> httpMethod. - APISummary map[string]map[string]MethodSnapshot `json:"apiSummary,omitempty"` - CreatedDate unixEpochTime `json:"createdDate"` - ID string `json:"id"` - RestAPIID string `json:"-"` - Description string `json:"description,omitempty"` + APISummary map[string]map[string]MethodSnapshot `json:"apiSummary,omitempty"` + // Config is the full invoke-time configuration snapshot (see + // DeploymentConfig's doc) -- internal state, never part of the wire + // response, matching the RestAPIID json:"-" convention below. + Config *DeploymentConfig `json:"-"` + CreatedDate unixEpochTime `json:"createdDate"` + ID string `json:"id"` + RestAPIID string `json:"-"` + Description string `json:"description,omitempty"` } // PutMethodInput is the input for PutMethod. diff --git a/services/apigateway/persistence.go b/services/apigateway/persistence.go index a5a323a8ed..afc98cd87f 100644 --- a/services/apigateway/persistence.go +++ b/services/apigateway/persistence.go @@ -75,11 +75,14 @@ func fromResourceSnapshot(v *resourceSnapshot) *Resource { } type deploymentSnapshot struct { - APISummary map[string]map[string]MethodSnapshot `json:"apiSummary,omitempty"` - CreatedDate unixEpochTime `json:"createdDate"` - ID string `json:"id"` - RestAPIID string `json:"restApiId"` - Description string `json:"description,omitempty"` + APISummary map[string]map[string]MethodSnapshot `json:"apiSummary,omitempty"` + // Config is additive: an older snapshot without it simply restores a + // deployment with a nil Config, same as any pre-this-feature deployment. + Config *DeploymentConfig `json:"config,omitempty"` + CreatedDate unixEpochTime `json:"createdDate"` + ID string `json:"id"` + RestAPIID string `json:"restApiId"` + Description string `json:"description,omitempty"` } func deploymentSnapshotKey(v *deploymentSnapshot) string { return deploymentKey(v.RestAPIID, v.ID) } @@ -91,6 +94,7 @@ func toDeploymentSnapshot(v *Deployment) *deploymentSnapshot { Description: v.Description, CreatedDate: v.CreatedDate, APISummary: v.APISummary, + Config: v.Config, } } @@ -101,6 +105,7 @@ func fromDeploymentSnapshot(v *deploymentSnapshot) *Deployment { Description: v.Description, CreatedDate: v.CreatedDate, APISummary: v.APISummary, + Config: v.Config, } } diff --git a/services/apigateway/persistence_test.go b/services/apigateway/persistence_test.go index da5c114d74..8827b5848f 100644 --- a/services/apigateway/persistence_test.go +++ b/services/apigateway/persistence_test.go @@ -256,6 +256,69 @@ func TestInMemoryBackend_SnapshotRestore_FullState(t *testing.T) { assert.Equal(t, vpcLink.Name, gotVpcLink.Name) } +// TestInMemoryBackend_SnapshotRestore_DeploymentConfig proves a deployment's +// invoke-time configuration snapshot (DeploymentConfig) survives a +// Snapshot/Restore round trip and a restored backend can still serve real +// stage traffic from it -- not just that the field decodes. +func TestInMemoryBackend_SnapshotRestore_DeploymentConfig(t *testing.T) { + t.Parallel() + + b := apigateway.NewInMemoryBackend() + + api, err := b.CreateRestAPI(apigateway.CreateRestAPIInput{Name: "snapshot-config-api"}) + require.NoError(t, err) + + resource, err := b.CreateResource(api.ID, api.RootResourceID, "widgets") + require.NoError(t, err) + + _, err = b.PutMethod(apigateway.PutMethodInput{ + RestAPIID: api.ID, ResourceID: resource.ID, HTTPMethod: http.MethodGet, AuthorizationType: "NONE", + }) + require.NoError(t, err) + + _, err = b.PutIntegration(api.ID, resource.ID, http.MethodGet, apigateway.PutIntegrationInput{Type: "MOCK"}) + require.NoError(t, err) + + _, err = b.PutIntegrationResponse( + api.ID, resource.ID, http.MethodGet, "200", + apigateway.PutIntegrationResponseInput{ + ResponseTemplates: map[string]string{"application/json": "snapshotted"}, + }, + ) + require.NoError(t, err) + + depl, err := b.CreateDeployment(api.ID, "prod", "") + require.NoError(t, err) + + // Editing the integration live after the snapshot must not leak into it. + _, err = b.PutIntegrationResponse( + api.ID, resource.ID, http.MethodGet, "200", + apigateway.PutIntegrationResponseInput{ + ResponseTemplates: map[string]string{"application/json": "live-edit"}, + }, + ) + require.NoError(t, err) + + snap := b.Snapshot(t.Context()) + require.NotNil(t, snap) + + fresh := apigateway.NewInMemoryBackend() + require.NoError(t, fresh.Restore(t.Context(), snap)) + + cfg, err := fresh.DeploymentConfig(api.ID, depl.ID) + require.NoError(t, err) + require.Len(t, cfg.Resources, 2) // root + widgets + + h := apigateway.NewHandler(fresh) + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/restapis/"+api.ID+"/prod/_user_request_/widgets", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + assert.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "snapshotted", rec.Body.String()) +} + func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { t.Parallel() diff --git a/services/apigateway/proxy.go b/services/apigateway/proxy.go index 65422fc273..e40de2d030 100644 --- a/services/apigateway/proxy.go +++ b/services/apigateway/proxy.go @@ -179,21 +179,40 @@ func (h *Handler) handleProxyRequest(apiID, stageName string) http.HandlerFunc { // 403 "Missing Authentication Token", not 404. Gate on that here so an API with // resources/methods/integrations configured but never deployed (or invoked with a // made-up stage name) cannot be routed to. - if _, err := h.Backend.GetStage(apiID, stageName); err != nil { + stage, err := h.Backend.GetStage(apiID, stageName) + if err != nil { + writeMissingAuthenticationTokenResponse(w) + + return + } + + // A stage with no deployment is exactly as uninvocable as an + // undeployed API: real API Gateway returns the same 403 "Missing + // Authentication Token" ("Why did I receive a 403 Missing + // Authentication Token error from an API Gateway API endpoint?" lists + // "you didn't deploy the API" alongside a bad resource path/method). + if stage.DeploymentID == "" { writeMissingAuthenticationTokenResponse(w) return } - // Resolve the routing trie (cached per resource-set version) and match. - trie, err := h.routingTrie(apiID) + // Resolve the stage's deployment snapshot: real API Gateway serves the + // resources/methods/integrations captured at CreateDeployment time, not + // whatever has been edited live since (api-gateway-basic-concept.html). + cfg, err := h.Backend.DeploymentConfig(apiID, stage.DeploymentID) if err != nil { - logger.Load(ctx).ErrorContext(ctx, "APIGateway proxy: failed to get resources", "error", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Load(ctx).ErrorContext(ctx, "APIGateway proxy: deployment snapshot missing", + "apiId", apiID, "deploymentId", stage.DeploymentID, "error", err) + writeMissingAuthenticationTokenResponse(w) return } + // Resolve the routing trie (cached per deployment, which is immutable + // once created) and match. + trie := h.routingTrie(stage.DeploymentID, cfg) + // Match request path to resource path, extracting any path parameters. resource, pathParams := matchResourceTrie(trie, r.URL.Path, stageName) if resource == nil { @@ -216,46 +235,50 @@ func (h *Handler) handleProxyRequest(apiID, stageName string) http.HandlerFunc { h.addCORSHeaders(w, r, resource.CorsConfiguration) } + method := deployedResourceMethod(resource, r.Method) + // Apply method-level access controls (throttle, authorizer, request validator). - denied := h.applyMethodControls( - ctx, w, r, apiID, stageName, resource.ID, resource.Path, pathParams, - ) + denied := h.applyMethodControls(ctx, w, r, apiID, stageName, cfg, method, resource.Path, pathParams) if denied { return } - // Get the integration. - integration, err := h.Backend.GetIntegration(apiID, resource.ID, r.Method) - if err != nil { - // Fall back to any method. - integration, err = h.Backend.GetIntegration(apiID, resource.ID, "ANY") - if err != nil { - writeMissingAuthenticationTokenResponse(w) + if method == nil || method.MethodIntegration == nil { + writeMissingAuthenticationTokenResponse(w) - return - } + return } - h.dispatchIntegration(ctx, w, r, apiID, stageName, resource, integration, pathParams) + h.dispatchIntegration(ctx, w, r, apiID, stageName, resource, method.MethodIntegration, pathParams) + } +} + +// deployedResourceMethod returns resource's method for httpMethod from the +// deployment snapshot, falling back to the catch-all ANY method AWS allows, +// or nil when neither is configured. +func deployedResourceMethod(resource *Resource, httpMethod string) *Method { + if m, ok := resource.ResourceMethods[httpMethod]; ok { + return m } + + return resource.ResourceMethods["ANY"] } // applyMethodControls runs the throttle, authorizer, and request validator checks for the -// matched method. Returns true if the request was denied and the response has already been -// written. +// matched method (nil when the resource has no method for this request's HTTP verb, in +// which case there is nothing to enforce). Returns true if the request was denied and the +// response has already been written. func (h *Handler) applyMethodControls( ctx context.Context, w http.ResponseWriter, r *http.Request, - apiID, stageName, resourceID, resourcePath string, + apiID, stageName string, + cfg *DeploymentConfig, + method *Method, + resourcePath string, pathParams map[string]string, ) bool { - method, methodErr := h.Backend.GetMethod(apiID, resourceID, r.Method) - if methodErr != nil { - method, methodErr = h.Backend.GetMethod(apiID, resourceID, "ANY") - } - - if methodErr != nil || method == nil { + if method == nil { return false } @@ -273,13 +296,13 @@ func (h *Handler) applyMethodControls( } if method.AuthorizerID != "" { - if h.runAuthorizer(ctx, w, r, apiID, stageName, method.AuthorizerID) { + if h.runAuthorizer(ctx, w, r, apiID, stageName, cfg, method.AuthorizerID) { return true } } if method.RequestValidatorID != "" { - if h.runRequestValidator(ctx, w, r, apiID, method, pathParams) { + if h.runRequestValidator(ctx, w, r, cfg, method, pathParams) { return true } } diff --git a/services/apigateway/proxy_authorizer.go b/services/apigateway/proxy_authorizer.go index a18be30dd7..69545d043d 100644 --- a/services/apigateway/proxy_authorizer.go +++ b/services/apigateway/proxy_authorizer.go @@ -156,10 +156,12 @@ func (h *Handler) runAuthorizer( ctx context.Context, w http.ResponseWriter, r *http.Request, - apiID, stageName, authorizerID string, + apiID, stageName string, + cfg *DeploymentConfig, + authorizerID string, ) bool { - auth, err := h.Backend.GetAuthorizer(apiID, authorizerID) - if err != nil { + auth, ok := cfg.Authorizers[authorizerID] + if !ok { logger.Load(ctx).WarnContext(ctx, "APIGateway proxy: authorizer not found", "authorizerId", authorizerID) http.Error(w, "Authorizer configuration error", http.StatusInternalServerError) diff --git a/services/apigateway/proxy_integrations.go b/services/apigateway/proxy_integrations.go index de878eb7b0..d7ad387201 100644 --- a/services/apigateway/proxy_integrations.go +++ b/services/apigateway/proxy_integrations.go @@ -59,7 +59,7 @@ func (h *Handler) handleAWSProxy( var lambdaResp LambdaProxyResponse if parseErr := json.Unmarshal(respBytes, &lambdaResp); parseErr != nil { w.WriteHeader(http.StatusOK) - _, _ = w.Write(respBytes) //nolint:gosec // local emulation: response passthrough is intentional + _, _ = w.Write(respBytes) return } @@ -86,19 +86,26 @@ func (h *Handler) handleAWSProxy( bodyBytes = []byte(lambdaResp.Body) } - bodyBytes = maybeCompressResponse(w, r, bodyBytes, h.minCompressSize(apiID)) + bodyBytes = maybeCompressResponse(w, r, bodyBytes, h.minCompressSize(apiID, stageName)) w.WriteHeader(statusCode) _, _ = w.Write(bodyBytes) } -// minCompressSize returns the MinimumCompressionSize for the given API (0 = disabled). -func (h *Handler) minCompressSize(apiID string) int { - api, err := h.Backend.GetRestAPI(apiID) - if err != nil || api == nil { +// minCompressSize returns the deployed MinimumCompressionSize for the stage (0 = +// disabled) -- like resources/methods/integrations, this RestApi-level setting is +// only read as of the stage's deployment snapshot, not live. +func (h *Handler) minCompressSize(apiID, stageName string) int { + stage, err := h.Backend.GetStage(apiID, stageName) + if err != nil || stage.DeploymentID == "" { return 0 } - return api.MinimumCompressionSize + cfg, err := h.Backend.DeploymentConfig(apiID, stage.DeploymentID) + if err != nil { + return 0 + } + + return cfg.MinimumCompressionSize } // handleAWSIntegration handles an AWS (non-proxy) integration using VTL templates. @@ -156,7 +163,7 @@ func (h *Handler) handleAWSIntegration( // Apply response mapping template using status-code pattern matching. responseBody, statusCode := h.applyResponseTemplate(respBytes, integration, vtlCtx.RequestID) - responseBody = maybeCompressResponse(w, r, responseBody, h.minCompressSize(apiID)) + responseBody = maybeCompressResponse(w, r, responseBody, h.minCompressSize(apiID, stageName)) w.Header().Set("Content-Type", contentTypeJSON) w.Header().Set("X-Content-Type-Options", "nosniff") w.WriteHeader(statusCode) @@ -260,11 +267,11 @@ func (h *Handler) handleAWSServiceIntegration( } responseBody, statusCode := h.applyResponseTemplate([]byte("{}"), integration, vtlCtx.RequestID) - responseBody = maybeCompressResponse(w, r, responseBody, h.minCompressSize(apiID)) + responseBody = maybeCompressResponse(w, r, responseBody, h.minCompressSize(apiID, stageName)) w.Header().Set("Content-Type", contentTypeJSON) w.Header().Set("X-Content-Type-Options", "nosniff") w.WriteHeader(statusCode) - _, _ = w.Write(responseBody) //nolint:gosec // local emulation: response passthrough is intentional + _, _ = w.Write(responseBody) } // sqsQueuePathSegments is the expected segment count of the path-style sqs @@ -567,7 +574,6 @@ func (h *Handler) handleHTTPProxy( r *http.Request, integration *Integration, ) { - //nolint:gosec // local emulation: integration URI is test-configured targetReq, err := http.NewRequestWithContext( ctx, r.Method, @@ -641,7 +647,7 @@ func (h *Handler) handleMockIntegration(w http.ResponseWriter, integration *Inte } w.WriteHeader(statusCode) - _, _ = w.Write([]byte(body)) //nolint:gosec // local emulation: mock integration body is test-configured + _, _ = w.Write([]byte(body)) } // mockResponseWithIR resolves the status code, body, and integration response for a MOCK integration. diff --git a/services/apigateway/proxy_internal_test.go b/services/apigateway/proxy_internal_test.go index 5482b7e460..e82c26e5bf 100644 --- a/services/apigateway/proxy_internal_test.go +++ b/services/apigateway/proxy_internal_test.go @@ -372,18 +372,23 @@ func TestDeleteRestAPI_EvictsTrieCache(t *testing.T) { api, err := backend.CreateRestAPI(CreateRestAPIInput{Name: "leak-api"}) require.NoError(t, err) - // Prime the trie cache the same way a proxied request would. - _, err = h.routingTrie(api.ID) + depl, err := backend.CreateDeployment(api.ID, "prod", "") + require.NoError(t, err) + + cfg, err := backend.DeploymentConfig(api.ID, depl.ID) require.NoError(t, err) - _, cached := h.trieCache.Load(api.ID) + // Prime the trie cache the same way a proxied request would. + h.routingTrie(depl.ID, cfg) + + _, cached := h.trieCache.Load(depl.ID) require.True(t, cached, "trie cache should hold an entry after routingTrie") status, _, err := h.deleteRestAPIAction([]byte(`{"restApiId":"` + api.ID + `"}`)) require.NoError(t, err) require.Equal(t, http.StatusAccepted, status) - _, stillCached := h.trieCache.Load(api.ID) + _, stillCached := h.trieCache.Load(depl.ID) assert.False(t, stillCached, "trie cache entry must be evicted on DeleteRestApi") } diff --git a/services/apigateway/proxy_routing.go b/services/apigateway/proxy_routing.go index d1f537d1c1..e12d16e21d 100644 --- a/services/apigateway/proxy_routing.go +++ b/services/apigateway/proxy_routing.go @@ -9,31 +9,22 @@ import ( "strings" ) -// trieCacheEntry pairs a built routing trie with the resource-set version it was built -// from, so the proxy can detect staleness cheaply. -type trieCacheEntry struct { - trie *resourcePathTrie - version uint64 -} - -// routingTrie returns the cached routing trie for the API, rebuilding it only when the -// backend reports a newer resource-set version. -func (h *Handler) routingTrie(apiID string) (*resourcePathTrie, error) { - resources, version, err := h.Backend.ResourcesForRouting(apiID) - if err != nil { - return nil, err - } - - if cached, ok := h.trieCache.Load(apiID); ok { - if entry, isEntry := cached.(*trieCacheEntry); isEntry && entry.version == version { - return entry.trie, nil +// routingTrie returns the cached routing trie for a deployment, building it once +// per deploymentID. A deployment's snapshot never changes after CreateDeployment, +// so unlike the live resource set this cache never needs version-based +// invalidation -- only eviction when the deployment itself is deleted (see +// deleteDeploymentAction / deleteRestAPIAction). +func (h *Handler) routingTrie(deploymentID string, cfg *DeploymentConfig) *resourcePathTrie { + if cached, ok := h.trieCache.Load(deploymentID); ok { + if trie, isTrie := cached.(*resourcePathTrie); isTrie { + return trie } } - trie := buildResourceTrie(resources) - h.trieCache.Store(apiID, &trieCacheEntry{trie: trie, version: version}) + trie := buildResourceTrie(cfg.Resources) + h.trieCache.Store(deploymentID, trie) - return trie, nil + return trie } // writeCORSPreflight writes an HTTP 200 response with CORS preflight headers. diff --git a/services/apigateway/proxy_validation.go b/services/apigateway/proxy_validation.go index eafb89e61b..a1040aaaf9 100644 --- a/services/apigateway/proxy_validation.go +++ b/services/apigateway/proxy_validation.go @@ -22,12 +22,12 @@ func (h *Handler) runRequestValidator( ctx context.Context, w http.ResponseWriter, r *http.Request, - apiID string, + cfg *DeploymentConfig, method *Method, pathParams map[string]string, ) bool { - rv, err := h.Backend.GetRequestValidator(apiID, method.RequestValidatorID) - if err != nil { + rv, ok := cfg.RequestValidators[method.RequestValidatorID] + if !ok { logger.Load(ctx).WarnContext(ctx, "APIGateway proxy: request validator not found", "validatorId", method.RequestValidatorID) @@ -43,7 +43,7 @@ func (h *Handler) runRequestValidator( } if rv.ValidateRequestBody { - if denied := h.validateRequestBody(ctx, w, r, apiID, method); denied { + if denied := h.validateRequestBody(ctx, w, r, cfg, method); denied { return true } } @@ -56,7 +56,7 @@ func (h *Handler) runRequestValidator( // schema when one is declared. Returns true when the AWS 400 body-validation response // has been written. func (h *Handler) validateRequestBody( - ctx context.Context, w http.ResponseWriter, r *http.Request, apiID string, method *Method, + ctx context.Context, w http.ResponseWriter, r *http.Request, cfg *DeploymentConfig, method *Method, ) bool { if r.Body == nil { return false @@ -78,14 +78,14 @@ func (h *Handler) validateRequestBody( return true } - schema := h.requestModelSchema(apiID, method, r.Header.Get("Content-Type")) + schema := requestModelSchema(cfg, method, r.Header.Get("Content-Type")) if schema == "" { return false } if err := validateJSONAgainstSchema(bodyBytes, schema); err != nil { logger.Load(ctx).InfoContext(ctx, "APIGateway proxy: request body schema validation failed", - "apiId", apiID, "error", err) + "error", err) writeValidationError(w, "Invalid request body") return true @@ -97,7 +97,7 @@ func (h *Handler) validateRequestBody( // requestModelSchema resolves the JSON Schema for the method's request model that // matches the request content type (falling back to application/json), or "" when the // method declares no usable model. -func (h *Handler) requestModelSchema(apiID string, method *Method, contentType string) string { +func requestModelSchema(cfg *DeploymentConfig, method *Method, contentType string) string { if len(method.RequestModels) == 0 { return "" } @@ -116,8 +116,8 @@ func (h *Handler) requestModelSchema(apiID string, method *Method, contentType s return "" } - model, err := h.Backend.GetModel(apiID, modelName, false) - if err != nil || model == nil { + model, ok := cfg.Models[modelName] + if !ok { return "" } diff --git a/services/apigateway/proxy_validation_test.go b/services/apigateway/proxy_validation_test.go index 3948bbe553..7e548fb38a 100644 --- a/services/apigateway/proxy_validation_test.go +++ b/services/apigateway/proxy_validation_test.go @@ -402,7 +402,7 @@ func TestProxy_StageMethodSettings_ZeroRateLimitMeansUnlimited(t *testing.T) { } } -func TestProxy_TrieCache_InvalidatesOnNewResource(t *testing.T) { +func TestProxy_TrieCache_PerDeployment(t *testing.T) { t.Parallel() backend := apigateway.NewInMemoryBackend() @@ -421,18 +421,26 @@ func TestProxy_TrieCache_InvalidatesOnNewResource(t *testing.T) { _, err = backend.CreateDeployment(api.ID, "prod", "v1") require.NoError(t, err) - // Prime the trie cache. + // Prime the trie cache for the first deployment's snapshot. assert.Equal(t, http.StatusOK, rawProxyGet(t, h, e, api.ID, "/first").Code) // Unmatched resource path on a deployed stage: AWS returns 403 "Missing // Authentication Token", not 404. assert.Equal(t, http.StatusForbidden, rawProxyGet(t, h, e, api.ID, "/second").Code) - // Add a new resource; the cached trie must be invalidated by the version bump. + // A resource added after the deployment is invisible to the stage until a + // new deployment captures it -- real API Gateway serves the snapshot taken + // at CreateDeployment time, not the live configuration. second, err := backend.CreateResource(api.ID, rootID, "second") require.NoError(t, err) wireMock(t, backend, api.ID, second.ID) + assert.Equal(t, http.StatusForbidden, rawProxyGet(t, h, e, api.ID, "/second").Code) + // Redeploying builds a fresh trie for the new snapshot; the old + // deployment's cached trie is untouched. + _, err = backend.CreateDeployment(api.ID, "prod", "v2") + require.NoError(t, err) assert.Equal(t, http.StatusOK, rawProxyGet(t, h, e, api.ID, "/second").Code) + assert.Equal(t, http.StatusOK, rawProxyGet(t, h, e, api.ID, "/first").Code) } func wireMock(t *testing.T, b *apigateway.InMemoryBackend, apiID, resourceID string) { diff --git a/services/apigateway/store.go b/services/apigateway/store.go index a883a50ad9..d2abbc8e0a 100644 --- a/services/apigateway/store.go +++ b/services/apigateway/store.go @@ -74,6 +74,10 @@ type StorageBackend interface { restAPIID, deploymentID string, input UpdateDeploymentInput, ) (*Deployment, error) + // DeploymentConfig returns the invoke-time configuration snapshot a + // deployment captured (see DeploymentConfig's doc). The data-plane proxy + // uses this instead of the live resource/method/integration state. + DeploymentConfig(restAPIID, deploymentID string) (*DeploymentConfig, error) // Stages GetStages(restAPIID string) ([]Stage, error) diff --git a/services/apigatewayv2/PARITY.md b/services/apigatewayv2/PARITY.md index 8c9b82a3a3..29b664e3e9 100644 --- a/services/apigatewayv2/PARITY.md +++ b/services/apigatewayv2/PARITY.md @@ -384,7 +384,7 @@ deferred: - ImportApi/ReimportApi basepath=split; failOnWarnings real effect (see gaps, bd gopherstack-jni0) - Quick-create DeleteRoute/DeleteStage/DeleteIntegration rejection (see gaps, bd gopherstack-2tx) - DeploymentID=="" gating for a never-deployed stage (see gaps, bd gopherstack-vli) -- per-deployment route/integration snapshotting itself was fixed 2026-09-06 (gopherstack-cfr1, see gaps and Notes #19) - - apigateway (v1)'s identical live-routing-vs-deployment-snapshot bug (bd gopherstack-fum) -- deliberately not fixed alongside v2's; v1's resource-tree/routingTrie data plane and lack of an autoDeploy model make it a distinctly larger effort, not a copy of this fix + - apigateway (v1)'s identical live-routing-vs-deployment-snapshot bug (bd gopherstack-fum) was NOT copied from this fix -- deliberately deferred at the time (v1's resource-tree/routingTrie data plane and lack of an autoDeploy model made it a distinctly larger effort) but has since been fixed independently, 2026-09-26; see services/apigateway/PARITY.md's CreateDeployment note and deployment_snapshot.go leaks: {status: clean, note: "portalProductSharingPolicies cleanup on DeletePortalProduct already covered by leak_internal_test.go from a prior sweep; authorizerCache entries are now purged on DeleteAuthorizer/DeleteApi (bd gopherstack-wmh, fixed and closed this pass -- see Notes #11), not merely TTL-bounded; no goroutines/janitors in this package"} --- diff --git a/services/apigatewayv2/apis.go b/services/apigatewayv2/apis.go index c6628214e8..6c230a1873 100644 --- a/services/apigatewayv2/apis.go +++ b/services/apigatewayv2/apis.go @@ -88,6 +88,7 @@ func (b *InMemoryBackend) CreateAPI(ctx context.Context, input CreateAPIInput) ( } cp := api + cp.Tags = copyTags(api.Tags) return &cp, nil } @@ -201,6 +202,7 @@ func (b *InMemoryBackend) GetAPI(apiID string) (*API, error) { } cp := *api + cp.Tags = copyTags(api.Tags) return &cp, nil } @@ -214,7 +216,9 @@ func (b *InMemoryBackend) GetAPIs() ([]API, error) { result := make([]API, 0, len(all)) for _, api := range all { - result = append(result, *api) + cp := *api + cp.Tags = copyTags(api.Tags) + result = append(result, cp) } sort.Slice(result, func(i, j int) bool { @@ -355,6 +359,7 @@ func (b *InMemoryBackend) UpdateAPI(apiID string, input UpdateAPIInput) (*API, e applyQuickCreateUpdateMutateLocked(route, integration, input) cp := *api + cp.Tags = copyTags(api.Tags) return &cp, nil } diff --git a/services/apigatewayv2/domain_names.go b/services/apigatewayv2/domain_names.go index 19fc71a8b7..faf29c3357 100644 --- a/services/apigatewayv2/domain_names.go +++ b/services/apigatewayv2/domain_names.go @@ -119,6 +119,7 @@ func (b *InMemoryBackend) CreateDomainName( b.domainNames.Put(dn) cp := *dn + cp.Tags = copyTags(dn.Tags) return &cp, nil } @@ -361,6 +362,7 @@ func (b *InMemoryBackend) GetDomainName(domainName string) (*DomainName, error) } cp := *dn + cp.Tags = copyTags(dn.Tags) return &cp, nil } @@ -374,7 +376,9 @@ func (b *InMemoryBackend) GetDomainNames() ([]DomainName, error) { result := make([]DomainName, 0, len(all)) for _, dn := range all { - result = append(result, *dn) + cp := *dn + cp.Tags = copyTags(dn.Tags) + result = append(result, cp) } sort.Slice(result, func(i, j int) bool { @@ -444,6 +448,7 @@ func (b *InMemoryBackend) UpdateDomainName(domainName string, input UpdateDomain } cp := *dn + cp.Tags = copyTags(dn.Tags) return &cp, nil } diff --git a/services/apigatewayv2/portals.go b/services/apigatewayv2/portals.go index 4c684c7f59..a3b42c3882 100644 --- a/services/apigatewayv2/portals.go +++ b/services/apigatewayv2/portals.go @@ -124,6 +124,7 @@ func (b *InMemoryBackend) CreatePortal(input CreatePortalInput) (*Portal, error) b.portals.Put(portal) cp := *portal + cp.Tags = copyTags(portal.Tags) return &cp, nil } @@ -174,6 +175,7 @@ func (b *InMemoryBackend) CreatePortalProduct(input CreatePortalProductInput) (* b.portalProducts.Put(product) cp := *product + cp.Tags = copyTags(product.Tags) return &cp, nil } @@ -404,6 +406,7 @@ func (b *InMemoryBackend) GetPortal(portalID string) (*Portal, error) { } cp := *p + cp.Tags = copyTags(p.Tags) return &cp, nil } @@ -417,7 +420,9 @@ func (b *InMemoryBackend) ListPortals() ([]Portal, error) { result := make([]Portal, 0, len(all)) for _, p := range all { - result = append(result, *p) + cp := *p + cp.Tags = copyTags(p.Tags) + result = append(result, cp) } sort.Slice(result, func(i, j int) bool { @@ -438,6 +443,7 @@ func (b *InMemoryBackend) GetPortalProduct(portalProductID string) (*PortalProdu } cp := *pp + cp.Tags = copyTags(pp.Tags) return &cp, nil } @@ -451,7 +457,9 @@ func (b *InMemoryBackend) ListPortalProducts() ([]PortalProduct, error) { result := make([]PortalProduct, 0, len(all)) for _, pp := range all { - result = append(result, *pp) + cp := *pp + cp.Tags = copyTags(pp.Tags) + result = append(result, cp) } sort.Slice(result, func(i, j int) bool { @@ -537,6 +545,7 @@ func (b *InMemoryBackend) UpdatePortal(portalID string, input UpdatePortalInput) p.LastModified = &now cp := *p + cp.Tags = copyTags(p.Tags) return &cp, nil } @@ -573,6 +582,7 @@ func (b *InMemoryBackend) UpdatePortalProduct( pp.LastModified = &now cp := *pp + cp.Tags = copyTags(pp.Tags) return &cp, nil } diff --git a/services/apigatewayv2/stages.go b/services/apigatewayv2/stages.go index e40e80bdd2..93b4e07383 100644 --- a/services/apigatewayv2/stages.go +++ b/services/apigatewayv2/stages.go @@ -43,6 +43,7 @@ func (b *InMemoryBackend) CreateStage(apiID string, input CreateStageInput) (*St b.stages.Put(stage) cp := *stage + cp.Tags = copyTags(stage.Tags) return &cp, nil } @@ -62,6 +63,7 @@ func (b *InMemoryBackend) GetStage(apiID, stageName string) (*Stage, error) { } cp := *s + cp.Tags = copyTags(s.Tags) return &cp, nil } @@ -79,7 +81,9 @@ func (b *InMemoryBackend) GetStages(apiID string) ([]Stage, error) { result := make([]Stage, 0, len(stages)) for _, s := range stages { - result = append(result, *s) + cp := *s + cp.Tags = copyTags(s.Tags) + result = append(result, cp) } sort.Slice(result, func(i, j int) bool { @@ -165,6 +169,7 @@ func (b *InMemoryBackend) UpdateStage(apiID, stageName string, input UpdateStage s.LastUpdatedDate = isoTime{time.Now()} cp := *s + cp.Tags = copyTags(s.Tags) return &cp, nil } diff --git a/services/apigatewayv2/tags_race_test.go b/services/apigatewayv2/tags_race_test.go new file mode 100644 index 0000000000..23bd764684 --- /dev/null +++ b/services/apigatewayv2/tags_race_test.go @@ -0,0 +1,111 @@ +package apigatewayv2_test + +import ( + "context" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/apigatewayv2" +) + +// TestResourceTagsConcurrentWithUntagResource proves Get/List for APIs and VPC +// links must not hand back a Tags map UntagResource mutates in place. +func TestResourceTagsConcurrentWithUntagResource(t *testing.T) { + t.Parallel() + + tests := []struct { + setup func(t *testing.T, b *apigatewayv2.InMemoryBackend) (resourceARN string) + reader func(b *apigatewayv2.InMemoryBackend) + name string + }{ + { + name: "GetAPIs races UntagResource", + setup: func(t *testing.T, b *apigatewayv2.InMemoryBackend) string { + t.Helper() + + api, err := b.CreateAPI(context.Background(), apigatewayv2.CreateAPIInput{ + Name: "race-api", + ProtocolType: "HTTP", + Tags: map[string]string{"env": "prod"}, + }) + require.NoError(t, err) + + return api.APIID + }, + reader: func(b *apigatewayv2.InMemoryBackend) { + apis, err := b.GetAPIs() + if err != nil { + return + } + + for _, api := range apis { + for k := range api.Tags { + _ = k + } + } + }, + }, + { + name: "GetVpcLinks races UntagResource", + setup: func(t *testing.T, b *apigatewayv2.InMemoryBackend) string { + t.Helper() + + vl, err := b.CreateVpcLink(apigatewayv2.CreateVpcLinkInput{ + Name: "race-vpc-link", + SubnetIDs: []string{"subnet-1"}, + Tags: map[string]string{"env": "prod"}, + }) + require.NoError(t, err) + + return "vpclinks/" + vl.VpcLinkID + }, + reader: func(b *apigatewayv2.InMemoryBackend) { + links, err := b.GetVpcLinks() + if err != nil { + return + } + + for _, vl := range links { + for k := range vl.Tags { + _ = k + } + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := apigatewayv2.NewInMemoryBackend() + resourceARN := tt.setup(t, b) + + const iterations = 500 + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + for range iterations { + tt.reader(b) + } + }() + + go func() { + defer wg.Done() + + for range iterations { + _ = b.TagResource(resourceARN, map[string]string{"env": "prod"}) + _ = b.UntagResource(resourceARN, []string{"env"}) + } + }() + + wg.Wait() + }) + } +} diff --git a/services/apigatewayv2/vpc_links.go b/services/apigatewayv2/vpc_links.go index cc09cc09d0..7df6908908 100644 --- a/services/apigatewayv2/vpc_links.go +++ b/services/apigatewayv2/vpc_links.go @@ -38,6 +38,7 @@ func (b *InMemoryBackend) CreateVpcLink(input CreateVpcLinkInput) (*VpcLink, err b.vpcLinks.Put(vpcLink) cp := *vpcLink + cp.Tags = copyTags(vpcLink.Tags) return &cp, nil } @@ -53,6 +54,7 @@ func (b *InMemoryBackend) GetVpcLink(vpcLinkID string) (*VpcLink, error) { } cp := *vpcLink + cp.Tags = copyTags(vpcLink.Tags) return &cp, nil } @@ -66,7 +68,9 @@ func (b *InMemoryBackend) GetVpcLinks() ([]VpcLink, error) { out := make([]VpcLink, 0, len(all)) for _, item := range all { - out = append(out, *item) + cp := *item + cp.Tags = copyTags(item.Tags) + out = append(out, cp) } sort.Slice(out, func(i, j int) bool { return out[i].VpcLinkID < out[j].VpcLinkID }) @@ -87,6 +91,7 @@ func (b *InMemoryBackend) UpdateVpcLink(vpcLinkID string, input UpdateVpcLinkInp } cp := *vpcLink + cp.Tags = copyTags(vpcLink.Tags) return &cp, nil } diff --git a/services/appconfig/bridge_test.go b/services/appconfig/bridge_test.go index c50fcceda6..0d950ca150 100644 --- a/services/appconfig/bridge_test.go +++ b/services/appconfig/bridge_test.go @@ -3,6 +3,7 @@ package appconfig_test import ( "strconv" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -153,16 +154,17 @@ func TestAppConfigDeploymentBridge_StateTransitions(t *testing.T) { dep := f.deployHostedContent(t, content, "growth-strat", 10, 5, 100) require.Equal(t, "DEPLOYING", dep.State, "a non-zero-duration strategy must not complete synchronously") - require.Eventually(t, func() bool { - d, err := f.ac.GetDeployment(f.appID, f.envID, dep.DeploymentNumber) + // deploymentStepDelay + deploymentBakeDelay are 8ms each; cross + // both plus a reconcile tick so the deployment reaches COMPLETE. + time.Sleep(50 * time.Millisecond) + synctest.Wait() - return err == nil && d.State == "COMPLETE" - }, 2*time.Second, 10*time.Millisecond, "deployment should reach COMPLETE") + d, err := f.ac.GetDeployment(f.appID, f.envID, dep.DeploymentNumber) + require.NoError(t, err) + require.Equal(t, "COMPLETE", d.State, "deployment should reach COMPLETE") wantID := strconv.FormatInt(int64(dep.DeploymentNumber), 10) - require.Eventually(t, func() bool { - return f.deploymentIDFor() == wantID - }, 2*time.Second, 10*time.Millisecond, "bridge should publish once the deployment completes") + assert.Equal(t, wantID, f.deploymentIDFor(), "bridge should publish once the deployment completes") gotContent, _, _ := f.pollLatestConfiguration(t) assert.Equal(t, content, gotContent) @@ -183,6 +185,12 @@ func TestAppConfigDeploymentBridge_StateTransitions(t *testing.T) { require.Equal(t, "ROLLED_BACK", final.State) assert.Empty(t, f.deploymentIDFor(), "a rolled-back deployment must never reach AppConfigData") + + // Let the reconciler goroutine's ticker fire once so it notices + // deploymentTimers is empty and self-terminates before the + // bubble ends. + time.Sleep(10 * time.Millisecond) + synctest.Wait() }, }, { @@ -211,7 +219,10 @@ func TestAppConfigDeploymentBridge_StateTransitions(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - tt.run(t, newBridgeFixture(t)) + + synctest.Test(t, func(t *testing.T) { + tt.run(t, newBridgeFixture(t)) + }) }) } } diff --git a/services/appconfig/deployments_test.go b/services/appconfig/deployments_test.go index 85496a8825..ef66be98cb 100644 --- a/services/appconfig/deployments_test.go +++ b/services/appconfig/deployments_test.go @@ -2,6 +2,7 @@ package appconfig_test import ( "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -51,64 +52,57 @@ func TestBackend_StartDeployment_ZeroDurationCompletesSynchronously(t *testing.T func TestBackend_StartDeployment_ProgressesThroughGrowthAndBake(t *testing.T) { t.Parallel() - b := appconfig.NewInMemoryBackend("123456789012", "us-east-1") - - app, err := b.CreateApplication("progress-app", "", nil) - require.NoError(t, err) - - env, err := b.CreateEnvironment(app.ID, "progress-env", "", nil, nil) - require.NoError(t, err) - - profile, err := b.CreateConfigurationProfile( - app.ID, "progress-profile", "", "hosted", "AWS.Freeform", "", "", nil, - nil, - ) - require.NoError(t, err) - - _, err = b.CreateHostedConfigurationVersion(app.ID, profile.ID, "application/json", "", "", []byte(`{}`), nil) - require.NoError(t, err) - - strategy, err := b.CreateDeploymentStrategy("progress-strat", "", 10, 5, 10, "LINEAR", "NONE", nil) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := appconfig.NewInMemoryBackend("123456789012", "us-east-1") - dep, err := b.StartDeployment(app.ID, env.ID, profile.ID, strategy.ID, "1", "", nil, nil, nil) - require.NoError(t, err) - assert.Equal(t, "DEPLOYING", dep.State, "a non-zero-duration strategy must not complete synchronously") - require.Len(t, dep.EventLog, 1) - assert.Equal(t, "DEPLOYMENT_STARTED", dep.EventLog[0].EventType) - - deadline := time.Now().Add(2 * time.Second) + app, err := b.CreateApplication("progress-app", "", nil) + require.NoError(t, err) - var final *appconfig.Deployment + env, err := b.CreateEnvironment(app.ID, "progress-env", "", nil, nil) + require.NoError(t, err) - for time.Now().Before(deadline) { - final, err = b.GetDeployment(app.ID, env.ID, dep.DeploymentNumber) + profile, err := b.CreateConfigurationProfile( + app.ID, "progress-profile", "", "hosted", "AWS.Freeform", "", "", nil, + nil, + ) require.NoError(t, err) - if final.State == "COMPLETE" { - break - } + _, err = b.CreateHostedConfigurationVersion(app.ID, profile.ID, "application/json", "", "", []byte(`{}`), nil) + require.NoError(t, err) - time.Sleep(time.Millisecond) - } + strategy, err := b.CreateDeploymentStrategy("progress-strat", "", 10, 5, 10, "LINEAR", "NONE", nil) + require.NoError(t, err) - require.NotNil(t, final) - assert.Equal(t, "COMPLETE", final.State) - assert.InDelta(t, float32(100), final.PercentageComplete, 0.001) - assert.Equal( - t, "DEPLOYMENT_COMPLETED", final.EventLog[0].EventType, - "EventLog must be ordered most-recent-first", - ) + dep, err := b.StartDeployment(app.ID, env.ID, profile.ID, strategy.ID, "1", "", nil, nil, nil) + require.NoError(t, err) + assert.Equal(t, "DEPLOYING", dep.State, "a non-zero-duration strategy must not complete synchronously") + require.Len(t, dep.EventLog, 1) + assert.Equal(t, "DEPLOYMENT_STARTED", dep.EventLog[0].EventType) - var sawStarted bool + // growthFactor 10 needs 10 steps (8ms each) to reach 100%, then an + // 8ms bake; cross all of it plus a reconcile tick. + time.Sleep(150 * time.Millisecond) + synctest.Wait() - for _, e := range final.EventLog { - if e.EventType == "DEPLOYMENT_STARTED" { - sawStarted = true + final, err := b.GetDeployment(app.ID, env.ID, dep.DeploymentNumber) + require.NoError(t, err) + assert.Equal(t, "COMPLETE", final.State) + assert.InDelta(t, float32(100), final.PercentageComplete, 0.001) + assert.Equal( + t, "DEPLOYMENT_COMPLETED", final.EventLog[0].EventType, + "EventLog must be ordered most-recent-first", + ) + + var sawStarted bool + + for _, e := range final.EventLog { + if e.EventType == "DEPLOYMENT_STARTED" { + sawStarted = true + } } - } - assert.True(t, sawStarted, "the original DEPLOYMENT_STARTED event must be preserved in history") + assert.True(t, sawStarted, "the original DEPLOYMENT_STARTED event must be preserved in history") + }) } // TestBackend_StartDeployment_UnknownHostedVersion_NotFound verifies the diff --git a/services/appconfig/handler_deployments_test.go b/services/appconfig/handler_deployments_test.go index c85f87b1d1..8933058c53 100644 --- a/services/appconfig/handler_deployments_test.go +++ b/services/appconfig/handler_deployments_test.go @@ -6,8 +6,8 @@ import ( "log/slog" "net/http" "net/http/httptest" - "strconv" "testing" + "testing/synctest" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -125,37 +125,6 @@ func doRequestWithHeader( return rec } -// waitForDeploymentTerminal polls GetDeployment until State reaches a -// terminal value (COMPLETE/ROLLED_BACK/REVERTED) or the deadline elapses, -// returning the last-observed deployment. -func waitForDeploymentTerminal( - t *testing.T, h *appconfig.Handler, appID, envID string, deploymentNumber int, -) appconfig.Deployment { - t.Helper() - - deadline := time.Now().Add(2 * time.Second) - - var dep appconfig.Deployment - - for time.Now().Before(deadline) { - rec := doRequest(t, h, http.MethodGet, - "/applications/"+appID+"/environments/"+envID+"/deployments/"+strconv.Itoa(deploymentNumber), nil) - require.Equal(t, http.StatusOK, rec.Code) - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &dep)) - - switch dep.State { - case "COMPLETE", "ROLLED_BACK", "REVERTED": - return dep - } - - time.Sleep(time.Millisecond) - } - - t.Fatalf("deployment did not reach a terminal state within the deadline, last state: %s", dep.State) - - return dep -} - // seedExperimentRunHTTP creates an application, environment, feature-flag // configuration profile, and experiment definition through the real router // path, returning the application ID and the created @@ -318,134 +287,140 @@ func TestHandler_ExperimentRun_HTTP_Errors(t *testing.T) { func TestHandler_Deployment_Lifecycle(t *testing.T) { t.Parallel() - h := newTestHandler(t) - - // Create app. - rec := doRequest(t, h, http.MethodPost, "/applications", []byte(`{"name":"deploy-app"}`)) - require.Equal(t, http.StatusCreated, rec.Code) - - var app appconfig.Application - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &app)) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - // Create env. - rec = doRequest( - t, - h, - http.MethodPost, - "/applications/"+app.ID+"/environments", - []byte(`{"name":"staging"}`), - ) - require.Equal(t, http.StatusCreated, rec.Code) - - var env appconfig.Environment - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &env)) - - // Create configuration profile (required by StartDeployment validation). - profBody := []byte(`{"name":"my-profile","locationUri":"hosted"}`) - rec = doRequest( - t, - h, - http.MethodPost, - "/applications/"+app.ID+"/configurationprofiles", - profBody, - ) - require.Equal(t, http.StatusCreated, rec.Code) - - var prof appconfig.ConfigurationProfile - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &prof)) - - // Create a hosted configuration version (required for StartDeployment - // to validate ConfigurationVersion against, for a "hosted" profile). - rec = doRequest( - t, h, http.MethodPost, - "/applications/"+app.ID+"/configurationprofiles/"+prof.ID+"/hostedconfigurationversions", - []byte(`{"content":"enabled"}`), - ) - require.Equal(t, http.StatusCreated, rec.Code) + // Create app. + rec := doRequest(t, h, http.MethodPost, "/applications", []byte(`{"name":"deploy-app"}`)) + require.Equal(t, http.StatusCreated, rec.Code) - // Create deployment strategy (required by StartDeployment validation). - // A non-zero duration and bake time exercise the real DEPLOYING -> - // BAKING -> COMPLETE state machine (see waitForDeploymentTerminal). - stratBody := []byte( - `{"name":"my-strategy","deploymentDurationInMinutes":10,"growthFactor":20,"finalBakeTimeInMinutes":5}`, - ) - rec = doRequest(t, h, http.MethodPost, "/deploymentstrategies", stratBody) - require.Equal(t, http.StatusCreated, rec.Code) + var app appconfig.Application + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &app)) + + // Create env. + rec = doRequest( + t, + h, + http.MethodPost, + "/applications/"+app.ID+"/environments", + []byte(`{"name":"staging"}`), + ) + require.Equal(t, http.StatusCreated, rec.Code) - var strat appconfig.DeploymentStrategy - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &strat)) + var env appconfig.Environment + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &env)) + + // Create configuration profile (required by StartDeployment validation). + profBody := []byte(`{"name":"my-profile","locationUri":"hosted"}`) + rec = doRequest( + t, + h, + http.MethodPost, + "/applications/"+app.ID+"/configurationprofiles", + profBody, + ) + require.Equal(t, http.StatusCreated, rec.Code) - // Start deployment. - depBodyStr := `{"configurationProfileId":"` + prof.ID + - `","deploymentStrategyId":"` + strat.ID + `","configurationVersion":"1"}` - rec = doRequest( - t, - h, - http.MethodPost, - "/applications/"+app.ID+"/environments/"+env.ID+"/deployments", - []byte(depBodyStr), - ) - require.Equal(t, http.StatusCreated, rec.Code) + var prof appconfig.ConfigurationProfile + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &prof)) - var dep appconfig.Deployment - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &dep)) - assert.Equal(t, int32(1), dep.DeploymentNumber) - assert.Equal(t, "DEPLOYING", dep.State, "a non-zero-duration strategy must not complete synchronously") - assert.NotEmpty(t, dep.EventLog, "StartDeployment must record a DEPLOYMENT_STARTED event") + // Create a hosted configuration version (required for StartDeployment + // to validate ConfigurationVersion against, for a "hosted" profile). + rec = doRequest( + t, h, http.MethodPost, + "/applications/"+app.ID+"/configurationprofiles/"+prof.ID+"/hostedconfigurationversions", + []byte(`{"content":"enabled"}`), + ) + require.Equal(t, http.StatusCreated, rec.Code) - final := waitForDeploymentTerminal(t, h, app.ID, env.ID, 1) - assert.Equal(t, "COMPLETE", final.State) - assert.InDelta(t, float32(100.0), final.PercentageComplete, 0.001) + // Create deployment strategy (required by StartDeployment validation). + // A non-zero duration and bake time exercise the real DEPLOYING -> + // BAKING -> COMPLETE state machine. + stratBody := []byte( + `{"name":"my-strategy","deploymentDurationInMinutes":10,"growthFactor":20,"finalBakeTimeInMinutes":5}`, + ) + rec = doRequest(t, h, http.MethodPost, "/deploymentstrategies", stratBody) + require.Equal(t, http.StatusCreated, rec.Code) - // Get deployment. - rec = doRequest( - t, - h, - http.MethodGet, - "/applications/"+app.ID+"/environments/"+env.ID+"/deployments/1", - nil, - ) - assert.Equal(t, http.StatusOK, rec.Code) + var strat appconfig.DeploymentStrategy + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &strat)) + + // Start deployment. + depBodyStr := `{"configurationProfileId":"` + prof.ID + + `","deploymentStrategyId":"` + strat.ID + `","configurationVersion":"1"}` + rec = doRequest( + t, + h, + http.MethodPost, + "/applications/"+app.ID+"/environments/"+env.ID+"/deployments", + []byte(depBodyStr), + ) + require.Equal(t, http.StatusCreated, rec.Code) - // List deployments. - rec = doRequest( - t, - h, - http.MethodGet, - "/applications/"+app.ID+"/environments/"+env.ID+"/deployments", - nil, - ) - assert.Equal(t, http.StatusOK, rec.Code) + var dep appconfig.Deployment + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &dep)) + assert.Equal(t, int32(1), dep.DeploymentNumber) + assert.Equal(t, "DEPLOYING", dep.State, "a non-zero-duration strategy must not complete synchronously") + assert.NotEmpty(t, dep.EventLog, "StartDeployment must record a DEPLOYMENT_STARTED event") + + // deploymentStepDelay + deploymentBakeDelay are 8ms each; cross both + // plus a reconcile tick so the deployment reaches a terminal state. + time.Sleep(50 * time.Millisecond) + synctest.Wait() + + rec = doRequest( + t, + h, + http.MethodGet, + "/applications/"+app.ID+"/environments/"+env.ID+"/deployments/1", + nil, + ) + require.Equal(t, http.StatusOK, rec.Code) + var final appconfig.Deployment + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &final)) + assert.Equal(t, "COMPLETE", final.State) + assert.InDelta(t, float32(100.0), final.PercentageComplete, 0.001) + + // List deployments. + rec = doRequest( + t, + h, + http.MethodGet, + "/applications/"+app.ID+"/environments/"+env.ID+"/deployments", + nil, + ) + assert.Equal(t, http.StatusOK, rec.Code) + + // Stopping a COMPLETE deployment without Allow-Revert is rejected -- + // real AWS only allows it via AllowRevert (see + // TestHandler_StopDeployment_AllowRevert for that path). + rec = doRequest( + t, + h, + http.MethodDelete, + "/applications/"+app.ID+"/environments/"+env.ID+"/deployments/1", + nil, + ) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + // Stop deployment with Allow-Revert reverts it. Real StopDeploymentOutput + // echoes the full post-stop deployment (appconfig@v1.48.4 + // api_op_StopDeployment.go) with 200, not an empty 204 body. + rec = doRequestWithHeader( + t, h, http.MethodDelete, + "/applications/"+app.ID+"/environments/"+env.ID+"/deployments/1", + "Allow-Revert", "true", nil, + ) + require.Equal(t, http.StatusOK, rec.Code) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &dep)) + assert.Equal(t, "REVERTED", dep.State, "StopDeploymentOutput itself must reflect the new state") - // Stopping a COMPLETE deployment without Allow-Revert is rejected -- - // real AWS only allows it via AllowRevert (see - // TestHandler_StopDeployment_AllowRevert for that path). - rec = doRequest( - t, - h, - http.MethodDelete, - "/applications/"+app.ID+"/environments/"+env.ID+"/deployments/1", - nil, - ) - assert.Equal(t, http.StatusBadRequest, rec.Code) - - // Stop deployment with Allow-Revert reverts it. Real StopDeploymentOutput - // echoes the full post-stop deployment (appconfig@v1.48.4 - // api_op_StopDeployment.go) with 200, not an empty 204 body. - rec = doRequestWithHeader( - t, h, http.MethodDelete, - "/applications/"+app.ID+"/environments/"+env.ID+"/deployments/1", - "Allow-Revert", "true", nil, - ) - require.Equal(t, http.StatusOK, rec.Code) - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &dep)) - assert.Equal(t, "REVERTED", dep.State, "StopDeploymentOutput itself must reflect the new state") - - rec = doRequest(t, h, http.MethodGet, - "/applications/"+app.ID+"/environments/"+env.ID+"/deployments/1", nil) - require.Equal(t, http.StatusOK, rec.Code) - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &dep)) - assert.Equal(t, "REVERTED", dep.State) + rec = doRequest(t, h, http.MethodGet, + "/applications/"+app.ID+"/environments/"+env.ID+"/deployments/1", nil) + require.Equal(t, http.StatusOK, rec.Code) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &dep)) + assert.Equal(t, "REVERTED", dep.State) + }) } func TestHandler_Deployment_HTTP_NotFound(t *testing.T) { diff --git a/services/appconfig/whitebox_test.go b/services/appconfig/whitebox_test.go index 6b2e284495..c29886a6d9 100644 --- a/services/appconfig/whitebox_test.go +++ b/services/appconfig/whitebox_test.go @@ -2,6 +2,7 @@ package appconfig import ( "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -137,51 +138,53 @@ func TestBackend_ExtensionAssociation_CascadeDeleteOnApplication(t *testing.T) { func TestDeploymentTimers_DrainToZero(t *testing.T) { t.Parallel() - b := NewInMemoryBackend("123456789012", "us-east-1") + synctest.Test(t, func(t *testing.T) { + b := NewInMemoryBackend("123456789012", "us-east-1") - app, err := b.CreateApplication("timer-leak-app", "", nil) - require.NoError(t, err) + app, err := b.CreateApplication("timer-leak-app", "", nil) + require.NoError(t, err) - env, err := b.CreateEnvironment(app.ID, "timer-leak-env", "", nil, nil) - require.NoError(t, err) + env, err := b.CreateEnvironment(app.ID, "timer-leak-env", "", nil, nil) + require.NoError(t, err) - profile, err := b.CreateConfigurationProfile( - app.ID, "timer-leak-profile", "", "hosted", "AWS.Freeform", "", "", nil, - nil, - ) - require.NoError(t, err) + profile, err := b.CreateConfigurationProfile( + app.ID, "timer-leak-profile", "", "hosted", "AWS.Freeform", "", "", nil, + nil, + ) + require.NoError(t, err) - _, err = b.CreateHostedConfigurationVersion( - app.ID, profile.ID, "application/json", "", "", []byte(`{}`), nil, - ) - require.NoError(t, err) + _, err = b.CreateHostedConfigurationVersion( + app.ID, profile.ID, "application/json", "", "", []byte(`{}`), nil, + ) + require.NoError(t, err) - // A non-zero duration and bake time forces real DEPLOYING -> BAKING - // progression (registers a timer), rather than the synchronous - // zero-duration path (which never touches deploymentTimers at all). - strategy, err := b.CreateDeploymentStrategy("timer-leak-strat", "", 10, 5, 25, "LINEAR", "NONE", nil) - require.NoError(t, err) + // A non-zero duration and bake time forces real DEPLOYING -> BAKING + // progression (registers a timer), rather than the synchronous + // zero-duration path (which never touches deploymentTimers at all). + strategy, err := b.CreateDeploymentStrategy("timer-leak-strat", "", 10, 5, 25, "LINEAR", "NONE", nil) + require.NoError(t, err) - const deployments = 5 + const deployments = 5 - for range deployments { - _, startErr := b.StartDeployment(app.ID, env.ID, profile.ID, strategy.ID, "1", "", nil, nil, nil) - require.NoError(t, startErr) - } + for range deployments { + _, startErr := b.StartDeployment(app.ID, env.ID, profile.ID, strategy.ID, "1", "", nil, nil, nil) + require.NoError(t, startErr) + } - deploymentTimerCount := func() int { - b.mu.RLock("test.deploymentTimerCount") - defer b.mu.RUnlock() + deploymentTimerCount := func() int { + b.mu.RLock("test.deploymentTimerCount") + defer b.mu.RUnlock() - return len(b.deploymentTimers) - } + return len(b.deploymentTimers) + } - assert.Positive(t, deploymentTimerCount(), "sanity: progression must actually register timers") + assert.Positive(t, deploymentTimerCount(), "sanity: progression must actually register timers") - deadline := time.Now().Add(2 * time.Second) - for deploymentTimerCount() > 0 && time.Now().Before(deadline) { - time.Sleep(time.Millisecond) - } + // growthFactor 25 needs 4 steps (8ms each) to reach 100%, then an 8ms + // bake; cross all of it plus a reconcile tick. + time.Sleep(150 * time.Millisecond) + synctest.Wait() - assert.Equal(t, 0, deploymentTimerCount(), "every deployment timer must drain once its deployment completes") + assert.Equal(t, 0, deploymentTimerCount(), "every deployment timer must drain once its deployment completes") + }) } diff --git a/services/appconfigdata/janitor_test.go b/services/appconfigdata/janitor_test.go index dbabf90e3c..e0caa49c4e 100644 --- a/services/appconfigdata/janitor_test.go +++ b/services/appconfigdata/janitor_test.go @@ -3,8 +3,11 @@ package appconfigdata_test import ( "context" "testing" + "testing/synctest" "time" + "github.com/stretchr/testify/assert" + "github.com/blackbirdworks/gopherstack/services/appconfigdata" ) @@ -43,47 +46,42 @@ func TestJanitor_RunExitsOnContextCancel(t *testing.T) { func TestJanitor_SweepsExpiredSessionsOnTick(t *testing.T) { t.Parallel() - b := appconfigdata.NewInMemoryBackend() - if err := b.SetConfiguration("app", "env", "p", `{}`, "application/json"); err != nil { - t.Fatalf("SetConfiguration failed: %v", err) - } - - token, err := b.StartSession("app", "env", "p", 0) - if err != nil { - t.Fatalf("StartSession failed: %v", err) - } + synctest.Test(t, func(t *testing.T) { + b := appconfigdata.NewInMemoryBackend() + if err := b.SetConfiguration("app", "env", "p", `{}`, "application/json"); err != nil { + t.Fatalf("SetConfiguration failed: %v", err) + } - if b.LookupSession(token) == nil { - t.Fatal("session must exist immediately after StartSession") - } + token, err := b.StartSession("app", "env", "p", 0) + if err != nil { + t.Fatalf("StartSession failed: %v", err) + } - j := appconfigdata.NewJanitor(b) - j.Interval = 5 * time.Millisecond - j.SessionTTL = 0 // every session is immediately idle-expired + if b.LookupSession(token) == nil { + t.Fatal("session must exist immediately after StartSession") + } - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() + j := appconfigdata.NewJanitor(b) + j.Interval = 5 * time.Millisecond + j.SessionTTL = 0 // every session is immediately idle-expired - done := make(chan struct{}) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() - go func() { - defer close(done) - j.Run(ctx) - }() + done := make(chan struct{}) - deadline := time.Now().Add(2 * time.Second) - for time.Now().Before(deadline) { - if b.LookupSession(token) == nil { - cancel() - <-done + go func() { + defer close(done) + j.Run(ctx) + }() - return - } + // j.Interval is 5ms; cross a tick so the janitor sweeps the session. + time.Sleep(10 * time.Millisecond) + synctest.Wait() - time.Sleep(5 * time.Millisecond) - } + assert.Nil(t, b.LookupSession(token), "janitor did not sweep the expired session") - cancel() - <-done - t.Fatal("janitor did not sweep the expired session within the deadline") + cancel() + <-done + }) } diff --git a/services/applicationautoscaling/PARITY.md b/services/applicationautoscaling/PARITY.md index b4e18edf97..1a9fca572d 100644 --- a/services/applicationautoscaling/PARITY.md +++ b/services/applicationautoscaling/PARITY.md @@ -4,12 +4,12 @@ last_audit_commit: d0f3046ef last_audit_date: 2026-09-04 overall: A # real, wire-breaking bugs found and fixed ops: - RegisterScalableTarget: {wire: fixed, errors: fixed, state: fixed, persist: ok, note: "upsert confirmed; RoleARN/Tags/SuspendedState correctly left unchanged when omitted on update. FIXED: over-tag-limit now reports LimitExceededException (RegisterScalableTarget's modeled error set has no TooManyTagsException, confirmed against the vendored SDK's deserializeOpErrorRegisterScalableTarget), not ValidationException. FIXED (gopherstack-8xo): MinCapacity/MaxCapacity were plain int32 on the wire and backend signature, so omitting either field on an update call (e.g. a client that only wants to change RoleARN) decoded as 0 and silently reset the scalable target's capacity to 0 -- real AWS models both as *int32, 'required when registering a new scalable target' only (api_op_RegisterScalableTarget.go field docs), and the op doc states 'Any parameters that you don't specify are not changed by this update request.' Changed MinCapacity/MaxCapacity to *int32 end to end (wire, RegisterScalableTarget, updateExistingTarget); omitted on update now correctly preserves the stored value, still required when registering a brand-new target."} - DeregisterScalableTarget: {wire: ok, errors: fixed, state: ok, persist: ok, note: "cascades delete to scaling policies + scheduled actions for the same (ns,resourceId,dimension), matching real AWS. FIXED: ObjectNotFoundException HTTP status was 404, now 400 (see notes)."} - DescribeScalableTargets: {wire: fixed, errors: fixed, state: ok, persist: ok, note: "FIXED: NextToken is now an opaque base64 cursor (was the raw sort key) and a malformed token now returns InvalidNextTokenException/400 (DescribeScalableTargets' modeled error set includes it). Added PredictedCapacity field (always omitted -- see notes)."} - PutScalingPolicy: {wire: fixed, errors: fixed, state: fixed, persist: ok, note: "FIXED (prior pass): default PolicyType was TargetTrackingScaling, real default is StepScaling; PolicyARN colon-vs-slash. FIXED prior pass: (1) now requires the scalable target to already be registered, raising ObjectNotFoundException otherwise -- PutScalingPolicy's modeled error set includes it and its doc text names 'any operation that depends on the existence of a scalable target'; a client could previously PutScalingPolicy against a namespace/resourceId/dimension that was never registered, which real AWS rejects. (2) PredictiveScalingPolicyConfiguration was accepted by the real API but silently dropped -- now captured, persisted, and echoed by DescribeScalingPolicies. (3) enforces the real, documented AWS quotas: 50 scaling policies/scalable target and 20 step adjustments/step-scaling-policy, raising LimitExceededException. DOWNGRADED this pass: Alarms (CloudWatch alarm references) is a real field on both PutScalingPolicy's and DescribeScalingPolicies' response shapes; the prior pass synthesized stable-looking Alarm name+ARN entries for it, but those ARNs pointed at CloudWatch alarms that do not actually exist anywhere (gopherstack's applicationautoscaling backend has no cross-service reference to the cloudwatch backend) -- a caller querying cloudwatch:DescribeAlarms with that ARN would get nothing back. That is exactly the invented-resource fabrication this project removes elsewhere, so Alarms is now honestly left empty (nil/omitted) for every policy type instead. See gaps/deferred."} + RegisterScalableTarget: {wire: fixed, errors: fixed, state: fixed, persist: ok, note: "upsert confirmed; RoleARN/Tags/SuspendedState correctly left unchanged when omitted on update. FIXED: over-tag-limit now reports LimitExceededException (RegisterScalableTarget's modeled error set has no TooManyTagsException, confirmed against the vendored SDK's deserializeOpErrorRegisterScalableTarget), not ValidationException. FIXED (gopherstack-8xo): MinCapacity/MaxCapacity were plain int32 on the wire and backend signature, so omitting either field on an update call (e.g. a client that only wants to change RoleARN) decoded as 0 and silently reset the scalable target's capacity to 0 -- real AWS models both as *int32, 'required when registering a new scalable target' only (api_op_RegisterScalableTarget.go field docs), and the op doc states 'Any parameters that you don't specify are not changed by this update request.' Changed MinCapacity/MaxCapacity to *int32 end to end (wire, RegisterScalableTarget, updateExistingTarget); omitted on update now correctly preserves the stored value, still required when registering a brand-new target. 2026-09-26: ServiceNamespace=dynamodb now validates table existence and pushes into DynamoDB's own autoscaling state -- see dynamodb_wiring family."} + DeregisterScalableTarget: {wire: ok, errors: fixed, state: ok, persist: ok, note: "cascades delete to scaling policies + scheduled actions for the same (ns,resourceId,dimension), matching real AWS. FIXED: ObjectNotFoundException HTTP status was 404, now 400 (see notes). 2026-09-26: does NOT clear DynamoDB-side autoscaling settings for ServiceNamespace=dynamodb -- disclosed, matches real AWS; see dynamodb_wiring family."} + DescribeScalableTargets: {wire: fixed, errors: fixed, state: ok, persist: ok, note: "FIXED: NextToken is now an opaque base64 cursor (was the raw sort key) and a malformed token now returns InvalidNextTokenException/400 (DescribeScalableTargets' modeled error set includes it). Added PredictedCapacity field (always omitted -- see notes). 2026-09-26: for ServiceNamespace=dynamodb, now also synthesizes rows for settings configured directly via DynamoDB's own UpdateTableReplicaAutoScaling -- see dynamodb_wiring family."} + PutScalingPolicy: {wire: fixed, errors: fixed, state: fixed, persist: ok, note: "FIXED (prior pass): default PolicyType was TargetTrackingScaling, real default is StepScaling; PolicyARN colon-vs-slash. FIXED prior pass: (1) now requires the scalable target to already be registered, raising ObjectNotFoundException otherwise -- PutScalingPolicy's modeled error set includes it and its doc text names 'any operation that depends on the existence of a scalable target'; a client could previously PutScalingPolicy against a namespace/resourceId/dimension that was never registered, which real AWS rejects. (2) PredictiveScalingPolicyConfiguration was accepted by the real API but silently dropped -- now captured, persisted, and echoed by DescribeScalingPolicies. (3) enforces the real, documented AWS quotas: 50 scaling policies/scalable target and 20 step adjustments/step-scaling-policy, raising LimitExceededException. DOWNGRADED this pass: Alarms (CloudWatch alarm references) is a real field on both PutScalingPolicy's and DescribeScalingPolicies' response shapes; the prior pass synthesized stable-looking Alarm name+ARN entries for it, but those ARNs pointed at CloudWatch alarms that do not actually exist anywhere (gopherstack's applicationautoscaling backend has no cross-service reference to the cloudwatch backend) -- a caller querying cloudwatch:DescribeAlarms with that ARN would get nothing back. That is exactly the invented-resource fabrication this project removes elsewhere, so Alarms is now honestly left empty (nil/omitted) for every policy type instead. See gaps/deferred. 2026-09-26: a TargetTrackingScaling policy against ServiceNamespace=dynamodb now also pushes into DynamoDB's own autoscaling state -- see dynamodb_wiring family."} DeleteScalingPolicy: {wire: ok, errors: ok, state: ok, persist: ok} - DescribeScalingPolicies: {wire: fixed, errors: fixed, state: ok, persist: ok, note: "FIXED (prior pass): deleted the invented PolicyARNs filter field/behavior -- confirmed against DescribeScalingPoliciesInput/its serializer in the vendored SDK, real AWS has no such filter (only PolicyNames/ResourceId/ScalableDimension/ServiceNamespace). FIXED (prior pass): NextToken is now opaque base64 with InvalidNextTokenException on malformed input. FIXED (prior pass): PredictiveScalingPolicyConfiguration now populated. DOWNGRADED this pass: Alarms now honestly empty (see PutScalingPolicy)."} + DescribeScalingPolicies: {wire: fixed, errors: fixed, state: ok, persist: ok, note: "FIXED (prior pass): deleted the invented PolicyARNs filter field/behavior -- confirmed against DescribeScalingPoliciesInput/its serializer in the vendored SDK, real AWS has no such filter (only PolicyNames/ResourceId/ScalableDimension/ServiceNamespace). FIXED (prior pass): NextToken is now opaque base64 with InvalidNextTokenException on malformed input. FIXED (prior pass): PredictiveScalingPolicyConfiguration now populated. DOWNGRADED this pass: Alarms now honestly empty (see PutScalingPolicy). 2026-09-26: for ServiceNamespace=dynamodb, now also synthesizes rows for policies configured directly via DynamoDB's own UpdateTableReplicaAutoScaling -- see dynamodb_wiring family."} DescribeScalingActivities: {wire: fixed, errors: fixed, state: ok, persist: n/a, note: "scalingActivities intentionally ephemeral; most-recent-first via slices.Backward; NextToken now opaque base64 with InvalidNextTokenException on malformed input. Added Details/NotScaledReasons wire fields (always empty/omitted -- see gaps, IncludeNotScaledActivities is accepted but vacuous)."} PutScheduledAction: {wire: fixed, errors: fixed, state: fixed, persist: ok, note: "FIXED (prior pass): StartTime/EndTime epoch-seconds; ARN colon-vs-slash. FIXED (prior pass): now requires the scalable target to already be registered (ObjectNotFoundException), same rationale as PutScalingPolicy. Enforces the real, documented, non-adjustable AWS quota of 200 scheduled actions/scalable target, raising LimitExceededException. FIXED (gopherstack-8xo): on update, StartTime/EndTime were only overwritten when the caller resent them, otherwise silently keeping the old values -- the exact opposite of the documented behavior: 'To update a scheduled action, specify the parameters that you want to change. If you don't specify start and end times, the old values are deleted.' Now always overwritten with whatever the caller sent (nil included), matching the doc. FIXED (gopherstack-8xo): Schedule was required on every PutScheduledAction call including updates, but PutScheduledActionInput does not mark it 'This member is required' (only ResourceId/ScalableDimension/ScheduledActionName/ServiceNamespace are), consistent with the same 'specify the parameters you want to change' update semantics -- Schedule is now only required when registering a brand-new action, and is left unchanged when omitted on an update."} DeleteScheduledAction: {wire: ok, errors: ok, state: ok, persist: ok} @@ -22,6 +22,7 @@ families: tagging: {status: ok, note: "TagResource/ListTagsForResource/UntagResource operate on scalable-target ARNs only, matching real AWS (Application Auto Scaling only supports tagging scalable targets)"} error_types: {status: fixed, note: "Every modeled AWS exception (ConcurrentUpdateException/FailedResourceAccessException/InternalServiceException/InvalidNextTokenException/LimitExceededException/ObjectNotFoundException/ResourceNotFoundException/TooManyTagsException/ValidationException) now has a distinct sentinel in errors.go and a correct HTTP status in handler.go's handleError, matching each type's ErrorFault() classification in the vendored SDK's types/errors.go: FaultServer (ConcurrentUpdateException, InternalServiceException) -> HTTP 500; FaultClient (everything else) -> HTTP 400. Previously ObjectNotFoundException incorrectly returned 404, ValidationException(ErrAlreadyExists) incorrectly returned 409, and TooManyTagsException/LimitExceededException/InvalidNextTokenException/ResourceNotFoundException/ConcurrentUpdateException/FailedResourceAccessException did not exist as distinct types at all (their scenarios either fell through to a generic ValidationException/404 or were simply unreachable). ConcurrentUpdateException/FailedResourceAccessException specifically remain without a backend-state trigger but are reachable via chaos fault injection -- see deferred."} quotas: {status: fixed, note: "FIXED this pass (gopherstack-cdxe): RegisterScalableTarget now enforces the real, documented per-account/per-region 'scalable targets per resource type' AWS quota (5,000 for dynamodb, 3,000 for ecs, 1,500 for cassandra/Keyspaces, 500 for every other ServiceNamespace -- see maxScalableTargetsForNamespace in scalable_targets.go), raising LimitExceededException once exhausted. Upserting an already-registered target does not consume additional quota. Combined with the prior pass's 50 scaling policies/target, 200 scheduled actions/target, and 20 step adjustments/policy quotas, every documented Application Auto Scaling quota is now enforced."} + dynamodb_wiring: {status: fixed, note: "2026-09-26 (gopherstack-101r, closes the cross-service decision this and dynamodb's PARITY.md both flagged NOT wired): RegisterScalableTarget/DeregisterScalableTarget/PutScalingPolicy/DescribeScalableTargets/DescribeScalingPolicies now wire to services/dynamodb for ServiceNamespace=dynamodb (dynamodb_bridge.go/dynamodb_bridge_describe.go/cross_service.go), via the SetAppConfig/siblingServices lazy-lookup pattern already used by grafana/ram/workspaces/resiliencehub/mgn (this service imports services/dynamodb; dynamodb has zero reference back, so there is no import cycle -- dynamodb remains the single source of truth for a table's autoscaling settings and needs no awareness of this service). RegisterScalableTarget parses ResourceId (table/ or table//index/) + ScalableDimension (dynamodb:{table,index}:{Read,Write}CapacityUnits), validates the table exists (ValidationException 'DynamoDB table does not exist: ' -- verbatim wording confirmed against a real-account error transcript, https://github.com/terraform-aws-modules/terraform-aws-dynamodb-table/issues/15, not an official AWS doc string, disclosed as such; real AWS performs this check by calling into the target service, per the general mechanism documented at security_iam_permission_validation.html), then pushes MinCapacity/MaxCapacity/RoleARN into dynamodb's own AutoScaling/ReplicaAutoScaling state via dynamodb's UpdateTableReplicaAutoScaling -- reading dynamodb's current stored settings first and carrying forward whatever this call doesn't touch (capacity vs. scaling policy), so a capacity-only RegisterScalableTarget call can never wipe out a scaling policy PutScalingPolicy configured earlier or vice versa (same clobber-bug class dynamodb's own gopherstack-1vv2 fixed for its two update paths). PutScalingPolicy pushes a TargetTrackingScaling policy's TargetValue/DisableScaleIn/ScaleInCooldown/ScaleOutCooldown the same way; StepScaling/PredictiveScaling policies against the dynamodb namespace are left local-only -- dynamodb's own UpdateTableReplicaAutoScaling models exactly one target-tracking-shaped ScalingPolicyUpdate per dimension and has no representation for the other two policy types. The reverse direction (a target/policy configured through dynamodb's own UpdateTableReplicaAutoScaling, never through this service) is handled by DescribeScalableTargets/DescribeScalingPolicies synthesizing a row from dynamodb's live state for any (resourceId, dimension) with no matching local row -- bounded scope, disclosed: when the caller gives explicit ResourceIds/ResourceId this covers both table- and index-level dimensions, but a filterless 'describe everything' call only probes table-level dimensions per table (via dynamodb's ListTables), not every index of every table. NOT wired, disclosed, matches real AWS (confirmed via Application Auto Scaling's docs and reports of the same behavior on a real account, not guessed): DeregisterScalableTarget does not clear the corresponding dynamodb-side settings, and this service has no hook on dynamodb table deletion -- real AWS does not automatically deregister/clean up a scalable target when its underlying resource is deleted either (deregistering, and any resulting cleanup, is documented as the caller's own responsibility); leaving both orphaned matches, rather than deviates from, real AWS. Tests: services/applicationautoscaling/dynamodb_bridge_test.go (real aws-sdk-go-v2 clients for both services, table-driven, t.Parallel(), covers push-through both dimensions, table-not-found ValidationException, the capacity/policy no-clobber case, both reverse-direction synthesis cases, and graceful no-op when the sibling isn't wired); cli_applicationautoscaling_dynamodb_wiring_test.go (root package, drives the real initializeServices composition root end to end, same shape as cli_dynamodb_kinesis_wiring_test.go). See services/dynamodb/PARITY.md's autoscaling family entry for the dynamodb-side half of this pass (a related, necessary fix to DescribeTableReplicaAutoScaling's Replicas list for plain, non-global tables). FIXED 2026-09-26 (gopherstack regression from this same pass, broke test/terraform/fixtures/s3tables-messaging-and-streaming.tf): RegisterScalableTarget/PutScalingPolicy against a PAY_PER_REQUEST table used to write straight through to dynamodb's UpdateTableReplicaAutoScaling, which correctly rejects on-demand tables but with its own ValidationException -- that error was leaking out through this service double-wrapped and carrying DynamoDB's namespace (`com.amazonaws.dynamodb.v20120810#ValidationException`), not this service's. Real AWS rejects the same case itself, before ever reaching DynamoDB: confirmed against a real account's error transcript in terraform-provider-aws#22784 (aws_appautoscaling_target against an on-demand DynamoDB table) -- `ValidationException: Validation failed for scalable target. Reason: PAY_PER_REQUEST table mode is not scalable.` Both RegisterScalableTarget (validateDynamoDBTargetExists) and PutScalingPolicy (pushDynamoDBTargetTrackingPolicy, re-checked in case billing mode changed after registration) now call dynamodb's DescribeTable and return this service's own ValidationException with that verbatim message when BillingModeSummary.BillingMode is PAY_PER_REQUEST, never reaching UpdateTableReplicaAutoScaling for such a table. Tests: TestRegisterScalableTarget_DynamoDB_PayPerRequestRejected/TestPutScalingPolicy_DynamoDB_PayPerRequestRejected in dynamodb_bridge_test.go."} gaps: [] items_still_open: - DescribeScalingActivities accepts IncludeNotScaledActivities (now threaded into the backend filter, and the response shape now has NotScaledReasons/Details fields) but it remains observably vacuous: gopherstack's mock backend never generates "not scaled" activities (no real metric evaluation loop exists to decide not-to-scale), so there is nothing to surface regardless of the flag's value. Verified vacuous, not a fabricated stub -- generating fake not-scaled events would be worse than reporting none. Re-confirmed this pass (gopherstack-cdxe): implementing this honestly would require a real metric-evaluation loop against real CloudWatch data, out of scope. diff --git a/services/applicationautoscaling/README.md b/services/applicationautoscaling/README.md index 3ccb792ced..f2843358fd 100644 --- a/services/applicationautoscaling/README.md +++ b/services/applicationautoscaling/README.md @@ -8,7 +8,7 @@ | Metric | Value | | --- | --- | | PARITY entries audited | 14 (14 ok) | -| Feature families | 3 (3 ok) | +| Feature families | 4 (4 ok) | | Known gaps | 4 | | Deferred items | 2 | | Resource leaks | clean | diff --git a/services/applicationautoscaling/cross_service.go b/services/applicationautoscaling/cross_service.go new file mode 100644 index 0000000000..7daa6246b1 --- /dev/null +++ b/services/applicationautoscaling/cross_service.go @@ -0,0 +1,33 @@ +package applicationautoscaling + +import ( + "github.com/blackbirdworks/gopherstack/pkgs/service" + + ddbbackend "github.com/blackbirdworks/gopherstack/services/dynamodb" +) + +// siblingServices is matched structurally against *CLI to avoid an import +// cycle; see services/grafana/cross_service.go for the reference pattern. +type siblingServices interface { + GetDynamoDBHandler() service.Registerable +} + +// SetAppConfig records ctx.Config so the DynamoDB backend can be resolved lazily. +func (b *InMemoryBackend) SetAppConfig(cfg any) { + b.appConfig = cfg +} + +// dynamoDBBackend returns the DynamoDB backend, if wired. +func (b *InMemoryBackend) dynamoDBBackend() (ddbbackend.StorageBackend, bool) { + s, ok := b.appConfig.(siblingServices) + if !ok { + return nil, false + } + + h, ok := s.GetDynamoDBHandler().(*ddbbackend.DynamoDBHandler) + if !ok || h == nil || h.Backend == nil { + return nil, false + } + + return h.Backend, true +} diff --git a/services/applicationautoscaling/dynamodb_bridge.go b/services/applicationautoscaling/dynamodb_bridge.go new file mode 100644 index 0000000000..51de3a959d --- /dev/null +++ b/services/applicationautoscaling/dynamodb_bridge.go @@ -0,0 +1,493 @@ +package applicationautoscaling + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + sdkddb "github.com/aws/aws-sdk-go-v2/service/dynamodb" + ddbtypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + + "github.com/blackbirdworks/gopherstack/pkgs/awsmeta" + ddbbackend "github.com/blackbirdworks/gopherstack/services/dynamodb" +) + +// dynamoDBOnDemandMessage is AAS's own on-demand rejection text, from a real-account +// transcript: github.com/hashicorp/terraform-provider-aws/issues/22784. +const dynamoDBOnDemandMessage = "Validation failed for scalable target. Reason: " + + "PAY_PER_REQUEST table mode is not scalable." + +// dynamoDBServiceNamespace routes a scalable target/scaling policy through this bridge. +const dynamoDBServiceNamespace = "dynamodb" + +// The four ScalableDimension values AWS models for DynamoDB. +const ( + dimTableRead = "dynamodb:table:ReadCapacityUnits" + dimTableWrite = "dynamodb:table:WriteCapacityUnits" + dimIndexRead = "dynamodb:index:ReadCapacityUnits" + dimIndexWrite = "dynamodb:index:WriteCapacityUnits" +) + +// ResourceId path segments: "table/" or "table//index/". +const ( + resourceIDTableSegment = "table" + resourceIDIndexSegment = "index" +) + +// dynamoDBTarget is the table/index/direction a ResourceId+ScalableDimension +// pair addresses -- the same selector dynamodb's own autoscaling.go uses internally. +type dynamoDBTarget struct { + tableName string + indexName string // empty for a table-level dimension + read bool // true for *ReadCapacityUnits, false for *WriteCapacityUnits +} + +// matched=false means scalableDimension isn't a DynamoDB dimension; callers +// skip the bridge rather than reject the request. +func parseDynamoDBTarget(resourceID, scalableDimension string) (dynamoDBTarget, bool, error) { + wantIndex, read, matched := classifyDynamoDBDimension(scalableDimension) + if !matched { + return dynamoDBTarget{}, false, nil + } + + tableName, indexName, ok := splitDynamoDBResourceID(resourceID) + if !ok || (indexName != "") != wantIndex { + return dynamoDBTarget{}, true, invalidDynamoDBResourceIDError(scalableDimension, wantIndex) + } + + return dynamoDBTarget{tableName: tableName, indexName: indexName, read: read}, true, nil +} + +// ok=false means id matches neither ResourceId shape. +func splitDynamoDBResourceID(id string) (string, string, bool) { + const tablePartCount = 2 + + const indexPartCount = 4 + + parts := strings.Split(id, "/") + + switch len(parts) { + case tablePartCount: + if parts[0] == resourceIDTableSegment && parts[1] != "" { + return parts[1], "", true + } + case indexPartCount: + if parts[0] == resourceIDTableSegment && parts[1] != "" && + parts[2] == resourceIDIndexSegment && parts[3] != "" { + return parts[1], parts[3], true + } + } + + return "", "", false +} + +func invalidDynamoDBResourceIDError(scalableDimension string, wantIndex bool) error { + if wantIndex { + return fmt.Errorf( + "%w: ResourceId must be table//index/ for ScalableDimension %s", + ErrValidation, scalableDimension, + ) + } + + return fmt.Errorf( + "%w: ResourceId must be table/ for ScalableDimension %s", + ErrValidation, scalableDimension, + ) +} + +// Returns (wantIndex, read, matched). +func classifyDynamoDBDimension(dimension string) (bool, bool, bool) { + switch dimension { + case dimTableRead: + return false, true, true + case dimTableWrite: + return false, false, true + case dimIndexRead: + return true, true, true + case dimIndexWrite: + return true, false, true + default: + return false, false, false + } +} + +// dynamoDBResourceID is the inverse of parseDynamoDBTarget. +func dynamoDBResourceID(target dynamoDBTarget) string { + if target.indexName == "" { + return "table/" + target.tableName + } + + return "table/" + target.tableName + "/index/" + target.indexName +} + +// dynamoDBDimension rebuilds the canonical ScalableDimension for target. +func dynamoDBDimension(target dynamoDBTarget) string { + switch { + case target.indexName == "" && target.read: + return dimTableRead + case target.indexName == "" && !target.read: + return dimTableWrite + case target.indexName != "" && target.read: + return dimIndexRead + default: + return dimIndexWrite + } +} + +// dynamoDBRequestContext carries this backend's own account/region so the +// sibling call resolves the same table/replica a same-region request would. +func (b *InMemoryBackend) dynamoDBRequestContext() context.Context { + return awsmeta.Set(context.Background(), &awsmeta.Metadata{Account: b.accountID, Region: b.region}) +} + +// registerDynamoDBScalableTarget validates the table exists, then pushes +// MinCapacity/MaxCapacity/RoleARN into DynamoDB's own autoscaling state. +func (b *InMemoryBackend) registerDynamoDBScalableTarget( + resourceID, scalableDimension string, minCapacity, maxCapacity *int32, roleARN string, +) error { + target, matched, parseErr := parseDynamoDBTarget(resourceID, scalableDimension) + if parseErr != nil { + return parseErr + } + + if !matched { + return nil + } + + if err := b.validateDynamoDBTargetExists(target, resourceID); err != nil { + return err + } + + if err := b.pushDynamoDBCapacity(target, minCapacity, maxCapacity, roleARN); err != nil { + return fmt.Errorf("%w: %s", ErrValidation, err.Error()) + } + + return nil +} + +// Existence verified against a real account: terraform-aws-modules/terraform-aws-dynamodb-table#15. +func (b *InMemoryBackend) validateDynamoDBTargetExists(target dynamoDBTarget, resourceID string) error { + ddb, ok := b.dynamoDBBackend() + if !ok { + return nil + } + + onDemand, exists := b.dynamoDBTableIsOnDemand(ddb, target.tableName) + if !exists { + return fmt.Errorf("%w: DynamoDB table does not exist: %s", ErrValidation, resourceID) + } + + if onDemand { + return fmt.Errorf("%w: %s", ErrValidation, dynamoDBOnDemandMessage) + } + + return nil +} + +// dynamoDBTableIsOnDemand reports PAY_PER_REQUEST billing; ok is false if the +// table can't be described. +func (b *InMemoryBackend) dynamoDBTableIsOnDemand(ddb ddbbackend.StorageBackend, tableName string) (bool, bool) { + out, err := ddb.DescribeTable(b.dynamoDBRequestContext(), &sdkddb.DescribeTableInput{ + TableName: aws.String(tableName), + }) + if err != nil { + return false, false + } + + return isPayPerRequestTable(out), true +} + +// isPayPerRequestTable reports on-demand billing; nil BillingModeSummary means PROVISIONED. +func isPayPerRequestTable(out *sdkddb.DescribeTableOutput) bool { + return out != nil && out.Table != nil && out.Table.BillingModeSummary != nil && + out.Table.BillingModeSummary.BillingMode == ddbtypes.BillingModePayPerRequest +} + +// ok=false means the sibling isn't wired or the table/replica/index can't be resolved. +func dynamoDBAutoScalingSettings( + ctx context.Context, ddb ddbbackend.StorageBackend, target dynamoDBTarget, region string, +) (*ddbtypes.AutoScalingSettingsDescription, bool) { + out, err := ddb.DescribeTableReplicaAutoScaling(ctx, &sdkddb.DescribeTableReplicaAutoScalingInput{ + TableName: aws.String(target.tableName), + }) + if err != nil || out.TableAutoScalingDescription == nil { + return nil, false + } + + for _, r := range out.TableAutoScalingDescription.Replicas { + if aws.ToString(r.RegionName) != region { + continue + } + + return replicaAutoScalingSettingsForTarget(r, target), true + } + + return nil, false +} + +func replicaAutoScalingSettingsForTarget( + r ddbtypes.ReplicaAutoScalingDescription, target dynamoDBTarget, +) *ddbtypes.AutoScalingSettingsDescription { + if target.indexName == "" { + if target.read { + return r.ReplicaProvisionedReadCapacityAutoScalingSettings + } + + return r.ReplicaProvisionedWriteCapacityAutoScalingSettings + } + + for _, g := range r.GlobalSecondaryIndexes { + if aws.ToString(g.IndexName) != target.indexName { + continue + } + + if target.read { + return g.ProvisionedReadCapacityAutoScalingSettings + } + + return g.ProvisionedWriteCapacityAutoScalingSettings + } + + return nil +} + +// dynamoDBAutoScalingUpdateInput wraps upd into the input shape matching +// target: table-level write, per-GSI write, per-replica read, or per-replica-per-GSI read. +func dynamoDBAutoScalingUpdateInput( + target dynamoDBTarget, region string, upd *ddbtypes.AutoScalingSettingsUpdate, +) *sdkddb.UpdateTableReplicaAutoScalingInput { + input := &sdkddb.UpdateTableReplicaAutoScalingInput{TableName: aws.String(target.tableName)} + + switch { + case target.indexName == "" && !target.read: + input.ProvisionedWriteCapacityAutoScalingUpdate = upd + case target.indexName != "" && !target.read: + input.GlobalSecondaryIndexUpdates = []ddbtypes.GlobalSecondaryIndexAutoScalingUpdate{ + {IndexName: aws.String(target.indexName), ProvisionedWriteCapacityAutoScalingUpdate: upd}, + } + case target.indexName == "" && target.read: + input.ReplicaUpdates = []ddbtypes.ReplicaAutoScalingUpdate{ + {RegionName: aws.String(region), ReplicaProvisionedReadCapacityAutoScalingUpdate: upd}, + } + default: // index + read + input.ReplicaUpdates = []ddbtypes.ReplicaAutoScalingUpdate{ + { + RegionName: aws.String(region), + ReplicaGlobalSecondaryIndexUpdates: []ddbtypes.ReplicaGlobalSecondaryIndexAutoScalingUpdate{ + {IndexName: aws.String(target.indexName), ProvisionedReadCapacityAutoScalingUpdate: upd}, + }, + }, + } + } + + return input +} + +// nil fields mean "leave unchanged"; an AutoScalingSettingsUpdate otherwise +// replaces the whole stored throughput, wiping omitted fields. +func carryForwardAutoScalingSettingsUpdate( + existing *ddbtypes.AutoScalingSettingsDescription, + minCapacity, maxCapacity *int64, + roleARN *string, +) *ddbtypes.AutoScalingSettingsUpdate { + upd := &ddbtypes.AutoScalingSettingsUpdate{ + MinimumUnits: minCapacity, + MaximumUnits: maxCapacity, + } + + if existing != nil { + if minCapacity == nil { + upd.MinimumUnits = existing.MinimumUnits + } + + if maxCapacity == nil { + upd.MaximumUnits = existing.MaximumUnits + } + + upd.AutoScalingRoleArn = existing.AutoScalingRoleArn + } + + if roleARN != nil && *roleARN != "" { + upd.AutoScalingRoleArn = roleARN + } + + if existing != nil { + upd.ScalingPolicyUpdate = carryForwardScalingPolicyUpdate(existing) + } + + return upd +} + +func carryForwardScalingPolicyUpdate( + existing *ddbtypes.AutoScalingSettingsDescription, +) *ddbtypes.AutoScalingPolicyUpdate { + if len(existing.ScalingPolicies) == 0 { + return nil + } + + p := existing.ScalingPolicies[0] + if p.TargetTrackingScalingPolicyConfiguration == nil { + return &ddbtypes.AutoScalingPolicyUpdate{PolicyName: p.PolicyName} + } + + t := p.TargetTrackingScalingPolicyConfiguration + + return &ddbtypes.AutoScalingPolicyUpdate{ + PolicyName: p.PolicyName, + TargetTrackingScalingPolicyConfiguration: &ddbtypes.AutoScalingTargetTrackingScalingPolicyConfigurationUpdate{ + TargetValue: t.TargetValue, + DisableScaleIn: t.DisableScaleIn, + ScaleInCooldown: t.ScaleInCooldown, + ScaleOutCooldown: t.ScaleOutCooldown, + }, + } +} + +func (b *InMemoryBackend) pushDynamoDBCapacity( + target dynamoDBTarget, minCapacity, maxCapacity *int32, roleARN string, +) error { + ddb, ok := b.dynamoDBBackend() + if !ok { + return nil + } + + ctx := b.dynamoDBRequestContext() + + existing, _ := dynamoDBAutoScalingSettings(ctx, ddb, target, b.region) + + var roleARNPtr *string + if roleARN != "" { + roleARNPtr = &roleARN + } + + upd := carryForwardAutoScalingSettingsUpdate(existing, toInt64Ptr(minCapacity), toInt64Ptr(maxCapacity), roleARNPtr) + + _, err := ddb.UpdateTableReplicaAutoScaling(ctx, dynamoDBAutoScalingUpdateInput(target, b.region, upd)) + + return err +} + +// DynamoDB models a single ScalingPolicyUpdate per dimension; StepScaling/ +// PredictiveScaling have no DynamoDB-side representation (see PARITY.md). +func (b *InMemoryBackend) pushDynamoDBTargetTrackingPolicy( + target dynamoDBTarget, policyName string, cfg map[string]any, +) error { + ddb, ok := b.dynamoDBBackend() + if !ok { + return nil + } + + // Billing mode can change after registration; re-check so DynamoDB's error never leaks. + if onDemand, exists := b.dynamoDBTableIsOnDemand(ddb, target.tableName); exists && onDemand { + return fmt.Errorf("%w: %s", ErrValidation, dynamoDBOnDemandMessage) + } + + ctx := b.dynamoDBRequestContext() + + existing, _ := dynamoDBAutoScalingSettings(ctx, ddb, target, b.region) + + upd := carryForwardAutoScalingSettingsUpdate(existing, nil, nil, nil) + upd.ScalingPolicyUpdate = &ddbtypes.AutoScalingPolicyUpdate{ + PolicyName: aws.String(policyName), + TargetTrackingScalingPolicyConfiguration: &ddbtypes.AutoScalingTargetTrackingScalingPolicyConfigurationUpdate{ + TargetValue: targetTrackingFloat(cfg, "TargetValue"), + DisableScaleIn: targetTrackingBool(cfg, "DisableScaleIn"), + ScaleInCooldown: targetTrackingInt32(cfg, "ScaleInCooldown"), + ScaleOutCooldown: targetTrackingInt32(cfg, "ScaleOutCooldown"), + }, + } + + _, err := ddb.UpdateTableReplicaAutoScaling(ctx, dynamoDBAutoScalingUpdateInput(target, b.region, upd)) + + return err +} + +// Only TargetTrackingScaling policies push into DynamoDB; see pushDynamoDBTargetTrackingPolicy. +func (b *InMemoryBackend) pushDynamoDBPolicyIfApplicable( + serviceNamespace, resourceID, scalableDimension, policyType, policyName string, + targetTrackingConfig map[string]any, +) error { + if serviceNamespace != dynamoDBServiceNamespace || policyType != policyTypeTargetTrackingScaling { + return nil + } + + if targetTrackingConfig == nil { + return nil + } + + target, matched, parseErr := parseDynamoDBTarget(resourceID, scalableDimension) + if parseErr != nil { + return parseErr + } + + if !matched { + return nil + } + + if err := b.pushDynamoDBTargetTrackingPolicy(target, policyName, targetTrackingConfig); err != nil { + if errors.Is(err, ErrValidation) { + return err + } + + return fmt.Errorf("%w: %s", ErrValidation, err.Error()) + } + + return nil +} + +func toInt64Ptr(v *int32) *int64 { + if v == nil { + return nil + } + + out := int64(*v) + + return &out +} + +// targetTrackingFloat/Bool/Int32 read a field from the passthrough config +// map; values may be float64 (decoded JSON) or a Go literal (tests). +func targetTrackingFloat(cfg map[string]any, key string) *float64 { + switch v := cfg[key].(type) { + case float64: + return &v + case float32: + f := float64(v) + + return &f + case int: + f := float64(v) + + return &f + default: + return nil + } +} + +func targetTrackingBool(cfg map[string]any, key string) *bool { + v, ok := cfg[key].(bool) + if !ok { + return nil + } + + return &v +} + +func targetTrackingInt32(cfg map[string]any, key string) *int32 { + switch v := cfg[key].(type) { + case float64: + i := int32(v) + + return &i + case int: + i := int32(v) //nolint:gosec // G115: cooldown seconds, never near int32 range + + return &i + case int32: + return &v + default: + return nil + } +} diff --git a/services/applicationautoscaling/dynamodb_bridge_describe.go b/services/applicationautoscaling/dynamodb_bridge_describe.go new file mode 100644 index 0000000000..cd24b2a52c --- /dev/null +++ b/services/applicationautoscaling/dynamodb_bridge_describe.go @@ -0,0 +1,226 @@ +package applicationautoscaling + +import ( + "context" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + sdkddb "github.com/aws/aws-sdk-go-v2/service/dynamodb" + ddbtypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + + "github.com/blackbirdworks/gopherstack/pkgs/arn" + ddbbackend "github.com/blackbirdworks/gopherstack/services/dynamodb" +) + +// dimensionsPerResource is the read+write dimension pair every DynamoDB +// resourceId (table or index) is probed for. +const dimensionsPerResource = 2 + +func capacityToInt32(v int64) int32 { + return int32(v) //nolint:gosec // G115: DynamoDB capacity units, never near int32 range +} + +// Synthesizes rows for settings configured directly via DynamoDB (known avoids +// duplicates); an empty f.ResourceIDs only probes table-level dimensions (PARITY.md). +func (b *InMemoryBackend) dynamodbSiblingScalableTargets( + f DescribeScalableTargetsFilter, known map[string]bool, +) []*ScalableTarget { + if f.ServiceNamespace != dynamoDBServiceNamespace { + return nil + } + + ddb, wired := b.dynamoDBBackend() + if !wired { + return nil + } + + ctx := b.dynamoDBRequestContext() + + out := make([]*ScalableTarget, 0) + + for _, target := range b.candidateDynamoDBTargets(ctx, ddb, f.ResourceIDs) { + dimension := dynamoDBDimension(target) + if f.ScalableDimension != "" && dimension != f.ScalableDimension { + continue + } + + resourceID := dynamoDBResourceID(target) + if known[scalableTargetKey(dynamoDBServiceNamespace, resourceID, dimension)] { + continue + } + + desc, ok := dynamoDBAutoScalingSettings(ctx, ddb, target, b.region) + if !ok || desc == nil || desc.MinimumUnits == nil || desc.MaximumUnits == nil { + continue + } + + out = append(out, b.synthesizeScalableTarget(target, desc)) + } + + return out +} + +// dynamodbSiblingScalingPolicies is the DescribeScalingPolicies analog of +// dynamodbSiblingScalableTargets; same scope/disclosure. +func (b *InMemoryBackend) dynamodbSiblingScalingPolicies( + f DescribeScalingPoliciesFilter, known map[string]bool, +) []*ScalingPolicy { + if f.ServiceNamespace != dynamoDBServiceNamespace { + return nil + } + + ddb, wired := b.dynamoDBBackend() + if !wired { + return nil + } + + ctx := b.dynamoDBRequestContext() + + var resourceIDs []string + if f.ResourceID != "" { + resourceIDs = []string{f.ResourceID} + } + + out := make([]*ScalingPolicy, 0) + + for _, target := range b.candidateDynamoDBTargets(ctx, ddb, resourceIDs) { + dimension := dynamoDBDimension(target) + if f.ScalableDimension != "" && dimension != f.ScalableDimension { + continue + } + + resourceID := dynamoDBResourceID(target) + + desc, ok := dynamoDBAutoScalingSettings(ctx, ddb, target, b.region) + if !ok || desc == nil || len(desc.ScalingPolicies) == 0 { + continue + } + + p := desc.ScalingPolicies[0] + policyName := aws.ToString(p.PolicyName) + + if known[policyNameKey(dynamoDBServiceNamespace, resourceID, dimension, policyName)] { + continue + } + + out = append(out, b.synthesizeScalingPolicy(target, policyName, p)) + } + + return out +} + +// Parses resourceIDs directly when given, else derives from the account's first ListTables page. +func (b *InMemoryBackend) candidateDynamoDBTargets( + ctx context.Context, ddb ddbbackend.StorageBackend, resourceIDs []string, +) []dynamoDBTarget { + if len(resourceIDs) > 0 { + return candidateTargetsForResourceIDs(resourceIDs) + } + + out, err := ddb.ListTables(ctx, &sdkddb.ListTablesInput{}) + if err != nil || out == nil { + return nil + } + + targets := make([]dynamoDBTarget, 0, len(out.TableNames)*dimensionsPerResource) + for _, name := range out.TableNames { + targets = append(targets, + dynamoDBTarget{tableName: name, read: true}, + dynamoDBTarget{tableName: name, read: false}, + ) + } + + return targets +} + +// A ResourceId matching neither shape is skipped, not rejected. +func candidateTargetsForResourceIDs(resourceIDs []string) []dynamoDBTarget { + targets := make([]dynamoDBTarget, 0, len(resourceIDs)*dimensionsPerResource) + + for _, id := range resourceIDs { + tableName, indexName, ok := splitDynamoDBResourceID(id) + if !ok { + continue + } + + targets = append(targets, + dynamoDBTarget{tableName: tableName, indexName: indexName, read: true}, + dynamoDBTarget{tableName: tableName, indexName: indexName, read: false}, + ) + } + + return targets +} + +// Stable (not random): there's no registration event here to mint a UUID +// from, and repeated Describe calls must return the same ARN. +func syntheticTargetARNSuffix(resourceID, dimension string) string { + r := strings.NewReplacer("/", "-", ":", "-") + + return r.Replace(resourceID) + "-" + r.Replace(dimension) +} + +func (b *InMemoryBackend) synthesizeScalableTarget( + target dynamoDBTarget, desc *ddbtypes.AutoScalingSettingsDescription, +) *ScalableTarget { + resourceID := dynamoDBResourceID(target) + dimension := dynamoDBDimension(target) + + return &ScalableTarget{ + ServiceNamespace: dynamoDBServiceNamespace, + ResourceID: resourceID, + ScalableDimension: dimension, + MinCapacity: capacityToInt32(aws.ToInt64(desc.MinimumUnits)), + MaxCapacity: capacityToInt32(aws.ToInt64(desc.MaximumUnits)), + RoleARN: aws.ToString(desc.AutoScalingRoleArn), + AccountID: b.accountID, + Region: b.region, + Tags: map[string]string{}, + ARN: arn.Build( + "application-autoscaling", b.region, b.accountID, + "scalable-target/"+syntheticTargetARNSuffix(resourceID, dimension), + ), + } +} + +// DynamoDB's own API has no metric-type field; the metric is implied by the dimension. +func dynamoDBPredefinedMetricType(target dynamoDBTarget) string { + if target.read { + return "DynamoDBReadCapacityUtilization" + } + + return "DynamoDBWriteCapacityUtilization" +} + +func (b *InMemoryBackend) synthesizeScalingPolicy( + target dynamoDBTarget, policyName string, p ddbtypes.AutoScalingPolicyDescription, +) *ScalingPolicy { + resourceID := dynamoDBResourceID(target) + dimension := dynamoDBDimension(target) + + cfg := map[string]any{ + "PredefinedMetricSpecification": map[string]any{ + "PredefinedMetricType": dynamoDBPredefinedMetricType(target), + }, + } + + if t := p.TargetTrackingScalingPolicyConfiguration; t != nil { + cfg["TargetValue"] = aws.ToFloat64(t.TargetValue) + cfg["DisableScaleIn"] = aws.ToBool(t.DisableScaleIn) + cfg["ScaleInCooldown"] = aws.ToInt32(t.ScaleInCooldown) + cfg["ScaleOutCooldown"] = aws.ToInt32(t.ScaleOutCooldown) + } + + return &ScalingPolicy{ + ServiceNamespace: dynamoDBServiceNamespace, + ResourceID: resourceID, + ScalableDimension: dimension, + PolicyName: policyName, + PolicyType: policyTypeTargetTrackingScaling, + TargetTrackingConfig: cfg, + ARN: arn.Build( + "autoscaling", b.region, b.accountID, + "scalingPolicy:"+syntheticTargetARNSuffix(resourceID, dimension)+":policyName/"+policyName, + ), + } +} diff --git a/services/applicationautoscaling/dynamodb_bridge_test.go b/services/applicationautoscaling/dynamodb_bridge_test.go new file mode 100644 index 0000000000..a5a46a51a4 --- /dev/null +++ b/services/applicationautoscaling/dynamodb_bridge_test.go @@ -0,0 +1,387 @@ +package applicationautoscaling_test + +import ( + "net/http/httptest" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awscfg "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + aassdk "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling" + aastypes "github.com/aws/aws-sdk-go-v2/service/applicationautoscaling/types" + ddbsdk "github.com/aws/aws-sdk-go-v2/service/dynamodb" + ddbtypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/applicationautoscaling" + ddbbackend "github.com/blackbirdworks/gopherstack/services/dynamodb" +) + +// fakeDynamoDBSibling satisfies siblingServices structurally, mirroring *CLI. +type fakeDynamoDBSibling struct { + ddbHandler service.Registerable +} + +func (f *fakeDynamoDBSibling) GetDynamoDBHandler() service.Registerable { return f.ddbHandler } + +// newTestDDBSDKClient stands up the real aws-sdk-go-v2 dynamodb client against +// an httptest server running h. +func newTestDDBSDKClient(t *testing.T, h *ddbbackend.DynamoDBHandler) *ddbsdk.Client { + t.Helper() + + e := echo.New() + registry := service.NewRegistry() + require.NoError(t, registry.Register(h)) + e.Use(service.NewServiceRouter(registry).RouteHandler()) + + srv := httptest.NewServer(e) + t.Cleanup(srv.Close) + + cfg, err := awscfg.LoadDefaultConfig( + t.Context(), + awscfg.WithRegion("us-east-1"), + awscfg.WithCredentialsProvider(credentials.NewStaticCredentialsProvider("test", "test", "")), + ) + require.NoError(t, err) + + return ddbsdk.NewFromConfig(cfg, func(o *ddbsdk.Options) { + o.BaseEndpoint = aws.String(srv.URL) + }) +} + +// newWiredBackends builds a DynamoDB and an ApplicationAutoScaling +// backend/client pair, wired together via SetAppConfig. +func newWiredBackends(t *testing.T) (*ddbsdk.Client, *aassdk.Client) { + t.Helper() + + ddbHandler := ddbbackend.NewHandler(ddbbackend.NewInMemoryDB()) + ddbClient := newTestDDBSDKClient(t, ddbHandler) + + aasBk := applicationautoscaling.NewInMemoryBackend("123456789012", "us-east-1") + aasBk.SetAppConfig(&fakeDynamoDBSibling{ddbHandler: ddbHandler}) + aasClient := newTestAASSDKClient(t, applicationautoscaling.NewHandler(aasBk)) + + return ddbClient, aasClient +} + +// createProvisionedTable creates a minimal PROVISIONED-billing table -- +// DynamoDB rejects autoscaling settings against a PAY_PER_REQUEST table. +func createProvisionedTable(t *testing.T, ddbClient *ddbsdk.Client, name string) { + t.Helper() + + _, err := ddbClient.CreateTable(t.Context(), &ddbsdk.CreateTableInput{ + TableName: aws.String(name), + KeySchema: []ddbtypes.KeySchemaElement{ + {AttributeName: aws.String("pk"), KeyType: ddbtypes.KeyTypeHash}, + }, + AttributeDefinitions: []ddbtypes.AttributeDefinition{ + {AttributeName: aws.String("pk"), AttributeType: ddbtypes.ScalarAttributeTypeS}, + }, + BillingMode: ddbtypes.BillingModeProvisioned, + ProvisionedThroughput: &ddbtypes.ProvisionedThroughput{ + ReadCapacityUnits: aws.Int64(5), + WriteCapacityUnits: aws.Int64(5), + }, + }) + require.NoError(t, err) +} + +// createOnDemandTable creates a minimal PAY_PER_REQUEST-billing table. +func createOnDemandTable(t *testing.T, ddbClient *ddbsdk.Client, name string) { + t.Helper() + + _, err := ddbClient.CreateTable(t.Context(), &ddbsdk.CreateTableInput{ + TableName: aws.String(name), + KeySchema: []ddbtypes.KeySchemaElement{ + {AttributeName: aws.String("pk"), KeyType: ddbtypes.KeyTypeHash}, + }, + AttributeDefinitions: []ddbtypes.AttributeDefinition{ + {AttributeName: aws.String("pk"), AttributeType: ddbtypes.ScalarAttributeTypeS}, + }, + BillingMode: ddbtypes.BillingModePayPerRequest, + }) + require.NoError(t, err) +} + +// On-demand tables get AAS's own ValidationException, not DynamoDB's wrapped error. +func TestRegisterScalableTarget_DynamoDB_PayPerRequestRejected(t *testing.T) { + t.Parallel() + + ddbClient, aasClient := newWiredBackends(t) + createOnDemandTable(t, ddbClient, "ondemand-table") + + _, err := aasClient.RegisterScalableTarget(t.Context(), &aassdk.RegisterScalableTargetInput{ + ServiceNamespace: aastypes.ServiceNamespaceDynamodb, + ResourceId: aws.String("table/ondemand-table"), + ScalableDimension: aastypes.ScalableDimensionDynamoDBTableWriteCapacityUnits, + MinCapacity: aws.Int32(5), + MaxCapacity: aws.Int32(500), + }) + require.Error(t, err) + + var vErr *aastypes.ValidationException + require.ErrorAs(t, err, &vErr) + assert.Contains(t, aws.ToString(vErr.Message), "PAY_PER_REQUEST table mode is not scalable") + assert.NotContains(t, err.Error(), "amazonaws.dynamodb", "must not leak DynamoDB's own error namespace") +} + +// A table switched to on-demand after registration is rejected at PutScalingPolicy. +func TestPutScalingPolicy_DynamoDB_PayPerRequestRejected(t *testing.T) { + t.Parallel() + + ddbClient, aasClient := newWiredBackends(t) + createProvisionedTable(t, ddbClient, "switched-table") + + ctx := t.Context() + + _, err := aasClient.RegisterScalableTarget(ctx, &aassdk.RegisterScalableTargetInput{ + ServiceNamespace: aastypes.ServiceNamespaceDynamodb, + ResourceId: aws.String("table/switched-table"), + ScalableDimension: aastypes.ScalableDimensionDynamoDBTableWriteCapacityUnits, + MinCapacity: aws.Int32(5), + MaxCapacity: aws.Int32(500), + }) + require.NoError(t, err) + + _, err = ddbClient.UpdateTable(ctx, &ddbsdk.UpdateTableInput{ + TableName: aws.String("switched-table"), + BillingMode: ddbtypes.BillingModePayPerRequest, + }) + require.NoError(t, err) + + _, err = aasClient.PutScalingPolicy(ctx, &aassdk.PutScalingPolicyInput{ + ServiceNamespace: aastypes.ServiceNamespaceDynamodb, + ResourceId: aws.String("table/switched-table"), + ScalableDimension: aastypes.ScalableDimensionDynamoDBTableWriteCapacityUnits, + PolicyName: aws.String("switched-policy"), + PolicyType: aastypes.PolicyTypeTargetTrackingScaling, + TargetTrackingScalingPolicyConfiguration: &aastypes.TargetTrackingScalingPolicyConfiguration{ + TargetValue: aws.Float64(70), + PredefinedMetricSpecification: &aastypes.PredefinedMetricSpecification{ + PredefinedMetricType: aastypes.MetricTypeDynamoDBWriteCapacityUtilization, + }, + }, + }) + require.Error(t, err) + + var vErr *aastypes.ValidationException + require.ErrorAs(t, err, &vErr) + assert.Contains(t, aws.ToString(vErr.Message), "PAY_PER_REQUEST table mode is not scalable") + assert.NotContains(t, err.Error(), "amazonaws.dynamodb", "must not leak DynamoDB's own error namespace") +} + +// RegisterScalableTarget(ns=dynamodb) must push capacity into DynamoDB's own +// autoscaling state, so DescribeTableReplicaAutoScaling agrees. +func TestRegisterScalableTarget_DynamoDB_ReflectsInDescribeTableReplicaAutoScaling(t *testing.T) { + t.Parallel() + + tests := []struct { + dimension aastypes.ScalableDimension + read bool + minCap int32 + maxCap int32 + }{ + {dimension: aastypes.ScalableDimensionDynamoDBTableWriteCapacityUnits, read: false, minCap: 5, maxCap: 500}, + {dimension: aastypes.ScalableDimensionDynamoDBTableReadCapacityUnits, read: true, minCap: 3, maxCap: 300}, + } + + for _, tt := range tests { + t.Run(string(tt.dimension), func(t *testing.T) { + t.Parallel() + + ddbClient, aasClient := newWiredBackends(t) + createProvisionedTable(t, ddbClient, "wire-table") + + _, err := aasClient.RegisterScalableTarget(t.Context(), &aassdk.RegisterScalableTargetInput{ + ServiceNamespace: aastypes.ServiceNamespaceDynamodb, + ResourceId: aws.String("table/wire-table"), + ScalableDimension: tt.dimension, + MinCapacity: aws.Int32(tt.minCap), + MaxCapacity: aws.Int32(tt.maxCap), + }) + require.NoError(t, err) + + desc, err := ddbClient.DescribeTableReplicaAutoScaling( + t.Context(), + &ddbsdk.DescribeTableReplicaAutoScalingInput{ + TableName: aws.String("wire-table"), + }, + ) + require.NoError(t, err) + require.Len(t, desc.TableAutoScalingDescription.Replicas, 1) + + replica := desc.TableAutoScalingDescription.Replicas[0] + + settings := replica.ReplicaProvisionedWriteCapacityAutoScalingSettings + if tt.read { + settings = replica.ReplicaProvisionedReadCapacityAutoScalingSettings + } + + require.NotNil(t, settings) + assert.Equal(t, int64(tt.minCap), aws.ToInt64(settings.MinimumUnits)) + assert.Equal(t, int64(tt.maxCap), aws.ToInt64(settings.MaximumUnits)) + }) + } +} + +func TestRegisterScalableTarget_DynamoDB_TableDoesNotExist(t *testing.T) { + t.Parallel() + + _, aasClient := newWiredBackends(t) + + _, err := aasClient.RegisterScalableTarget(t.Context(), &aassdk.RegisterScalableTargetInput{ + ServiceNamespace: aastypes.ServiceNamespaceDynamodb, + ResourceId: aws.String("table/no-such-table"), + ScalableDimension: aastypes.ScalableDimensionDynamoDBTableReadCapacityUnits, + MinCapacity: aws.Int32(1), + MaxCapacity: aws.Int32(10), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "DynamoDB table does not exist: table/no-such-table") + + var vErr *aastypes.ValidationException + require.ErrorAs(t, err, &vErr) +} + +// A capacity-only RegisterScalableTarget call must not wipe out a policy +// PutScalingPolicy configured earlier (same clobber-bug class as gopherstack-1vv2). +func TestRegisterScalableTarget_DynamoDB_DoesNotClobberScalingPolicy(t *testing.T) { + t.Parallel() + + ddbClient, aasClient := newWiredBackends(t) + createProvisionedTable(t, ddbClient, "carry-forward-table") + + ctx := t.Context() + + _, err := aasClient.RegisterScalableTarget(ctx, &aassdk.RegisterScalableTargetInput{ + ServiceNamespace: aastypes.ServiceNamespaceDynamodb, + ResourceId: aws.String("table/carry-forward-table"), + ScalableDimension: aastypes.ScalableDimensionDynamoDBTableWriteCapacityUnits, + MinCapacity: aws.Int32(5), + MaxCapacity: aws.Int32(500), + }) + require.NoError(t, err) + + _, err = aasClient.PutScalingPolicy(ctx, &aassdk.PutScalingPolicyInput{ + ServiceNamespace: aastypes.ServiceNamespaceDynamodb, + ResourceId: aws.String("table/carry-forward-table"), + ScalableDimension: aastypes.ScalableDimensionDynamoDBTableWriteCapacityUnits, + PolicyName: aws.String("carry-forward-policy"), + PolicyType: aastypes.PolicyTypeTargetTrackingScaling, + TargetTrackingScalingPolicyConfiguration: &aastypes.TargetTrackingScalingPolicyConfiguration{ + TargetValue: aws.Float64(70), + PredefinedMetricSpecification: &aastypes.PredefinedMetricSpecification{ + PredefinedMetricType: aastypes.MetricTypeDynamoDBWriteCapacityUtilization, + }, + }, + }) + require.NoError(t, err) + + _, err = aasClient.RegisterScalableTarget(ctx, &aassdk.RegisterScalableTargetInput{ + ServiceNamespace: aastypes.ServiceNamespaceDynamodb, + ResourceId: aws.String("table/carry-forward-table"), + ScalableDimension: aastypes.ScalableDimensionDynamoDBTableWriteCapacityUnits, + MinCapacity: aws.Int32(10), + MaxCapacity: aws.Int32(1000), + }) + require.NoError(t, err) + + desc, err := ddbClient.DescribeTableReplicaAutoScaling(ctx, &ddbsdk.DescribeTableReplicaAutoScalingInput{ + TableName: aws.String("carry-forward-table"), + }) + require.NoError(t, err) + require.Len(t, desc.TableAutoScalingDescription.Replicas, 1) + + settings := desc.TableAutoScalingDescription.Replicas[0].ReplicaProvisionedWriteCapacityAutoScalingSettings + require.NotNil(t, settings) + assert.Equal(t, int64(10), aws.ToInt64(settings.MinimumUnits)) + assert.Equal(t, int64(1000), aws.ToInt64(settings.MaximumUnits)) + require.Len(t, settings.ScalingPolicies, 1, "the earlier PutScalingPolicy call must survive") + assert.Equal(t, "carry-forward-policy", aws.ToString(settings.ScalingPolicies[0].PolicyName)) +} + +// The reverse direction: settings set via DynamoDB's own API must show up on +// DescribeScalableTargets. +func TestUpdateTableReplicaAutoScaling_DynamoDB_ReflectsInDescribeScalableTargets(t *testing.T) { + t.Parallel() + + ddbClient, aasClient := newWiredBackends(t) + createProvisionedTable(t, ddbClient, "vv-table") + + _, err := ddbClient.UpdateTableReplicaAutoScaling(t.Context(), &ddbsdk.UpdateTableReplicaAutoScalingInput{ + TableName: aws.String("vv-table"), + ProvisionedWriteCapacityAutoScalingUpdate: &ddbtypes.AutoScalingSettingsUpdate{ + MinimumUnits: aws.Int64(3), + MaximumUnits: aws.Int64(300), + }, + }) + require.NoError(t, err) + + out, err := aasClient.DescribeScalableTargets(t.Context(), &aassdk.DescribeScalableTargetsInput{ + ServiceNamespace: aastypes.ServiceNamespaceDynamodb, + ResourceIds: []string{"table/vv-table"}, + }) + require.NoError(t, err) + require.Len(t, out.ScalableTargets, 1) + + target := out.ScalableTargets[0] + assert.Equal(t, "table/vv-table", aws.ToString(target.ResourceId)) + assert.Equal(t, aastypes.ScalableDimensionDynamoDBTableWriteCapacityUnits, target.ScalableDimension) + assert.Equal(t, int32(3), aws.ToInt32(target.MinCapacity)) + assert.Equal(t, int32(300), aws.ToInt32(target.MaxCapacity)) +} + +func TestUpdateTableReplicaAutoScaling_DynamoDB_ReflectsInDescribeScalingPolicies(t *testing.T) { + t.Parallel() + + ddbClient, aasClient := newWiredBackends(t) + createProvisionedTable(t, ddbClient, "vv-policy-table") + + _, err := ddbClient.UpdateTableReplicaAutoScaling(t.Context(), &ddbsdk.UpdateTableReplicaAutoScalingInput{ + TableName: aws.String("vv-policy-table"), + ProvisionedWriteCapacityAutoScalingUpdate: &ddbtypes.AutoScalingSettingsUpdate{ + MinimumUnits: aws.Int64(1), + MaximumUnits: aws.Int64(100), + ScalingPolicyUpdate: &ddbtypes.AutoScalingPolicyUpdate{ + PolicyName: aws.String("native-policy"), + TargetTrackingScalingPolicyConfiguration: &ddbtypes.AutoScalingTargetTrackingScalingPolicyConfigurationUpdate{ + TargetValue: aws.Float64(65), + }, + }, + }, + }) + require.NoError(t, err) + + out, err := aasClient.DescribeScalingPolicies(t.Context(), &aassdk.DescribeScalingPoliciesInput{ + ServiceNamespace: aastypes.ServiceNamespaceDynamodb, + ResourceId: aws.String("table/vv-policy-table"), + ScalableDimension: aastypes.ScalableDimensionDynamoDBTableWriteCapacityUnits, + }) + require.NoError(t, err) + require.Len(t, out.ScalingPolicies, 1) + + p := out.ScalingPolicies[0] + assert.Equal(t, "native-policy", aws.ToString(p.PolicyName)) + require.NotNil(t, p.TargetTrackingScalingPolicyConfiguration) + assert.InDelta(t, 65.0, aws.ToFloat64(p.TargetTrackingScalingPolicyConfiguration.TargetValue), 0.001) +} + +// The bridge degrades gracefully when the DynamoDB sibling isn't wired. +func TestRegisterScalableTarget_DynamoDB_NoSiblingWired(t *testing.T) { + t.Parallel() + + aasBk := applicationautoscaling.NewInMemoryBackend("123456789012", "us-east-1") + aasClient := newTestAASSDKClient(t, applicationautoscaling.NewHandler(aasBk)) + + _, err := aasClient.RegisterScalableTarget(t.Context(), &aassdk.RegisterScalableTargetInput{ + ServiceNamespace: aastypes.ServiceNamespaceDynamodb, + ResourceId: aws.String("table/unwired-table"), + ScalableDimension: aastypes.ScalableDimensionDynamoDBTableReadCapacityUnits, + MinCapacity: aws.Int32(1), + MaxCapacity: aws.Int32(10), + }) + require.NoError(t, err) +} diff --git a/services/applicationautoscaling/provider.go b/services/applicationautoscaling/provider.go index 34a8cb24c9..780be63636 100644 --- a/services/applicationautoscaling/provider.go +++ b/services/applicationautoscaling/provider.go @@ -17,6 +17,7 @@ func (p *Provider) Init(ctx *service.AppContext) (service.Registerable, error) { accountID, region := service.AccountRegionOrDefault(ctx) backend := NewInMemoryBackend(accountID, region) + backend.SetAppConfig(ctx.Config) handler := NewHandler(backend) return handler, nil diff --git a/services/applicationautoscaling/scalable_targets.go b/services/applicationautoscaling/scalable_targets.go index 9e27073d2c..63de5bacb6 100644 --- a/services/applicationautoscaling/scalable_targets.go +++ b/services/applicationautoscaling/scalable_targets.go @@ -58,32 +58,21 @@ func (b *InMemoryBackend) scalableTargetsForNamespaceLocked(serviceNamespace str return count } -// RegisterScalableTarget upserts a scalable target (creates or updates). -// minCapacity and maxCapacity are *int32, not int32: real AWS's -// RegisterScalableTargetInput models both as optional pointers, required only -// "when registering a new scalable target" (api_op_RegisterScalableTarget.go), -// and the operation doc states "Any parameters that you don't specify are not -// changed by this update request." A plain int32 could not distinguish -// "caller omitted MinCapacity" from "caller explicitly set MinCapacity to 0" -// (0 is a documented valid capacity for several namespaces), so omitting it on -// an update would silently reset capacity to 0 -- see updateExistingTarget. -func (b *InMemoryBackend) RegisterScalableTarget( - serviceNamespace, resourceID, scalableDimension string, - minCapacity, maxCapacity *int32, - tags map[string]string, - roleARN string, - suspendedState *SuspendedState, -) (*ScalableTarget, error) { +// validateRegisterScalableTargetBasics checks the fields required regardless +// of namespace and the tag-count quota. +func validateRegisterScalableTargetBasics( + serviceNamespace, resourceID, scalableDimension string, tags map[string]string, +) error { if serviceNamespace == "" { - return nil, fmt.Errorf("%w: ServiceNamespace is required", ErrValidation) + return fmt.Errorf("%w: ServiceNamespace is required", ErrValidation) } if resourceID == "" { - return nil, fmt.Errorf("%w: ResourceId is required", ErrValidation) + return fmt.Errorf("%w: ResourceId is required", ErrValidation) } if scalableDimension == "" { - return nil, fmt.Errorf("%w: ScalableDimension is required", ErrValidation) + return fmt.Errorf("%w: ScalableDimension is required", ErrValidation) } // RegisterScalableTarget's modeled error set has LimitExceededException @@ -91,13 +80,44 @@ func (b *InMemoryBackend) RegisterScalableTarget( // ErrTooManyTags's doc comment), so an over-limit tag count here is // reported as LimitExceededException. if len(tags) > maxTagsPerResource { - return nil, fmt.Errorf( + return fmt.Errorf( "%w: too many tags; maximum allowed is %d", ErrLimitExceeded, maxTagsPerResource, ) } + return nil +} + +// RegisterScalableTarget upserts a scalable target (creates or updates). +// minCapacity and maxCapacity are *int32, not int32: real AWS's +// RegisterScalableTargetInput models both as optional pointers, required only +// "when registering a new scalable target" (api_op_RegisterScalableTarget.go), +// and the operation doc states "Any parameters that you don't specify are not +// changed by this update request." A plain int32 could not distinguish +// "caller omitted MinCapacity" from "caller explicitly set MinCapacity to 0" +// (0 is a documented valid capacity for several namespaces), so omitting it on +// an update would silently reset capacity to 0 -- see updateExistingTarget. +func (b *InMemoryBackend) RegisterScalableTarget( + serviceNamespace, resourceID, scalableDimension string, + minCapacity, maxCapacity *int32, + tags map[string]string, + roleARN string, + suspendedState *SuspendedState, +) (*ScalableTarget, error) { + if err := validateRegisterScalableTargetBasics(serviceNamespace, resourceID, scalableDimension, tags); err != nil { + return nil, err + } + + if serviceNamespace == dynamoDBServiceNamespace { + if err := b.registerDynamoDBScalableTarget( + resourceID, scalableDimension, minCapacity, maxCapacity, roleARN, + ); err != nil { + return nil, err + } + } + b.mu.Lock("RegisterScalableTarget") defer b.mu.Unlock() @@ -327,7 +347,6 @@ type DescribeScalableTargetsFilter struct { // Returns ErrInvalidNextToken if f.NextToken fails to decode. func (b *InMemoryBackend) DescribeScalableTargets(f DescribeScalableTargetsFilter) ([]*ScalableTarget, string, error) { b.mu.RLock("DescribeScalableTargets") - defer b.mu.RUnlock() var idSet map[string]bool if len(f.ResourceIDs) > 0 { @@ -338,7 +357,11 @@ func (b *InMemoryBackend) DescribeScalableTargets(f DescribeScalableTargetsFilte } list := make([]*ScalableTarget, 0, b.scalableTargets.Len()) + known := make(map[string]bool, b.scalableTargets.Len()) + for _, t := range b.scalableTargets.All() { + known[scalableTargetKey(t.ServiceNamespace, t.ResourceID, t.ScalableDimension)] = true + if f.ServiceNamespace != "" && t.ServiceNamespace != f.ServiceNamespace { continue } @@ -357,6 +380,11 @@ func (b *InMemoryBackend) DescribeScalableTargets(f DescribeScalableTargetsFilte list = append(list, &cp) } + b.mu.RUnlock() + + // A target set via DynamoDB's own API must show up here too. + list = append(list, b.dynamodbSiblingScalableTargets(f, known)...) + return paginate(list, f.MaxResults, f.NextToken, func(t *ScalableTarget) string { return t.ResourceID + "|" + t.ScalableDimension }) diff --git a/services/applicationautoscaling/scaling_policies.go b/services/applicationautoscaling/scaling_policies.go index 931ffe69ab..49271ae878 100644 --- a/services/applicationautoscaling/scaling_policies.go +++ b/services/applicationautoscaling/scaling_policies.go @@ -71,37 +71,51 @@ func stepAdjustmentCount(stepScalingConfig map[string]any) int { return len(list) } -// PutScalingPolicy upserts a scaling policy (update if policyName matches for resource, create otherwise). -func (b *InMemoryBackend) PutScalingPolicy( +// validatePutScalingPolicyBasics checks the fields required regardless of +// policy type. +func validatePutScalingPolicyBasics( serviceNamespace, resourceID, scalableDimension, policyName, policyType string, - targetTrackingConfig, stepScalingConfig, predictiveScalingConfig map[string]any, -) (*ScalingPolicy, error) { +) error { if serviceNamespace == "" { - return nil, fmt.Errorf("%w: ServiceNamespace is required", ErrValidation) + return fmt.Errorf("%w: ServiceNamespace is required", ErrValidation) } if resourceID == "" { - return nil, fmt.Errorf("%w: ResourceId is required", ErrValidation) + return fmt.Errorf("%w: ResourceId is required", ErrValidation) } if scalableDimension == "" { - return nil, fmt.Errorf("%w: ScalableDimension is required", ErrValidation) + return fmt.Errorf("%w: ScalableDimension is required", ErrValidation) } if policyName == "" { - return nil, fmt.Errorf("%w: PolicyName is required", ErrValidation) + return fmt.Errorf("%w: PolicyName is required", ErrValidation) } - // Validate PolicyType if provided; do not default yet — defaulting only - // applies when creating a brand-new policy (see below). + // Validate PolicyType if provided; do not default yet -- defaulting only + // applies when creating a brand-new policy (see PutScalingPolicy). if policyType != "" && !isValidPolicyType(policyType) { - return nil, fmt.Errorf( + return fmt.Errorf( "%w: invalid PolicyType %q; must be one of StepScaling, TargetTrackingScaling, PredictiveScaling", ErrValidation, policyType, ) } + return nil +} + +// PutScalingPolicy upserts a scaling policy (update if policyName matches for resource, create otherwise). +func (b *InMemoryBackend) PutScalingPolicy( + serviceNamespace, resourceID, scalableDimension, policyName, policyType string, + targetTrackingConfig, stepScalingConfig, predictiveScalingConfig map[string]any, +) (*ScalingPolicy, error) { + if err := validatePutScalingPolicyBasics( + serviceNamespace, resourceID, scalableDimension, policyName, policyType, + ); err != nil { + return nil, err + } + if stepAdjustmentCount(stepScalingConfig) > maxStepAdjustmentsPerPolicy { return nil, fmt.Errorf( "%w: too many step adjustments; maximum allowed is %d", @@ -135,6 +149,12 @@ func (b *InMemoryBackend) PutScalingPolicy( p.PolicyType = policyType } + if err := b.pushDynamoDBPolicyIfApplicable( + serviceNamespace, resourceID, scalableDimension, p.PolicyType, policyName, targetTrackingConfig, + ); err != nil { + return nil, err + } + p.TargetTrackingConfig = maps.Clone(targetTrackingConfig) p.StepScalingConfig = maps.Clone(stepScalingConfig) p.PredictiveScalingConfig = maps.Clone(predictiveScalingConfig) @@ -161,6 +181,12 @@ func (b *InMemoryBackend) PutScalingPolicy( policyType = policyTypeStepScaling } + if err := b.pushDynamoDBPolicyIfApplicable( + serviceNamespace, resourceID, scalableDimension, policyType, policyName, targetTrackingConfig, + ); err != nil { + return nil, err + } + // Real AWS policy ARNs separate the policyName segment from the // resource/namespace/resourceId segment with a colon, not a slash: // scalingPolicy:{uuid}:resource/{namespace}/{resourceId}:policyName/{name}. @@ -305,17 +331,25 @@ func policyMatchesFilter(p *ScalingPolicy, f DescribeScalingPoliciesFilter, name // Returns ErrInvalidNextToken if f.NextToken fails to decode. func (b *InMemoryBackend) DescribeScalingPolicies(f DescribeScalingPoliciesFilter) ([]*ScalingPolicy, string, error) { b.mu.RLock("DescribeScalingPolicies") - defer b.mu.RUnlock() nameSet := buildStringSet(f.PolicyNames) list := make([]*ScalingPolicy, 0, b.scalingPolicies.Len()) + known := make(map[string]bool, b.scalingPolicies.Len()) + for _, p := range b.scalingPolicies.All() { + known[policyNameKey(p.ServiceNamespace, p.ResourceID, p.ScalableDimension, p.PolicyName)] = true + if policyMatchesFilter(p, f, nameSet) { list = append(list, cloneScalingPolicy(p)) } } + b.mu.RUnlock() + + // A policy set via DynamoDB's own API must show up here too. + list = append(list, b.dynamodbSiblingScalingPolicies(f, known)...) + return paginate(list, f.MaxResults, f.NextToken, func(p *ScalingPolicy) string { return p.ARN }) diff --git a/services/applicationautoscaling/store.go b/services/applicationautoscaling/store.go index eb376796f8..26e249a668 100644 --- a/services/applicationautoscaling/store.go +++ b/services/applicationautoscaling/store.go @@ -48,6 +48,9 @@ type InMemoryBackend struct { mu *lockmetrics.RWMutex accountID string region string + // appConfig is the service.AppContext.Config value from Provider.Init, + // used to reach the DynamoDB backend lazily -- see cross_service.go. + appConfig any // scalingActivities is append-order-sensitive: DescribeScalingActivities // returns entries most-recent-first via slices.Backward over this exact // slice. store.Table has no defined insertion order (see pkgs/store's diff --git a/services/appsync/events.go b/services/appsync/events.go index 73f9b05a30..021d38a630 100644 --- a/services/appsync/events.go +++ b/services/appsync/events.go @@ -2,6 +2,7 @@ package appsync import ( "fmt" + "maps" "slices" "strings" @@ -37,6 +38,7 @@ func (b *InMemoryBackend) CreateAPI( b.eventAPIs.Put(api) cp := *api + cp.Tags = maps.Clone(api.Tags) return &cp, nil } @@ -52,6 +54,7 @@ func (b *InMemoryBackend) GetAPI(apiID string) (*API, error) { } cp := *api + cp.Tags = maps.Clone(api.Tags) return &cp, nil } @@ -66,6 +69,7 @@ func (b *InMemoryBackend) ListAPIs() ([]*API, error) { for _, api := range apis { cp := *api + cp.Tags = maps.Clone(api.Tags) out = append(out, &cp) } @@ -117,6 +121,7 @@ func (b *InMemoryBackend) UpdateAPI(apiID, name, ownerContact string, eventConfi } cp := *api + cp.Tags = maps.Clone(api.Tags) return &cp, nil } diff --git a/services/athena/PARITY.md b/services/athena/PARITY.md index 6f933d419f..3ec2ef2036 100644 --- a/services/athena/PARITY.md +++ b/services/athena/PARITY.md @@ -20,7 +20,7 @@ ops: BatchGetPreparedStatement: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED — request field was StatementNames, real wire is PreparedStatementNames; response field was UnprocessedStatementNames, real wire is UnprocessedPreparedStatementNames. Op was silently non-functional for real SDK clients (request always parsed as an empty name list)."} GetSessionEndpoint: {wire: ok, errors: ok, state: ok, persist: n/a, note: "FIXED — response was {SessionEndpoint: url}; real shape is {EndpointUrl, AuthToken, AuthTokenExpirationTime} (all three required). Client previously got a fully empty result."} CreatePresignedNotebookUrl: {wire: ok, errors: ok, state: ok, persist: n/a, note: "FIXED — response was {NotebookSessionUrl: url}; real shape is {NotebookUrl, AuthToken, AuthTokenExpirationTime} (all three required). Same class of bug as GetSessionEndpoint; both now share backend.newSessionAuthToken()."} - GetResourceDashboard: {wire: ok, errors: ok, state: ok, persist: n/a, note: "FIXED — was a disguised no-op ignoring the required ResourceARN input and returning {ResourceDashboard: {}}; real shape is {Url: string}. Now validates ResourceARN is non-empty (InvalidRequestException otherwise) and returns a synthesized dashboard URL."} + GetResourceDashboard: {wire: ok, errors: ok, state: ok, persist: n/a, note: "FIXED — was a disguised no-op ignoring the required ResourceARN input and returning {ResourceDashboard: {}}; real shape is {Url: string}. Now validates ResourceARN is non-empty (InvalidRequestException otherwise) and returns a synthesized dashboard URL. FIXED 2026-09-26 (gopherstack parity sweep): the prior fix still fabricated a dashboard URL for a session that does not exist -- only the empty-string case was rejected. Real GetResourceDashboard declares ResourceNotFoundException (api_op_GetResourceDashboard.go); now looks up the session and returns that error when it is not found. See wire_get_resource_dashboard_test.go."} StartQueryExecution: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED (gopherstack-zgfq) — ResultConfiguration.OutputLocation and EncryptionConfiguration were validated, stored, and echoed back, but no S3 object was ever written, so a client that ran a query and then fetched the result file from OutputLocation found nothing. Added athena.S3Storer + SetS3Backend, wired in cli.go's wireAthenaS3 from services/s3's real PutObject (no adapter needed -- s3.InMemoryBackend.PutObject already matches the interface). On a succeeded execution, writes an object to \"/.csv\": DISCLOSED APPROXIMATION, not a verified wire shape -- the pinned SDK (types.ResultConfiguration.OutputLocation doc) states only that results are stored under that S3 location, and documents neither an object key nor a file format. The .csv body is a plain header-row-then-rows encoding/csv dump of the query's result columns; SSE_S3/SSE_KMS map to the object's ServerSideEncryption/SSEKMSKeyId, CSE_KMS (client-side) is accepted but not actually encrypted (no KMS simulation exists to encrypt against). When S3 is unwired (every test that constructs the backend directly), writeResultObject is a no-op and StartQueryExecution's existing store/echo behavior is unchanged."} StopQueryExecution: {wire: ok, errors: ok, state: ok, persist: ok} GetQueryExecution: {wire: ok, errors: ok, state: ok, persist: ok, note: "Query lifecycle is synchronous (QUEUED/RUNNING never observed) — StartQueryExecution runs the statement inline and stores a terminal SUCCEEDED/FAILED state before returning, so SDK poll loops never hang."} @@ -58,6 +58,16 @@ leaks: {status: clean, note: "janitor uses pkgs/worker.Group with proper ctx.Don ## Notes +### 2026-09-26 parity sweep: GetResourceDashboard fabricated a URL for a nonexistent session + +The prior fix (see op row) validated ResourceARN was non-empty but never checked +the referenced session actually existed, so any garbage ARN got back a fake +dashboard URL instead of the SDK-declared ResourceNotFoundException +(`api_op_GetResourceDashboard.go`). Fixed by looking the session up in +`b.sessions` before synthesizing the URL. `wire_get_resource_dashboard_test.go` +proves both the not-found and found-session paths with a real +aws-sdk-go-v2/service/athena client. + ### 2026-09-23 lakeformation-appsync-neptune-and-athena terraform coverage `aws_athena_database`'s real create flow (StartQueryExecution "create database" DDL, then GetDatabase) failed: execCreateDatabase/execDropDatabase always wrote to Athena's own simulated `b.databases` map, never to the wired Glue backend, while GetDatabase/ListDatabases route a GLUE-type catalog (AwsDataCatalog is one by default) straight to Glue -- so a DDL-created database was invisible. Fixed: both DDL paths now check `isGlueBacked` and call through the (now read+write) `GlueMetadataSource` interface. diff --git a/services/athena/handler_sessions_test.go b/services/athena/handler_sessions_test.go index b0e3388ef5..4d684ae137 100644 --- a/services/athena/handler_sessions_test.go +++ b/services/athena/handler_sessions_test.go @@ -2,6 +2,7 @@ package athena_test import ( "encoding/json" + "fmt" "net/http" "strings" "testing" @@ -568,20 +569,29 @@ func TestHandler_GetResourceDashboard(t *testing.T) { t.Parallel() tests := []struct { - name string - body string - wantStatus int + name string + body string + wantBodyContains string + wantStatus int + createSession bool }{ { - name: "success", - body: `{"ResourceARN":"arn:aws:athena:us-east-1:000000000000:session/sess-1"}`, - wantStatus: http.StatusOK, + name: "success", + body: `{"ResourceARN":"arn:aws:athena:us-east-1:000000000000:session/%s"}`, + createSession: true, + wantStatus: http.StatusOK, }, { name: "missing_resource_arn_rejected", body: `{}`, wantStatus: http.StatusBadRequest, }, + { + name: "unknown_session_not_found", + body: `{"ResourceARN":"arn:aws:athena:us-east-1:000000000000:session/does-not-exist"}`, + wantStatus: http.StatusBadRequest, + wantBodyContains: "ResourceNotFoundException", + }, } for _, tt := range tests { @@ -589,9 +599,19 @@ func TestHandler_GetResourceDashboard(t *testing.T) { t.Parallel() h := newTestHandler(t) - rec := doRequest(t, h, "GetResourceDashboard", tt.body) + + body := tt.body + if tt.createSession { + body = fmt.Sprintf(body, startSession(t, h)) + } + + rec := doRequest(t, h, "GetResourceDashboard", body) assert.Equal(t, tt.wantStatus, rec.Code) + if tt.wantBodyContains != "" { + assert.Contains(t, rec.Body.String(), tt.wantBodyContains) + } + if tt.wantStatus == http.StatusOK { url := jsonField(t, rec.Body.Bytes(), "Url") assert.Contains(t, url, "athena.") diff --git a/services/athena/sessions.go b/services/athena/sessions.go index 7cc0f1b960..72c8218cc8 100644 --- a/services/athena/sessions.go +++ b/services/athena/sessions.go @@ -311,13 +311,29 @@ func (b *InMemoryBackend) ListApplicationDPUSizes() []ApplicationDPUSizes { } } -// GetResourceDashboard returns the Live UI/Persistence UI dashboard URL for a -// resource (session) ARN, matching the real GetResourceDashboard response's -// single required "Url" field. +// GetResourceDashboard returns a session's dashboard URL. Real AWS declares +// ResourceNotFoundException for an unknown session (api_op_GetResourceDashboard.go). func (b *InMemoryBackend) GetResourceDashboard(resourceARN string) (string, error) { if resourceARN == "" { return "", fmt.Errorf("%w: ResourceARN is required", ErrValidation) } + sessionID := resourceARN + if kind, id, ok := resourceKindFromARN(resourceARN); ok { + if kind != "session" { + return "", fmt.Errorf("%w: unsupported resource kind %q", ErrResourceNotFound, kind) + } + + sessionID = id + } + + b.mu.RLock("GetResourceDashboard") + _, ok := b.sessions.Get(sessionID) + b.mu.RUnlock() + + if !ok { + return "", fmt.Errorf("%w: session %q not found", ErrResourceNotFound, resourceARN) + } + return fmt.Sprintf("https://athena.%s.amazonaws.com/dashboards/%s", b.region, randomID()), nil } diff --git a/services/athena/wire_get_resource_dashboard_test.go b/services/athena/wire_get_resource_dashboard_test.go new file mode 100644 index 0000000000..3e6212336a --- /dev/null +++ b/services/athena/wire_get_resource_dashboard_test.go @@ -0,0 +1,62 @@ +package athena_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + athenasdk "github.com/aws/aws-sdk-go-v2/service/athena" + "github.com/aws/aws-sdk-go-v2/service/athena/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/config" + "github.com/blackbirdworks/gopherstack/services/athena" +) + +// TestGetResourceDashboard_UnknownSession verifies an unknown ResourceARN +// returns ResourceNotFoundException instead of a fabricated dashboard URL. +func TestGetResourceDashboard_UnknownSession(t *testing.T) { + t.Parallel() + + b := athena.NewInMemoryBackend(config.DefaultAccountID, config.DefaultRegion) + h := athena.NewHandler(b) + client := newTestAthenaClient(t, h) + + _, err := client.GetResourceDashboard(t.Context(), &athenasdk.GetResourceDashboardInput{ + ResourceARN: aws.String("does-not-exist"), + }) + require.Error(t, err) + + var apiErr *types.ResourceNotFoundException + require.ErrorAs(t, err, &apiErr, "expected ResourceNotFoundException, got %v", err) + + var genericErr *smithy.GenericAPIError + assert.NotErrorAs(t, err, &genericErr, "must not fall back to an untyped error") +} + +// TestGetResourceDashboard_ExistingSession verifies the happy path still +// returns a dashboard URL for a session that actually exists. +func TestGetResourceDashboard_ExistingSession(t *testing.T) { + t.Parallel() + + b := athena.NewInMemoryBackend(config.DefaultAccountID, config.DefaultRegion) + h := athena.NewHandler(b) + client := newTestAthenaClient(t, h) + + require.NoError(t, b.CreateWorkGroup("wg", "", "ENABLED", athena.WorkGroupConfiguration{}, nil)) + sessionID, _, err := b.StartSession( + "wg", "", "", + athena.EngineConfiguration{}, + athena.SessionConfiguration{}, + athena.MonitoringConfiguration{}, + "", + ) + require.NoError(t, err) + + out, err := client.GetResourceDashboard(t.Context(), &athenasdk.GetResourceDashboardInput{ + ResourceARN: aws.String(sessionID), + }) + require.NoError(t, err) + assert.NotEmpty(t, aws.ToString(out.Url)) +} diff --git a/services/autoscaling/auto_scaling_groups.go b/services/autoscaling/auto_scaling_groups.go index 3223eaf859..3ab415fe3a 100644 --- a/services/autoscaling/auto_scaling_groups.go +++ b/services/autoscaling/auto_scaling_groups.go @@ -2,6 +2,7 @@ package autoscaling import ( "fmt" + "slices" "strings" "time" @@ -11,6 +12,13 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/store" ) +// cloneGroupMutableSlices deep-copies Instances and Tags before a caller sees them. +// Other locked methods mutate their elements in place, so a shallow "cp := *g" would share backing arrays. +func cloneGroupMutableSlices(g *AutoScalingGroup) { + g.Instances = slices.Clone(g.Instances) + g.Tags = slices.Clone(g.Tags) +} + // lcInstanceType returns the InstanceType from the named launch configuration, or // "t2.micro" if the launch configuration is not found (preserving previous default). func lcInstanceType(lcs *store.Table[LaunchConfiguration], lcName string) string { @@ -201,6 +209,7 @@ func (b *InMemoryBackend) CreateAutoScalingGroup(input CreateAutoScalingGroupInp ) cp := *group + cloneGroupMutableSlices(&cp) return &cp, nil } @@ -220,6 +229,10 @@ func (b *InMemoryBackend) DescribeAutoScalingGroups(names []string, filters []Ta groups := describeByNames(b.groups, names, func(a, c *AutoScalingGroup) bool { return a.AutoScalingGroupName < c.AutoScalingGroupName }) + for i := range groups { + cloneGroupMutableSlices(&groups[i]) + } + if len(filters) == 0 { return groups, nil } @@ -562,6 +575,7 @@ func (b *InMemoryBackend) UpdateAutoScalingGroup(input UpdateAutoScalingGroupInp } cp := *g + cloneGroupMutableSlices(&cp) return &cp, nil } diff --git a/services/autoscaling/describe_race_test.go b/services/autoscaling/describe_race_test.go new file mode 100644 index 0000000000..ee103aeb9c --- /dev/null +++ b/services/autoscaling/describe_race_test.go @@ -0,0 +1,105 @@ +package autoscaling_test + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/autoscaling" +) + +// TestInMemoryBackend_DescribeAutoScalingGroups_RacesWithMutation guards cloneGroupMutableSlices. +// Each case pairs a field with the in-place mutator that used to share its backing array with a live group. +func TestInMemoryBackend_DescribeAutoScalingGroups_RacesWithMutation(t *testing.T) { + t.Parallel() + + tests := []struct { + mutate func(b *autoscaling.InMemoryBackend, instanceIDs []string, i int) + read func(g autoscaling.AutoScalingGroup) + name string + }{ + { + name: "instances", + mutate: func(b *autoscaling.InMemoryBackend, instanceIDs []string, i int) { + _ = b.SetInstanceProtection("race-group", instanceIDs, i%2 == 0) + }, + read: func(g autoscaling.AutoScalingGroup) { + for _, inst := range g.Instances { + _ = inst.ProtectedFromScaleIn + } + }, + }, + { + name: "tags", + mutate: func(b *autoscaling.InMemoryBackend, _ []string, _ int) { + _ = b.CreateOrUpdateTags([]autoscaling.ResourceTag{ + {ResourceID: "race-group", ResourceType: "auto-scaling-group", Key: "env", Value: "prod"}, + }) + }, + read: func(g autoscaling.AutoScalingGroup) { + for _, tag := range g.Tags { + _ = tag.Value + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := autoscaling.NewInMemoryBackend() + + _, err := b.CreateAutoScalingGroup(autoscaling.CreateAutoScalingGroupInput{ + AutoScalingGroupName: "race-group", + MinSize: 0, + MaxSize: 10, + DesiredCapacity: 3, + Tags: []autoscaling.Tag{{Key: "env", Value: "dev"}}, + }) + require.NoError(t, err) + + groups, err := b.DescribeAutoScalingGroups(nil, nil) + require.NoError(t, err) + require.Len(t, groups, 1) + require.NotEmpty(t, groups[0].Instances) + + instanceIDs := make([]string, len(groups[0].Instances)) + for i, inst := range groups[0].Instances { + instanceIDs[i] = inst.InstanceID + } + + const iterations = 300 + + var wg sync.WaitGroup + + wg.Add(2) + + go func() { + defer wg.Done() + + for range iterations { + gs, describeErr := b.DescribeAutoScalingGroups(nil, nil) + if describeErr != nil { + continue + } + + for _, g := range gs { + tt.read(g) + } + } + }() + + go func() { + defer wg.Done() + + for i := range iterations { + tt.mutate(b, instanceIDs, i) + } + }() + + wg.Wait() + }) + } +} diff --git a/services/autoscaling/scheduled_action_scheduler_test.go b/services/autoscaling/scheduled_action_scheduler_test.go index f666d81b82..4041469bcf 100644 --- a/services/autoscaling/scheduled_action_scheduler_test.go +++ b/services/autoscaling/scheduled_action_scheduler_test.go @@ -3,6 +3,7 @@ package autoscaling import ( "context" "testing" + "testing/synctest" "time" ) @@ -288,73 +289,59 @@ func TestApplyDueScheduledActions_InvalidCapacityDoesNotPanic(t *testing.T) { func TestScheduledActionScheduler_RunFiresAndStopsCleanly(t *testing.T) { t.Parallel() - b := NewInMemoryBackend() - t.Cleanup(b.Close) + synctest.Test(t, func(t *testing.T) { + b := NewInMemoryBackend() + t.Cleanup(b.Close) - _, err := b.CreateAutoScalingGroup(CreateAutoScalingGroupInput{ - AutoScalingGroupName: "sched-run-asg", - MinSize: 0, - MaxSize: 10, - DesiredCapacity: 1, - }) - if err != nil { - t.Fatalf("CreateAutoScalingGroup: %v", err) - } + _, err := b.CreateAutoScalingGroup(CreateAutoScalingGroupInput{ + AutoScalingGroupName: "sched-run-asg", + MinSize: 0, + MaxSize: 10, + DesiredCapacity: 1, + }) + if err != nil { + t.Fatalf("CreateAutoScalingGroup: %v", err) + } - desired := int32(4) + desired := int32(4) - err = b.PutScheduledUpdateGroupAction("sched-run-asg", ScheduledUpdateGroupAction{ - ScheduledActionName: "scale-run", - StartTime: time.Now().UTC().Add(-time.Minute), - DesiredCapacity: &desired, - }) - if err != nil { - t.Fatalf("PutScheduledUpdateGroupAction: %v", err) - } + err = b.PutScheduledUpdateGroupAction("sched-run-asg", ScheduledUpdateGroupAction{ + ScheduledActionName: "scale-run", + StartTime: time.Now().UTC().Add(-time.Minute), + DesiredCapacity: &desired, + }) + if err != nil { + t.Fatalf("PutScheduledUpdateGroupAction: %v", err) + } - const tickInterval = 10 * time.Millisecond + const tickInterval = 10 * time.Millisecond - sched := NewScheduledActionScheduler(b, tickInterval) + sched := NewScheduledActionScheduler(b, tickInterval) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) + done := make(chan struct{}) - go func() { - defer close(done) + go func() { + defer close(done) - sched.Run(ctx) - }() + sched.Run(ctx) + }() - deadline := time.Now().Add(2 * time.Second) + // tickInterval is 10ms; cross a tick so Run applies the due action. + time.Sleep(20 * time.Millisecond) + synctest.Wait() - for time.Now().Before(deadline) { - groups, describeErr := b.DescribeAutoScalingGroups([]string{"sched-run-asg"}, nil) - if describeErr != nil { - t.Fatalf("DescribeAutoScalingGroups: %v", describeErr) + groups, err := b.DescribeAutoScalingGroups([]string{"sched-run-asg"}, nil) + if err != nil { + t.Fatalf("DescribeAutoScalingGroups: %v", err) } - if groups[0].DesiredCapacity == desired { - break + if got := groups[0].DesiredCapacity; got != desired { + t.Fatalf("DesiredCapacity = %d, want %d (Run() never applied the due action)", got, desired) } - time.Sleep(tickInterval) - } - - groups, err := b.DescribeAutoScalingGroups([]string{"sched-run-asg"}, nil) - if err != nil { - t.Fatalf("DescribeAutoScalingGroups: %v", err) - } - - if got := groups[0].DesiredCapacity; got != desired { - t.Fatalf("DesiredCapacity = %d, want %d (Run() never applied the due action)", got, desired) - } - - cancel() - - select { - case <-done: - case <-time.After(2 * time.Second): - t.Fatal("Run() did not return within 2s of context cancellation") - } + cancel() + <-done + }) } diff --git a/services/batch/handler.go b/services/batch/handler.go index 358c2bc5f8..65c5a1299a 100644 --- a/services/batch/handler.go +++ b/services/batch/handler.go @@ -35,6 +35,15 @@ const ( // the /v1/ prefix; exclude them to avoid routing Kafka requests to Batch. kafkaClustersPrefix = "/v1/clusters" kafkaConfigurationsPrefix = "/v1/configurations" + // kafkaConnectConnectorPrefix covers MSK Connect's /v1/connectors, + // /v1/connectors/{arn}(/operations), and /v1/connectorOperations/{arn}. + // kafkaConnectPluginPrefix and kafkaConnectWorkerPrefix cover its + // /v1/custom-plugins and /v1/worker-configurations resources. All share + // the /v1/ prefix; exclude them to avoid routing MSK Connect requests to + // Batch (both are PriorityPathVersioned, and Batch registers first). + kafkaConnectConnectorPrefix = "/v1/connector" + kafkaConnectPluginPrefix = "/v1/custom-plugins" + kafkaConnectWorkerPrefix = "/v1/worker-configurations" ) // Handler is the Echo HTTP handler for AWS Batch operations. @@ -145,8 +154,9 @@ func (h *Handler) ChaosRegions() []string { return []string{h.Backend.Region()} // RouteMatcher returns a function that matches Batch requests. // It matches /v1/ paths but explicitly excludes /v1/apis (AppSync), -// CodeArtifact paths, and Kafka paths to prevent routing conflicts when -// multiple services use PriorityPathVersioned. The tags path is scoped by +// CodeArtifact paths, Kafka (MSK) paths, and MSK Connect paths to prevent +// routing conflicts when multiple services use PriorityPathVersioned. The +// tags path is scoped by // ARN via isBatchTagPath instead of excluded outright, since Batch owns its // own ARNs there too (see isAppSyncTagPath in services/appsync/handler.go // for the mirrored guard that stops AppSync's tag-path matcher from @@ -176,6 +186,12 @@ func (h *Handler) RouteMatcher() service.Matcher { strings.HasPrefix(path, kafkaConfigurationsPrefix) { return false } + // Exclude MSK Connect paths which share the /v1/ prefix. + if strings.HasPrefix(path, kafkaConnectConnectorPrefix) || + strings.HasPrefix(path, kafkaConnectPluginPrefix) || + strings.HasPrefix(path, kafkaConnectWorkerPrefix) { + return false + } return strings.HasPrefix(path, v1Prefix) } diff --git a/services/ce/anomalies.go b/services/ce/anomalies.go index 1360b3bf85..be65c0375c 100644 --- a/services/ce/anomalies.go +++ b/services/ce/anomalies.go @@ -112,6 +112,7 @@ func (b *InMemoryBackend) CreateAnomalyMonitor( b.anomalyMonitors.Put(mon) out := *mon + out.Tags = cloneCETags(mon.Tags) return &out, nil } @@ -147,6 +148,7 @@ func (b *InMemoryBackend) GetAnomalyMonitors( result = make([]*AnomalyMonitor, 0, len(all)) for _, mon := range all { out := *mon + out.Tags = cloneCETags(mon.Tags) result = append(result, &out) } } else { @@ -164,6 +166,7 @@ func (b *InMemoryBackend) GetAnomalyMonitors( for _, mon := range b.anomalyMonitors.All() { if _, ok := set[mon.MonitorARN]; ok { out := *mon + out.Tags = cloneCETags(mon.Tags) result = append(result, &out) } } @@ -198,6 +201,7 @@ func (b *InMemoryBackend) UpdateAnomalyMonitor( mon.LastUpdatedDate = time.Now().UTC() out := *mon + out.Tags = cloneCETags(mon.Tags) return &out, nil } @@ -254,6 +258,7 @@ func (b *InMemoryBackend) CreateAnomalySubscription( b.anomalySubscriptions.Put(sub) out := *sub + out.Tags = cloneCETags(sub.Tags) return &out, nil } @@ -310,6 +315,7 @@ func (b *InMemoryBackend) GetAnomalySubscriptions( } out := *sub + out.Tags = cloneCETags(sub.Tags) result = append(result, &out) } @@ -390,6 +396,7 @@ func (b *InMemoryBackend) UpdateAnomalySubscription( } out := *sub + out.Tags = cloneCETags(sub.Tags) return &out, nil } diff --git a/services/ce/cost_categories.go b/services/ce/cost_categories.go index f2537a369a..c3e1da7425 100644 --- a/services/ce/cost_categories.go +++ b/services/ce/cost_categories.go @@ -66,6 +66,7 @@ func (b *InMemoryBackend) CreateCostCategoryDefinition( out.Rules = make([]CostCategoryRule, len(cat.Rules)) copy(out.Rules, cat.Rules) out.SplitChargeRules = copySplitChargeRules(cat.SplitChargeRules) + out.Tags = cloneCETags(cat.Tags) return &out, nil } @@ -101,6 +102,7 @@ func (b *InMemoryBackend) DeleteCostCategoryDefinition(catARN string) (*CostCate b.costCategories.Delete(catARN) out := *cat + out.Tags = cloneCETags(cat.Tags) return &out, nil } @@ -116,6 +118,7 @@ func (b *InMemoryBackend) DescribeCostCategoryDefinition(catARN string) (*CostCa } out := *cat + out.Tags = cloneCETags(cat.Tags) return &out, nil } @@ -147,6 +150,7 @@ func (b *InMemoryBackend) ListCostCategoryDefinitions( } out := *cat + out.Tags = cloneCETags(cat.Tags) result = append(result, &out) } @@ -194,6 +198,7 @@ func (b *InMemoryBackend) UpdateCostCategoryDefinition( out.Rules = make([]CostCategoryRule, len(cat.Rules)) copy(out.Rules, cat.Rules) out.SplitChargeRules = copySplitChargeRules(cat.SplitChargeRules) + out.Tags = cloneCETags(cat.Tags) return &out, nil } diff --git a/services/ce/tags_race_test.go b/services/ce/tags_race_test.go new file mode 100644 index 0000000000..4c7700e26d --- /dev/null +++ b/services/ce/tags_race_test.go @@ -0,0 +1,105 @@ +package ce_test + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/ce" +) + +// TestTaggedResourceConcurrentWithUntag proves Describe/List for cost +// categories and anomaly monitors must not alias a Tags map UntagResource mutates. +func TestTaggedResourceConcurrentWithUntag(t *testing.T) { + t.Parallel() + + tests := []struct { + setup func(t *testing.T, b *ce.InMemoryBackend) (arn string) + reader func(b *ce.InMemoryBackend, arn string) + name string + }{ + { + name: "DescribeCostCategoryDefinition races UntagResource", + setup: func(t *testing.T, b *ce.InMemoryBackend) string { + t.Helper() + + cat, err := b.CreateCostCategoryDefinition( + "race-cat", "CostCategoryExpression.v1", "unassigned", + nil, map[string]string{"env": "prod"}, nil, "", + ) + require.NoError(t, err) + + return cat.ARN + }, + reader: func(b *ce.InMemoryBackend, arn string) { + cat, err := b.DescribeCostCategoryDefinition(arn) + if err != nil { + return + } + + for k := range cat.Tags { + _ = k + } + }, + }, + { + name: "GetAnomalyMonitors races UntagResource", + setup: func(t *testing.T, b *ce.InMemoryBackend) string { + t.Helper() + + mon, err := b.CreateAnomalyMonitor( + "race-mon", "DIMENSIONAL", "SERVICE", nil, map[string]string{"env": "prod"}, + ) + require.NoError(t, err) + + return mon.MonitorARN + }, + reader: func(b *ce.InMemoryBackend, arn string) { + mons, _, err := b.GetAnomalyMonitors([]string{arn}, 0, "") + if err != nil { + return + } + + for _, m := range mons { + for k := range m.Tags { + _ = k + } + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := ce.NewInMemoryBackend("000000000000", "us-east-1") + arn := tt.setup(t, b) + + const iterations = 500 + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + for range iterations { + tt.reader(b, arn) + } + }() + + go func() { + defer wg.Done() + + for range iterations { + _ = b.TagResource(arn, map[string]string{"env": "prod"}) + _ = b.UntagResource(arn, []string{"env"}) + } + }() + + wg.Wait() + }) + } +} diff --git a/services/cloudformation/PARITY.md b/services/cloudformation/PARITY.md index 3201edfd0b..c597427316 100644 --- a/services/cloudformation/PARITY.md +++ b/services/cloudformation/PARITY.md @@ -1,7 +1,53 @@ --- service: cloudformation sdk_module: aws-sdk-go-v2/service/cloudformation@v1.76.1 -last_audit_commit: 57873cfd3 # 2026-09-24 25 new resource types added: Glue +last_audit_commit: 54869319e # 2026-09-25 24 new resource types added: EC2 + # PlacementGroup/VPNGateway/VPNConnection/ + # PrefixList/NetworkInterfacePermission/ + # VPCEndpointService/TransitGatewayVpcAttachment/ + # TransitGatewayPeeringAttachment/ + # TransitGatewayMulticastDomain/ + # TrafficMirrorFilter/TrafficMirrorFilterRule/ + # TrafficMirrorTarget/TrafficMirrorSession/ + # RouteServer/RouteServerEndpoint/ + # RouteServerPeer/NetworkInsightsPath/ + # VerifiedAccessInstance (18 types); IAM + # SAMLProvider/VirtualMFADevice (2 types); + # ElastiCache User (1 type); ApiGatewayV2 + # VpcLink (1 type); the real CFN type names + # AWS::CertificateManager::Certificate and + # AWS::OpenSearchService::Domain added as + # aliases for the existing (undocumented) + # AWS::ACM::Certificate/AWS::OpenSearch::Domain + # handlers, same pattern as the existing + # AWS::KinesisFirehose::DeliveryStream alias + # (2 types) (405 -> 429 supported types); no + # new services wired into the CloudFormation + # backend -- EC2/IAM/ElastiCache/ApiGatewayV2/ + # ACM/OpenSearch were all already wired; + # cfn_attributes_gen.go regenerated + # (cmd/cfnattrgen) -- most new attribute names + # were excluded by its goconst-safety rule + # (already common literals elsewhere in the + # package, e.g. "Id"/"Arn"/"State"), which is + # documented conservative behavior, not a + # regression: an excluded attribute falls back + # to pre-existing permissive resolution rather + # than being wrongly rejected. Skipped (ops + # missing or no real backend): AWS::EC2::Fleet/ + # SpotFleet/CapacityReservationFleet (would + # need broker-side instance-launch simulation + # beyond this sweep's scope), AWS::EC2::Ipam* + # family (complex nested scope/pool graph), + # AWS::EC2::LocalGateway* (Outposts-only, + # no realistic test env), AWS::ECS:: + # ContainerInstance/Task/ServiceRevision (not + # documented CFN resource types in the current + # public TemplateReference), AWS::RDS:: + # DBSecurityGroup (EC2-Classic-only, no VPC + # equivalent to back it), AWS::S3::AccessPoint + # family (no backend Create/DeleteAccessPoint); + # prior: 57873cfd3 # 2026-09-24 25 new resource types added: Glue # Blueprint/CustomEntityType/Workflow (3 types); # DataSync Agent/LocationS3/Task (3 types); # Transfer Profile/Workflow (2 types); AppConfig @@ -101,7 +147,7 @@ last_audit_commit: 57873cfd3 # 2026-09-24 25 new resource types added: Glue # StreamConsumer, the real AWS::KinesisFirehose::DeliveryStream type # name, ECS CapacityProvider/ClusterCapacityProviderAssociations/ # TaskSet/PrimaryTaskSet); prior: 05eeb3af7 -last_audit_date: 2026-09-24 # prior: 2026-09-24 (28-type IAM/ECR/ElastiCache/Neptune/DocDB/Backup/Glue/CodeBuild/Kinesis/Lambda pass earlier same day) +last_audit_date: 2026-09-25 # prior: 2026-09-24 (25-type Glue/DataSync/Transfer/AppConfig/Macie/GuardDuty/AccessAnalyzer/Amplify/Batch/EFS/Redshift pass) overall: A # This pass closed out all 4 documented gaps and independently re-verified/acted # on all 6 documented deferred items (see gaps:/deferred: below for exact # disposition of each -- some fixed, some reclassified to ok after @@ -155,9 +201,9 @@ ops: DeleteStackSet: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: now idempotent (no-op, not StackSetNotFoundException) — SDK's DeleteStackSet error deserializer models only {OperationInProgressException, StackSetNotEmptyException}, no not-found case, mirroring the already-fixed DeleteStack precedent"} DescribeStackSet: {wire: ok, errors: ok, state: ok, persist: ok, note: "FIXED this pass (was the #1 named gap): full field set now returned, field-diffed against awsAwsquery_deserializeDocumentStackSet -- Parameters, Capabilities, Tags, StackSetARN, AdministrationRoleARN, ExecutionRoleName, PermissionModel, OrganizationalUnitIds, AutoDeployment{Enabled,RetainStacksOnAccountRemoval}, ManagedExecution{Active}. CreateStackSet/UpdateStackSet now accept these via a new StackSetOptions struct (signature change, all callers updated). Regions is intentionally NOT stored on StackSet -- it's computed live from stack instances each call (StackSetRegions) to avoid a second source of truth, mirroring the driftByStackID rationale below. Verified via TestStackSet_DescribeFieldCompleteness"} ListStackSets: {wire: ok, errors: ok, state: ok, persist: ok, note: "this pass (constraint-parameter audit): fixed -- Status (cloudformation@v1.76.1 api_op_ListStackSets.go:75-76) was read nowhere, so a real client's Status=DELETED filter silently fell back to returning every StackSet instead of the empty list real AWS would return (DeleteStackSet hard-deletes its row, so no DELETED-status StackSet can ever exist in this backend -- an unfiltered call and a Status=ACTIVE-filtered call are behaviorally identical; only Status=DELETED was actually wrong). Now applies the filter (exact match against StackSetSummary.Status)."} - CreateStackInstances: {wire: ok, errors: ok, state: ok, persist: ok, note: "real per-account/region child stacks are provisioned (provisionStackInstance), not just recorded rows — verified correct. gopherstack-g7b5: now also accepts DeploymentTargets.OrganizationalUnitIds.member.N (serializers.go's DeploymentTargets/OrganizationalUnitIdList encoders) and resolves each OU to its real member accounts via a wired Organizations backend (services/cloudformation/organizations_directory.go's OrganizationsDirectory interface, satisfied by organizations.InMemoryBackend.ResolveAccountIDsUnderParent, wired in cli.go's wireCloudFormationOrganizations). Requires PermissionModel=SERVICE_MANAGED and ActivateOrganizationsAccess; errors clearly otherwise rather than silently expanding to zero accounts. gopherstack-nirx: DeploymentTargets.AccountFilterType was documented as rejected but the field was never read by the handler (silently dropped, computing a union of Accounts and OU-resolved accounts regardless of the requested filter) — now handler_stack_sets.go's unsupportedAccountFilterType actually rejects INTERSECTION/DIFFERENCE/UNION with ValidationError; only unset/NONE (the union case) is honoured. See TestStackInstances_AccountFilterType"} - DeleteStackInstances: {wire: fixed, errors: ok, state: fixed, persist: ok, note: "tears down provisioned child stacks via deleteStackLocked — verified correct. gopherstack-g7b5: also accepts DeploymentTargets.OrganizationalUnitIds, same resolution path as CreateStackInstances. CORRECTION 2026-09-11 (required-member sweep pass 4a): 'verified correct' missed that RetainStacks (required, api_op_DeleteStackInstances.go) was never read at all -- the handler always tore down the child stack via deleteStackLocked regardless of what the caller asked. Fixed: RetainStacks is now required and presence-validated; when true, deleteMatchingStackInstances drops only the stack-instance association and leaves the child stack alive. See TestDeleteStackInstances_RetainStacksKeepsChildStack."} - UpdateStackInstances: {wire: ok, errors: ok, state: ok, persist: ok, note: "gopherstack-g7b5: also accepts DeploymentTargets.OrganizationalUnitIds"} + CreateStackInstances: {wire: ok, errors: ok, state: ok, persist: ok, note: "real per-account/region child stacks are provisioned (provisionStackInstance), not just recorded rows — verified correct. gopherstack-g7b5: now also accepts DeploymentTargets.OrganizationalUnitIds.member.N (serializers.go's DeploymentTargets/OrganizationalUnitIdList encoders) and resolves each OU to its real member accounts via a wired Organizations backend (services/cloudformation/organizations_directory.go's OrganizationsDirectory interface, satisfied by organizations.InMemoryBackend.ResolveAccountIDsUnderParent, wired in cli.go's wireCloudFormationOrganizations). Requires PermissionModel=SERVICE_MANAGED and ActivateOrganizationsAccess; errors clearly otherwise rather than silently expanding to zero accounts. FIXED 2026-09-26 (was: gopherstack-nirx's INTERSECTION/DIFFERENCE/UNION-rejected-outright state): DeploymentTargets.AccountFilterType now implements the full documented enum (API_DeploymentTargets.html) -- NONE (OU accounts only, Accounts ignored), INTERSECTION (Accounts ∩ OU accounts), DIFFERENCE (OU accounts minus Accounts), UNION (OU accounts plus Accounts, the wire default when unset) -- via resolveInstanceTargets/combineAccountFilter (stack_instances.go). Also enforces the two Create-specific documented rules: UNION is rejected with ValidationError ('UNION is not supported for CreateStackInstances operations'), and specifying both Accounts and OrganizationalUnitIds without an explicit AccountFilterType is rejected ('you must specify DeploymentTargets.AccountFilterType...'). AccountsUrl (S3-hosted account list) is accepted on the wire but not fetched -- no S3 client wired for it, same structural gap as TemplateURL not being fetched elsewhere in this service. See TestStackInstances_AccountFilterType_Create/_UnionRejectedAtCreate/_RequiredWhenBothGivenAtCreate/_InvalidValue"} + DeleteStackInstances: {wire: fixed, errors: ok, state: fixed, persist: ok, note: "tears down provisioned child stacks via deleteStackLocked — verified correct. gopherstack-g7b5: also accepts DeploymentTargets.OrganizationalUnitIds, same resolution path as CreateStackInstances. CORRECTION 2026-09-11 (required-member sweep pass 4a): 'verified correct' missed that RetainStacks (required, api_op_DeleteStackInstances.go) was never read at all -- the handler always tore down the child stack via deleteStackLocked regardless of what the caller asked. Fixed: RetainStacks is now required and presence-validated; when true, deleteMatchingStackInstances drops only the stack-instance association and leaves the child stack alive. See TestDeleteStackInstances_RetainStacksKeepsChildStack. FIXED 2026-09-26: AccountFilterType (NONE/INTERSECTION/DIFFERENCE/UNION) now determines exactly which accounts' instances are torn down, same combineAccountFilter as CreateStackInstances/UpdateStackInstances (previously always used the union of Accounts and OU-resolved accounts regardless of the requested filter). See TestStackInstances_AccountFilterType_DeleteDifference"} + UpdateStackInstances: {wire: ok, errors: ok, state: ok, persist: ok, note: "gopherstack-g7b5: also accepts DeploymentTargets.OrganizationalUnitIds. FIXED 2026-09-26: AccountFilterType (NONE/INTERSECTION/DIFFERENCE/UNION, UNION allowed here unlike Create) now determines the touched account set recorded via ListStackSetOperationResults; note UpdateStackInstances updates existing stack instances rather than provisioning new ones for a previously-untargeted account (pre-existing structural behavior, unaffected by this fix). See TestStackInstances_AccountFilterType_UpdateUnion"} ListStackInstances: {wire: ok, errors: ok, state: ok, persist: ok, note: "this pass (constraint-parameter audit): fixed -- handleListStackInstances read only StackSetName/NextToken; StackInstanceAccount, StackInstanceRegion, and Filters (cloudformation@v1.76.1 api_op_ListStackInstances.go) were parsed nowhere, so every call returned every instance in the StackSet regardless of the filter sent. Now applies StackInstanceAccount/StackInstanceRegion (exact match) and Filters entries named DRIFT_STATUS/LAST_OPERATION_ID (matched against StackInstance.DriftStatus/LastOperationID). DETAILED_STATUS is accepted on the wire but left unenforced and documented as a gap: this backend tracks no field distinct from Status, and DetailedStatus's real values (PENDING/RUNNING/SUCCEEDED/FAILED/CANCELLED/INOPERABLE/SKIPPED_SUSPENDED_ACCOUNT) don't correspond to StackInstanceStatus's (CURRENT/OUTDATED/INOPERABLE) closely enough to map one onto the other without fabricating data."} DescribeStackInstance: {wire: ok, errors: ok, state: ok, persist: ok} DetectStackDrift: {wire: ok, errors: ok, state: ok, persist: ok, note: "2026-08-22 (gopherstack-r80d batch 26, NEW ops: row -- had no prior entry): required output StackDriftDetectionId always a real uuid, field-diffed against DetectStackDriftOutput; 0 bugs"} @@ -225,7 +271,7 @@ gaps: [] items_still_open: - "changeset_diff.go requiresRecreation() covers only a curated subset of resource types' replacement-forcing properties — expanding it is future work under gopherstack-e5h, not a regression (re-verified 2026-09-18)" - "SetTypeConfiguration accepts configuration for any type name without prior registration — intentional permissiveness for first-party AWS types this emulator doesn't catalog fully (bd: gopherstack-e5h; re-verified 2026-09-18)" - - "StackSets DeploymentTargets.AccountFilterType INTERSECTION/DIFFERENCE/UNION and AccountsUrl are not implemented (only unset/NONE is honoured; other values are rejected with ValidationError, not silently dropped) — no account-filter graph to compute them against (bd: gopherstack-g7b5, gopherstack-nirx; re-verified 2026-09-18)" + - "StackSets DeploymentTargets.AccountsUrl (S3-hosted account list) is accepted on the wire but not fetched — no S3 client wired for it, same structural gap as TemplateURL not being fetched elsewhere in this service (bd: gopherstack-g7b5; AccountFilterType INTERSECTION/DIFFERENCE/UNION themselves were fixed 2026-09-26, see ops: CreateStackInstances/UpdateStackInstances/DeleteStackInstances)" - "ImportStacksToStackSet doesn't tag imported instances with a real OU — ImportStacksToStackSetInput has no DeploymentTargets to source one from (structural, unaffected by the gopherstack-g7b5 OU work; re-verified 2026-09-18)" - "StackSetOperations complete synchronously as SUCCEEDED (RUNNING/STOPPING unreachable) — deliberate: cloudformation has no clock/janitor-driven lifecycle anywhere, every op resolves inside its own handler call (gopherstack-b3pm; see families: stacksets for the full writeup and tests; re-verified 2026-09-18)" - "Stack policy enforcement doesn't implement NotAction/NotResource (disclosed, not approximated), treats Replacement=='Conditionally' as Update:Replace (errs protective), doesn't model StackPolicyBody/URL at Create/UpdateStack time, and doesn't check parameter-only updates (no TemplateBody diff to compute) — see families: stack_policy_enforcement (gopherstack-cqy3; re-verified 2026-09-18)" @@ -239,6 +285,189 @@ leaks: {status: clean, note: "no goroutines/janitors/tickers introduced this pas ## Notes +### 2026-09-26: StackSets DeploymentTargets.AccountFilterType (NONE/INTERSECTION/DIFFERENCE/UNION) + +Previously INTERSECTION/DIFFERENCE/UNION were rejected outright with +ValidationError (gopherstack-nirx) and only unset/NONE was honoured, both +computed identically as the union of Accounts and OU-resolved accounts. +Implemented the full enum per +docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_DeploymentTargets.html: +NONE (OU accounts only, Accounts ignored), INTERSECTION (Accounts ∩ OU +accounts), DIFFERENCE (OU accounts minus Accounts), UNION (OU accounts plus +Accounts, the documented default when AccountFilterType is unset) — +`resolveInstanceTargets`/`combineAccountFilter` (stack_instances.go), threaded +through CreateStackInstances, UpdateStackInstances, and DeleteStackInstances +(all three gained a `filterType` parameter; every direct backend-call test +site was updated to pass `""`, preserving prior union-default behavior). + +Also enforces the two Create-specific validation rules the API reference +documents: "UNION is not supported for CreateStackInstances operations", and +"When performing create operations, if you specify both +OrganizationalUnitIds and Accounts, you must also specify the +AccountFilterType property" — both return ValidationError +(`parseAccountFilterType`, handler_stack_sets.go). An unrecognized +AccountFilterType value is also rejected with ValidationError. + +AccountsUrl remains unfetched (see items_still_open) — same structural class +as TemplateURL not being fetched elsewhere in this service. + +Tests: `stack_instances_account_filter_test.go`, table-driven through the +real aws-sdk-go-v2 client, asserting `ListStackInstances` (Create) or +`ListStackSetOperationResults` (Update, which updates existing instances +rather than provisioning new ones) returns exactly the expected accounts for +each filter type, plus the two Create-only validation rules and an invalid +enum value. + +### 2026-09-26 (parity sweep): 6 new resource types (429 -> 435), 3 new backend families wired + +Added real create+delete support for 6 `AWS::*` resource types across 3 +newly-landed service families, each backed by a genuine `InMemoryBackend` +call (no stubs), with Ref/Fn::GetAtt verified against the live AWS +CloudFormation Template Reference docs (fetched this pass) and, where the +Template Reference itself left `Ref` undocumented, cross-checked against the +AWS-published resource-provider schemas +(`aws-cloudformation-resource-providers-kafkaconnect`'s `primaryIdentifier`) +and the legacy `CloudFormationResourceSpecification.json`. + +- **KinesisVideo** (2 types): Stream, SignalingChannel + (`resources_kinesisvideo.go`) -- `Ref`/`Fn::GetAtt Arn` both return the + resource ARN; the Template Reference page leaves `Ref` undocumented for + both, stating only the `Arn` attribute (same undocumented-`Ref`-equals- + sole-ARN-attribute pattern independently confirmed for KafkaConnect + Connector below via its published resource-provider schema) +- **ECRPublic** (1 type): PublicRepository (`resources_ecrpublic.go`) -- + `Ref` returns the repository name (documented), `Fn::GetAtt Arn` returns + the repository ARN (documented); delete does not force-empty the + repository (`AWS::ECR::PublicRepository` has no `EmptyOnDelete` property, + unlike `AWS::ECR::Repository`), matching real AWS's less convenient + behavior for public repos +- **KafkaConnect** (3 types): Connector, CustomPlugin, WorkerConfiguration + (`resources_kafkaconnect.go`) -- each type's `Ref` returns its ARN, + confirmed via the resource-provider schema's `primaryIdentifier` (equal to + its sole `readOnlyProperty`) since the Template Reference page leaves + `Ref` undocumented for all three; CustomPlugin/WorkerConfiguration also + expose a documented `Revision` `Fn::GetAtt` attribute + +All three backends were newly wired into the CloudFormation backend: +`ServiceBackends` (`resources.go`) gained `KinesisVideo`/`ECRPublic`/ +`KafkaConnect` fields, `BackendsProvider` (`provider.go`) gained the matching +`Get*Handler` methods, and `extractAllServiceBackends` wires them from the +handlers -- `cli.go` already had `GetKinesisVideoHandler`/ +`GetECRPublicHandler`/`GetKafkaConnectHandler` getters (added when those +services first landed), so no `cli.go` change was needed. Dispatch wiring +chains `createKinesisVideoResource`/`createECRPublicResource`/ +`createKafkaConnectResource` (and their `delete*` counterparts) off the end +of `createNewestSupplementalResource`/`deleteNewestSupplementalResource` in +`resources_newest_dispatch.go`. + +**Fn::GetAtt side-channel stashing.** All 6 types stash their real ARN (and, +for CustomPlugin/WorkerConfiguration, `Revision`) into +`physicalIDs[logicalID+"/AttrName"]` at create time; their `resTypeXxx` +constants were added to `resolveGetAtt`'s existing custom-resource-style +whitelist in `template.go` so those stashed values are read back instead of +falling through to the default `return physID`. This mattered concretely for +ECRPublic (`Ref` is the repository *name*, but `Arn` differs) and for +CustomPlugin/WorkerConfiguration's `Revision` (an integer, never equal to +the ARN `Ref` returns) -- both were caught by the new integration tests +before being added to the whitelist (ECRPublic's `Arn` output resolved to +the bare repository name, and `Revision` resolved to the full ARN). + +`cfn_attributes_gen.go` was regenerated (`cmd/cfnattrgen`) against a fresh +download of the legacy `CloudFormationResourceSpecification.json`. All 6 new +types' documented attributes were narrow enough to clear the generator's +goconst-safety rule for KinesisVideo::Stream/SignalingChannel (`Arn`), +ECRPublic::PublicRepository (`Arn`), and KafkaConnect::Connector +(`ConnectorArn`); KafkaConnect::CustomPlugin/WorkerConfiguration were +excluded whole because `Revision` already clears golangci-lint's `goconst` +threshold elsewhere in the package -- per the generator's documented +contract this is conservative, not lossy (an excluded type falls back to +today's permissive `Fn::GetAtt` resolution, which the stash-and-whitelist +fix above already makes correct regardless of table membership). Unrelated +to this pass: regenerating against today's spec download also dropped +`AWS::EC2::PrefixList` and `AWS::SageMaker::ImageVersion` from the table -- +independently reproduced against the pre-existing (unmodified) source, so +this is drift in the package's own literal-occurrence counts since the last +generation, not something this pass's new code caused. Both types keep +working via the same permissive fallback. + +Task B (separate, `services/ecrpublic`): fixed an unrelated +`InitiateLayerUpload` session leak -- see that service's own PARITY.md Notes +entry. Task C (separate, `services/kafkaconnect`): implemented +`RestartConnector` -- see that service's own PARITY.md. + +### 2026-09-25 (parity sweep): 24 new resource types (405 -> 429), no new backend families + +Added real create+delete support for 24 `AWS::*` resource types, each backed +by a genuine `InMemoryBackend` call (no stubs), with Ref/Fn::GetAtt verified +against the live AWS CloudFormation Template Reference docs (fetched this +pass). Every backing service (EC2, IAM, ElastiCache, ApiGatewayV2, ACM, +OpenSearch) was already wired into the CloudFormation backend, so no +`cli.go`/`provider.go` changes were needed this pass. + +- **EC2** (18 types): PlacementGroup, VPNGateway, VPNConnection, PrefixList, + NetworkInterfacePermission, VPCEndpointService + (`resources_ec2_networking_extras.go`, `resources_ec2_vpn.go`); + TransitGatewayVpcAttachment, TransitGatewayPeeringAttachment, + TransitGatewayMulticastDomain (`resources_ec2_transitgateway_more.go`); + TrafficMirrorFilter, TrafficMirrorFilterRule, TrafficMirrorTarget, + TrafficMirrorSession (`resources_ec2_trafficmirror.go`); RouteServer, + RouteServerEndpoint, RouteServerPeer (`resources_ec2_routeserver.go`); + NetworkInsightsPath (`resources_ec2_networkinsights.go`); + VerifiedAccessInstance (`resources_ec2_networking_extras.go`) +- **IAM** (2 types): SAMLProvider, VirtualMFADevice (`resources_iam_extras.go`) +- **ElastiCache** (1 type): User (`resources_elasticache_user.go`) +- **ApiGatewayV2** (1 type): VpcLink (`resources_apigatewayv2_vpclink.go`) +- **Type-name aliases** (2 types): `AWS::CertificateManager::Certificate` + and `AWS::OpenSearchService::Domain` are the real CFN type names for the + existing (undocumented) `AWS::ACM::Certificate`/`AWS::OpenSearch::Domain` + handlers -- added as aliases in `resources.go`'s `createMiscLegacyResource`/ + `deleteComputeStorageResource`/`deleteAppNetworkResource`, same pattern as + the existing `AWS::KinesisFirehose::DeliveryStream` alias + (`resources_type_aliases.go` holds the two new constants) + +Dispatch wiring lives in a new `createEC2AdvancedNetworkingResource`/ +`deleteEC2AdvancedNetworkingResource` pair in `resources_newest_dispatch.go`, +chained off the end of `createNewestSupplementalResource`/ +`deleteNewestSupplementalResource`. + +**Fn::GetAtt side-channel stashing.** PrefixList (Arn/OwnerId/Version), +TransitGatewayPeeringAttachment (State), TransitGatewayMulticastDomain +(CreationTime/State/Arn), RouteServer/RouteServerEndpoint/RouteServerPeer +(Arn plus their real ENI/VPC/subnet fields), NetworkInsightsPath +(NetworkInsightsPathArn/SourceArn/DestinationArn), and ElastiCache User +(Arn/Status) all stash real backend values into `physicalIDs[logicalID+ +"/AttrName"]` at create time; their `resTypeXxx` constants were added to +`resolveGetAtt`'s existing custom-resource-style whitelist in `template.go` +so those stashed values are actually read back instead of falling through to +the default `return physID` (documented next to `getExtraResourceAttribute`). +Where the backend has no honest value to stash (e.g. IAM SAMLProvider's +SamlProviderUUID, EC2 VerifiedAccessInstance's CreationTime/LastUpdatedTime, +EC2 TransitGatewayPeeringAttachment's CreationTime, EC2 PlacementGroup's +GroupId as distinct from GroupName), the attribute is left on the default +physID fallback rather than fabricated -- called out in each file's +`---- AWS::Xxx::Yyy ----` doc comment. + +**Skipped (real gaps, not fixed this pass).** AWS::EC2::Fleet/SpotFleet/ +CapacityReservationFleet (would need broker-side instance-launch simulation); +the AWS::EC2::Ipam* family (complex nested scope/pool/resource-discovery +graph); AWS::EC2::LocalGateway* (Outposts-only, no realistic local test +environment); AWS::ECS::ContainerInstance/Task/ServiceRevision (not +documented CFN resource types in the current public TemplateReference, so +Ref/GetAtt can't be verified against docs); AWS::RDS::DBSecurityGroup +(EC2-Classic-only, no VPC equivalent to back it honestly); the +AWS::S3::AccessPoint family (this backend has no +Create/DeleteAccessPoint). AWS::EC2::VPCEndpointService's +PrivateDnsNameConfiguration.* Fn::GetAtt attributes are left unimplemented: +the backend's `CreateVpcEndpointServiceConfiguration` doesn't model private +DNS name verification at all. + +cfn_attributes_gen.go was regenerated (`cmd/cfnattrgen`); most of the new +attribute names above were excluded by its goconst-safety rule (already +common literals elsewhere in the package, e.g. "Id"/"Arn"/"State"/ +"VpcId"/"SubnetId") -- documented conservative behavior, not a regression: +an excluded attribute falls back to the pre-existing permissive resolution +rather than being wrongly rejected. + ### 2026-09-24 (parity sweep): 25 new resource types (380 -> 405), 6 new backend families wired Added real create+delete support for 25 `AWS::*` resource types across 11 diff --git a/services/cloudformation/README.md b/services/cloudformation/README.md index 7fdfe2ac8d..dedb74cfbf 100644 --- a/services/cloudformation/README.md +++ b/services/cloudformation/README.md @@ -1,7 +1,7 @@ # CloudFormation -**Parity grade: A** · SDK `aws-sdk-go-v2/service/cloudformation@v1.76.1` · last audited 2026-09-24 (`57873cfd3`) +**Parity grade: A** · SDK `aws-sdk-go-v2/service/cloudformation@v1.76.1` · last audited 2026-09-25 (`54869319e`) ## Coverage @@ -17,7 +17,7 @@ - changeset_diff.go requiresRecreation() covers only a curated subset of resource types' replacement-forcing properties — expanding it is future work under gopherstack-e5h, not a regression (re-verified 2026-09-18) - SetTypeConfiguration accepts configuration for any type name without prior registration — intentional permissiveness for first-party AWS types this emulator doesn't catalog fully (bd: gopherstack-e5h; re-verified 2026-09-18) -- StackSets DeploymentTargets.AccountFilterType INTERSECTION/DIFFERENCE/UNION and AccountsUrl are not implemented (only unset/NONE is honoured; other values are rejected with ValidationError, not silently dropped) — no account-filter graph to compute them against (bd: gopherstack-g7b5, gopherstack-nirx; re-verified 2026-09-18) +- StackSets DeploymentTargets.AccountsUrl (S3-hosted account list) is accepted on the wire but not fetched — no S3 client wired for it, same structural gap as TemplateURL not being fetched elsewhere in this service (bd: gopherstack-g7b5; AccountFilterType INTERSECTION/DIFFERENCE/UNION themselves were fixed 2026-09-26, see ops: CreateStackInstances/UpdateStackInstances/DeleteStackInstances) - ImportStacksToStackSet doesn't tag imported instances with a real OU — ImportStacksToStackSetInput has no DeploymentTargets to source one from (structural, unaffected by the gopherstack-g7b5 OU work; re-verified 2026-09-18) - StackSetOperations complete synchronously as SUCCEEDED (RUNNING/STOPPING unreachable) — deliberate: cloudformation has no clock/janitor-driven lifecycle anywhere, every op resolves inside its own handler call (gopherstack-b3pm; see families: stacksets for the full writeup and tests; re-verified 2026-09-18) - Stack policy enforcement doesn't implement NotAction/NotResource (disclosed, not approximated), treats Replacement=='Conditionally' as Update:Replace (errs protective), doesn't model StackPolicyBody/URL at Create/UpdateStack time, and doesn't check parameter-only updates (no TemplateBody diff to compute) — see families: stack_policy_enforcement (gopherstack-cqy3; re-verified 2026-09-18) diff --git a/services/cloudformation/cfn_attributes.go b/services/cloudformation/cfn_attributes.go new file mode 100644 index 0000000000..a5e74beea3 --- /dev/null +++ b/services/cloudformation/cfn_attributes.go @@ -0,0 +1,45 @@ +package cloudformation + +import ( + _ "embed" + "encoding/json" + "sync" +) + +// cfn_attributes.json is generated by cmd/cfnattrgen from the CloudFormation +// resource specification; DO NOT EDIT. Regenerate with `make cfn-attrs-gen`. +// +//go:embed cfn_attributes.json +var cfnAttributesJSON []byte + +//nolint:gochecknoglobals // lazily populated cache of the embedded attribute table +var ( + cfnResourceAttributes map[string]map[string]struct{} + cfnResourceAttributesOnce sync.Once +) + +// resourceAttributeTable parses cfnAttributesJSON once. JSON, not Go source, +// keeps attribute-name literals out of goconst's package-wide count. +func resourceAttributeTable() map[string]map[string]struct{} { + cfnResourceAttributesOnce.Do(func() { + var raw map[string][]string + if err := json.Unmarshal(cfnAttributesJSON, &raw); err != nil { + panic("cloudformation: invalid embedded cfn_attributes.json: " + err.Error()) + } + + table := make(map[string]map[string]struct{}, len(raw)) + + for resType, attrs := range raw { + set := make(map[string]struct{}, len(attrs)) + for _, a := range attrs { + set[a] = struct{}{} + } + + table[resType] = set + } + + cfnResourceAttributes = table + }) + + return cfnResourceAttributes +} diff --git a/services/cloudformation/cfn_attributes.json b/services/cloudformation/cfn_attributes.json new file mode 100644 index 0000000000..7cdea439bf --- /dev/null +++ b/services/cloudformation/cfn_attributes.json @@ -0,0 +1,1041 @@ +{ + "AWS::AccessAnalyzer::Analyzer": [ + "Arn" + ], + "AWS::AccessAnalyzer::ArchiveRule": [ + "Arn", + "CreatedAt", + "UpdatedAt" + ], + "AWS::Amplify::App": [ + "AppId", + "AppName", + "Arn", + "DefaultDomain" + ], + "AWS::Amplify::Branch": [ + "Arn", + "BranchName" + ], + "AWS::ApiGatewayV2::Integration": [ + "IntegrationId" + ], + "AWS::ApiGatewayV2::Route": [ + "RouteId" + ], + "AWS::ApiGatewayV2::VpcLink": [ + "VpcLinkId" + ], + "AWS::AppConfig::Application": [ + "ApplicationId" + ], + "AWS::AppConfig::ConfigurationProfile": [ + "ConfigurationProfileId", + "KmsKeyArn" + ], + "AWS::AppConfig::DeploymentStrategy": [ + "Id" + ], + "AWS::AppConfig::Environment": [ + "EnvironmentId" + ], + "AWS::AppSync::ChannelNamespace": [ + "ChannelNamespaceArn" + ], + "AWS::AppSync::DomainName": [ + "AppSyncDomainName", + "DomainName", + "DomainNameArn", + "HostedZoneId" + ], + "AWS::Athena::CapacityReservation": [ + "AllocatedDpus", + "Arn", + "CreationTime", + "LastSuccessfulAllocationTime", + "Status" + ], + "AWS::Athena::NamedQuery": [ + "NamedQueryId" + ], + "AWS::Athena::WorkGroup": [ + "CreationTime", + "WorkGroupConfiguration.EngineVersion.EffectiveEngineVersion" + ], + "AWS::AutoScaling::ScalingPolicy": [ + "Arn", + "PolicyName" + ], + "AWS::AutoScaling::ScheduledAction": [ + "ScheduledActionName" + ], + "AWS::Backup::Framework": [ + "CreationTime", + "DeploymentStatus", + "FrameworkArn", + "FrameworkStatus" + ], + "AWS::Backup::ReportPlan": [ + "ReportPlanArn" + ], + "AWS::Batch::SchedulingPolicy": [ + "Arn" + ], + "AWS::Batch::ServiceEnvironment": [ + "ServiceEnvironmentArn" + ], + "AWS::CertificateManager::Certificate": [ + "CertificateArn" + ], + "AWS::CloudFormation::WaitCondition": [ + "Data" + ], + "AWS::CloudFront::CloudFrontOriginAccessIdentity": [ + "Id", + "S3CanonicalUserId" + ], + "AWS::CloudFront::ContinuousDeploymentPolicy": [ + "Id", + "LastModifiedTime" + ], + "AWS::CloudFront::KeyGroup": [ + "Id", + "LastModifiedTime" + ], + "AWS::CloudFront::KeyValueStore": [ + "Arn", + "Id", + "Status" + ], + "AWS::CloudFront::OriginRequestPolicy": [ + "Id", + "LastModifiedTime" + ], + "AWS::CloudFront::PublicKey": [ + "CreatedTime", + "Id" + ], + "AWS::CloudTrail::Channel": [ + "ChannelArn" + ], + "AWS::CloudTrail::EventDataStore": [ + "CreatedTimestamp", + "EventDataStoreArn", + "Status", + "UpdatedTimestamp" + ], + "AWS::CloudWatch::Alarm": [ + "Arn" + ], + "AWS::CloudWatch::InsightRule": [ + "Arn", + "RuleName" + ], + "AWS::CloudWatch::MetricStream": [ + "Arn", + "CreationDate", + "LastUpdateDate", + "State" + ], + "AWS::CodeArtifact::Domain": [ + "Arn", + "EncryptionKey", + "Name", + "Owner" + ], + "AWS::CodeArtifact::PackageGroup": [ + "Arn" + ], + "AWS::CodeArtifact::Repository": [ + "Arn", + "DomainName", + "DomainOwner", + "Name" + ], + "AWS::CodeBuild::ReportGroup": [ + "Arn" + ], + "AWS::Config::AggregationAuthorization": [ + "AggregationAuthorizationArn" + ], + "AWS::Config::ConfigRule": [ + "Arn", + "Compliance.Type", + "ConfigRuleId" + ], + "AWS::Config::ConfigurationAggregator": [ + "ConfigurationAggregatorArn" + ], + "AWS::Config::ConformancePack": [ + "ConformancePackArn" + ], + "AWS::Config::StoredQuery": [ + "QueryArn", + "QueryId" + ], + "AWS::DataSync::Agent": [ + "AgentArn", + "EndpointType" + ], + "AWS::DataSync::LocationS3": [ + "LocationArn", + "LocationUri" + ], + "AWS::DataSync::Task": [ + "DestinationNetworkInterfaceArns", + "SourceNetworkInterfaceArns", + "Status", + "TaskArn" + ], + "AWS::DocDB::GlobalCluster": [ + "GlobalClusterArn", + "GlobalClusterResourceId" + ], + "AWS::DynamoDB::Table": [ + "Arn", + "StreamArn" + ], + "AWS::EC2::CapacityReservation": [ + "AvailabilityZone", + "AvailableInstanceCount", + "CapacityAllocationSet", + "CapacityReservationArn", + "CapacityReservationFleetId", + "CommitmentInfo", + "CommitmentInfo.CommitmentEndDate", + "CommitmentInfo.CommittedInstanceCount", + "CreateDate", + "DeliveryPreference", + "Id", + "InstanceType", + "OwnerId", + "ReservationType", + "StartDate", + "State", + "Tenancy", + "TotalInstanceCount" + ], + "AWS::EC2::CarrierGateway": [ + "CarrierGatewayId", + "OwnerId", + "State" + ], + "AWS::EC2::CustomerGateway": [ + "CustomerGatewayId" + ], + "AWS::EC2::DHCPOptions": [ + "DhcpOptionsId" + ], + "AWS::EC2::EIPAssociation": [ + "Id" + ], + "AWS::EC2::EgressOnlyInternetGateway": [ + "Id" + ], + "AWS::EC2::Host": [ + "HostId" + ], + "AWS::EC2::IPAM": [ + "Arn", + "DefaultResourceDiscoveryAssociationId", + "DefaultResourceDiscoveryId", + "IpamId", + "PrivateDefaultScopeId", + "PublicDefaultScopeId", + "ResourceDiscoveryAssociationCount", + "ScopeCount" + ], + "AWS::EC2::IPAMPool": [ + "Arn", + "IpamArn", + "IpamPoolId", + "IpamScopeArn", + "IpamScopeType", + "PoolDepth", + "State", + "StateMessage" + ], + "AWS::EC2::IPAMPoolCidr": [ + "IpamPoolCidrId", + "State" + ], + "AWS::EC2::IPAMScope": [ + "Arn", + "IpamArn", + "IpamScopeId", + "IpamScopeType", + "IsDefault", + "PoolCount" + ], + "AWS::EC2::Instance": [ + "AvailabilityZone", + "InstanceId", + "PrivateDnsName", + "PrivateIp", + "PublicDnsName", + "PublicIp", + "State", + "State.Code", + "State.Name", + "VpcId" + ], + "AWS::EC2::InstanceConnectEndpoint": [ + "AvailabilityZone", + "AvailabilityZoneId", + "CreatedAt", + "Id", + "InstanceConnectEndpointArn", + "NetworkInterfaceIds", + "OwnerId", + "PublicDnsNames", + "PublicDnsNames.Dualstack", + "PublicDnsNames.Dualstack.DnsName", + "PublicDnsNames.Dualstack.FipsDnsName", + "PublicDnsNames.Ipv4", + "PublicDnsNames.Ipv4.DnsName", + "PublicDnsNames.Ipv4.FipsDnsName", + "State", + "StateMessage", + "VpcId" + ], + "AWS::EC2::LaunchTemplate": [ + "DefaultVersionNumber", + "LatestVersionNumber", + "LaunchTemplateId" + ], + "AWS::EC2::NetworkInsightsPath": [ + "CreatedDate", + "DestinationArn", + "NetworkInsightsPathArn", + "NetworkInsightsPathId", + "SourceArn" + ], + "AWS::EC2::NetworkInterface": [ + "Id", + "PrimaryIpv6Address", + "PrimaryPrivateIpAddress", + "PublicIpDnsNameOptions", + "PublicIpDnsNameOptions.DnsHostnameType", + "PublicIpDnsNameOptions.PublicDualStackDnsName", + "PublicIpDnsNameOptions.PublicIpv4DnsName", + "PublicIpDnsNameOptions.PublicIpv6DnsName", + "SecondaryPrivateIpAddresses", + "VpcId" + ], + "AWS::EC2::PlacementGroup": [ + "GroupId", + "GroupName" + ], + "AWS::EC2::PrefixList": [ + "Arn", + "OwnerId", + "PrefixListId", + "Version" + ], + "AWS::EC2::RouteServer": [ + "Arn", + "Id" + ], + "AWS::EC2::RouteServerEndpoint": [ + "Arn", + "EniAddress", + "EniId", + "Id", + "VpcId" + ], + "AWS::EC2::RouteServerPeer": [ + "Arn", + "EndpointEniAddress", + "EndpointEniId", + "Id", + "RouteServerId", + "SubnetId", + "VpcId" + ], + "AWS::EC2::SecurityGroup": [ + "GroupId", + "Id", + "VpcId" + ], + "AWS::EC2::TrafficMirrorFilter": [ + "Id" + ], + "AWS::EC2::TrafficMirrorFilterRule": [ + "TrafficMirrorFilterRuleId" + ], + "AWS::EC2::TrafficMirrorSession": [ + "Id" + ], + "AWS::EC2::TrafficMirrorTarget": [ + "Id" + ], + "AWS::EC2::TransitGateway": [ + "EncryptionSupportState", + "Id", + "TransitGatewayArn" + ], + "AWS::EC2::TransitGatewayAttachment": [ + "Id" + ], + "AWS::EC2::TransitGatewayMulticastDomain": [ + "CreationTime", + "State", + "TransitGatewayMulticastDomainArn", + "TransitGatewayMulticastDomainId" + ], + "AWS::EC2::TransitGatewayPeeringAttachment": [ + "CreationTime", + "State", + "Status", + "Status.Code", + "Status.Message", + "TransitGatewayAttachmentId" + ], + "AWS::EC2::TransitGatewayRouteTable": [ + "TransitGatewayRouteTableId" + ], + "AWS::EC2::TransitGatewayVpcAttachment": [ + "Id" + ], + "AWS::EC2::VPC": [ + "CidrBlock", + "CidrBlockAssociations", + "DefaultNetworkAcl", + "DefaultSecurityGroup", + "Ipv6CidrBlocks", + "VpcEncryptionControl.ResourceExclusions", + "VpcEncryptionControl.ResourceExclusions.EgressOnlyInternetGateway", + "VpcEncryptionControl.ResourceExclusions.EgressOnlyInternetGateway.State", + "VpcEncryptionControl.ResourceExclusions.EgressOnlyInternetGateway.StateMessage", + "VpcEncryptionControl.ResourceExclusions.ElasticFileSystem", + "VpcEncryptionControl.ResourceExclusions.ElasticFileSystem.State", + "VpcEncryptionControl.ResourceExclusions.ElasticFileSystem.StateMessage", + "VpcEncryptionControl.ResourceExclusions.InternetGateway", + "VpcEncryptionControl.ResourceExclusions.InternetGateway.State", + "VpcEncryptionControl.ResourceExclusions.InternetGateway.StateMessage", + "VpcEncryptionControl.ResourceExclusions.Lambda", + "VpcEncryptionControl.ResourceExclusions.Lambda.State", + "VpcEncryptionControl.ResourceExclusions.Lambda.StateMessage", + "VpcEncryptionControl.ResourceExclusions.NatGateway", + "VpcEncryptionControl.ResourceExclusions.NatGateway.State", + "VpcEncryptionControl.ResourceExclusions.NatGateway.StateMessage", + "VpcEncryptionControl.ResourceExclusions.VirtualPrivateGateway", + "VpcEncryptionControl.ResourceExclusions.VirtualPrivateGateway.State", + "VpcEncryptionControl.ResourceExclusions.VirtualPrivateGateway.StateMessage", + "VpcEncryptionControl.ResourceExclusions.VpcLattice", + "VpcEncryptionControl.ResourceExclusions.VpcLattice.State", + "VpcEncryptionControl.ResourceExclusions.VpcLattice.StateMessage", + "VpcEncryptionControl.ResourceExclusions.VpcPeering", + "VpcEncryptionControl.ResourceExclusions.VpcPeering.State", + "VpcEncryptionControl.ResourceExclusions.VpcPeering.StateMessage", + "VpcEncryptionControl.State", + "VpcEncryptionControl.StateMessage", + "VpcEncryptionControl.VpcEncryptionControlId", + "VpcEncryptionControl.VpcId", + "VpcId" + ], + "AWS::EC2::VPCEndpoint": [ + "CreationTimestamp", + "DnsEntries", + "Id", + "NetworkInterfaceIds" + ], + "AWS::EC2::VPCEndpointService": [ + "PrivateDnsNameConfiguration.Name", + "PrivateDnsNameConfiguration.State", + "PrivateDnsNameConfiguration.Type", + "PrivateDnsNameConfiguration.Value", + "ServiceId" + ], + "AWS::EC2::VPNConnection": [ + "VpnConnectionId" + ], + "AWS::EC2::VPNGateway": [ + "VPNGatewayId" + ], + "AWS::EC2::VerifiedAccessInstance": [ + "CidrEndpointsCustomSubDomainNameServers", + "CreationTime", + "LastUpdatedTime", + "VerifiedAccessInstanceId" + ], + "AWS::EC2::Volume": [ + "VolumeId" + ], + "AWS::ECR::PublicRepository": [ + "Arn" + ], + "AWS::ECR::RegistryPolicy": [ + "RegistryId" + ], + "AWS::ECR::RepositoryCreationTemplate": [ + "CreatedAt", + "UpdatedAt" + ], + "AWS::ECS::Cluster": [ + "Arn" + ], + "AWS::ECS::TaskSet": [ + "Id" + ], + "AWS::EFS::AccessPoint": [ + "AccessPointId", + "Arn" + ], + "AWS::EKS::AccessEntry": [ + "AccessEntryArn" + ], + "AWS::EKS::Addon": [ + "Arn" + ], + "AWS::EKS::FargateProfile": [ + "Arn" + ], + "AWS::EKS::IdentityProviderConfig": [ + "IdentityProviderConfigArn" + ], + "AWS::EKS::PodIdentityAssociation": [ + "AssociationArn", + "AssociationId", + "ExternalId" + ], + "AWS::ElastiCache::GlobalReplicationGroup": [ + "GlobalReplicationGroupId", + "Status" + ], + "AWS::ElastiCache::ParameterGroup": [ + "CacheParameterGroupName" + ], + "AWS::ElastiCache::User": [ + "Arn", + "Status" + ], + "AWS::ElastiCache::UserGroup": [ + "Arn", + "Status" + ], + "AWS::ElasticLoadBalancingV2::LoadBalancer": [ + "CanonicalHostedZoneID", + "DNSName", + "LoadBalancerArn", + "LoadBalancerFullName", + "LoadBalancerName", + "SecurityGroups" + ], + "AWS::ElasticLoadBalancingV2::TargetGroup": [ + "LoadBalancerArns", + "TargetGroupArn", + "TargetGroupFullName", + "TargetGroupName" + ], + "AWS::Events::Connection": [ + "Arn", + "ArnForPolicy", + "AuthParameters.ConnectivityParameters.ResourceParameters.ResourceAssociationArn", + "InvocationConnectivityParameters.ResourceParameters.ResourceAssociationArn", + "SecretArn" + ], + "AWS::Events::Endpoint": [ + "Arn", + "EndpointId", + "EndpointUrl", + "State", + "StateReason" + ], + "AWS::Glue::Blueprint": [ + "Arn", + "CreatedOn", + "LastModifiedOn", + "ParameterSpec", + "Status" + ], + "AWS::Glue::Classifier": [ + "Name" + ], + "AWS::Glue::Registry": [ + "Arn" + ], + "AWS::Glue::Schema": [ + "Arn", + "InitialSchemaVersionId" + ], + "AWS::GuardDuty::Detector": [ + "Id" + ], + "AWS::GuardDuty::IPSet": [ + "Id" + ], + "AWS::IAM::AccessKey": [ + "SecretAccessKey" + ], + "AWS::IAM::OIDCProvider": [ + "Arn" + ], + "AWS::IAM::Role": [ + "Arn", + "RoleId" + ], + "AWS::IAM::SAMLProvider": [ + "Arn", + "SamlProviderUUID" + ], + "AWS::IAM::ServerCertificate": [ + "Arn" + ], + "AWS::IAM::ServiceLinkedRole": [ + "RoleName" + ], + "AWS::IAM::VirtualMFADevice": [ + "SerialNumber" + ], + "AWS::IoT::Authorizer": [ + "Arn" + ], + "AWS::IoT::BillingGroup": [ + "Arn", + "Id" + ], + "AWS::IoT::Certificate": [ + "Arn", + "Id" + ], + "AWS::IoT::CustomMetric": [ + "MetricArn" + ], + "AWS::IoT::Dimension": [ + "Arn" + ], + "AWS::IoT::DomainConfiguration": [ + "Arn", + "DomainType", + "ServerCertificates" + ], + "AWS::IoT::FleetMetric": [ + "CreationDate", + "LastModifiedDate", + "MetricArn", + "Version" + ], + "AWS::IoT::JobTemplate": [ + "Arn" + ], + "AWS::IoT::MitigationAction": [ + "MitigationActionArn", + "MitigationActionId" + ], + "AWS::IoT::Policy": [ + "Arn", + "Id" + ], + "AWS::IoT::ProvisioningTemplate": [ + "TemplateArn" + ], + "AWS::IoT::RoleAlias": [ + "RoleAliasArn" + ], + "AWS::IoT::ScheduledAudit": [ + "ScheduledAuditArn" + ], + "AWS::IoT::SecurityProfile": [ + "SecurityProfileArn" + ], + "AWS::IoT::ThingGroup": [ + "Arn", + "Id" + ], + "AWS::IoT::ThingType": [ + "Arn", + "Id" + ], + "AWS::IoT::TopicRuleDestination": [ + "Arn", + "StatusReason" + ], + "AWS::KMS::Key": [ + "Arn", + "KeyId" + ], + "AWS::KafkaConnect::Connector": [ + "ConnectorArn" + ], + "AWS::KafkaConnect::CustomPlugin": [ + "CustomPluginArn", + "FileDescription", + "FileDescription.FileMd5", + "FileDescription.FileSize", + "Revision" + ], + "AWS::KafkaConnect::WorkerConfiguration": [ + "Revision", + "WorkerConfigurationArn" + ], + "AWS::Kinesis::StreamConsumer": [ + "ConsumerARN", + "ConsumerCreationTimestamp", + "ConsumerName", + "ConsumerStatus", + "StreamARN" + ], + "AWS::KinesisFirehose::DeliveryStream": [ + "Arn" + ], + "AWS::KinesisVideo::SignalingChannel": [ + "Arn" + ], + "AWS::KinesisVideo::Stream": [ + "Arn" + ], + "AWS::Lambda::CodeSigningConfig": [ + "CodeSigningConfigArn", + "CodeSigningConfigId" + ], + "AWS::Lambda::Function": [ + "Arn", + "SnapStartResponse", + "SnapStartResponse.ApplyOn", + "SnapStartResponse.OptimizationStatus" + ], + "AWS::Logs::Delivery": [ + "Arn", + "DeliveryDestinationType", + "DeliveryId" + ], + "AWS::Logs::DeliveryDestination": [ + "Arn" + ], + "AWS::Logs::DeliverySource": [ + "Arn", + "ResourceArns", + "Service", + "Status", + "StatusReason" + ], + "AWS::Logs::Destination": [ + "Arn" + ], + "AWS::Logs::Integration": [ + "IntegrationStatus" + ], + "AWS::Logs::LogAnomalyDetector": [ + "AnomalyDetectorArn", + "AnomalyDetectorStatus", + "CreationTimeStamp", + "LastModifiedTimeStamp" + ], + "AWS::Logs::LogGroup": [ + "Arn" + ], + "AWS::Logs::ScheduledQuery": [ + "CreationTime", + "LastExecutionStatus", + "LastTriggeredTime", + "LastUpdatedTime", + "ScheduledQueryArn" + ], + "AWS::Macie::AllowList": [ + "Arn", + "Id", + "Status" + ], + "AWS::Macie::FindingsFilter": [ + "Arn", + "Id" + ], + "AWS::MemoryDB::ACL": [ + "Arn", + "Status" + ], + "AWS::MemoryDB::Cluster": [ + "ARN", + "ClusterEndpoint.Address", + "ClusterEndpoint.Port", + "ParameterGroupStatus", + "Status" + ], + "AWS::MemoryDB::ParameterGroup": [ + "ARN" + ], + "AWS::MemoryDB::SubnetGroup": [ + "ARN", + "SupportedNetworkTypes" + ], + "AWS::MemoryDB::User": [ + "Arn", + "Status" + ], + "AWS::OpenSearchService::Domain": [ + "AdvancedSecurityOptions.AnonymousAuthDisableDate", + "Arn", + "DomainArn", + "DomainEndpoint", + "DomainEndpointV2", + "DomainEndpoints", + "Id", + "IdentityCenterOptions.IdentityCenterApplicationARN", + "IdentityCenterOptions.IdentityStoreId", + "ServiceSoftwareOptions", + "ServiceSoftwareOptions.AutomatedUpdateDate", + "ServiceSoftwareOptions.Cancellable", + "ServiceSoftwareOptions.CurrentVersion", + "ServiceSoftwareOptions.Description", + "ServiceSoftwareOptions.NewVersion", + "ServiceSoftwareOptions.OptionalDeployment", + "ServiceSoftwareOptions.UpdateAvailable", + "ServiceSoftwareOptions.UpdateStatus" + ], + "AWS::RDS::DBInstance": [ + "AutomaticRestartTime", + "CertificateDetails", + "CertificateDetails.CAIdentifier", + "CertificateDetails.ValidTill", + "DBInstanceArn", + "DBInstanceStatus", + "DBSystemId", + "DbiResourceId", + "Endpoint", + "Endpoint.Address", + "Endpoint.HostedZoneId", + "Endpoint.Port", + "InstanceCreateTime", + "IsStorageConfigUpgradeAvailable", + "LatestRestorableTime", + "ListenerEndpoint", + "ListenerEndpoint.Address", + "ListenerEndpoint.HostedZoneId", + "ListenerEndpoint.Port", + "MasterUserSecret.SecretArn", + "PercentProgress", + "ReadReplicaDBClusterIdentifiers", + "ReadReplicaDBInstanceIdentifiers", + "ResumeFullAutomationModeTime", + "SecondaryAvailabilityZone", + "StatusInfos", + "StorageOperationPercentProgress", + "StorageOperationStatus" + ], + "AWS::RDS::DBProxy": [ + "DBProxyArn", + "Endpoint", + "VpcId" + ], + "AWS::RDS::DBProxyEndpoint": [ + "DBProxyEndpointArn", + "Endpoint", + "IsDefault", + "VpcId" + ], + "AWS::Redshift::ClusterSubnetGroup": [ + "ClusterSubnetGroupName" + ], + "AWS::Route53::HostedZone": [ + "Id", + "NameServers" + ], + "AWS::Route53Resolver::FirewallDomainList": [ + "Arn", + "CreationTime", + "CreatorRequestId", + "DomainCount", + "Id", + "ManagedOwnerName", + "ModificationTime", + "Status", + "StatusMessage" + ], + "AWS::Route53Resolver::FirewallRuleGroup": [ + "Arn", + "CreationTime", + "CreatorRequestId", + "Id", + "ModificationTime", + "OwnerId", + "RuleCount", + "ShareStatus", + "Status", + "StatusMessage" + ], + "AWS::Route53Resolver::FirewallRuleGroupAssociation": [ + "Arn", + "CreationTime", + "CreatorRequestId", + "Id", + "ManagedOwnerName", + "ModificationTime", + "Status", + "StatusMessage" + ], + "AWS::Route53Resolver::OutpostResolver": [ + "Arn", + "CreationTime", + "CreatorRequestId", + "Id", + "ModificationTime", + "Status", + "StatusMessage" + ], + "AWS::Route53Resolver::ResolverQueryLoggingConfig": [ + "Arn", + "AssociationCount", + "CreationTime", + "CreatorRequestId", + "Id", + "OwnerId", + "ShareStatus", + "Status" + ], + "AWS::Route53Resolver::ResolverQueryLoggingConfigAssociation": [ + "CreationTime", + "Error", + "ErrorMessage", + "Id", + "Status" + ], + "AWS::Route53Resolver::ResolverRuleAssociation": [ + "Name", + "ResolverRuleAssociationId", + "ResolverRuleId", + "VPCId" + ], + "AWS::S3::Bucket": [ + "Arn", + "DomainName", + "DualStackDomainName", + "MetadataConfiguration.AnnotationTableConfiguration.TableArn", + "MetadataConfiguration.AnnotationTableConfiguration.TableName", + "MetadataConfiguration.Destination", + "MetadataConfiguration.Destination.TableBucketArn", + "MetadataConfiguration.Destination.TableBucketType", + "MetadataConfiguration.Destination.TableNamespace", + "MetadataConfiguration.InventoryTableConfiguration.TableArn", + "MetadataConfiguration.InventoryTableConfiguration.TableName", + "MetadataConfiguration.JournalTableConfiguration.TableArn", + "MetadataConfiguration.JournalTableConfiguration.TableName", + "MetadataTableConfiguration.S3TablesDestination.TableArn", + "MetadataTableConfiguration.S3TablesDestination.TableNamespace", + "RegionalDomainName", + "WebsiteURL" + ], + "AWS::SNS::Topic": [ + "TopicArn", + "TopicName" + ], + "AWS::SQS::Queue": [ + "Arn", + "QueueName", + "QueueUrl" + ], + "AWS::SSM::MaintenanceWindowTarget": [ + "WindowTargetId" + ], + "AWS::SSM::MaintenanceWindowTask": [ + "WindowTaskId" + ], + "AWS::SSM::ResourcePolicy": [ + "PolicyHash", + "PolicyId" + ], + "AWS::SageMaker::CodeRepository": [ + "CodeRepositoryArn", + "CodeRepositoryName" + ], + "AWS::SageMaker::Domain": [ + "DomainArn", + "DomainId", + "HomeEfsFileSystemId", + "SecurityGroupIdForDomainBoundary", + "SingleSignOnApplicationArn", + "SingleSignOnManagedApplicationInstanceId", + "Url" + ], + "AWS::SageMaker::Endpoint": [ + "EndpointArn", + "EndpointName" + ], + "AWS::SageMaker::EndpointConfig": [ + "EndpointConfigArn", + "EndpointConfigName" + ], + "AWS::SageMaker::FeatureGroup": [ + "CreationTime", + "FeatureGroupStatus" + ], + "AWS::SageMaker::Image": [ + "ImageArn" + ], + "AWS::SageMaker::ImageVersion": [ + "ContainerImage", + "ImageArn", + "ImageVersionArn", + "Version" + ], + "AWS::SageMaker::Model": [ + "ModelArn", + "ModelName" + ], + "AWS::SageMaker::ModelPackageGroup": [ + "CreationTime", + "ModelPackageGroupArn", + "ModelPackageGroupStatus" + ], + "AWS::SageMaker::NotebookInstance": [ + "NotebookInstanceArn", + "NotebookInstanceName" + ], + "AWS::SageMaker::NotebookInstanceLifecycleConfig": [ + "NotebookInstanceLifecycleConfigName" + ], + "AWS::SageMaker::Project": [ + "CreationTime", + "ProjectArn", + "ProjectId", + "ProjectStatus" + ], + "AWS::SageMaker::Workteam": [ + "WorkteamName" + ], + "AWS::Scheduler::ScheduleGroup": [ + "Arn", + "CreationDate", + "LastModificationDate", + "State" + ], + "AWS::SecretsManager::Secret": [ + "Id" + ], + "AWS::ServiceDiscovery::HttpNamespace": [ + "Arn", + "Id" + ], + "AWS::ServiceDiscovery::PrivateDnsNamespace": [ + "Arn", + "HostedZoneId", + "Id" + ], + "AWS::ServiceDiscovery::PublicDnsNamespace": [ + "Arn", + "HostedZoneId", + "Id" + ], + "AWS::ServiceDiscovery::Service": [ + "Arn", + "Id", + "Name" + ], + "AWS::StepFunctions::Activity": [ + "Arn", + "Name" + ], + "AWS::StepFunctions::StateMachine": [ + "Arn", + "Name", + "StateMachineRevisionId" + ], + "AWS::Transfer::Profile": [ + "Arn", + "ProfileId" + ], + "AWS::Transfer::Workflow": [ + "Arn", + "WorkflowId" + ] +} diff --git a/services/cloudformation/cfn_attributes_gen.go b/services/cloudformation/cfn_attributes_gen.go deleted file mode 100644 index 9ca30145ec..0000000000 --- a/services/cloudformation/cfn_attributes_gen.go +++ /dev/null @@ -1,328 +0,0 @@ -// Code generated by cmd/cfnattrgen from the CloudFormation resource specification; DO NOT EDIT. -package cloudformation - -//nolint:gochecknoglobals // generated static lookup table -var cfnResourceAttributes = map[string]map[string]struct{}{ - resTypeAccessAnalyzerAnalyzer: { - attrNameArn: {}, - }, - resTypeAccessAnalyzerArchiveRule: { - attrNameArn: {}, - "CreatedAt": {}, - "UpdatedAt": {}, - }, - resTypeAPIGatewayV2Integ: { - "IntegrationId": {}, - }, - resTypeAPIGatewayV2Route: { - "RouteId": {}, - }, - resTypeAppConfigEnvironment: { - "EnvironmentId": {}, - }, - resTypeAppSyncChannelNamespace: { - "ChannelNamespaceArn": {}, - }, - resTypeAthenaNamedQuery: { - "NamedQueryId": {}, - }, - resTypeAthenaWorkGroup: { - "CreationTime": {}, - "WorkGroupConfiguration.EngineVersion.EffectiveEngineVersion": {}, - }, - resTypeASGScheduledActn: { - "ScheduledActionName": {}, - }, - resTypeBackupFramework: { - "CreationTime": {}, - "DeploymentStatus": {}, - "FrameworkArn": {}, - "FrameworkStatus": {}, - }, - resTypeBackupReportPlan: { - "ReportPlanArn": {}, - }, - resTypeBatchSchedulingPolicy: { - attrNameArn: {}, - }, - resTypeBatchServiceEnvironment: { - "ServiceEnvironmentArn": {}, - }, - resTypeCloudTrailChannel: { - "ChannelArn": {}, - }, - resTypeCloudWatchAlarm: { - attrNameArn: {}, - }, - resTypeCodeArtifactPackageGroup: { - attrNameArn: {}, - }, - resTypeCodeBuildReportGroup: { - attrNameArn: {}, - }, - resTypeConfigAggregationAuthorization: { - "AggregationAuthorizationArn": {}, - }, - resTypeConfigConfigRule: { - attrNameArn: {}, - "Compliance.Type": {}, - "ConfigRuleId": {}, - }, - resTypeConfigConfigurationAggregator: { - "ConfigurationAggregatorArn": {}, - }, - resTypeConfigConformancePack: { - "ConformancePackArn": {}, - }, - resTypeConfigStoredQuery: { - "QueryArn": {}, - "QueryId": {}, - }, - resTypeDataSyncAgent: { - "AgentArn": {}, - "EndpointType": {}, - }, - resTypeDataSyncLocationS3: { - "LocationArn": {}, - "LocationUri": {}, - }, - resTypeDocDBGlobalCluster: { - "GlobalClusterArn": {}, - "GlobalClusterResourceId": {}, - }, - resTypeEC2CustomerGateway: { - "CustomerGatewayId": {}, - }, - resTypeEC2DHCPOptions: { - "DhcpOptionsId": {}, - }, - resTypeEC2Host: { - "HostId": {}, - }, - resTypeEC2IPAMScope: { - attrNameArn: {}, - "IpamArn": {}, - "IpamScopeId": {}, - "IpamScopeType": {}, - "IsDefault": {}, - "PoolCount": {}, - }, - resTypeEC2LaunchTemplate: { - "DefaultVersionNumber": {}, - "LatestVersionNumber": {}, - "LaunchTemplateId": {}, - }, - resTypeEC2TGWRouteTable: { - "TransitGatewayRouteTableId": {}, - }, - resTypeECRRegistryPolicy: { - "RegistryId": {}, - }, - resTypeECRRepositoryCreationTemplate: { - "CreatedAt": {}, - "UpdatedAt": {}, - }, - resTypeECSCluster: { - attrNameArn: {}, - }, - resTypeEFSAccessPoint: { - "AccessPointId": {}, - attrNameArn: {}, - }, - resTypeEKSAccessEntry: { - "AccessEntryArn": {}, - }, - resTypeEKSAddon: { - attrNameArn: {}, - }, - resTypeEKSFargateProfile: { - attrNameArn: {}, - }, - resTypeEKSIdentityProviderConfig: { - "IdentityProviderConfigArn": {}, - }, - resTypeElastiCacheParameterGroup: { - "CacheParameterGroupName": {}, - }, - resTypeGlueClassifier: { - attrNameName: {}, - }, - resTypeGlueRegistry: { - attrNameArn: {}, - }, - resTypeGlueSchema: { - attrNameArn: {}, - "InitialSchemaVersionId": {}, - }, - resTypeIAMAccessKey: { - "SecretAccessKey": {}, - }, - resTypeIAMOIDCProvider: { - attrNameArn: {}, - }, - resTypeIAMRole: { - attrNameArn: {}, - "RoleId": {}, - }, - resTypeIAMServerCertificate: { - attrNameArn: {}, - }, - resTypeIoTAuthorizer: { - attrNameArn: {}, - }, - resTypeIoTDimension: { - attrNameArn: {}, - }, - resTypeIoTDomainConfiguration: { - attrNameArn: {}, - "DomainType": {}, - "ServerCertificates": {}, - }, - resTypeIoTJobTemplate: { - attrNameArn: {}, - }, - resTypeIoTMitigationAction: { - "MitigationActionArn": {}, - "MitigationActionId": {}, - }, - resTypeIoTProvisioningTemplate: { - "TemplateArn": {}, - }, - resTypeIoTRoleAlias: { - "RoleAliasArn": {}, - }, - resTypeIoTScheduledAudit: { - "ScheduledAuditArn": {}, - }, - resTypeIoTSecurityProfile: { - "SecurityProfileArn": {}, - }, - resTypeIoTTopicRuleDestination: { - attrNameArn: {}, - "StatusReason": {}, - }, - resTypeFirehoseDeliveryStream: { - attrNameArn: {}, - }, - resTypeLambdaCodeSigningConfig: { - "CodeSigningConfigArn": {}, - "CodeSigningConfigId": {}, - }, - resTypeLambdaFunction: { - attrNameArn: {}, - "SnapStartResponse": {}, - "SnapStartResponse.ApplyOn": {}, - "SnapStartResponse.OptimizationStatus": {}, - }, - resTypeLogsDelivery: { - attrNameArn: {}, - "DeliveryDestinationType": {}, - "DeliveryId": {}, - }, - resTypeLogsDeliveryDestination: { - attrNameArn: {}, - }, - resTypeLogsDestination: { - attrNameArn: {}, - }, - resTypeLogsIntegration: { - "IntegrationStatus": {}, - }, - resTypeLogsAnomalyDetector: { - "AnomalyDetectorArn": {}, - "AnomalyDetectorStatus": {}, - "CreationTimeStamp": {}, - "LastModifiedTimeStamp": {}, - }, - resTypeLogGroup: { - attrNameArn: {}, - }, - resTypeLogsScheduledQuery: { - "CreationTime": {}, - "LastExecutionStatus": {}, - "LastTriggeredTime": {}, - "LastUpdatedTime": {}, - "ScheduledQueryArn": {}, - }, - resTypeMemoryDBParameterGroup: { - "ARN": {}, - }, - resTypeMemoryDBSubnetGroup: { - "ARN": {}, - "SupportedNetworkTypes": {}, - }, - resTypeRedshiftClusterSubnetGroup: { - "ClusterSubnetGroupName": {}, - }, - resTypeR53RResolverRuleAssoc: { - attrNameName: {}, - "ResolverRuleAssociationId": {}, - "ResolverRuleId": {}, - "VPCId": {}, - }, - resTypeSSMMaintenanceWindowTarget: { - "WindowTargetId": {}, - }, - resTypeSSMMaintenanceWindowTask: { - "WindowTaskId": {}, - }, - resTypeSSMResourcePolicy: { - "PolicyHash": {}, - "PolicyId": {}, - }, - resTypeSageMakerCodeRepository: { - "CodeRepositoryArn": {}, - "CodeRepositoryName": {}, - }, - resTypeSageMakerFeatureGroup: { - "CreationTime": {}, - "FeatureGroupStatus": {}, - }, - resTypeSageMakerImage: { - "ImageArn": {}, - }, - resTypeSageMakerImageVersion: { - "ContainerImage": {}, - "ImageArn": {}, - "ImageVersionArn": {}, - "Version": {}, - }, - resTypeSageMakerModelPackageGroup: { - "CreationTime": {}, - "ModelPackageGroupArn": {}, - "ModelPackageGroupStatus": {}, - }, - resTypeSageMakerNotebookInstance: { - "NotebookInstanceArn": {}, - "NotebookInstanceName": {}, - }, - resTypeSageMakerNotebookInstanceLifecycleConfig: { - "NotebookInstanceLifecycleConfigName": {}, - }, - resTypeSageMakerProject: { - "CreationTime": {}, - "ProjectArn": {}, - "ProjectId": {}, - "ProjectStatus": {}, - }, - resTypeSageMakerWorkteam: { - "WorkteamName": {}, - }, - resTypeStepFunctionsActivity: { - attrNameArn: {}, - attrNameName: {}, - }, - resTypeStepFunctionsStateMachine: { - attrNameArn: {}, - attrNameName: {}, - "StateMachineRevisionId": {}, - }, - resTypeTransferProfile: { - attrNameArn: {}, - "ProfileId": {}, - }, - resTypeTransferWorkflow: { - attrNameArn: {}, - "WorkflowId": {}, - }, -} diff --git a/services/cloudformation/handler_stack_sets.go b/services/cloudformation/handler_stack_sets.go index 41f0aa3ae0..44d541548e 100644 --- a/services/cloudformation/handler_stack_sets.go +++ b/services/cloudformation/handler_stack_sets.go @@ -487,20 +487,50 @@ func (h *Handler) handleListStackSets(form url.Values, c *echo.Context) error { ) } -// unsupportedAccountFilterType returns the requested -// DeploymentTargets.AccountFilterType value if it's one this backend doesn't -// implement, or "" if the request should proceed. Only NONE (the union of -// Accounts and resolved OrganizationalUnitIds, this backend's only supported -// mode) passes; INTERSECTION/DIFFERENCE/UNION are rejected explicitly rather -// than silently computed as NONE (botocore cloudformation service-2.json -// AccountFilterType enum, botocore 1.43.56). -func unsupportedAccountFilterType(form url.Values) string { - switch ft := form.Get("DeploymentTargets.AccountFilterType"); ft { - case "", valueNone: - return "" - default: - return ft +// validAccountFilterTypes is the documented DeploymentTargets.AccountFilterType +// enum (API_DeploymentTargets.html; "" is the wire default, equivalent to UNION). +var validAccountFilterTypes = map[string]bool{ //nolint:gochecknoglobals // read-only lookup + "": true, + valueNone: true, + accountFilterIntersection: true, + accountFilterDifference: true, + accountFilterUnion: true, +} + +// parseAccountFilterType validates DeploymentTargets.AccountFilterType and +// returns it, or the ValidationError message text CloudFormation returns for +// an invalid or unsupported combination +// (docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/ +// API_DeploymentTargets.html). isCreate gates the two rules the API +// reference documents as specific to CreateStackInstances: UNION is not +// supported there, and specifying both OrganizationalUnitIds and Accounts +// requires an explicit AccountFilterType. +func parseAccountFilterType(form url.Values, isCreate bool) (string, string) { + filterType := form.Get("DeploymentTargets.AccountFilterType") + if !validAccountFilterTypes[filterType] { + return "", fmt.Sprintf( + "DeploymentTargets.AccountFilterType %s is not a valid value; must be one of "+ + "NONE, INTERSECTION, DIFFERENCE, UNION", filterType, + ) + } + + if !isCreate { + return filterType, "" + } + + if filterType == accountFilterUnion { + return "", "AccountFilterType UNION is not supported for CreateStackInstances operations" } + + hasAccounts := len(parseStackInstanceAccounts(form)) > 0 + hasOUs := len(parseMemberList(form, "DeploymentTargets.OrganizationalUnitIds.")) > 0 + + if filterType == "" && hasAccounts && hasOUs { + return "", "you must specify DeploymentTargets.AccountFilterType when specifying " + + "both Accounts and OrganizationalUnitIds" + } + + return filterType, "" } // parseStackInstanceAccounts returns the union of the legacy top-level @@ -514,16 +544,18 @@ func parseStackInstanceAccounts(form url.Values) []string { } // stackInstancesOp is CreateStackInstances or DeleteStackInstances -- same -// request shape (accounts/OU targets/regions in, an operation ID out). +// request shape (accounts/OU targets/regions/filter type in, an operation ID out). type stackInstancesOp func( - ctx context.Context, stackSetName string, accounts, ouIDs, regions []string, + ctx context.Context, stackSetName string, accounts, ouIDs, regions []string, filterType string, ) (string, error) // handleStackInstancesOp parses the shared CreateStackInstances/ // DeleteStackInstances request shape, invokes op, and writes the shared -// {OperationId} response envelope under responseElem/resultElem. +// {OperationId} response envelope under responseElem/resultElem. isCreate +// gates the CreateStackInstances-only AccountFilterType validation rules +// (see parseAccountFilterType). func (h *Handler) handleStackInstancesOp( - form url.Values, c *echo.Context, responseElem, resultElem string, op stackInstancesOp, + form url.Values, c *echo.Context, responseElem, resultElem string, isCreate bool, op stackInstancesOp, ) error { name := form.Get("StackSetName") if name == "" { @@ -534,14 +566,15 @@ func (h *Handler) handleStackInstancesOp( return h.xmlError(c, "ValidationError", err.Error()) } - if ft := unsupportedAccountFilterType(form); ft != "" { - return h.xmlError(c, "ValidationError", - fmt.Sprintf("DeploymentTargets.AccountFilterType %s is not supported", ft)) + filterType, filterErrMsg := parseAccountFilterType(form, isCreate) + if filterErrMsg != "" { + return h.xmlError(c, "ValidationError", filterErrMsg) } + accounts := parseStackInstanceAccounts(form) ouIDs := parseMemberList(form, "DeploymentTargets.OrganizationalUnitIds.") regions := parseMemberList(form, "Regions.") - opID, err := op(c.Request().Context(), name, accounts, ouIDs, regions) + opID, err := op(c.Request().Context(), name, accounts, ouIDs, regions, filterType) if err != nil { return h.xmlError(c, stackInstancesErrorCode(err), err.Error()) } @@ -566,7 +599,7 @@ func (h *Handler) handleStackInstancesOp( func (h *Handler) handleCreateStackInstances(form url.Values, c *echo.Context) error { return h.handleStackInstancesOp( - form, c, "CreateStackInstancesResponse", "CreateStackInstancesResult", h.Backend.CreateStackInstances, + form, c, "CreateStackInstancesResponse", "CreateStackInstancesResult", true, h.Backend.CreateStackInstances, ) } @@ -576,11 +609,13 @@ func (h *Handler) handleDeleteStackInstances(form url.Values, c *echo.Context) e return h.xmlError(c, "ValidationError", "RetainStacks is required") } retainStacks := retainStr == boolTrue - op := func(ctx context.Context, stackSetName string, accounts, ouIDs, regions []string) (string, error) { - return h.Backend.DeleteStackInstances(ctx, stackSetName, accounts, ouIDs, regions, retainStacks) + op := func( + ctx context.Context, stackSetName string, accounts, ouIDs, regions []string, filterType string, + ) (string, error) { + return h.Backend.DeleteStackInstances(ctx, stackSetName, accounts, ouIDs, regions, retainStacks, filterType) } - return h.handleStackInstancesOp(form, c, "DeleteStackInstancesResponse", "DeleteStackInstancesResult", op) + return h.handleStackInstancesOp(form, c, "DeleteStackInstancesResponse", "DeleteStackInstancesResult", false, op) } func (h *Handler) handleUpdateStackInstances(form url.Values, c *echo.Context) error { @@ -593,14 +628,15 @@ func (h *Handler) handleUpdateStackInstances(form url.Values, c *echo.Context) e return h.xmlError(c, "ValidationError", err.Error()) } - if ft := unsupportedAccountFilterType(form); ft != "" { - return h.xmlError(c, "ValidationError", - fmt.Sprintf("DeploymentTargets.AccountFilterType %s is not supported", ft)) + filterType, filterErrMsg := parseAccountFilterType(form, false) + if filterErrMsg != "" { + return h.xmlError(c, "ValidationError", filterErrMsg) } + accounts := parseStackInstanceAccounts(form) ouIDs := parseMemberList(form, "DeploymentTargets.OrganizationalUnitIds.") regions := parseMemberList(form, "Regions.") - opID, err := h.Backend.UpdateStackInstances(name, accounts, ouIDs, regions) + opID, err := h.Backend.UpdateStackInstances(name, accounts, ouIDs, regions, filterType) if err != nil { return h.xmlError(c, stackInstancesErrorCode(err), err.Error()) } diff --git a/services/cloudformation/intrinsics_getatt_attribute_test.go b/services/cloudformation/intrinsics_getatt_attribute_test.go index 0ce975c4df..f185e66eb4 100644 --- a/services/cloudformation/intrinsics_getatt_attribute_test.go +++ b/services/cloudformation/intrinsics_getatt_attribute_test.go @@ -16,7 +16,7 @@ import ( // Real CreateStack rejects this synchronously with a ValidationError // ("Template error: resource does not support attribute type in // Fn::GetAtt"); AWS::Lambda::CodeSigningConfig is one of the table's listed -// types (cfn_attributes_gen.go), with CodeSigningConfigArn/CodeSigningConfigId +// types (cfn_attributes.json), with CodeSigningConfigArn/CodeSigningConfigId // as its only documented attributes. func TestCreateStack_GetAttAttributeValidation(t *testing.T) { t.Parallel() @@ -67,16 +67,15 @@ func TestCreateStack_GetAttAttributeValidation(t *testing.T) { _, client := newNewerTypesTestClient(t) - // AWS::CodeArtifact::Domain isn't in cfn_attributes_gen.go's table - // (see PARITY.md's gopherstack-p7pvq note: it's dropped whole rather - // than partially, since not every attribute this backend stashes for - // it can be safely re-quoted there) -- any attribute on it must still - // fall back to the resource's physical ID, not error. + // AWS::CodeDeploy::DeploymentConfig has no Attributes documented in + // the CloudFormation spec at all, so it isn't in cfn_attributes.json + // -- any attribute on it must still fall back to the resource's + // physical ID, not error. tmpl := `{ "Resources": { - "Dom": {"Type": "AWS::CodeArtifact::Domain", "Properties": {"DomainName": "gaa-domain"}} + "DC": {"Type": "AWS::CodeDeploy::DeploymentConfig", "Properties": {"DeploymentConfigName": "gaa-config"}} }, -"Outputs": {"Fallback": {"Value": {"Fn::GetAtt": ["Dom", "SomeFieldThisBackendDoesNotModel"]}}} +"Outputs": {"Fallback": {"Value": {"Fn::GetAtt": ["DC", "SomeFieldThisBackendDoesNotModel"]}}} }` outputs := createStackAndGetOutputs(t, client, "gaa-fallback-stack", tmpl) diff --git a/services/cloudformation/intrinsics_validate.go b/services/cloudformation/intrinsics_validate.go index fb1ae3fd7a..41558989e9 100644 --- a/services/cloudformation/intrinsics_validate.go +++ b/services/cloudformation/intrinsics_validate.go @@ -260,7 +260,7 @@ func validateGetAttAttribute(logicalID, resType, attrName string) error { return nil } - attrs, known := cfnResourceAttributes[resType] + attrs, known := resourceAttributeTable()[resType] if !known { return nil } diff --git a/services/cloudformation/list_maxresults_sd1a7_test.go b/services/cloudformation/list_maxresults_sd1a7_test.go index 5169127ed6..c0e5d5fcd1 100644 --- a/services/cloudformation/list_maxresults_sd1a7_test.go +++ b/services/cloudformation/list_maxresults_sd1a7_test.go @@ -215,7 +215,7 @@ func TestListStackInstances_MaxResults(t *testing.T) { require.NoError(t, err) accounts := []string{"111111111111", "222222222222", "333333333333"} - _, err = backend.CreateStackInstances(ctx, "maxres-inst-ss", accounts, nil, []string{"us-east-1"}) + _, err = backend.CreateStackInstances(ctx, "maxres-inst-ss", accounts, nil, []string{"us-east-1"}, "") require.NoError(t, err) page1, err := client.ListStackInstances(ctx, &cfnsdk.ListStackInstancesInput{ diff --git a/services/cloudformation/list_pagination_v8jl_test.go b/services/cloudformation/list_pagination_v8jl_test.go index 3cdc24f946..4e0047e02d 100644 --- a/services/cloudformation/list_pagination_v8jl_test.go +++ b/services/cloudformation/list_pagination_v8jl_test.go @@ -322,7 +322,7 @@ func TestListStackSetOperationResults_Pagination(t *testing.T) { require.NoError(t, err) accounts := []string{"111111111111", "222222222222", "333333333333"} - _, err = backend.CreateStackInstances(ctx, "opresults-ss", accounts, nil, []string{"us-east-1"}) + _, err = backend.CreateStackInstances(ctx, "opresults-ss", accounts, nil, []string{"us-east-1"}, "") require.NoError(t, err) opsOut, err := client.ListStackSetOperations(ctx, &cfnsdk.ListStackSetOperationsInput{ @@ -379,7 +379,7 @@ func TestListStackSetAutoDeploymentTargets_Pagination(t *testing.T) { require.NoError(t, err) accounts := []string{"111111111111", "222222222222", "333333333333"} - _, err = backend.CreateStackInstances(ctx, "autotargets-ss", accounts, nil, []string{"us-east-1"}) + _, err = backend.CreateStackInstances(ctx, "autotargets-ss", accounts, nil, []string{"us-east-1"}, "") require.NoError(t, err) page1, err := client.ListStackSetAutoDeploymentTargets(ctx, &cfnsdk.ListStackSetAutoDeploymentTargetsInput{ diff --git a/services/cloudformation/list_summary_shapes_test.go b/services/cloudformation/list_summary_shapes_test.go index 5e77209e5f..48fccacf85 100644 --- a/services/cloudformation/list_summary_shapes_test.go +++ b/services/cloudformation/list_summary_shapes_test.go @@ -91,7 +91,14 @@ func testListStackInstanceResourceDriftsNarrowShape(t *testing.T) { _, err := backend.CreateStackSet("drift-summary-ss", "desc", simpleTemplate, cloudformation.StackSetOptions{}) require.NoError(t, err) - _, err = backend.CreateStackInstances(ctx, "drift-summary-ss", []string{"111111111111"}, nil, []string{"us-east-1"}) + _, err = backend.CreateStackInstances( + ctx, + "drift-summary-ss", + []string{"111111111111"}, + nil, + []string{"us-east-1"}, + "", + ) require.NoError(t, err) instances, err := backend.ListStackInstances( diff --git a/services/cloudformation/persistence_test.go b/services/cloudformation/persistence_test.go index 228a3f2257..f7f0dbdaf5 100644 --- a/services/cloudformation/persistence_test.go +++ b/services/cloudformation/persistence_test.go @@ -97,7 +97,7 @@ func TestInMemoryBackend_SnapshotRestore_PlainMapFields(t *testing.T) { require.NoError(t, err) opID, err := original.CreateStackInstances( - ctx, "test-set", []string{"111111111111"}, nil, []string{"us-east-1"}, + ctx, "test-set", []string{"111111111111"}, nil, []string{"us-east-1"}, "", ) require.NoError(t, err) require.NotEmpty(t, opID) diff --git a/services/cloudformation/provider.go b/services/cloudformation/provider.go index 5e3b2c95e1..11572b8034 100644 --- a/services/cloudformation/provider.go +++ b/services/cloudformation/provider.go @@ -30,6 +30,7 @@ import ( ddbbackend "github.com/blackbirdworks/gopherstack/services/dynamodb" ec2backend "github.com/blackbirdworks/gopherstack/services/ec2" ecrbackend "github.com/blackbirdworks/gopherstack/services/ecr" + ecrpublicbackend "github.com/blackbirdworks/gopherstack/services/ecrpublic" ecsbackend "github.com/blackbirdworks/gopherstack/services/ecs" efsbackend "github.com/blackbirdworks/gopherstack/services/efs" eksbackend "github.com/blackbirdworks/gopherstack/services/eks" @@ -42,7 +43,9 @@ import ( iambackend "github.com/blackbirdworks/gopherstack/services/iam" iotbackend "github.com/blackbirdworks/gopherstack/services/iot" kafkabackend "github.com/blackbirdworks/gopherstack/services/kafka" + kafkaconnectbackend "github.com/blackbirdworks/gopherstack/services/kafkaconnect" kinesisbackend "github.com/blackbirdworks/gopherstack/services/kinesis" + kinesisvideobackend "github.com/blackbirdworks/gopherstack/services/kinesisvideo" kmsbackend "github.com/blackbirdworks/gopherstack/services/kms" lambdabackend "github.com/blackbirdworks/gopherstack/services/lambda" macie2backend "github.com/blackbirdworks/gopherstack/services/macie2" @@ -154,6 +157,9 @@ type BackendsProvider interface { GetGuardDutyHandler() service.Registerable GetAccessAnalyzerHandler() service.Registerable GetAmplifyHandler() service.Registerable + GetKinesisVideoHandler() service.Registerable + GetECRPublicHandler() service.Registerable + GetKafkaConnectHandler() service.Registerable GetGlobalConfig() *config.GlobalConfig } @@ -205,6 +211,9 @@ func extractCoreBackends(bp BackendsProvider, backends *ServiceBackends) { backends.GuardDuty, _ = getHandler[*guarddutybackend.Handler](bp.GetGuardDutyHandler()) backends.AccessAnalyzer, _ = getHandler[*accessanalyzerbackend.Handler](bp.GetAccessAnalyzerHandler()) backends.Amplify, _ = getHandler[*amplifybackend.Handler](bp.GetAmplifyHandler()) + backends.KinesisVideo, _ = getHandler[*kinesisvideobackend.Handler](bp.GetKinesisVideoHandler()) + backends.ECRPublic, _ = getHandler[*ecrpublicbackend.Handler](bp.GetECRPublicHandler()) + backends.KafkaConnect, _ = getHandler[*kafkaconnectbackend.Handler](bp.GetKafkaConnectHandler()) } // extractAllServiceBackends populates all extended and phase-2 service backends. diff --git a/services/cloudformation/resources.go b/services/cloudformation/resources.go index fcf245117d..70e96f3aee 100644 --- a/services/cloudformation/resources.go +++ b/services/cloudformation/resources.go @@ -77,8 +77,11 @@ import ( backupbackend "github.com/blackbirdworks/gopherstack/services/backup" "github.com/blackbirdworks/gopherstack/services/bedrockruntime" datasyncbackend "github.com/blackbirdworks/gopherstack/services/datasync" + ecrpublicbackend "github.com/blackbirdworks/gopherstack/services/ecrpublic" elbv2backend "github.com/blackbirdworks/gopherstack/services/elbv2" guarddutybackend "github.com/blackbirdworks/gopherstack/services/guardduty" + kafkaconnectbackend "github.com/blackbirdworks/gopherstack/services/kafkaconnect" + kinesisvideobackend "github.com/blackbirdworks/gopherstack/services/kinesisvideo" macie2backend "github.com/blackbirdworks/gopherstack/services/macie2" "github.com/blackbirdworks/gopherstack/services/memorydb" wafv2backend "github.com/blackbirdworks/gopherstack/services/wafv2" @@ -164,8 +167,12 @@ type ServiceBackends struct { GuardDuty *guarddutybackend.Handler AccessAnalyzer *accessanalyzerbackend.Handler Amplify *amplifybackend.Handler - AccountID string - Region string + // Phase-7 backends + KinesisVideo *kinesisvideobackend.Handler + ECRPublic *ecrpublicbackend.Handler + KafkaConnect *kafkaconnectbackend.Handler + AccountID string + Region string } // NestedStackCreator is a callback used to create and delete nested CloudFormation stacks. @@ -967,7 +974,10 @@ func (rc *ResourceCreator) createMiscLegacyResource( physID, err := rc.createRedshiftCluster(logicalID, props, params, physicalIDs) return physID, true, err - case "AWS::OpenSearch::Domain": + case "AWS::OpenSearch::Domain", resTypeOpenSearchServiceDomain: + // AWS::OpenSearchService::Domain is the real CFN type name; the old + // AWS::OpenSearch::Domain name is kept as an alias since existing + // tests/templates in this repo use it. physID, err := rc.createOpenSearchDomain(logicalID, props, params, physicalIDs) return physID, true, err @@ -1004,7 +1014,10 @@ func (rc *ResourceCreator) createMiscLegacyResource( physID, err := rc.createSESEmailIdentity(logicalID, props, params, physicalIDs) return physID, true, err - case "AWS::ACM::Certificate": + case "AWS::ACM::Certificate", resTypeCertificateManagerCertificate: + // AWS::CertificateManager::Certificate is the real CFN type name; + // the old AWS::ACM::Certificate name is kept as an alias since + // existing tests/templates in this repo use it. physID, err := rc.createACMCertificate(ctx, logicalID, props, params, physicalIDs) return physID, true, err @@ -1389,6 +1402,8 @@ func (b *InMemoryBackend) deleteResolveContext(stack *Stack) map[string]string { return out } +// Delete deletes a single resource by type and physical ID. An already-gone target counts +// as deleted, as CloudFormation's handler contract treats NotFound on delete. func (rc *ResourceCreator) Delete( ctx context.Context, resourceType, physicalID string, @@ -1399,6 +1414,32 @@ func (rc *ResourceCreator) Delete( return nil } + err := rc.deleteResource(ctx, resourceType, physicalID, props, stackPhysicalIDs) + if isResourceGoneError(err) { + return nil + } + + return err +} + +// isResourceGoneError reports whether delErr is a NotFound-class error; every such error +// in this codebase names itself "not found" or "NotFound". +func isResourceGoneError(delErr error) bool { + if delErr == nil { + return false + } + + msg := strings.ToLower(delErr.Error()) + + return strings.Contains(msg, "not found") || strings.Contains(msg, "notfound") +} + +func (rc *ResourceCreator) deleteResource( + ctx context.Context, + resourceType, physicalID string, + props map[string]any, + stackPhysicalIDs map[string]string, +) error { if rc.deleteHook != nil { rc.deleteHook(resourceType) } @@ -1892,7 +1933,7 @@ func (rc *ResourceCreator) deleteComputeStorageResource( case "AWS::Redshift::Cluster": return true, rc.deleteRedshiftCluster(physicalID) - case "AWS::OpenSearch::Domain": + case "AWS::OpenSearch::Domain", resTypeOpenSearchServiceDomain: return true, rc.deleteOpenSearchDomain(physicalID) } @@ -1946,7 +1987,7 @@ func (rc *ResourceCreator) deleteAppNetworkResource(ctx context.Context, physica case "AWS::SES::EmailIdentity": return rc.deleteSESEmailIdentity(physicalID) - case "AWS::ACM::Certificate": + case "AWS::ACM::Certificate", resTypeCertificateManagerCertificate: return rc.deleteACMCertificate(ctx, physicalID) case "AWS::Cognito::UserPool": diff --git a/services/cloudformation/resources_apigatewayv2.go b/services/cloudformation/resources_apigatewayv2.go index 61729b178e..0d4252c216 100644 --- a/services/cloudformation/resources_apigatewayv2.go +++ b/services/cloudformation/resources_apigatewayv2.go @@ -2,7 +2,6 @@ package cloudformation import ( "context" - "errors" "fmt" "strings" @@ -198,12 +197,7 @@ func (rc *ResourceCreator) deleteAPIGatewayV2Stage(physicalID string) error { apiID := physicalID[:idx] stageName := physicalID[idx+1:] - err := rc.backends.APIGatewayV2.Backend.DeleteStage(apiID, stageName) - if errors.Is(err, apigatewayv2backend.ErrStageNotFound) || errors.Is(err, apigatewayv2backend.ErrAPINotFound) { - return nil - } - - return err + return rc.backends.APIGatewayV2.Backend.DeleteStage(apiID, stageName) } func (rc *ResourceCreator) createAPIGatewayV2Integration( @@ -249,13 +243,7 @@ func (rc *ResourceCreator) deleteAPIGatewayV2Integration(physicalID string) erro apiID := physicalID[:idx] integrationID := physicalID[idx+1:] - err := rc.backends.APIGatewayV2.Backend.DeleteIntegration(apiID, integrationID) - if errors.Is(err, apigatewayv2backend.ErrIntegrationNotFound) || - errors.Is(err, apigatewayv2backend.ErrAPINotFound) { - return nil - } - - return err + return rc.backends.APIGatewayV2.Backend.DeleteIntegration(apiID, integrationID) } func (rc *ResourceCreator) createAPIGatewayV2Route( @@ -298,12 +286,7 @@ func (rc *ResourceCreator) deleteAPIGatewayV2Route(physicalID string) error { apiID := physicalID[:idx] routeID := physicalID[idx+1:] - err := rc.backends.APIGatewayV2.Backend.DeleteRoute(apiID, routeID) - if errors.Is(err, apigatewayv2backend.ErrRouteNotFound) || errors.Is(err, apigatewayv2backend.ErrAPINotFound) { - return nil - } - - return err + return rc.backends.APIGatewayV2.Backend.DeleteRoute(apiID, routeID) } // ---- API Gateway v2 supplemental ---- diff --git a/services/cloudformation/resources_apigatewayv2_test.go b/services/cloudformation/resources_apigatewayv2_test.go index 1baf518925..40d22b74eb 100644 --- a/services/cloudformation/resources_apigatewayv2_test.go +++ b/services/cloudformation/resources_apigatewayv2_test.go @@ -147,3 +147,78 @@ func apiGatewayV2ChildProperties(resourceType string) string { return `"ApiId":{"Ref":"MyApi"},"StageName":"prod"` } } + +// TestDeleteStack_APIGatewayV2FullStack: an Api stack reaches DELETE_COMPLETE even when its +// Integration, Route and Stage were already cascade-deleted. +func TestDeleteStack_APIGatewayV2FullStack(t *testing.T) { + t.Parallel() + + tests := []struct { + predelete func(t *testing.T, apigw *apigatewayv2backend.InMemoryBackend, apiID string) + name string + }{ + {name: "clean_delete"}, + { + name: "children_already_gone", + predelete: func(t *testing.T, apigw *apigatewayv2backend.InMemoryBackend, apiID string) { + t.Helper() + + integs, err := apigw.GetIntegrations(apiID) + require.NoError(t, err) + require.Len(t, integs, 1) + require.NoError(t, apigw.DeleteIntegration(apiID, integs[0].IntegrationID)) + + routes, err := apigw.GetRoutes(apiID) + require.NoError(t, err) + require.Len(t, routes, 1) + require.NoError(t, apigw.DeleteRoute(apiID, routes[0].RouteID)) + + require.NoError(t, apigw.DeleteStage(apiID, "prod")) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backends := newDependentServiceBackends(t) + apigw, ok := backends.APIGatewayV2.Backend.(*apigatewayv2backend.InMemoryBackend) + require.True(t, ok) + + b := cloudformation.NewInMemoryBackendWithConfig( + "000000000000", "us-east-1", cloudformation.NewResourceCreator(backends), + ) + + tmpl := `{"AWSTemplateFormatVersion":"2010-09-09","Resources":{` + + `"MyApi":{"Type":"AWS::ApiGatewayV2::Api","Properties":{"Name":"fullstack","ProtocolType":"HTTP"}},` + + `"Integration":{"Type":"AWS::ApiGatewayV2::Integration","Properties":{` + + apiGatewayV2ChildProperties("AWS::ApiGatewayV2::Integration") + `}},` + + `"Route":{"Type":"AWS::ApiGatewayV2::Route","Properties":{` + + apiGatewayV2ChildProperties("AWS::ApiGatewayV2::Route") + `}},` + + `"Stage":{"Type":"AWS::ApiGatewayV2::Stage","Properties":{` + + apiGatewayV2ChildProperties("AWS::ApiGatewayV2::Stage") + `}}` + + `}}` + + stackName := "apigwv2-fullstack-" + tc.name + + stack, err := b.CreateStack(t.Context(), stackName, tmpl, nil, cloudformation.StackOptions{}) + require.NoError(t, err) + require.Equal(t, "CREATE_COMPLETE", stack.StackStatus) + + apiRes, err := b.DescribeStackResource(stackName, "MyApi") + require.NoError(t, err) + + if tc.predelete != nil { + tc.predelete(t, apigw, apiRes.PhysicalID) + } + + require.NoError(t, b.DeleteStack(t.Context(), stackName)) + + final, err := b.DescribeStack(stackName) + require.NoError(t, err) + assert.Equal(t, "DELETE_COMPLETE", final.StackStatus) + assert.Empty(t, final.StackStatusReason) + }) + } +} diff --git a/services/cloudformation/resources_apigatewayv2_vpclink.go b/services/cloudformation/resources_apigatewayv2_vpclink.go new file mode 100644 index 0000000000..d7d78411e1 --- /dev/null +++ b/services/cloudformation/resources_apigatewayv2_vpclink.go @@ -0,0 +1,71 @@ +package cloudformation + +import ( + "fmt" + + apigatewayv2backend "github.com/blackbirdworks/gopherstack/services/apigatewayv2" +) + +const resTypeAPIGatewayV2VpcLink = "AWS::ApiGatewayV2::VpcLink" + +// createAPIGatewayV2VpcLinkResource handles AWS::ApiGatewayV2::VpcLink +// creation. Returns handled=false otherwise. +func (rc *ResourceCreator) createAPIGatewayV2VpcLinkResource( + logicalID, resourceType string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, bool, error) { + if resourceType != resTypeAPIGatewayV2VpcLink { + return "", false, nil + } + + id, err := rc.createAPIGatewayV2VpcLink(logicalID, props, params, physicalIDs) + + return id, true, err +} + +// deleteAPIGatewayV2VpcLinkResource handles deletion for the type created above. +func (rc *ResourceCreator) deleteAPIGatewayV2VpcLinkResource(resourceType, physicalID string) (bool, error) { + if resourceType != resTypeAPIGatewayV2VpcLink { + return false, nil + } + + if rc.backends.APIGatewayV2 == nil { + return true, nil + } + + return true, ignoreNotFound( + rc.backends.APIGatewayV2.Backend.DeleteVpcLink(physicalID), apigatewayv2backend.ErrVpcLinkNotFound, + ) +} + +// ---- AWS::ApiGatewayV2::VpcLink ---- +// Ref returns the VPC link's ID (documented). Fn::GetAtt.VpcLinkId is the +// same value, so no side-channel stash is needed. + +func (rc *ResourceCreator) createAPIGatewayV2VpcLink( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.APIGatewayV2 == nil { + return logicalID + "-stub", nil + } + + name := strProp(props, "Name", params, physicalIDs) + if name == "" { + name = logicalID + } + + link, err := rc.backends.APIGatewayV2.Backend.CreateVpcLink(apigatewayv2backend.CreateVpcLinkInput{ + Name: name, + SecurityGroupIDs: strSliceProp(props["SecurityGroupIds"], params, physicalIDs), + SubnetIDs: strSliceProp(props["SubnetIds"], params, physicalIDs), + Tags: tagListProp(props, params, physicalIDs), + }) + if err != nil { + return "", fmt.Errorf("create API Gateway V2 VPC link %s: %w", name, err) + } + + return link.VpcLinkID, nil +} diff --git a/services/cloudformation/resources_apigatewayv2_vpclink_test.go b/services/cloudformation/resources_apigatewayv2_vpclink_test.go new file mode 100644 index 0000000000..7f10cac38d --- /dev/null +++ b/services/cloudformation/resources_apigatewayv2_vpclink_test.go @@ -0,0 +1,45 @@ +package cloudformation_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + cfnsdk "github.com/aws/aws-sdk-go-v2/service/cloudformation" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateStack_APIGatewayV2VpcLink(t *testing.T) { + t.Parallel() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "VPC": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.0.0.0/16"}}, + "Subnet": {"Type": "AWS::EC2::Subnet", + "Properties": {"VpcId": {"Ref": "VPC"}, "CidrBlock": "10.0.1.0/24"}}, + "Link": { + "Type": "AWS::ApiGatewayV2::VpcLink", + "Properties": {"Name": "unit-vpc-link", "SubnetIds": [{"Ref": "Subnet"}]} + } +}, +"Outputs": { + "Ref": {"Value": {"Ref": "Link"}}, + "Id": {"Value": {"Fn::GetAtt": ["Link", "VpcLinkId"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "apigwv2-vpclink-stack", tmpl) + assert.Equal(t, outputs["Ref"], outputs["Id"]) + + link, err := backends.APIGatewayV2.Backend.GetVpcLink(outputs["Ref"]) + require.NoError(t, err) + assert.Equal(t, "unit-vpc-link", link.Name) + + _, err = client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("apigwv2-vpclink-stack")}) + require.NoError(t, err) + + _, err = backends.APIGatewayV2.Backend.GetVpcLink(outputs["Ref"]) + require.Error(t, err) +} diff --git a/services/cloudformation/resources_ec2_networking_extras.go b/services/cloudformation/resources_ec2_networking_extras.go new file mode 100644 index 0000000000..f190936dfb --- /dev/null +++ b/services/cloudformation/resources_ec2_networking_extras.go @@ -0,0 +1,236 @@ +package cloudformation + +import ( + "fmt" + "strconv" + + ec2backend "github.com/blackbirdworks/gopherstack/services/ec2" +) + +const ( + resTypeEC2PlacementGroup = "AWS::EC2::PlacementGroup" + resTypeEC2NetIfacePermission = "AWS::EC2::NetworkInterfacePermission" + resTypeEC2PrefixList = "AWS::EC2::PrefixList" + resTypeEC2VPCEndpointService = "AWS::EC2::VPCEndpointService" + resTypeEC2VerifiedAccessInst = "AWS::EC2::VerifiedAccessInstance" +) + +// createEC2NetworkingExtrasResource handles the standalone EC2 networking +// types listed above (no shared props). Returns handled=false otherwise. +func (rc *ResourceCreator) createEC2NetworkingExtrasResource( + logicalID, resourceType string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, bool, error) { + switch resourceType { + case resTypeEC2PlacementGroup: + id, err := rc.createEC2PlacementGroup(logicalID, props, params, physicalIDs) + + return id, true, err + case resTypeEC2NetIfacePermission: + id, err := rc.createEC2NetworkInterfacePermission(props, params, physicalIDs) + + return id, true, err + case resTypeEC2PrefixList: + id, err := rc.createEC2PrefixList(logicalID, props, params, physicalIDs) + + return id, true, err + case resTypeEC2VPCEndpointService: + id, err := rc.createEC2VPCEndpointService(props, params, physicalIDs) + + return id, true, err + case resTypeEC2VerifiedAccessInst: + id, err := rc.createEC2VerifiedAccessInstance(props, params, physicalIDs) + + return id, true, err + default: + return "", false, nil + } +} + +// deleteEC2NetworkingExtrasResource handles deletion for the types created above. +func (rc *ResourceCreator) deleteEC2NetworkingExtrasResource(resourceType, physicalID string) (bool, error) { + if rc.backends.EC2 == nil { + switch resourceType { + case resTypeEC2PlacementGroup, resTypeEC2NetIfacePermission, resTypeEC2PrefixList, + resTypeEC2VPCEndpointService, resTypeEC2VerifiedAccessInst: + return true, nil + default: + return false, nil + } + } + + switch resourceType { + case resTypeEC2PlacementGroup: + return true, ignoreNotFound( + rc.backends.EC2.Backend.DeletePlacementGroup(physicalID), ec2backend.ErrPlacementGroupNotFound, + ) + case resTypeEC2NetIfacePermission: + return true, ignoreNotFound( + rc.backends.EC2.Backend.DeleteNetworkInterfacePermission(physicalID), + ec2backend.ErrNetworkInterfacePermissionNotFound, + ) + case resTypeEC2PrefixList: + _, err := rc.backends.EC2.Backend.DeleteManagedPrefixList(physicalID) + + return true, ignoreNotFound(err, ec2backend.ErrManagedPrefixListNotFound) + case resTypeEC2VPCEndpointService: + return true, rc.backends.EC2.Backend.DeleteVpcEndpointServiceConfigurations([]string{physicalID}) + case resTypeEC2VerifiedAccessInst: + _, err := rc.backends.EC2.Backend.DeleteVerifiedAccessInstance(physicalID) + + return true, ignoreNotFound(err, ec2backend.ErrVerifiedAccessInstanceNotFound) + default: + return false, nil + } +} + +// ---- AWS::EC2::PlacementGroup ---- +// Ref returns the placement group name (documented). GroupId has no +// separate identifier in this backend (PlacementGroup has no ID field +// distinct from its name), so Fn::GetAtt.GroupId falls back to the name too. + +func (rc *ResourceCreator) createEC2PlacementGroup( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return logicalID + "-stub", nil + } + + name := strProp(props, "GroupName", params, physicalIDs) + if name == "" { + name = logicalID + } + + pg, err := rc.backends.EC2.Backend.CreatePlacementGroup( + name, strProp(props, "Strategy", params, physicalIDs), tagListProp(props, params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create placement group %s: %w", name, err) + } + + return pg.Name, nil +} + +// ---- AWS::EC2::NetworkInterfacePermission ---- +// Ref returns the permission's resource name (documented, undocumented +// attribute list -- no Fn::GetAtt section on the docs page). + +func (rc *ResourceCreator) createEC2NetworkInterfacePermission( + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return "eni-perm-stub", nil + } + + perm, err := rc.backends.EC2.Backend.CreateNetworkInterfacePermission( + strProp(props, "NetworkInterfaceId", params, physicalIDs), + strProp(props, "AwsAccountId", params, physicalIDs), + strProp(props, "AwsService", params, physicalIDs), + strProp(props, "Permission", params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create network interface permission: %w", err) + } + + return perm.PermissionID, nil +} + +// ---- AWS::EC2::PrefixList ---- +// Ref returns the prefix list ID (documented). Arn, OwnerId, and Version are +// stashed at create time and read back through resolveGetAtt's +// stack-props side channel (see getExtraResourceAttribute's prefix-list case). + +func (rc *ResourceCreator) createEC2PrefixList( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return logicalID + "-stub", nil + } + + name := strProp(props, "PrefixListName", params, physicalIDs) + if name == "" { + name = logicalID + } + + var entries []ec2backend.PrefixListEntry + if raw, ok := props["Entries"].([]any); ok { + for _, e := range raw { + em, isMap := e.(map[string]any) + if !isMap { + continue + } + + entries = append(entries, ec2backend.PrefixListEntry{ + Cidr: strProp(em, "Cidr", params, physicalIDs), + Description: strProp(em, "Description", params, physicalIDs), + }) + } + } + + pl, err := rc.backends.EC2.Backend.CreateManagedPrefixList( + name, strProp(props, "AddressFamily", params, physicalIDs), intProp(props, "MaxEntries"), entries, + ) + if err != nil { + return "", fmt.Errorf("create managed prefix list %s: %w", name, err) + } + + physicalIDs[logicalID+"/Arn"] = pl.PrefixListArn + physicalIDs[logicalID+"/OwnerId"] = pl.OwnerID + physicalIDs[logicalID+"/Version"] = strconv.FormatInt(pl.Version, 10) + + return pl.PrefixListID, nil +} + +// ---- AWS::EC2::VPCEndpointService ---- +// Ref returns the ID of the VPC endpoint service configuration (documented). +// PrivateDnsNameConfiguration is not modeled by this backend (private DNS +// name verification is not implemented), so those Fn::GetAtt attributes +// are left unimplemented rather than fabricated. + +func (rc *ResourceCreator) createEC2VPCEndpointService( + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return "vpce-svc-stub", nil + } + + cfg, err := rc.backends.EC2.Backend.CreateVpcEndpointServiceConfiguration( + boolProp(props, "AcceptanceRequired"), strSliceProp(props["NetworkLoadBalancerArns"], params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create VPC endpoint service configuration: %w", err) + } + + return cfg.ServiceID, nil +} + +// ---- AWS::EC2::VerifiedAccessInstance ---- +// Ref returns the ID of the Verified Access instance (documented). +// CreationTime, LastUpdatedTime, and CidrEndpointsCustomSubDomainNameServers +// are not tracked by this backend, so those attributes fall back to the +// instance ID rather than being fabricated. + +func (rc *ResourceCreator) createEC2VerifiedAccessInstance( + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return "vai-stub", nil + } + + inst, err := rc.backends.EC2.Backend.CreateVerifiedAccessInstance( + strProp(props, "Description", params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create Verified Access instance: %w", err) + } + + return inst.VerifiedAccessInstanceID, nil +} diff --git a/services/cloudformation/resources_ec2_networking_extras_test.go b/services/cloudformation/resources_ec2_networking_extras_test.go new file mode 100644 index 0000000000..a464136929 --- /dev/null +++ b/services/cloudformation/resources_ec2_networking_extras_test.go @@ -0,0 +1,186 @@ +package cloudformation_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + cfnsdk "github.com/aws/aws-sdk-go-v2/service/cloudformation" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateStack_EC2NetworkingExtrasTypes(t *testing.T) { + t.Parallel() + + cases := []struct { + run func(t *testing.T) + name string + }{ + {testEC2PlacementGroup, "placement_group"}, + {testEC2PrefixList, "prefix_list"}, + {testEC2VPCEndpointService, "vpc_endpoint_service"}, + {testEC2VerifiedAccessInstance, "verified_access_instance"}, + {testEC2NetworkInterfacePermission, "network_interface_permission"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.run(t) + }) + } +} + +func testEC2PlacementGroup(t *testing.T) { + t.Helper() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "PG": {"Type": "AWS::EC2::PlacementGroup", "Properties": {"GroupName": "unit-pg", "Strategy": "spread"}} +}, +"Outputs": { + "Ref": {"Value": {"Ref": "PG"}}, + "GroupName": {"Value": {"Fn::GetAtt": ["PG", "GroupName"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "ec2-pg-stack", tmpl) + assert.Equal(t, "unit-pg", outputs["Ref"]) + assert.Equal(t, "unit-pg", outputs["GroupName"]) + require.Len(t, backends.EC2.Backend.DescribePlacementGroups([]string{"unit-pg"}), 1) + + _, err := client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("ec2-pg-stack")}) + require.NoError(t, err) + assert.Empty(t, backends.EC2.Backend.DescribePlacementGroups([]string{"unit-pg"})) +} + +func testEC2PrefixList(t *testing.T) { + t.Helper() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "PL": { + "Type": "AWS::EC2::PrefixList", + "Properties": { + "PrefixListName": "unit-pl", + "AddressFamily": "IPv4", + "MaxEntries": 5, + "Entries": [{"Cidr": "10.0.0.0/24", "Description": "office"}] + } + } +}, +"Outputs": { + "Ref": {"Value": {"Ref": "PL"}}, + "Arn": {"Value": {"Fn::GetAtt": ["PL", "Arn"]}}, + "OwnerId": {"Value": {"Fn::GetAtt": ["PL", "OwnerId"]}}, + "Version": {"Value": {"Fn::GetAtt": ["PL", "Version"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "ec2-pl-stack", tmpl) + assert.Contains(t, outputs["Ref"], "pl-") + assert.Contains(t, outputs["Arn"], "prefix-list/"+outputs["Ref"]) + assert.Equal(t, "000000000000", outputs["OwnerId"]) + assert.Equal(t, "1", outputs["Version"]) + require.Len(t, backends.EC2.Backend.DescribeManagedPrefixLists([]string{outputs["Ref"]}), 1) + + _, err := client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("ec2-pl-stack")}) + require.NoError(t, err) + assert.Empty(t, backends.EC2.Backend.DescribeManagedPrefixLists([]string{outputs["Ref"]})) +} + +func testEC2VPCEndpointService(t *testing.T) { + t.Helper() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "Svc": { + "Type": "AWS::EC2::VPCEndpointService", + "Properties": { + "AcceptanceRequired": false, + "NetworkLoadBalancerArns": ["arn:aws:elasticloadbalancing:us-east-1:000000000000:loadbalancer/net/nlb/abc"] + } + } +}, +"Outputs": { + "Ref": {"Value": {"Ref": "Svc"}}, + "ServiceId": {"Value": {"Fn::GetAtt": ["Svc", "ServiceId"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "ec2-vpces-stack", tmpl) + assert.Equal(t, outputs["Ref"], outputs["ServiceId"]) + assert.Contains(t, outputs["Ref"], "vpce-svc-") + require.Len(t, backends.EC2.Backend.DescribeVpcEndpointServiceConfigurations([]string{outputs["Ref"]}), 1) + + _, err := client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("ec2-vpces-stack")}) + require.NoError(t, err) + assert.Empty(t, backends.EC2.Backend.DescribeVpcEndpointServiceConfigurations([]string{outputs["Ref"]})) +} + +func testEC2VerifiedAccessInstance(t *testing.T) { + t.Helper() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "VAI": {"Type": "AWS::EC2::VerifiedAccessInstance", "Properties": {"Description": "unit test instance"}} +}, +"Outputs": { + "Ref": {"Value": {"Ref": "VAI"}}, + "Id": {"Value": {"Fn::GetAtt": ["VAI", "VerifiedAccessInstanceId"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "ec2-vai-stack", tmpl) + assert.Equal(t, outputs["Ref"], outputs["Id"]) + assert.Contains(t, outputs["Ref"], "vai-") + require.Len(t, backends.EC2.Backend.DescribeVerifiedAccessInstances([]string{outputs["Ref"]}), 1) + + _, err := client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("ec2-vai-stack")}) + require.NoError(t, err) + assert.Empty(t, backends.EC2.Backend.DescribeVerifiedAccessInstances([]string{outputs["Ref"]})) +} + +func testEC2NetworkInterfacePermission(t *testing.T) { + t.Helper() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "VPC": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.0.0.0/16"}}, + "Subnet": {"Type": "AWS::EC2::Subnet", + "Properties": {"VpcId": {"Ref": "VPC"}, "CidrBlock": "10.0.1.0/24"}}, + "ENI": {"Type": "AWS::EC2::NetworkInterface", "Properties": {"SubnetId": {"Ref": "Subnet"}}}, + "Perm": { + "Type": "AWS::EC2::NetworkInterfacePermission", + "Properties": { + "NetworkInterfaceId": {"Ref": "ENI"}, + "AwsAccountId": "111111111111", + "AwsService": "ec2.amazonaws.com", + "Permission": "INSTANCE-ATTACH" + } + } +}, +"Outputs": { + "ENIRef": {"Value": {"Ref": "ENI"}}, + "PermRef": {"Value": {"Ref": "Perm"}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "ec2-enip-stack", tmpl) + assert.Contains(t, outputs["PermRef"], "eni-perm-") + require.Len(t, backends.EC2.Backend.DescribeNetworkInterfacePermissions([]string{outputs["ENIRef"]}), 1) + + _, err := client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("ec2-enip-stack")}) + require.NoError(t, err) + assert.Empty(t, backends.EC2.Backend.DescribeNetworkInterfacePermissions([]string{outputs["ENIRef"]})) +} diff --git a/services/cloudformation/resources_ec2_networkinsights.go b/services/cloudformation/resources_ec2_networkinsights.go new file mode 100644 index 0000000000..8fcd59e608 --- /dev/null +++ b/services/cloudformation/resources_ec2_networkinsights.go @@ -0,0 +1,71 @@ +package cloudformation + +import ( + "fmt" + + ec2backend "github.com/blackbirdworks/gopherstack/services/ec2" +) + +const resTypeEC2NetworkInsightsPath = "AWS::EC2::NetworkInsightsPath" + +// createEC2NetworkInsightsResource handles AWS::EC2::NetworkInsightsPath +// creation. Returns handled=false otherwise. +func (rc *ResourceCreator) createEC2NetworkInsightsResource( + logicalID, resourceType string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, bool, error) { + if resourceType != resTypeEC2NetworkInsightsPath { + return "", false, nil + } + + id, err := rc.createEC2NetworkInsightsPath(logicalID, props, params, physicalIDs) + + return id, true, err +} + +// deleteEC2NetworkInsightsResource handles deletion for the type created above. +func (rc *ResourceCreator) deleteEC2NetworkInsightsResource(resourceType, physicalID string) (bool, error) { + if resourceType != resTypeEC2NetworkInsightsPath { + return false, nil + } + + if rc.backends.EC2 == nil { + return true, nil + } + + return true, ignoreNotFound( + rc.backends.EC2.Backend.DeleteNetworkInsightsPath(physicalID), ec2backend.ErrNetworkInsightsPathNotFound, + ) +} + +// ---- AWS::EC2::NetworkInsightsPath ---- +// Ref returns the ID of the path (documented). NetworkInsightsPathArn, +// SourceArn, and DestinationArn are stashed; CreatedDate is not tracked by +// this backend so it is left unimplemented rather than fabricated. + +func (rc *ResourceCreator) createEC2NetworkInsightsPath( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return logicalID + "-stub", nil + } + + source := strProp(props, "Source", params, physicalIDs) + destination := strProp(props, "Destination", params, physicalIDs) + + p, err := rc.backends.EC2.Backend.CreateNetworkInsightsPath( + source, destination, strProp(props, "Protocol", params, physicalIDs), intProp(props, "DestinationPort"), + ) + if err != nil { + return "", fmt.Errorf("create network insights path: %w", err) + } + + physicalIDs[logicalID+"/NetworkInsightsPathArn"] = p.NetworkInsightsPathArn + physicalIDs[logicalID+"/SourceArn"] = source + physicalIDs[logicalID+"/DestinationArn"] = destination + + return p.NetworkInsightsPathID, nil +} diff --git a/services/cloudformation/resources_ec2_networkinsights_test.go b/services/cloudformation/resources_ec2_networkinsights_test.go new file mode 100644 index 0000000000..a8f176a8ca --- /dev/null +++ b/services/cloudformation/resources_ec2_networkinsights_test.go @@ -0,0 +1,53 @@ +package cloudformation_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + cfnsdk "github.com/aws/aws-sdk-go-v2/service/cloudformation" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateStack_EC2NetworkInsightsPath(t *testing.T) { + t.Parallel() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "VPC": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.0.0.0/16"}}, + "Subnet": {"Type": "AWS::EC2::Subnet", + "Properties": {"VpcId": {"Ref": "VPC"}, "CidrBlock": "10.0.1.0/24"}}, + "Source": {"Type": "AWS::EC2::NetworkInterface", "Properties": {"SubnetId": {"Ref": "Subnet"}}}, + "Dest": {"Type": "AWS::EC2::NetworkInterface", "Properties": {"SubnetId": {"Ref": "Subnet"}}}, + "Path": { + "Type": "AWS::EC2::NetworkInsightsPath", + "Properties": { + "Source": {"Ref": "Source"}, + "Destination": {"Ref": "Dest"}, + "Protocol": "tcp", + "DestinationPort": 443 + } + } +}, +"Outputs": { + "Ref": {"Value": {"Ref": "Path"}}, + "Id": {"Value": {"Fn::GetAtt": ["Path", "NetworkInsightsPathId"]}}, + "Arn": {"Value": {"Fn::GetAtt": ["Path", "NetworkInsightsPathArn"]}}, + "SourceArn": {"Value": {"Fn::GetAtt": ["Path", "SourceArn"]}}, + "DestinationArn": {"Value": {"Fn::GetAtt": ["Path", "DestinationArn"]}}, + "SourceRef": {"Value": {"Ref": "Source"}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "ec2-nip-stack", tmpl) + assert.Equal(t, outputs["Ref"], outputs["Id"]) + assert.Contains(t, outputs["Arn"], "network-insights-path/"+outputs["Ref"]) + assert.Equal(t, outputs["SourceRef"], outputs["SourceArn"]) + require.Len(t, backends.EC2.Backend.DescribeNetworkInsightsPaths([]string{outputs["Ref"]}), 1) + + _, err := client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("ec2-nip-stack")}) + require.NoError(t, err) + assert.Empty(t, backends.EC2.Backend.DescribeNetworkInsightsPaths([]string{outputs["Ref"]})) +} diff --git a/services/cloudformation/resources_ec2_routeserver.go b/services/cloudformation/resources_ec2_routeserver.go new file mode 100644 index 0000000000..a9e7e1d948 --- /dev/null +++ b/services/cloudformation/resources_ec2_routeserver.go @@ -0,0 +1,166 @@ +package cloudformation + +import ( + "fmt" + + "github.com/blackbirdworks/gopherstack/pkgs/arn" + ec2backend "github.com/blackbirdworks/gopherstack/services/ec2" +) + +const ( + resTypeEC2RouteServer = "AWS::EC2::RouteServer" + resTypeEC2RouteServerEndpoint = "AWS::EC2::RouteServerEndpoint" + resTypeEC2RouteServerPeer = "AWS::EC2::RouteServerPeer" +) + +// createEC2RouteServerResource handles the route server resource types +// listed above. Returns handled=false otherwise. +func (rc *ResourceCreator) createEC2RouteServerResource( + logicalID, resourceType string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, bool, error) { + switch resourceType { + case resTypeEC2RouteServer: + id, err := rc.createEC2RouteServer(logicalID, props, params, physicalIDs) + + return id, true, err + case resTypeEC2RouteServerEndpoint: + id, err := rc.createEC2RouteServerEndpoint(logicalID, props, params, physicalIDs) + + return id, true, err + case resTypeEC2RouteServerPeer: + id, err := rc.createEC2RouteServerPeer(logicalID, props, params, physicalIDs) + + return id, true, err + default: + return "", false, nil + } +} + +// deleteEC2RouteServerResource handles deletion for the types created above. +func (rc *ResourceCreator) deleteEC2RouteServerResource(resourceType, physicalID string) (bool, error) { + if rc.backends.EC2 == nil { + switch resourceType { + case resTypeEC2RouteServer, resTypeEC2RouteServerEndpoint, resTypeEC2RouteServerPeer: + return true, nil + default: + return false, nil + } + } + + switch resourceType { + case resTypeEC2RouteServer: + _, err := rc.backends.EC2.Backend.DeleteRouteServer(physicalID) + + return true, ignoreNotFound(err, ec2backend.ErrRouteServerNotFound) + case resTypeEC2RouteServerEndpoint: + _, err := rc.backends.EC2.Backend.DeleteRouteServerEndpoint(physicalID) + + return true, ignoreNotFound(err, ec2backend.ErrRouteServerEndpointNotFound) + case resTypeEC2RouteServerPeer: + _, err := rc.backends.EC2.Backend.DeleteRouteServerPeer(physicalID) + + return true, ignoreNotFound(err, ec2backend.ErrRouteServerPeerNotFound) + default: + return false, nil + } +} + +// ---- AWS::EC2::RouteServer ---- +// Ref returns the route server ID (documented). Arn is stashed since this +// backend has no dedicated ARN field on RouteServer. + +func (rc *ResourceCreator) createEC2RouteServer( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return logicalID + "-stub", nil + } + + rs, err := rc.backends.EC2.Backend.CreateRouteServer( + int64Prop(props, "AmazonSideAsn", params, physicalIDs), + strProp(props, "PersistRoutesState", params, physicalIDs), + int64Prop(props, "PersistRoutesDuration", params, physicalIDs), + boolProp(props, "SnsNotificationsEnabled"), + ) + if err != nil { + return "", fmt.Errorf("create route server: %w", err) + } + + physicalIDs[logicalID+"/Arn"] = arn.Build( + "ec2", rc.backends.Region, rc.backends.AccountID, "route-server/"+rs.RouteServerID, + ) + + return rs.RouteServerID, nil +} + +// ---- AWS::EC2::RouteServerEndpoint ---- +// Ref returns the endpoint ID (documented). Arn, EniAddress, EniId, and +// VpcId are stashed from the backend's real values. + +func (rc *ResourceCreator) createEC2RouteServerEndpoint( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return logicalID + "-stub", nil + } + + ep, err := rc.backends.EC2.Backend.CreateRouteServerEndpoint( + strProp(props, "RouteServerId", params, physicalIDs), strProp(props, "SubnetId", params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create route server endpoint: %w", err) + } + + physicalIDs[logicalID+"/Arn"] = arn.Build( + "ec2", rc.backends.Region, rc.backends.AccountID, "route-server-endpoint/"+ep.RouteServerEndpointID, + ) + physicalIDs[logicalID+"/EniAddress"] = ep.EniAddress + physicalIDs[logicalID+"/EniId"] = ep.EniID + physicalIDs[logicalID+"/VpcId"] = ep.VpcID + + return ep.RouteServerEndpointID, nil +} + +// ---- AWS::EC2::RouteServerPeer ---- +// Ref returns the peer ID (documented). Arn, EndpointEniAddress, +// EndpointEniId, RouteServerId, SubnetId, and VpcId are stashed from the +// backend's real values. + +func (rc *ResourceCreator) createEC2RouteServerPeer( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return logicalID + "-stub", nil + } + + bgpOptions, _ := props["BgpOptions"].(map[string]any) + + peer, err := rc.backends.EC2.Backend.CreateRouteServerPeer( + strProp(props, "RouteServerEndpointId", params, physicalIDs), + strProp(props, "PeerAddress", params, physicalIDs), + int64Prop(bgpOptions, "PeerAsn", params, physicalIDs), + strProp(bgpOptions, "PeerLivenessDetection", params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create route server peer: %w", err) + } + + physicalIDs[logicalID+"/Arn"] = arn.Build( + "ec2", rc.backends.Region, rc.backends.AccountID, "route-server-peer/"+peer.RouteServerPeerID, + ) + physicalIDs[logicalID+"/EndpointEniAddress"] = peer.EniAddress + physicalIDs[logicalID+"/EndpointEniId"] = peer.EniID + physicalIDs[logicalID+"/RouteServerId"] = peer.RouteServerID + physicalIDs[logicalID+"/SubnetId"] = peer.SubnetID + physicalIDs[logicalID+"/VpcId"] = peer.VpcID + + return peer.RouteServerPeerID, nil +} diff --git a/services/cloudformation/resources_ec2_routeserver_test.go b/services/cloudformation/resources_ec2_routeserver_test.go new file mode 100644 index 0000000000..28123fe509 --- /dev/null +++ b/services/cloudformation/resources_ec2_routeserver_test.go @@ -0,0 +1,70 @@ +package cloudformation_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + cfnsdk "github.com/aws/aws-sdk-go-v2/service/cloudformation" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateStack_EC2RouteServerTypes(t *testing.T) { + t.Parallel() + t.Run("server_endpoint_peer", testEC2RouteServerChain) +} + +func testEC2RouteServerChain(t *testing.T) { + t.Parallel() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "VPC": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.0.0.0/16"}}, + "Subnet": {"Type": "AWS::EC2::Subnet", + "Properties": {"VpcId": {"Ref": "VPC"}, "CidrBlock": "10.0.1.0/24"}}, + "RS": { + "Type": "AWS::EC2::RouteServer", + "Properties": {"AmazonSideAsn": 65000, "PersistRoutesState": "enabled"} + }, + "Endpoint": { + "Type": "AWS::EC2::RouteServerEndpoint", + "Properties": {"RouteServerId": {"Ref": "RS"}, "SubnetId": {"Ref": "Subnet"}} + }, + "Peer": { + "Type": "AWS::EC2::RouteServerPeer", + "Properties": { + "RouteServerEndpointId": {"Ref": "Endpoint"}, + "PeerAddress": "10.0.1.100", + "BgpOptions": {"PeerAsn": 65001} + } + } +}, +"Outputs": { + "RSRef": {"Value": {"Ref": "RS"}}, + "RSArn": {"Value": {"Fn::GetAtt": ["RS", "Arn"]}}, + "EndpointRef": {"Value": {"Ref": "Endpoint"}}, + "EndpointVpcId": {"Value": {"Fn::GetAtt": ["Endpoint", "VpcId"]}}, + "PeerRef": {"Value": {"Ref": "Peer"}}, + "PeerRouteServerId": {"Value": {"Fn::GetAtt": ["Peer", "RouteServerId"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "ec2-rs-stack", tmpl) + assert.Contains(t, outputs["RSRef"], "rs-") + assert.Contains(t, outputs["RSArn"], "route-server/"+outputs["RSRef"]) + assert.NotEmpty(t, outputs["EndpointVpcId"]) + assert.Equal(t, outputs["RSRef"], outputs["PeerRouteServerId"]) + + require.Len(t, backends.EC2.Backend.DescribeRouteServers([]string{outputs["RSRef"]}), 1) + require.Len(t, backends.EC2.Backend.DescribeRouteServerEndpoints([]string{outputs["EndpointRef"]}), 1) + require.Len(t, backends.EC2.Backend.DescribeRouteServerPeers([]string{outputs["PeerRef"]}), 1) + + _, err := client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("ec2-rs-stack")}) + require.NoError(t, err) + + assert.Empty(t, backends.EC2.Backend.DescribeRouteServerPeers([]string{outputs["PeerRef"]})) + assert.Empty(t, backends.EC2.Backend.DescribeRouteServerEndpoints([]string{outputs["EndpointRef"]})) + assert.Empty(t, backends.EC2.Backend.DescribeRouteServers([]string{outputs["RSRef"]})) +} diff --git a/services/cloudformation/resources_ec2_trafficmirror.go b/services/cloudformation/resources_ec2_trafficmirror.go new file mode 100644 index 0000000000..9d6acc92b3 --- /dev/null +++ b/services/cloudformation/resources_ec2_trafficmirror.go @@ -0,0 +1,181 @@ +package cloudformation + +import ( + "fmt" + + ec2backend "github.com/blackbirdworks/gopherstack/services/ec2" +) + +const ( + resTypeEC2TrafficMirrorFilter = "AWS::EC2::TrafficMirrorFilter" + resTypeEC2TrafficMirrorFilterRule = "AWS::EC2::TrafficMirrorFilterRule" + resTypeEC2TrafficMirrorTarget = "AWS::EC2::TrafficMirrorTarget" + resTypeEC2TrafficMirrorSession = "AWS::EC2::TrafficMirrorSession" +) + +// createEC2TrafficMirrorResource handles the Traffic Mirror resource types +// listed above. Returns handled=false otherwise. +func (rc *ResourceCreator) createEC2TrafficMirrorResource( + _, resourceType string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, bool, error) { + switch resourceType { + case resTypeEC2TrafficMirrorFilter: + id, err := rc.createEC2TrafficMirrorFilter(props, params, physicalIDs) + + return id, true, err + case resTypeEC2TrafficMirrorFilterRule: + id, err := rc.createEC2TrafficMirrorFilterRule(props, params, physicalIDs) + + return id, true, err + case resTypeEC2TrafficMirrorTarget: + id, err := rc.createEC2TrafficMirrorTarget(props, params, physicalIDs) + + return id, true, err + case resTypeEC2TrafficMirrorSession: + id, err := rc.createEC2TrafficMirrorSession(props, params, physicalIDs) + + return id, true, err + default: + return "", false, nil + } +} + +// deleteEC2TrafficMirrorResource handles deletion for the types created above. +func (rc *ResourceCreator) deleteEC2TrafficMirrorResource(resourceType, physicalID string) (bool, error) { + if rc.backends.EC2 == nil { + switch resourceType { + case resTypeEC2TrafficMirrorFilter, resTypeEC2TrafficMirrorFilterRule, + resTypeEC2TrafficMirrorTarget, resTypeEC2TrafficMirrorSession: + return true, nil + default: + return false, nil + } + } + + switch resourceType { + case resTypeEC2TrafficMirrorFilter: + return true, ignoreNotFound( + rc.backends.EC2.Backend.DeleteTrafficMirrorFilter(physicalID), ec2backend.ErrTrafficMirrorFilterNotFound, + ) + case resTypeEC2TrafficMirrorFilterRule: + return true, ignoreNotFound( + rc.backends.EC2.Backend.DeleteTrafficMirrorFilterRule(physicalID), + ec2backend.ErrTrafficMirrorFilterRuleNotFound, + ) + case resTypeEC2TrafficMirrorTarget: + return true, ignoreNotFound( + rc.backends.EC2.Backend.DeleteTrafficMirrorTarget(physicalID), ec2backend.ErrTrafficMirrorTargetNotFound, + ) + case resTypeEC2TrafficMirrorSession: + return true, ignoreNotFound( + rc.backends.EC2.Backend.DeleteTrafficMirrorSession(physicalID), ec2backend.ErrTrafficMirrorSessionNotFound, + ) + default: + return false, nil + } +} + +// ---- AWS::EC2::TrafficMirrorFilter ---- +// Ref returns the ID of the filter (documented, no Fn::GetAtt section). + +func (rc *ResourceCreator) createEC2TrafficMirrorFilter( + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return "tmf-stub", nil + } + + f, err := rc.backends.EC2.Backend.CreateTrafficMirrorFilter( + strProp(props, "Description", params, physicalIDs), tagListProp(props, params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create traffic mirror filter: %w", err) + } + + return f.TrafficMirrorFilterID, nil +} + +// ---- AWS::EC2::TrafficMirrorFilterRule ---- +// Ref returns the ID of the filter rule (documented, Fn::GetAtt returns the +// same ID). + +func (rc *ResourceCreator) createEC2TrafficMirrorFilterRule( + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return "tmfr-stub", nil + } + + rule, err := rc.backends.EC2.Backend.CreateTrafficMirrorFilterRule( + strProp(props, "TrafficMirrorFilterId", params, physicalIDs), + strProp(props, "TrafficDirection", params, physicalIDs), + strProp(props, "RuleAction", params, physicalIDs), + strProp(props, "SourceCidrBlock", params, physicalIDs), + strProp(props, "DestinationCidrBlock", params, physicalIDs), + strProp(props, "Description", params, physicalIDs), + intProp(props, "RuleNumber"), + intProp(props, "Protocol"), + tagListProp(props, params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create traffic mirror filter rule: %w", err) + } + + return rule.TrafficMirrorFilterRuleID, nil +} + +// ---- AWS::EC2::TrafficMirrorTarget ---- +// Ref returns the ID of the target (documented, no Fn::GetAtt section). + +func (rc *ResourceCreator) createEC2TrafficMirrorTarget( + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return "tmt-stub", nil + } + + t, err := rc.backends.EC2.Backend.CreateTrafficMirrorTarget( + strProp(props, "NetworkInterfaceId", params, physicalIDs), + strProp(props, "NetworkLoadBalancerArn", params, physicalIDs), + strProp(props, "Description", params, physicalIDs), + tagListProp(props, params, physicalIDs), + strProp(props, "GatewayLoadBalancerEndpointId", params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create traffic mirror target: %w", err) + } + + return t.TrafficMirrorTargetID, nil +} + +// ---- AWS::EC2::TrafficMirrorSession ---- +// Ref returns the ID of the session (documented, no Fn::GetAtt section). + +func (rc *ResourceCreator) createEC2TrafficMirrorSession( + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return "tms-stub", nil + } + + s, err := rc.backends.EC2.Backend.CreateTrafficMirrorSession( + strProp(props, "NetworkInterfaceId", params, physicalIDs), + strProp(props, "TrafficMirrorTargetId", params, physicalIDs), + strProp(props, "TrafficMirrorFilterId", params, physicalIDs), + strProp(props, "Description", params, physicalIDs), + intProp(props, "SessionNumber"), + tagListProp(props, params, physicalIDs), + intProp(props, "PacketLength"), + ) + if err != nil { + return "", fmt.Errorf("create traffic mirror session: %w", err) + } + + return s.TrafficMirrorSessionID, nil +} diff --git a/services/cloudformation/resources_ec2_trafficmirror_test.go b/services/cloudformation/resources_ec2_trafficmirror_test.go new file mode 100644 index 0000000000..79188adbc1 --- /dev/null +++ b/services/cloudformation/resources_ec2_trafficmirror_test.go @@ -0,0 +1,86 @@ +package cloudformation_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + cfnsdk "github.com/aws/aws-sdk-go-v2/service/cloudformation" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateStack_EC2TrafficMirrorTypes(t *testing.T) { + t.Parallel() + t.Run("filter_rule_target_session", testEC2TrafficMirrorChain) +} + +func testEC2TrafficMirrorChain(t *testing.T) { + t.Parallel() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "VPC": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.0.0.0/16"}}, + "Subnet": {"Type": "AWS::EC2::Subnet", + "Properties": {"VpcId": {"Ref": "VPC"}, "CidrBlock": "10.0.1.0/24"}}, + "ENI": {"Type": "AWS::EC2::NetworkInterface", "Properties": {"SubnetId": {"Ref": "Subnet"}}}, + "Filter": {"Type": "AWS::EC2::TrafficMirrorFilter", "Properties": {"Description": "unit filter"}}, + "Rule": { + "Type": "AWS::EC2::TrafficMirrorFilterRule", + "Properties": { + "TrafficMirrorFilterId": {"Ref": "Filter"}, + "TrafficDirection": "ingress", + "RuleAction": "accept", + "SourceCidrBlock": "0.0.0.0/0", + "DestinationCidrBlock": "0.0.0.0/0", + "RuleNumber": 1, + "Protocol": 6 + } + }, + "Target": { + "Type": "AWS::EC2::TrafficMirrorTarget", + "Properties": {"NetworkInterfaceId": {"Ref": "ENI"}, "Description": "unit target"} + }, + "Session": { + "Type": "AWS::EC2::TrafficMirrorSession", + "Properties": { + "NetworkInterfaceId": {"Ref": "ENI"}, + "TrafficMirrorTargetId": {"Ref": "Target"}, + "TrafficMirrorFilterId": {"Ref": "Filter"}, + "SessionNumber": 1 + } + } +}, +"Outputs": { + "FilterRef": {"Value": {"Ref": "Filter"}}, + "RuleRef": {"Value": {"Ref": "Rule"}}, + "RuleId": {"Value": {"Fn::GetAtt": ["Rule", "TrafficMirrorFilterRuleId"]}}, + "TargetRef": {"Value": {"Ref": "Target"}}, + "SessionRef": {"Value": {"Ref": "Session"}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "ec2-tm-stack", tmpl) + assert.Contains(t, outputs["FilterRef"], "tmf-") + assert.Equal(t, outputs["RuleRef"], outputs["RuleId"]) + assert.Contains(t, outputs["TargetRef"], "tmt-") + assert.Contains(t, outputs["SessionRef"], "tms-") + + require.Len(t, backends.EC2.Backend.DescribeTrafficMirrorFilters([]string{outputs["FilterRef"]}), 1) + + rules, err := backends.EC2.Backend.DescribeTrafficMirrorFilterRules(outputs["FilterRef"]) + require.NoError(t, err) + require.Len(t, rules, 1) + assert.Equal(t, outputs["RuleRef"], rules[0].TrafficMirrorFilterRuleID) + + require.Len(t, backends.EC2.Backend.DescribeTrafficMirrorTargets([]string{outputs["TargetRef"]}), 1) + require.Len(t, backends.EC2.Backend.DescribeTrafficMirrorSessions([]string{outputs["SessionRef"]}), 1) + + _, err = client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("ec2-tm-stack")}) + require.NoError(t, err) + + assert.Empty(t, backends.EC2.Backend.DescribeTrafficMirrorSessions([]string{outputs["SessionRef"]})) + assert.Empty(t, backends.EC2.Backend.DescribeTrafficMirrorTargets([]string{outputs["TargetRef"]})) + assert.Empty(t, backends.EC2.Backend.DescribeTrafficMirrorFilters([]string{outputs["FilterRef"]})) +} diff --git a/services/cloudformation/resources_ec2_transitgateway_attachments.go b/services/cloudformation/resources_ec2_transitgateway_attachments.go new file mode 100644 index 0000000000..7539a8ac0e --- /dev/null +++ b/services/cloudformation/resources_ec2_transitgateway_attachments.go @@ -0,0 +1,160 @@ +package cloudformation + +import ( + "fmt" + "time" + + "github.com/blackbirdworks/gopherstack/pkgs/arn" + ec2backend "github.com/blackbirdworks/gopherstack/services/ec2" +) + +const ( + resTypeEC2TGWVpcAttachment = "AWS::EC2::TransitGatewayVpcAttachment" + resTypeEC2TGWPeeringAttachment = "AWS::EC2::TransitGatewayPeeringAttachment" + resTypeEC2TGWMulticastDomain = "AWS::EC2::TransitGatewayMulticastDomain" +) + +// createEC2TransitGatewayMoreResource handles the transit gateway resource +// types listed above. Returns handled=false otherwise. +func (rc *ResourceCreator) createEC2TransitGatewayMoreResource( + logicalID, resourceType string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, bool, error) { + switch resourceType { + case resTypeEC2TGWVpcAttachment: + id, err := rc.createEC2TGWVpcAttachment(props, params, physicalIDs) + + return id, true, err + case resTypeEC2TGWPeeringAttachment: + id, err := rc.createEC2TGWPeeringAttachment(logicalID, props, params, physicalIDs) + + return id, true, err + case resTypeEC2TGWMulticastDomain: + id, err := rc.createEC2TGWMulticastDomain(logicalID, props, params, physicalIDs) + + return id, true, err + default: + return "", false, nil + } +} + +// deleteEC2TransitGatewayMoreResource handles deletion for the types created above. +func (rc *ResourceCreator) deleteEC2TransitGatewayMoreResource(resourceType, physicalID string) (bool, error) { + if rc.backends.EC2 == nil { + switch resourceType { + case resTypeEC2TGWVpcAttachment, resTypeEC2TGWPeeringAttachment, resTypeEC2TGWMulticastDomain: + return true, nil + default: + return false, nil + } + } + + switch resourceType { + case resTypeEC2TGWVpcAttachment: + return true, ignoreNotFound( + rc.backends.EC2.Backend.DeleteTransitGatewayVpcAttachment(physicalID), ec2backend.ErrTGWAttachmentNotFound, + ) + case resTypeEC2TGWPeeringAttachment: + _, err := rc.backends.EC2.Backend.DeleteTransitGatewayPeeringAttachment(physicalID) + + return true, ignoreNotFound(err, ec2backend.ErrTransitGatewayAttachmentNotFound) + case resTypeEC2TGWMulticastDomain: + return true, ignoreNotFound( + rc.backends.EC2.Backend.DeleteTransitGatewayMulticastDomain(physicalID), + ec2backend.ErrTGWMulticastDomainNotFound, + ) + default: + return false, nil + } +} + +// ---- AWS::EC2::TransitGatewayVpcAttachment ---- +// Ref returns the ID of the attachment (documented). Fn::GetAtt.Id is the +// same value, so no side-channel stash is needed. + +func (rc *ResourceCreator) createEC2TGWVpcAttachment( + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return "tgw-attach-stub", nil + } + + att, err := rc.backends.EC2.Backend.CreateTransitGatewayVpcAttachment( + strProp(props, "TransitGatewayId", params, physicalIDs), + strProp(props, "VpcId", params, physicalIDs), + strSliceProp(props["SubnetIds"], params, physicalIDs), + tagListProp(props, params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create transit gateway VPC attachment: %w", err) + } + + return att.TransitGatewayAttachmentID, nil +} + +// ---- AWS::EC2::TransitGatewayPeeringAttachment ---- +// Ref returns the ID of the attachment (documented). State is stashed since +// the backend sets a real value ("pendingAcceptance"); CreationTime is left +// unimplemented since the backend never populates it (zero time.Time would +// be a fabricated value). + +func (rc *ResourceCreator) createEC2TGWPeeringAttachment( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return logicalID + "-stub", nil + } + + att, err := rc.backends.EC2.Backend.CreateTransitGatewayPeeringAttachment( + strProp(props, "TransitGatewayId", params, physicalIDs), + strProp(props, "PeerTransitGatewayId", params, physicalIDs), + strProp(props, "PeerAccountId", params, physicalIDs), + strProp(props, "PeerRegion", params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create transit gateway peering attachment: %w", err) + } + + physicalIDs[logicalID+"/State"] = att.State + + return att.TransitGatewayAttachmentID, nil +} + +// ---- AWS::EC2::TransitGatewayMulticastDomain ---- +// Ref returns the multicast domain ID (documented). CreationTime, State, +// and the ARN are all stashed since the backend provides real values. + +func (rc *ResourceCreator) createEC2TGWMulticastDomain( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return logicalID + "-stub", nil + } + + options, _ := props["Options"].(map[string]any) + + domain, err := rc.backends.EC2.Backend.CreateTransitGatewayMulticastDomain( + strProp(props, "TransitGatewayId", params, physicalIDs), + strProp(options, "AutoAcceptSharedAssociations", params, physicalIDs), + strProp(options, "Igmpv2Support", params, physicalIDs), + strProp(options, "StaticSourcesSupport", params, physicalIDs), + tagListProp(props, params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create transit gateway multicast domain: %w", err) + } + + physicalIDs[logicalID+"/CreationTime"] = domain.CreationTime.UTC().Format(time.RFC3339) + physicalIDs[logicalID+"/State"] = domain.State + physicalIDs[logicalID+"/TransitGatewayMulticastDomainArn"] = arn.Build( + "ec2", rc.backends.Region, rc.backends.AccountID, "transit-gateway-multicast-domain/"+domain.ID, + ) + + return domain.ID, nil +} diff --git a/services/cloudformation/resources_ec2_transitgateway_attachments_test.go b/services/cloudformation/resources_ec2_transitgateway_attachments_test.go new file mode 100644 index 0000000000..a3edd22816 --- /dev/null +++ b/services/cloudformation/resources_ec2_transitgateway_attachments_test.go @@ -0,0 +1,146 @@ +package cloudformation_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + cfnsdk "github.com/aws/aws-sdk-go-v2/service/cloudformation" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateStack_EC2TransitGatewayMoreTypes(t *testing.T) { + t.Parallel() + + cases := []struct { + run func(t *testing.T) + name string + }{ + {testEC2TGWVpcAttachment, "vpc_attachment"}, + {testEC2TGWPeeringAttachment, "peering_attachment"}, + {testEC2TGWMulticastDomain, "multicast_domain"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.run(t) + }) + } +} + +func testEC2TGWVpcAttachment(t *testing.T) { + t.Helper() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "VPC": {"Type": "AWS::EC2::VPC", "Properties": {"CidrBlock": "10.0.0.0/16"}}, + "Subnet": {"Type": "AWS::EC2::Subnet", + "Properties": {"VpcId": {"Ref": "VPC"}, "CidrBlock": "10.0.1.0/24"}}, + "TGW": {"Type": "AWS::EC2::TransitGateway", "Properties": {"Description": "test tgw"}}, + "Att": { + "Type": "AWS::EC2::TransitGatewayVpcAttachment", + "Properties": {"TransitGatewayId": {"Ref": "TGW"}, "VpcId": {"Ref": "VPC"}, "SubnetIds": [{"Ref": "Subnet"}]} + } +}, +"Outputs": { + "Ref": {"Value": {"Ref": "Att"}}, + "Id": {"Value": {"Fn::GetAtt": ["Att", "Id"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "ec2-tgwvpcatt-stack", tmpl) + assert.Equal(t, outputs["Ref"], outputs["Id"]) + require.Len(t, backends.EC2.Backend.DescribeTransitGatewayVpcAttachments([]string{outputs["Ref"]}), 1) + + _, err := client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("ec2-tgwvpcatt-stack")}) + require.NoError(t, err) + + found := false + + for _, a := range backends.EC2.Backend.DescribeTransitGatewayVpcAttachments(nil) { + if a.TransitGatewayAttachmentID == outputs["Ref"] { + found = true + } + } + + assert.False(t, found, "deleted TGW VPC attachment must not appear in an unfiltered Describe") +} + +func testEC2TGWPeeringAttachment(t *testing.T) { + t.Helper() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "TGW": {"Type": "AWS::EC2::TransitGateway", "Properties": {"Description": "test tgw"}}, + "Peer": { + "Type": "AWS::EC2::TransitGatewayPeeringAttachment", + "Properties": { + "TransitGatewayId": {"Ref": "TGW"}, + "PeerTransitGatewayId": "tgw-peer12345", + "PeerAccountId": "222222222222", + "PeerRegion": "us-west-2" + } + } +}, +"Outputs": { + "Ref": {"Value": {"Ref": "Peer"}}, + "Id": {"Value": {"Fn::GetAtt": ["Peer", "TransitGatewayAttachmentId"]}}, + "State": {"Value": {"Fn::GetAtt": ["Peer", "State"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "ec2-tgwpeer-stack", tmpl) + assert.Equal(t, outputs["Ref"], outputs["Id"]) + assert.Equal(t, "pendingAcceptance", outputs["State"]) + require.Len(t, backends.EC2.Backend.DescribeTransitGatewayPeeringAttachments([]string{outputs["Ref"]}), 1) + + _, err := client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("ec2-tgwpeer-stack")}) + require.NoError(t, err) + + found := false + + for _, a := range backends.EC2.Backend.DescribeTransitGatewayPeeringAttachments(nil) { + if a.TransitGatewayAttachmentID == outputs["Ref"] { + found = true + } + } + + assert.False(t, found, "deleted TGW peering attachment must not appear in an unfiltered Describe") +} + +func testEC2TGWMulticastDomain(t *testing.T) { + t.Helper() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "TGW": {"Type": "AWS::EC2::TransitGateway", "Properties": {"Description": "test tgw"}}, + "Domain": { + "Type": "AWS::EC2::TransitGatewayMulticastDomain", + "Properties": {"TransitGatewayId": {"Ref": "TGW"}, "Options": {"Igmpv2Support": "enable"}} + } +}, +"Outputs": { + "Ref": {"Value": {"Ref": "Domain"}}, + "Id": {"Value": {"Fn::GetAtt": ["Domain", "TransitGatewayMulticastDomainId"]}}, + "Arn": {"Value": {"Fn::GetAtt": ["Domain", "TransitGatewayMulticastDomainArn"]}}, + "State": {"Value": {"Fn::GetAtt": ["Domain", "State"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "ec2-tgwmcast-stack", tmpl) + assert.Equal(t, outputs["Ref"], outputs["Id"]) + assert.Contains(t, outputs["Arn"], "transit-gateway-multicast-domain/"+outputs["Ref"]) + assert.NotEmpty(t, outputs["State"]) + require.Len(t, backends.EC2.Backend.DescribeTransitGatewayMulticastDomains([]string{outputs["Ref"]}), 1) + + _, err := client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("ec2-tgwmcast-stack")}) + require.NoError(t, err) + assert.Empty(t, backends.EC2.Backend.DescribeTransitGatewayMulticastDomains([]string{outputs["Ref"]})) +} diff --git a/services/cloudformation/resources_ec2_vpn.go b/services/cloudformation/resources_ec2_vpn.go new file mode 100644 index 0000000000..a7e53f7a35 --- /dev/null +++ b/services/cloudformation/resources_ec2_vpn.go @@ -0,0 +1,106 @@ +package cloudformation + +import ( + "fmt" + + ec2backend "github.com/blackbirdworks/gopherstack/services/ec2" +) + +const ( + resTypeEC2VPNGateway = "AWS::EC2::VPNGateway" + resTypeEC2VPNConnection = "AWS::EC2::VPNConnection" +) + +// createEC2VPNResource handles AWS::EC2::VPNGateway and +// AWS::EC2::VPNConnection creation. Returns handled=false otherwise. +func (rc *ResourceCreator) createEC2VPNResource( + _, resourceType string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, bool, error) { + switch resourceType { + case resTypeEC2VPNGateway: + id, err := rc.createEC2VPNGateway(props, params, physicalIDs) + + return id, true, err + case resTypeEC2VPNConnection: + id, err := rc.createEC2VPNConnection(props, params, physicalIDs) + + return id, true, err + default: + return "", false, nil + } +} + +// deleteEC2VPNResource handles deletion for the types created above. +func (rc *ResourceCreator) deleteEC2VPNResource(resourceType, physicalID string) (bool, error) { + if rc.backends.EC2 == nil { + switch resourceType { + case resTypeEC2VPNGateway, resTypeEC2VPNConnection: + return true, nil + default: + return false, nil + } + } + + switch resourceType { + case resTypeEC2VPNGateway: + return true, ignoreNotFound( + rc.backends.EC2.Backend.DeleteVpnGateway(physicalID), ec2backend.ErrVpnGatewayNotFound, + ) + case resTypeEC2VPNConnection: + return true, ignoreNotFound( + rc.backends.EC2.Backend.DeleteVpnConnection(physicalID), ec2backend.ErrVpnConnectionNotFound, + ) + default: + return false, nil + } +} + +// ---- AWS::EC2::VPNGateway ---- +// Ref returns the ID of the VPN gateway (documented). + +func (rc *ResourceCreator) createEC2VPNGateway( + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return "vgw-stub", nil + } + + gatewayType := strProp(props, "Type", params, physicalIDs) + asn := int64Prop(props, "AmazonSideAsn", params, physicalIDs) + + vgw, err := rc.backends.EC2.Backend.CreateVpnGateway(gatewayType, asn) + if err != nil { + return "", fmt.Errorf("create VPN gateway: %w", err) + } + + return vgw.VpnGatewayID, nil +} + +// ---- AWS::EC2::VPNConnection ---- +// Ref returns the ID of the VPN connection (documented). Only the +// VpnGatewayId form is supported: the backend's CreateVpnConnection +// requires a VpnGatewayId, so a template using TransitGatewayId instead +// fails honestly rather than being silently accepted. + +func (rc *ResourceCreator) createEC2VPNConnection( + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.EC2 == nil { + return "vpn-stub", nil + } + + connType := strProp(props, "Type", params, physicalIDs) + customerGatewayID := strProp(props, "CustomerGatewayId", params, physicalIDs) + vpnGatewayID := strProp(props, "VpnGatewayId", params, physicalIDs) + + conn, err := rc.backends.EC2.Backend.CreateVpnConnection(connType, customerGatewayID, vpnGatewayID) + if err != nil { + return "", fmt.Errorf("create VPN connection: %w", err) + } + + return conn.VpnConnectionID, nil +} diff --git a/services/cloudformation/resources_ec2_vpn_test.go b/services/cloudformation/resources_ec2_vpn_test.go new file mode 100644 index 0000000000..aa5f4ba86f --- /dev/null +++ b/services/cloudformation/resources_ec2_vpn_test.go @@ -0,0 +1,86 @@ +package cloudformation_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + cfnsdk "github.com/aws/aws-sdk-go-v2/service/cloudformation" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateStack_EC2VPNTypes(t *testing.T) { + t.Parallel() + + cases := []struct { + run func(t *testing.T) + name string + }{ + {testEC2VPNGatewayAndConnection, "vpn_gateway_and_connection"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.run(t) + }) + } +} + +func testEC2VPNGatewayAndConnection(t *testing.T) { + t.Helper() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "VGW": { + "Type": "AWS::EC2::VPNGateway", + "Properties": {"Type": "ipsec.1"} + }, + "CGW": { + "Type": "AWS::EC2::CustomerGateway", + "Properties": {"Type": "ipsec.1", "IpAddress": "203.0.113.1", "BgpAsn": "65000"} + }, + "Conn": { + "Type": "AWS::EC2::VPNConnection", + "Properties": { + "Type": "ipsec.1", + "CustomerGatewayId": {"Ref": "CGW"}, + "VpnGatewayId": {"Ref": "VGW"} + } + } +}, +"Outputs": { + "VGWRef": {"Value": {"Ref": "VGW"}}, + "VGWId": {"Value": {"Fn::GetAtt": ["VGW", "VPNGatewayId"]}}, + "ConnRef": {"Value": {"Ref": "Conn"}}, + "ConnId": {"Value": {"Fn::GetAtt": ["Conn", "VpnConnectionId"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "ec2-vpn-stack", tmpl) + assert.Equal(t, outputs["VGWRef"], outputs["VGWId"]) + assert.Contains(t, outputs["VGWRef"], "vgw-") + assert.Equal(t, outputs["ConnRef"], outputs["ConnId"]) + assert.Contains(t, outputs["ConnRef"], "vpn-") + + require.Len(t, backends.EC2.Backend.DescribeVpnGateways([]string{outputs["VGWRef"]}), 1) + require.Len(t, backends.EC2.Backend.DescribeVpnConnections([]string{outputs["ConnRef"]}), 1) + + _, err := client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("ec2-vpn-stack")}) + require.NoError(t, err) + + assert.Empty(t, backends.EC2.Backend.DescribeVpnGateways([]string{outputs["VGWRef"]})) + // DeleteVpnConnection keeps a tombstone reachable by explicit ID; an + // unfiltered Describe never surfaces it (see DescribeVpnConnections). + found := false + + for _, c := range backends.EC2.Backend.DescribeVpnConnections(nil) { + if c.VpnConnectionID == outputs["ConnRef"] { + found = true + } + } + + assert.False(t, found, "deleted VPN connection must not appear in an unfiltered Describe") +} diff --git a/services/cloudformation/resources_ecrpublic.go b/services/cloudformation/resources_ecrpublic.go new file mode 100644 index 0000000000..d028baff56 --- /dev/null +++ b/services/cloudformation/resources_ecrpublic.go @@ -0,0 +1,90 @@ +package cloudformation + +import ( + "fmt" + + ecrpublicbackend "github.com/blackbirdworks/gopherstack/services/ecrpublic" +) + +const resTypeECRPublicRepository = "AWS::ECR::PublicRepository" + +// createECRPublicResource handles the ECR Public resource type listed above. +func (rc *ResourceCreator) createECRPublicResource( + logicalID, resourceType string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, bool, error) { + if resourceType != resTypeECRPublicRepository { + return "", false, nil + } + + id, err := rc.createECRPublicRepository(logicalID, props, params, physicalIDs) + + return id, true, err +} + +// deleteECRPublicResource handles deletion for the type created above. +func (rc *ResourceCreator) deleteECRPublicResource(resourceType, physicalID string) (bool, error) { + if resourceType != resTypeECRPublicRepository { + return false, nil + } + + if rc.backends.ECRPublic == nil { + return true, nil + } + + // No force: AWS::ECR::PublicRepository has no EmptyOnDelete property + // (unlike AWS::ECR::Repository), so a non-empty repository fails to + // delete here exactly as it does in real CloudFormation. + _, err := rc.backends.ECRPublic.Backend.DeleteRepository(rc.backends.AccountID, physicalID, false) + + return true, ignoreNotFound(err, ecrpublicbackend.ErrRepositoryNotFound) +} + +// ---- AWS::ECR::PublicRepository ---- +// Ref returns the repository name (documented). Fn::GetAtt Arn returns the +// repository ARN (documented). + +func (rc *ResourceCreator) createECRPublicRepository( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.ECRPublic == nil { + return logicalID + "-stub", nil + } + + name := strProp(props, "RepositoryName", params, physicalIDs) + if name == "" { + name = logicalID + } + + catalogData, _ := props["RepositoryCatalogData"].(map[string]any) + + repo, err := rc.backends.ECRPublic.Backend.CreateRepository( + name, + &ecrpublicbackend.CatalogData{ + AboutText: strProp(catalogData, "AboutText", params, physicalIDs), + Description: strProp(catalogData, "RepositoryDescription", params, physicalIDs), + UsageText: strProp(catalogData, "UsageText", params, physicalIDs), + Architectures: strSliceProp(catalogData["Architectures"], params, physicalIDs), + OperatingSystems: strSliceProp(catalogData["OperatingSystems"], params, physicalIDs), + }, + tagListProp(props, params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create ECR Public repository %s: %w", name, err) + } + + if policyText := jsonProp(props, "RepositoryPolicyText"); policyText != "" { + if _, setErr := rc.backends.ECRPublic.Backend.SetRepositoryPolicy( + rc.backends.AccountID, repo.RepositoryName, policyText, + ); setErr != nil { + return "", fmt.Errorf("set ECR Public repository policy %s: %w", name, setErr) + } + } + + physicalIDs[logicalID+"/Arn"] = repo.RepositoryArn + + return repo.RepositoryName, nil +} diff --git a/services/cloudformation/resources_ecrpublic_test.go b/services/cloudformation/resources_ecrpublic_test.go new file mode 100644 index 0000000000..419a4d927a --- /dev/null +++ b/services/cloudformation/resources_ecrpublic_test.go @@ -0,0 +1,63 @@ +package cloudformation_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + cfnsdk "github.com/aws/aws-sdk-go-v2/service/cloudformation" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/cloudformation" + ecrpublicbackend "github.com/blackbirdworks/gopherstack/services/ecrpublic" +) + +// newECRPublicTestClient wires a real aws-sdk-go-v2 CloudFormation client +// against a backend with ECRPublic (among the other backends +// newMoreTypesServiceBackends already wires) set to a real in-memory service +// backend. +func newECRPublicTestClient(t *testing.T) (*cloudformation.ServiceBackends, *cfnsdk.Client) { + t.Helper() + + backends := newMoreTypesServiceBackends(t) + backends.ECRPublic = ecrpublicbackend.NewHandler(ecrpublicbackend.NewInMemoryBackend("000000000000", "us-east-1")) + + creator := cloudformation.NewResourceCreator(backends) + backend := cloudformation.NewInMemoryBackendWithConfig("000000000000", "us-east-1", creator) + client := newTestClientForBackend(t, backend) + + return backends, client +} + +func TestCreateStack_ECRPublicRepository(t *testing.T) { + t.Parallel() + + backends, client := newECRPublicTestClient(t) + + tmpl := `{ +"Resources": {"Repo": {"Type": "AWS::ECR::PublicRepository", "Properties": { + "RepositoryName": "my-repo", + "RepositoryCatalogData": {"AboutText": "about text", "RepositoryDescription": "short desc"} +}}}, +"Outputs": { + "Ref": {"Value": {"Ref": "Repo"}}, + "Arn": {"Value": {"Fn::GetAtt": ["Repo", "Arn"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "ecrpublic-stack", tmpl) + assert.Equal(t, "my-repo", outputs["Ref"]) + assert.Contains(t, outputs["Arn"], "repository/my-repo") + + repos, err := backends.ECRPublic.Backend.DescribeRepositories("", []string{"my-repo"}) + require.NoError(t, err) + require.Len(t, repos, 1) + assert.Equal(t, "about text", repos[0].CatalogData.AboutText) + assert.Equal(t, "short desc", repos[0].CatalogData.Description) + + _, err = client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("ecrpublic-stack")}) + require.NoError(t, err) + + _, err = backends.ECRPublic.Backend.DescribeRepositories("", []string{"my-repo"}) + require.Error(t, err) +} diff --git a/services/cloudformation/resources_elasticache_user.go b/services/cloudformation/resources_elasticache_user.go new file mode 100644 index 0000000000..443beb5cc5 --- /dev/null +++ b/services/cloudformation/resources_elasticache_user.go @@ -0,0 +1,81 @@ +package cloudformation + +import ( + "context" + "fmt" + + elasticachebackend "github.com/blackbirdworks/gopherstack/services/elasticache" +) + +const resTypeElastiCacheUser = "AWS::ElastiCache::User" + +// createElastiCacheUserResource handles AWS::ElastiCache::User creation. +// Returns handled=false otherwise. +func (rc *ResourceCreator) createElastiCacheUserResource( + ctx context.Context, + logicalID, resourceType string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, bool, error) { + if resourceType != resTypeElastiCacheUser { + return "", false, nil + } + + id, err := rc.createElastiCacheUser(ctx, logicalID, props, params, physicalIDs) + + return id, true, err +} + +// deleteElastiCacheUserResource handles deletion for the type created above. +func (rc *ResourceCreator) deleteElastiCacheUserResource( + ctx context.Context, resourceType, physicalID string, +) (bool, error) { + if resourceType != resTypeElastiCacheUser { + return false, nil + } + + if rc.backends.ElastiCache == nil { + return true, nil + } + + _, err := rc.backends.ElastiCache.Backend.DeleteUser(ctx, physicalID) + + return true, ignoreNotFound(err, elasticachebackend.ErrUserNotFound) +} + +// ---- AWS::ElastiCache::User ---- +// Ref returns the resource name (the UserId, documented). Arn and Status +// are stashed from the backend's real values. + +func (rc *ResourceCreator) createElastiCacheUser( + ctx context.Context, + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.ElastiCache == nil { + return logicalID + "-stub", nil + } + + userID := strProp(props, "UserId", params, physicalIDs) + if userID == "" { + userID = logicalID + } + + u, err := rc.backends.ElastiCache.Backend.CreateUser( + ctx, + userID, + strProp(props, "UserName", params, physicalIDs), + strProp(props, "AccessString", params, physicalIDs), + strProp(props, "Engine", params, physicalIDs), + boolProp(props, "NoPasswordRequired"), + ) + if err != nil { + return "", fmt.Errorf("create ElastiCache user %s: %w", userID, err) + } + + physicalIDs[logicalID+"/Arn"] = u.ARN + physicalIDs[logicalID+"/Status"] = u.Status + + return u.UserID, nil +} diff --git a/services/cloudformation/resources_elasticache_user_test.go b/services/cloudformation/resources_elasticache_user_test.go new file mode 100644 index 0000000000..ba3da7ea3d --- /dev/null +++ b/services/cloudformation/resources_elasticache_user_test.go @@ -0,0 +1,51 @@ +package cloudformation_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + cfnsdk "github.com/aws/aws-sdk-go-v2/service/cloudformation" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateStack_ElastiCacheUser(t *testing.T) { + t.Parallel() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "User": { + "Type": "AWS::ElastiCache::User", + "Properties": { + "UserId": "unit-ec-user", + "UserName": "unit-user", + "Engine": "redis", + "AccessString": "on ~* +@all", + "NoPasswordRequired": true + } + } +}, +"Outputs": { + "Ref": {"Value": {"Ref": "User"}}, + "Arn": {"Value": {"Fn::GetAtt": ["User", "Arn"]}}, + "Status": {"Value": {"Fn::GetAtt": ["User", "Status"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "ec-user-stack", tmpl) + assert.Equal(t, "unit-ec-user", outputs["Ref"]) + assert.NotEmpty(t, outputs["Arn"]) + assert.NotEmpty(t, outputs["Status"]) + + u, err := backends.ElastiCache.Backend.DescribeUsers(t.Context(), "unit-ec-user", "", "", 0, nil) + require.NoError(t, err) + require.Len(t, u.Data, 1) + + _, err = client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("ec-user-stack")}) + require.NoError(t, err) + + _, err = backends.ElastiCache.Backend.DescribeUsers(t.Context(), "unit-ec-user", "", "", 0, nil) + require.Error(t, err) +} diff --git a/services/cloudformation/resources_extended_types_test.go b/services/cloudformation/resources_extended_types_test.go index cb28337bec..cf0f91cefe 100644 --- a/services/cloudformation/resources_extended_types_test.go +++ b/services/cloudformation/resources_extended_types_test.go @@ -3,6 +3,7 @@ package cloudformation_test import ( "log/slog" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -151,6 +152,8 @@ func TestStreamNameFromARN(t *testing.T) { // Exercise streamNameFromARN indirectly via Kinesis delete path. backends := newExtendedServiceBackends() + fakeNow := time.Now() + withFakeClockedKinesis(backends, &fakeNow) rc := cloudformation.NewResourceCreator(backends) streamName := tt.want @@ -164,6 +167,8 @@ func TestStreamNameFromARN(t *testing.T) { deleteID = tt.input // pass plain name so fallback branch is hit } + fakeNow = fakeNow.Add(kinesisStreamSettleWait) + err = rc.Delete(t.Context(), "AWS::Kinesis::Stream", deleteID, nil, nil) require.NoError(t, err) }) diff --git a/services/cloudformation/resources_extensibility_test.go b/services/cloudformation/resources_extensibility_test.go index 89ab390de0..43b8083f68 100644 --- a/services/cloudformation/resources_extensibility_test.go +++ b/services/cloudformation/resources_extensibility_test.go @@ -6,6 +6,7 @@ import ( "fmt" "sync" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -123,24 +124,26 @@ func TestWaitConditionStore_SignalAndWait(t *testing.T) { func TestWaitConditionStore_AsyncSignal(t *testing.T) { t.Parallel() - store := cloudformation.NewWaitConditionStore() - token := "async-token" + synctest.Test(t, func(t *testing.T) { + store := cloudformation.NewWaitConditionStore() + token := "async-token" - // Signal from a goroutine after a short delay. - go func() { - time.Sleep(20 * time.Millisecond) - store.Signal( - token, - cloudformation.WCSignal{UniqueID: "u1", Status: "SUCCESS", Data: "data"}, - ) - }() + // Signal from a goroutine after a short delay. + go func() { + time.Sleep(20 * time.Millisecond) + store.Signal( + token, + cloudformation.WCSignal{UniqueID: "u1", Status: "SUCCESS", Data: "data"}, + ) + }() - ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) - defer cancel() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() - // Use a large emulator timeout so we wait for the goroutine signal. - err := store.Wait(ctx, token, 1, 2*time.Second) - require.NoError(t, err) + // Use a large emulator timeout so we wait for the goroutine signal. + err := store.Wait(ctx, token, 1, 2*time.Second) + require.NoError(t, err) + }) } func TestWaitConditionStore_ContextCancel(t *testing.T) { diff --git a/services/cloudformation/resources_iam_extras.go b/services/cloudformation/resources_iam_extras.go new file mode 100644 index 0000000000..e9dc469e75 --- /dev/null +++ b/services/cloudformation/resources_iam_extras.go @@ -0,0 +1,113 @@ +package cloudformation + +import ( + "fmt" + + iambackend "github.com/blackbirdworks/gopherstack/services/iam" +) + +const ( + resTypeIAMSAMLProvider = "AWS::IAM::SAMLProvider" + resTypeIAMVirtualMFADevice = "AWS::IAM::VirtualMFADevice" +) + +// createIAMExtrasResource handles AWS::IAM::SAMLProvider and +// AWS::IAM::VirtualMFADevice creation. Returns handled=false otherwise. +func (rc *ResourceCreator) createIAMExtrasResource( + logicalID, resourceType string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, bool, error) { + switch resourceType { + case resTypeIAMSAMLProvider: + id, err := rc.createIAMSAMLProvider(logicalID, props, params, physicalIDs) + + return id, true, err + case resTypeIAMVirtualMFADevice: + id, err := rc.createIAMVirtualMFADevice(logicalID, props, params, physicalIDs) + + return id, true, err + default: + return "", false, nil + } +} + +// deleteIAMExtrasResource handles deletion for the types created above. +func (rc *ResourceCreator) deleteIAMExtrasResource(resourceType, physicalID string) (bool, error) { + if rc.backends.IAM == nil { + switch resourceType { + case resTypeIAMSAMLProvider, resTypeIAMVirtualMFADevice: + return true, nil + default: + return false, nil + } + } + + switch resourceType { + case resTypeIAMSAMLProvider: + return true, ignoreNotFound( + rc.backends.IAM.Backend.DeleteSAMLProvider(physicalID), iambackend.ErrSAMLProviderNotFound, + ) + case resTypeIAMVirtualMFADevice: + return true, ignoreNotFound( + rc.backends.IAM.Backend.DeleteVirtualMFADevice(physicalID), iambackend.ErrUserNotFound, + ) + default: + return false, nil + } +} + +// ---- AWS::IAM::SAMLProvider ---- +// Ref returns the ARN (documented). SamlProviderUUID has no backend field +// to stash, so it falls back to the ARN rather than being fabricated. + +func (rc *ResourceCreator) createIAMSAMLProvider( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.IAM == nil { + return logicalID + "-stub", nil + } + + name := strProp(props, "Name", params, physicalIDs) + if name == "" { + name = logicalID + } + + metadata := strProp(props, "SamlMetadataDocument", params, physicalIDs) + + p, err := rc.backends.IAM.Backend.CreateSAMLProvider(name, metadata) + if err != nil { + return "", fmt.Errorf("create SAML provider %s: %w", name, err) + } + + return p.Arn, nil +} + +// ---- AWS::IAM::VirtualMFADevice ---- +// Ref returns the SerialNumber (documented). Associating the device with +// the Users property isn't performed: real AWS requires an MFA token/code +// to enable a device, which CFN's synchronous create can't supply. + +func (rc *ResourceCreator) createIAMVirtualMFADevice( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.IAM == nil { + return logicalID + "-stub", nil + } + + name := strProp(props, "VirtualMfaDeviceName", params, physicalIDs) + if name == "" { + name = logicalID + } + + dev, err := rc.backends.IAM.Backend.CreateVirtualMFADevice(name, strProp(props, "Path", params, physicalIDs)) + if err != nil { + return "", fmt.Errorf("create virtual MFA device %s: %w", name, err) + } + + return dev.SerialNumber, nil +} diff --git a/services/cloudformation/resources_iam_extras_test.go b/services/cloudformation/resources_iam_extras_test.go new file mode 100644 index 0000000000..5114b65511 --- /dev/null +++ b/services/cloudformation/resources_iam_extras_test.go @@ -0,0 +1,95 @@ +package cloudformation_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + cfnsdk "github.com/aws/aws-sdk-go-v2/service/cloudformation" + cfntypes "github.com/aws/aws-sdk-go-v2/service/cloudformation/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateStack_IAMExtrasTypes(t *testing.T) { + t.Parallel() + + cases := []struct { + run func(t *testing.T) + name string + }{ + {testIAMSAMLProvider, "saml_provider"}, + {testIAMVirtualMFADevice, "virtual_mfa_device"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.run(t) + }) + } +} + +func testIAMSAMLProvider(t *testing.T) { + t.Helper() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "Provider": { + "Type": "AWS::IAM::SAMLProvider", + "Properties": {"Name": "unit-saml-provider", "SamlMetadataDocument": ""} + } +}, +"Outputs": { + "Ref": {"Value": {"Ref": "Provider"}}, + "Arn": {"Value": {"Fn::GetAtt": ["Provider", "Arn"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "iam-saml-stack", tmpl, cfntypes.CapabilityCapabilityIam) + assert.Equal(t, outputs["Ref"], outputs["Arn"]) + assert.Contains(t, outputs["Ref"], "saml-provider/unit-saml-provider") + + p, err := backends.IAM.Backend.GetSAMLProvider(outputs["Ref"]) + require.NoError(t, err) + assert.NotNil(t, p) + + _, err = client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("iam-saml-stack")}) + require.NoError(t, err) + + _, err = backends.IAM.Backend.GetSAMLProvider(outputs["Ref"]) + require.Error(t, err) +} + +func testIAMVirtualMFADevice(t *testing.T) { + t.Helper() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "Device": { + "Type": "AWS::IAM::VirtualMFADevice", + "Properties": {"VirtualMfaDeviceName": "unit-mfa-device", "Path": "/", "Users": ["unit-user"]} + } +}, +"Outputs": { + "Ref": {"Value": {"Ref": "Device"}}, + "SerialNumber": {"Value": {"Fn::GetAtt": ["Device", "SerialNumber"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "iam-mfa-stack", tmpl, cfntypes.CapabilityCapabilityIam) + assert.Equal(t, outputs["Ref"], outputs["SerialNumber"]) + assert.Contains(t, outputs["Ref"], "mfa/unit-mfa-device") + + _, _, err := backends.IAM.Backend.GetVirtualMFADevice(outputs["Ref"]) + require.NoError(t, err) + + _, err = client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("iam-mfa-stack")}) + require.NoError(t, err) + + _, _, err = backends.IAM.Backend.GetVirtualMFADevice(outputs["Ref"]) + require.Error(t, err) +} diff --git a/services/cloudformation/resources_kafkaconnect.go b/services/cloudformation/resources_kafkaconnect.go new file mode 100644 index 0000000000..5768bd2482 --- /dev/null +++ b/services/cloudformation/resources_kafkaconnect.go @@ -0,0 +1,316 @@ +package cloudformation + +import ( + "fmt" + "strconv" + + kafkaconnectbackend "github.com/blackbirdworks/gopherstack/services/kafkaconnect" +) + +const ( + resTypeKafkaConnectConnector = "AWS::KafkaConnect::Connector" + resTypeKafkaConnectCustomPlugin = "AWS::KafkaConnect::CustomPlugin" + resTypeKafkaConnectWorkerConfiguration = "AWS::KafkaConnect::WorkerConfiguration" +) + +// createKafkaConnectResource handles the MSK Connect resource types listed above. +func (rc *ResourceCreator) createKafkaConnectResource( + logicalID, resourceType string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, bool, error) { + switch resourceType { + case resTypeKafkaConnectConnector: + id, err := rc.createKafkaConnectConnector(logicalID, props, params, physicalIDs) + + return id, true, err + case resTypeKafkaConnectCustomPlugin: + id, err := rc.createKafkaConnectCustomPlugin(logicalID, props, params, physicalIDs) + + return id, true, err + case resTypeKafkaConnectWorkerConfiguration: + id, err := rc.createKafkaConnectWorkerConfiguration(logicalID, props, params, physicalIDs) + + return id, true, err + default: + return "", false, nil + } +} + +// deleteKafkaConnectResource handles deletion for the types created above. +// All three types delete by their ARN-shaped physicalID directly. +func (rc *ResourceCreator) deleteKafkaConnectResource(resourceType, physicalID string) (bool, error) { + if rc.backends.KafkaConnect == nil { + switch resourceType { + case resTypeKafkaConnectConnector, resTypeKafkaConnectCustomPlugin, resTypeKafkaConnectWorkerConfiguration: + return true, nil + default: + return false, nil + } + } + + switch resourceType { + case resTypeKafkaConnectConnector: + _, err := rc.backends.KafkaConnect.Backend.DeleteConnector(physicalID, "") + + return true, ignoreNotFound(err, kafkaconnectbackend.ErrConnectorNotFound) + case resTypeKafkaConnectCustomPlugin: + _, err := rc.backends.KafkaConnect.Backend.DeleteCustomPlugin(physicalID) + + return true, ignoreNotFound(err, kafkaconnectbackend.ErrCustomPluginNotFound) + case resTypeKafkaConnectWorkerConfiguration: + _, err := rc.backends.KafkaConnect.Backend.DeleteWorkerConfiguration(physicalID) + + return true, ignoreNotFound(err, kafkaconnectbackend.ErrWorkerConfigNotFound) + default: + return false, nil + } +} + +// ---- AWS::KafkaConnect::Connector ---- +// Ref and Fn::GetAtt ConnectorArn both return the connector ARN: confirmed +// against the AWS-published resource-provider schema +// (aws-cloudformation-resource-providers-kafkaconnect, +// primaryIdentifier == /properties/ConnectorArn == its sole readOnlyProperty), +// since the CloudFormation Template Reference page itself leaves Ref +// undocumented and states only the ConnectorArn Fn::GetAtt attribute. + +func (rc *ResourceCreator) createKafkaConnectConnector( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.KafkaConnect == nil { + return logicalID + "-stub", nil + } + + name := strProp(props, "ConnectorName", params, physicalIDs) + if name == "" { + name = logicalID + } + + capacity, _ := props["Capacity"].(map[string]any) + kafkaCluster, _ := props["KafkaCluster"].(map[string]any) + clientAuth, _ := props["KafkaClusterClientAuthentication"].(map[string]any) + encryption, _ := props["KafkaClusterEncryptionInTransit"].(map[string]any) + logDelivery, _ := props["LogDelivery"].(map[string]any) + workerConfig, _ := props["WorkerConfiguration"].(map[string]any) + + spec := kafkaconnectbackend.ConnectorSpec{ + Name: name, + Description: strProp(props, "ConnectorDescription", params, physicalIDs), + ConnectorConfiguration: stringMapProp(props["ConnectorConfiguration"], params, physicalIDs), + Capacity: kafkaConnectCapacityFromProps(capacity, params, physicalIDs), + ApacheKafkaCluster: kafkaConnectApacheKafkaClusterFromProps(kafkaCluster, params, physicalIDs), + KafkaClusterClientAuthentication: strProp(clientAuth, "AuthenticationType", params, physicalIDs), + KafkaClusterEncryptionInTransit: strProp(encryption, "EncryptionType", params, physicalIDs), + KafkaConnectVersion: strProp(props, "KafkaConnectVersion", params, physicalIDs), + ServiceExecutionRoleArn: strProp(props, "ServiceExecutionRoleArn", params, physicalIDs), + NetworkType: strProp(props, "NetworkType", params, physicalIDs), + Plugins: kafkaConnectPluginsFromProps(props["Plugins"], params, physicalIDs), + WorkerLogDelivery: kafkaConnectWorkerLogDeliveryFromProps(logDelivery, params, physicalIDs), + Tags: tagListProp(props, params, physicalIDs), + } + + if workerConfig != nil { + spec.WorkerConfiguration = &kafkaconnectbackend.WorkerConfigRef{ + Arn: strProp(workerConfig, "WorkerConfigurationArn", params, physicalIDs), + Revision: int64Prop(workerConfig, "Revision", params, physicalIDs), + } + } + + c, err := rc.backends.KafkaConnect.Backend.CreateConnector(rc.backends.AccountID, rc.backends.Region, spec) + if err != nil { + return "", fmt.Errorf("create MSK Connect connector %s: %w", name, err) + } + + physicalIDs[logicalID+"/ConnectorArn"] = c.ARN + + return c.ARN, nil +} + +func kafkaConnectCapacityFromProps( + v map[string]any, params, physicalIDs map[string]string, +) kafkaconnectbackend.Capacity { + var capacity kafkaconnectbackend.Capacity + + if as, ok := v["AutoScaling"].(map[string]any); ok { + scaleIn, _ := as["ScaleInPolicy"].(map[string]any) + scaleOut, _ := as["ScaleOutPolicy"].(map[string]any) + + capacity.AutoScaling = &kafkaconnectbackend.AutoScaling{ + MinWorkerCount: int32Prop(as, "MinWorkerCount", params, physicalIDs), + MaxWorkerCount: int32Prop(as, "MaxWorkerCount", params, physicalIDs), + McuCount: int32Prop(as, "McuCount", params, physicalIDs), + MaxAutoscalingTaskCount: int32Prop(as, "MaxAutoscalingTaskCount", params, physicalIDs), + ScaleInCPUPercent: int32Prop(scaleIn, "CpuUtilizationPercentage", params, physicalIDs), + ScaleOutCPUPercent: int32Prop(scaleOut, "CpuUtilizationPercentage", params, physicalIDs), + } + } + + if pc, ok := v["ProvisionedCapacity"].(map[string]any); ok { + capacity.Provisioned = &kafkaconnectbackend.ProvisionedCapacity{ + McuCount: int32Prop(pc, "McuCount", params, physicalIDs), + WorkerCount: int32Prop(pc, "WorkerCount", params, physicalIDs), + } + } + + return capacity +} + +func kafkaConnectApacheKafkaClusterFromProps( + v map[string]any, params, physicalIDs map[string]string, +) kafkaconnectbackend.ApacheKafkaCluster { + akc, _ := v["ApacheKafkaCluster"].(map[string]any) + vpc, _ := akc["Vpc"].(map[string]any) + + return kafkaconnectbackend.ApacheKafkaCluster{ + BootstrapServers: strProp(akc, "BootstrapServers", params, physicalIDs), + Vpc: kafkaconnectbackend.Vpc{ + SecurityGroups: strSliceProp(vpc["SecurityGroups"], params, physicalIDs), + Subnets: strSliceProp(vpc["Subnets"], params, physicalIDs), + }, + } +} + +func kafkaConnectPluginsFromProps(v any, params, physicalIDs map[string]string) []kafkaconnectbackend.PluginRef { + list, ok := v.([]any) + if !ok { + return nil + } + + out := make([]kafkaconnectbackend.PluginRef, 0, len(list)) + + for _, item := range list { + m, isMap := item.(map[string]any) + if !isMap { + continue + } + + cp, _ := m["CustomPlugin"].(map[string]any) + out = append(out, kafkaconnectbackend.PluginRef{ + CustomPluginArn: strProp(cp, "CustomPluginArn", params, physicalIDs), + Revision: int64Prop(cp, "Revision", params, physicalIDs), + }) + } + + return out +} + +func kafkaConnectWorkerLogDeliveryFromProps( + v map[string]any, params, physicalIDs map[string]string, +) *kafkaconnectbackend.WorkerLogDelivery { + wld, ok := v["WorkerLogDelivery"].(map[string]any) + if !ok { + return nil + } + + out := &kafkaconnectbackend.WorkerLogDelivery{} + + if cw, isMap := wld["CloudWatchLogs"].(map[string]any); isMap { + out.CloudWatchLogs = &kafkaconnectbackend.CloudWatchLogsDelivery{ + Enabled: boolProp(cw, "Enabled"), + LogGroup: strProp(cw, "LogGroup", params, physicalIDs), + } + } + + if fh, isMap := wld["Firehose"].(map[string]any); isMap { + out.Firehose = &kafkaconnectbackend.FirehoseDelivery{ + DeliveryStream: strProp(fh, "DeliveryStream", params, physicalIDs), + Enabled: boolProp(fh, "Enabled"), + } + } + + if s3, isMap := wld["S3"].(map[string]any); isMap { + out.S3 = &kafkaconnectbackend.S3LogDelivery{ + Bucket: strProp(s3, "Bucket", params, physicalIDs), + Prefix: strProp(s3, "Prefix", params, physicalIDs), + Enabled: boolProp(s3, "Enabled"), + } + } + + return out +} + +// ---- AWS::KafkaConnect::CustomPlugin ---- +// Ref and Fn::GetAtt CustomPluginArn both return the custom plugin ARN (see +// the Connector doc comment above for the Ref-attribution basis; confirmed +// separately for CustomPlugin against its own resource-provider schema). +// Revision is a documented Fn::GetAtt attribute. + +func (rc *ResourceCreator) createKafkaConnectCustomPlugin( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.KafkaConnect == nil { + return logicalID + "-stub", nil + } + + name := strProp(props, "Name", params, physicalIDs) + if name == "" { + name = logicalID + } + + location, _ := props["Location"].(map[string]any) + s3Location, _ := location["S3Location"].(map[string]any) + + p, err := rc.backends.KafkaConnect.Backend.CreateCustomPlugin( + rc.backends.AccountID, + rc.backends.Region, + name, + strProp(props, "Description", params, physicalIDs), + strProp(props, "ContentType", params, physicalIDs), + strProp(s3Location, "BucketArn", params, physicalIDs), + strProp(s3Location, "FileKey", params, physicalIDs), + strProp(s3Location, "ObjectVersion", params, physicalIDs), + tagListProp(props, params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create MSK Connect custom plugin %s: %w", name, err) + } + + physicalIDs[logicalID+"/CustomPluginArn"] = p.ARN + physicalIDs[logicalID+"/Revision"] = strconv.FormatInt(p.Revision, 10) + + return p.ARN, nil +} + +// ---- AWS::KafkaConnect::WorkerConfiguration ---- +// Ref and Fn::GetAtt WorkerConfigurationArn both return the worker +// configuration ARN (see the Connector doc comment above for the +// Ref-attribution basis; confirmed separately for WorkerConfiguration +// against its own resource-provider schema). Revision is a documented +// Fn::GetAtt attribute. + +func (rc *ResourceCreator) createKafkaConnectWorkerConfiguration( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.KafkaConnect == nil { + return logicalID + "-stub", nil + } + + name := strProp(props, "Name", params, physicalIDs) + if name == "" { + name = logicalID + } + + w, err := rc.backends.KafkaConnect.Backend.CreateWorkerConfiguration( + rc.backends.AccountID, + rc.backends.Region, + name, + strProp(props, "Description", params, physicalIDs), + strProp(props, "PropertiesFileContent", params, physicalIDs), + tagListProp(props, params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create MSK Connect worker configuration %s: %w", name, err) + } + + physicalIDs[logicalID+"/WorkerConfigurationArn"] = w.ARN + physicalIDs[logicalID+"/Revision"] = strconv.FormatInt(w.LatestRevision.Revision, 10) + + return w.ARN, nil +} diff --git a/services/cloudformation/resources_kafkaconnect_test.go b/services/cloudformation/resources_kafkaconnect_test.go new file mode 100644 index 0000000000..cb672e6854 --- /dev/null +++ b/services/cloudformation/resources_kafkaconnect_test.go @@ -0,0 +1,161 @@ +package cloudformation_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + cfnsdk "github.com/aws/aws-sdk-go-v2/service/cloudformation" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/cloudformation" + kafkaconnectbackend "github.com/blackbirdworks/gopherstack/services/kafkaconnect" +) + +// newKafkaConnectTestClient wires a real aws-sdk-go-v2 CloudFormation client +// against a backend with KafkaConnect (among the other backends +// newMoreTypesServiceBackends already wires) set to a real in-memory service +// backend. +func newKafkaConnectTestClient(t *testing.T) (*cloudformation.ServiceBackends, *cfnsdk.Client) { + t.Helper() + + backends := newMoreTypesServiceBackends(t) + backends.KafkaConnect = kafkaconnectbackend.NewHandler(kafkaconnectbackend.NewInMemoryBackend()) + + creator := cloudformation.NewResourceCreator(backends) + backend := cloudformation.NewInMemoryBackendWithConfig("000000000000", "us-east-1", creator) + client := newTestClientForBackend(t, backend) + + return backends, client +} + +func TestCreateStack_KafkaConnectTypes(t *testing.T) { + t.Parallel() + + cases := []struct { + run func(t *testing.T) + name string + }{ + {testKafkaConnectConnector, "connector"}, + {testKafkaConnectCustomPlugin, "custom_plugin"}, + {testKafkaConnectWorkerConfiguration, "worker_configuration"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.run(t) + }) + } +} + +func testKafkaConnectConnector(t *testing.T) { + t.Helper() + + backends, client := newKafkaConnectTestClient(t) + + tmpl := `{ +"Resources": {"Connector": {"Type": "AWS::KafkaConnect::Connector", "Properties": { + "Capacity": {"ProvisionedCapacity": {"McuCount": 1, "WorkerCount": 1}}, + "ConnectorConfiguration": {"connector.class": "com.example.Connector"}, + "ConnectorName": "test-connector", + "KafkaCluster": {"ApacheKafkaCluster": { + "BootstrapServers": "broker1:9092,broker2:9092", + "Vpc": {"SecurityGroups": ["sg-1"], "Subnets": ["subnet-1", "subnet-2"]} + }}, + "KafkaClusterClientAuthentication": {"AuthenticationType": "NONE"}, + "KafkaClusterEncryptionInTransit": {"EncryptionType": "PLAINTEXT"}, + "KafkaConnectVersion": "2.7.1", + "Plugins": [{"CustomPlugin": { + "CustomPluginArn": "arn:aws:kafkaconnect:us-east-1:000000000000:custom-plugin/p/abc", + "Revision": 1 + }}], + "ServiceExecutionRoleArn": "arn:aws:iam::000000000000:role/connect-role" +}}}, +"Outputs": { + "Ref": {"Value": {"Ref": "Connector"}}, + "ConnectorArn": {"Value": {"Fn::GetAtt": ["Connector", "ConnectorArn"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "kc-connector-stack", tmpl) + assert.Equal(t, outputs["Ref"], outputs["ConnectorArn"]) + assert.Contains(t, outputs["ConnectorArn"], "connector/test-connector/") + + c, err := backends.KafkaConnect.Backend.DescribeConnector(outputs["ConnectorArn"]) + require.NoError(t, err) + assert.Equal(t, "broker1:9092,broker2:9092", c.ApacheKafkaCluster.BootstrapServers) + + _, err = client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("kc-connector-stack")}) + require.NoError(t, err) + + _, err = backends.KafkaConnect.Backend.DescribeConnector(outputs["ConnectorArn"]) + require.Error(t, err) +} + +func testKafkaConnectCustomPlugin(t *testing.T) { + t.Helper() + + backends, client := newKafkaConnectTestClient(t) + + tmpl := `{ +"Resources": {"Plugin": {"Type": "AWS::KafkaConnect::CustomPlugin", "Properties": { + "ContentType": "ZIP", + "Name": "test-plugin", + "Location": {"S3Location": {"BucketArn": "arn:aws:s3:::my-bucket", "FileKey": "plugin.zip"}} +}}}, +"Outputs": { + "Ref": {"Value": {"Ref": "Plugin"}}, + "CustomPluginArn": {"Value": {"Fn::GetAtt": ["Plugin", "CustomPluginArn"]}}, + "Revision": {"Value": {"Fn::GetAtt": ["Plugin", "Revision"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "kc-plugin-stack", tmpl) + assert.Equal(t, outputs["Ref"], outputs["CustomPluginArn"]) + assert.Contains(t, outputs["CustomPluginArn"], "custom-plugin/test-plugin/") + assert.Equal(t, "1", outputs["Revision"]) + + p, err := backends.KafkaConnect.Backend.DescribeCustomPlugin(outputs["CustomPluginArn"]) + require.NoError(t, err) + assert.Equal(t, "arn:aws:s3:::my-bucket", p.BucketArn) + + _, err = client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("kc-plugin-stack")}) + require.NoError(t, err) + + _, err = backends.KafkaConnect.Backend.DescribeCustomPlugin(outputs["CustomPluginArn"]) + require.Error(t, err) +} + +func testKafkaConnectWorkerConfiguration(t *testing.T) { + t.Helper() + + backends, client := newKafkaConnectTestClient(t) + + tmpl := `{ +"Resources": {"WC": {"Type": "AWS::KafkaConnect::WorkerConfiguration", "Properties": { + "Name": "test-worker-config", + "PropertiesFileContent": "a2V5PXZhbHVl" +}}}, +"Outputs": { + "Ref": {"Value": {"Ref": "WC"}}, + "WorkerConfigurationArn": {"Value": {"Fn::GetAtt": ["WC", "WorkerConfigurationArn"]}}, + "Revision": {"Value": {"Fn::GetAtt": ["WC", "Revision"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "kc-workerconfig-stack", tmpl) + assert.Equal(t, outputs["Ref"], outputs["WorkerConfigurationArn"]) + assert.Contains(t, outputs["WorkerConfigurationArn"], "worker-configuration/test-worker-config/") + assert.Equal(t, "1", outputs["Revision"]) + + w, err := backends.KafkaConnect.Backend.DescribeWorkerConfiguration(outputs["WorkerConfigurationArn"]) + require.NoError(t, err) + assert.Equal(t, "a2V5PXZhbHVl", w.LatestRevision.PropertiesFileContent) + + _, err = client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("kc-workerconfig-stack")}) + require.NoError(t, err) + + _, err = backends.KafkaConnect.Backend.DescribeWorkerConfiguration(outputs["WorkerConfigurationArn"]) + require.Error(t, err) +} diff --git a/services/cloudformation/resources_kinesisvideo.go b/services/cloudformation/resources_kinesisvideo.go new file mode 100644 index 0000000000..cf05064706 --- /dev/null +++ b/services/cloudformation/resources_kinesisvideo.go @@ -0,0 +1,129 @@ +package cloudformation + +import "fmt" + +const ( + resTypeKinesisVideoStream = "AWS::KinesisVideo::Stream" + resTypeKinesisVideoSignalingChannel = "AWS::KinesisVideo::SignalingChannel" +) + +// createKinesisVideoResource handles the KinesisVideo resource types listed +// above. +func (rc *ResourceCreator) createKinesisVideoResource( + logicalID, resourceType string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, bool, error) { + switch resourceType { + case resTypeKinesisVideoStream: + id, err := rc.createKinesisVideoStream(logicalID, props, params, physicalIDs) + + return id, true, err + case resTypeKinesisVideoSignalingChannel: + id, err := rc.createKinesisVideoSignalingChannel(logicalID, props, params, physicalIDs) + + return id, true, err + default: + return "", false, nil + } +} + +// deleteKinesisVideoResource handles deletion for the types created above. +// Both types delete by their ARN-shaped physicalID directly. +func (rc *ResourceCreator) deleteKinesisVideoResource(resourceType, physicalID string) (bool, error) { + if rc.backends.KinesisVideo == nil { + switch resourceType { + case resTypeKinesisVideoStream, resTypeKinesisVideoSignalingChannel: + return true, nil + default: + return false, nil + } + } + + switch resourceType { + case resTypeKinesisVideoStream: + return true, rc.backends.KinesisVideo.Backend.DeleteStream(physicalID, "") + case resTypeKinesisVideoSignalingChannel: + return true, rc.backends.KinesisVideo.Backend.DeleteSignalingChannel(physicalID, "") + default: + return false, nil + } +} + +// ---- AWS::KinesisVideo::Stream ---- +// Ref and Fn::GetAtt Arn both return the stream ARN: the CloudFormation +// Template Reference's Return values section documents only the Arn +// attribute and leaves Ref undocumented, the same pattern the AWS-published +// resource-provider schema confirms for AWS::KafkaConnect::Connector (Ref == +// its sole ARN attribute, primaryIdentifier == readOnlyProperties[0]). + +func (rc *ResourceCreator) createKinesisVideoStream( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.KinesisVideo == nil { + return logicalID + "-stub", nil + } + + name := strProp(props, "Name", params, physicalIDs) + if name == "" { + name = logicalID + } + + storageConfig, _ := props["StreamStorageConfiguration"].(map[string]any) + + s, err := rc.backends.KinesisVideo.Backend.CreateStream( + rc.backends.AccountID, + rc.backends.Region, + name, + strProp(props, "DeviceName", params, physicalIDs), + strProp(props, "MediaType", params, physicalIDs), + strProp(props, "KmsKeyId", params, physicalIDs), + strProp(storageConfig, "DefaultStorageTier", params, physicalIDs), + int32Prop(props, "DataRetentionInHours", params, physicalIDs), + tagListProp(props, params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create KinesisVideo stream %s: %w", name, err) + } + + physicalIDs[logicalID+"/Arn"] = s.ARN + + return s.ARN, nil +} + +// ---- AWS::KinesisVideo::SignalingChannel ---- +// Ref and Fn::GetAtt Arn both return the channel ARN (see the Stream doc +// comment above for the Ref-attribution basis). + +func (rc *ResourceCreator) createKinesisVideoSignalingChannel( + logicalID string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, error) { + if rc.backends.KinesisVideo == nil { + return logicalID + "-stub", nil + } + + name := strProp(props, "Name", params, physicalIDs) + if name == "" { + name = logicalID + } + + c, err := rc.backends.KinesisVideo.Backend.CreateSignalingChannel( + rc.backends.AccountID, + rc.backends.Region, + name, + strProp(props, "Type", params, physicalIDs), + int32Prop(props, "MessageTtlSeconds", params, physicalIDs), + tagListProp(props, params, physicalIDs), + ) + if err != nil { + return "", fmt.Errorf("create KinesisVideo signaling channel %s: %w", name, err) + } + + physicalIDs[logicalID+"/Arn"] = c.ARN + + return c.ARN, nil +} diff --git a/services/cloudformation/resources_kinesisvideo_test.go b/services/cloudformation/resources_kinesisvideo_test.go new file mode 100644 index 0000000000..ae1daaa0ff --- /dev/null +++ b/services/cloudformation/resources_kinesisvideo_test.go @@ -0,0 +1,111 @@ +package cloudformation_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + cfnsdk "github.com/aws/aws-sdk-go-v2/service/cloudformation" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/cloudformation" + kinesisvideobackend "github.com/blackbirdworks/gopherstack/services/kinesisvideo" +) + +// newKinesisVideoTestClient wires a real aws-sdk-go-v2 CloudFormation client +// against a backend with KinesisVideo (among the other backends +// newMoreTypesServiceBackends already wires) set to a real in-memory service +// backend. +func newKinesisVideoTestClient(t *testing.T) (*cloudformation.ServiceBackends, *cfnsdk.Client) { + t.Helper() + + backends := newMoreTypesServiceBackends(t) + backends.KinesisVideo = kinesisvideobackend.NewHandler(kinesisvideobackend.NewInMemoryBackend()) + + creator := cloudformation.NewResourceCreator(backends) + backend := cloudformation.NewInMemoryBackendWithConfig("000000000000", "us-east-1", creator) + client := newTestClientForBackend(t, backend) + + return backends, client +} + +func TestCreateStack_KinesisVideoTypes(t *testing.T) { + t.Parallel() + + cases := []struct { + run func(t *testing.T) + name string + }{ + {testKinesisVideoStream, "stream"}, + {testKinesisVideoSignalingChannel, "signaling_channel"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.run(t) + }) + } +} + +func testKinesisVideoStream(t *testing.T) { + t.Helper() + + backends, client := newKinesisVideoTestClient(t) + + tmpl := `{ +"Resources": {"Stream": {"Type": "AWS::KinesisVideo::Stream", "Properties": { + "Name": "test-stream", + "DataRetentionInHours": 24, + "MediaType": "video/h264" +}}}, +"Outputs": { + "Ref": {"Value": {"Ref": "Stream"}}, + "Arn": {"Value": {"Fn::GetAtt": ["Stream", "Arn"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "kvs-stream-stack", tmpl) + assert.Equal(t, outputs["Ref"], outputs["Arn"]) + assert.Contains(t, outputs["Arn"], "test-stream") + + s, err := backends.KinesisVideo.Backend.DescribeStream("test-stream", "") + require.NoError(t, err) + assert.Equal(t, int32(24), s.DataRetentionInHours) + + _, err = client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("kvs-stream-stack")}) + require.NoError(t, err) + + _, err = backends.KinesisVideo.Backend.DescribeStream("test-stream", "") + require.Error(t, err) +} + +func testKinesisVideoSignalingChannel(t *testing.T) { + t.Helper() + + backends, client := newKinesisVideoTestClient(t) + + tmpl := `{ +"Resources": {"Channel": {"Type": "AWS::KinesisVideo::SignalingChannel", "Properties": { + "Name": "test-channel", + "Type": "SINGLE_MASTER" +}}}, +"Outputs": { + "Ref": {"Value": {"Ref": "Channel"}}, + "Arn": {"Value": {"Fn::GetAtt": ["Channel", "Arn"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "kvs-channel-stack", tmpl) + assert.Equal(t, outputs["Ref"], outputs["Arn"]) + assert.Contains(t, outputs["Arn"], "test-channel") + + _, err := backends.KinesisVideo.Backend.DescribeSignalingChannel("test-channel", "") + require.NoError(t, err) + + _, err = client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("kvs-channel-stack")}) + require.NoError(t, err) + + _, err = backends.KinesisVideo.Backend.DescribeSignalingChannel("test-channel", "") + require.Error(t, err) +} diff --git a/services/cloudformation/resources_newest_dispatch.go b/services/cloudformation/resources_newest_dispatch.go index 3d0f13728c..43133cfc67 100644 --- a/services/cloudformation/resources_newest_dispatch.go +++ b/services/cloudformation/resources_newest_dispatch.go @@ -53,8 +53,59 @@ func (rc *ResourceCreator) createNewestSupplementalResource( if id, ok, err := rc.createRedshiftMoreResource(logicalID, resourceType, props, params, physicalIDs); ok { return id, true, err } + if id, ok, err := rc.createKinesisVideoResource(logicalID, resourceType, props, params, physicalIDs); ok { + return id, true, err + } + if id, ok, err := rc.createECRPublicResource(logicalID, resourceType, props, params, physicalIDs); ok { + return id, true, err + } + if id, ok, err := rc.createKafkaConnectResource(logicalID, resourceType, props, params, physicalIDs); ok { + return id, true, err + } + + return rc.createEC2AdvancedNetworkingResource(ctx, logicalID, resourceType, props, params, physicalIDs) +} + +// createEC2AdvancedNetworkingResource chains the EC2 VPN/networking-extras/ +// transit-gateway/traffic-mirror/route-server/network-insights families +// ahead of IAM, ElastiCache, and API Gateway V2's own new-type families +// added in this sweep. +func (rc *ResourceCreator) createEC2AdvancedNetworkingResource( + ctx context.Context, + logicalID, resourceType string, + props map[string]any, + params, physicalIDs map[string]string, +) (string, bool, error) { + if id, ok, err := rc.createEC2VPNResource(logicalID, resourceType, props, params, physicalIDs); ok { + return id, true, err + } + if id, ok, err := rc.createEC2NetworkingExtrasResource(logicalID, resourceType, props, params, physicalIDs); ok { + return id, true, err + } + if id, ok, err := rc.createEC2TransitGatewayMoreResource( + logicalID, resourceType, props, params, physicalIDs, + ); ok { + return id, true, err + } + if id, ok, err := rc.createEC2TrafficMirrorResource(logicalID, resourceType, props, params, physicalIDs); ok { + return id, true, err + } + if id, ok, err := rc.createEC2RouteServerResource(logicalID, resourceType, props, params, physicalIDs); ok { + return id, true, err + } + if id, ok, err := rc.createEC2NetworkInsightsResource(logicalID, resourceType, props, params, physicalIDs); ok { + return id, true, err + } + if id, ok, err := rc.createIAMExtrasResource(logicalID, resourceType, props, params, physicalIDs); ok { + return id, true, err + } + if id, ok, err := rc.createElastiCacheUserResource( + ctx, logicalID, resourceType, props, params, physicalIDs, + ); ok { + return id, true, err + } - return "", false, nil + return rc.createAPIGatewayV2VpcLinkResource(logicalID, resourceType, props, params, physicalIDs) } // deleteNewestSupplementalResource mirrors createNewestSupplementalResource @@ -95,8 +146,49 @@ func (rc *ResourceCreator) deleteNewestSupplementalResource( if handled, err := rc.deleteRedshiftMoreResource(resourceType, physicalID); handled { return true, err } + if handled, err := rc.deleteKinesisVideoResource(resourceType, physicalID); handled { + return true, err + } + if handled, err := rc.deleteECRPublicResource(resourceType, physicalID); handled { + return true, err + } + if handled, err := rc.deleteKafkaConnectResource(resourceType, physicalID); handled { + return true, err + } + + return rc.deleteEC2AdvancedNetworkingResource(ctx, resourceType, physicalID) +} + +// deleteEC2AdvancedNetworkingResource mirrors createEC2AdvancedNetworkingResource. +func (rc *ResourceCreator) deleteEC2AdvancedNetworkingResource( + ctx context.Context, resourceType, physicalID string, +) (bool, error) { + if handled, err := rc.deleteEC2VPNResource(resourceType, physicalID); handled { + return true, err + } + if handled, err := rc.deleteEC2NetworkingExtrasResource(resourceType, physicalID); handled { + return true, err + } + if handled, err := rc.deleteEC2TransitGatewayMoreResource(resourceType, physicalID); handled { + return true, err + } + if handled, err := rc.deleteEC2TrafficMirrorResource(resourceType, physicalID); handled { + return true, err + } + if handled, err := rc.deleteEC2RouteServerResource(resourceType, physicalID); handled { + return true, err + } + if handled, err := rc.deleteEC2NetworkInsightsResource(resourceType, physicalID); handled { + return true, err + } + if handled, err := rc.deleteIAMExtrasResource(resourceType, physicalID); handled { + return true, err + } + if handled, err := rc.deleteElastiCacheUserResource(ctx, resourceType, physicalID); handled { + return true, err + } - return false, nil + return rc.deleteAPIGatewayV2VpcLinkResource(resourceType, physicalID) } // deleteNewestPropsBasedResource handles deletes needing sibling CFN diff --git a/services/cloudformation/resources_storage_test.go b/services/cloudformation/resources_storage_test.go index 34386b0253..d463c4c747 100644 --- a/services/cloudformation/resources_storage_test.go +++ b/services/cloudformation/resources_storage_test.go @@ -2,13 +2,38 @@ package cloudformation_test import ( "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/blackbirdworks/gopherstack/services/cloudformation" + kinesisbackend "github.com/blackbirdworks/gopherstack/services/kinesis" ) +// kinesisStreamSettleWait safely exceeds the kinesis package's internal +// CREATING/UPDATING/DELETING transition delay (streamTransitionDelay, +// 250ms), so a fake-clocked stream created via ResourceCreator.Create is +// ACTIVE by the time a later call needs it (e.g. ResourceCreator.Delete's +// DeleteStream, which real AWS -- and now this backend -- only accepts on +// an ACTIVE stream). +const kinesisStreamSettleWait = time.Second + +// withFakeClockedKinesis replaces backends.Kinesis with a freshly built +// handler whose backend's clock is the caller-controlled fakeNow, so tests +// that call ResourceCreator.Create then .Delete back-to-back (no real +// elapsed time, unlike a genuine CloudFormation stack lifecycle where a +// delete always follows a create by a real, separate API call) can move +// the stream's lazy CREATING->ACTIVE deadline forward without a real +// time.Sleep. Single-goroutine use only (ResourceCreator.Create/Delete are +// called directly, synchronously, never through a real HTTP server here). +func withFakeClockedKinesis(backends *cloudformation.ServiceBackends, fakeNow *time.Time) { + backends.Kinesis = kinesisbackend.NewHandler( + kinesisbackend.NewInMemoryBackendWithConfig("000000000000", "us-east-1"). + WithClock(func() time.Time { return *fakeNow }), + ) +} + func TestResourceCreator_S3Bucket(t *testing.T) { t.Parallel() @@ -339,6 +364,8 @@ func TestResourceCreator_KinesisStream(t *testing.T) { t.Parallel() backends := newExtendedServiceBackends() + fakeNow := time.Now() + withFakeClockedKinesis(backends, &fakeNow) rc := cloudformation.NewResourceCreator(backends) physID, err := rc.Create( @@ -356,6 +383,8 @@ func TestResourceCreator_KinesisStream(t *testing.T) { assert.Contains(t, physID, tt.wantContains) } + fakeNow = fakeNow.Add(kinesisStreamSettleWait) + err = rc.Delete(t.Context(), "AWS::Kinesis::Stream", physID, nil, nil) require.NoError(t, err) }) diff --git a/services/cloudformation/resources_type_aliases.go b/services/cloudformation/resources_type_aliases.go new file mode 100644 index 0000000000..c35d77e000 --- /dev/null +++ b/services/cloudformation/resources_type_aliases.go @@ -0,0 +1,14 @@ +package cloudformation + +// These are the real CloudFormation type names for two families whose +// legacy (undocumented) names are already wired in resources.go: +// AWS::ACM::Certificate has no such entry in the CFN resource +// specification -- the real name is AWS::CertificateManager::Certificate -- +// and AWS::OpenSearch::Domain's real name is +// AWS::OpenSearchService::Domain. Both aliases dispatch to the same +// creators/deleters as their legacy counterparts (see createMiscLegacyResource, +// deleteComputeStorageResource, and deleteAppNetworkResource in resources.go). +const ( + resTypeCertificateManagerCertificate = "AWS::CertificateManager::Certificate" + resTypeOpenSearchServiceDomain = "AWS::OpenSearchService::Domain" +) diff --git a/services/cloudformation/resources_type_aliases_test.go b/services/cloudformation/resources_type_aliases_test.go new file mode 100644 index 0000000000..8c20694bc3 --- /dev/null +++ b/services/cloudformation/resources_type_aliases_test.go @@ -0,0 +1,92 @@ +package cloudformation_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + cfnsdk "github.com/aws/aws-sdk-go-v2/service/cloudformation" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateStack_TypeAliases(t *testing.T) { + t.Parallel() + + cases := []struct { + run func(t *testing.T) + name string + }{ + {testCertificateManagerCertificateAlias, "certificatemanager_certificate"}, + {testOpenSearchServiceDomainAlias, "opensearchservice_domain"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.run(t) + }) + } +} + +func testCertificateManagerCertificateAlias(t *testing.T) { + t.Helper() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "Cert": { + "Type": "AWS::CertificateManager::Certificate", + "Properties": {"DomainName": "unit-cm.example.com", "ValidationMethod": "DNS"} + } +}, +"Outputs": { + "Ref": {"Value": {"Ref": "Cert"}}, + "CertificateArn": {"Value": {"Fn::GetAtt": ["Cert", "CertificateArn"]}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "cm-cert-stack", tmpl) + assert.Equal(t, outputs["Ref"], outputs["CertificateArn"]) + assert.Contains(t, outputs["Ref"], "arn:aws:acm:") + + cert, err := backends.ACM.Backend.DescribeCertificate(t.Context(), outputs["Ref"]) + require.NoError(t, err) + assert.NotNil(t, cert) + + _, err = client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("cm-cert-stack")}) + require.NoError(t, err) + + _, err = backends.ACM.Backend.DescribeCertificate(t.Context(), outputs["Ref"]) + require.Error(t, err) +} + +func testOpenSearchServiceDomainAlias(t *testing.T) { + t.Helper() + + backends, client := newMoreResourcesTestClient(t) + + tmpl := `{ +"Resources": { + "Domain": { + "Type": "AWS::OpenSearchService::Domain", + "Properties": {"DomainName": "unit-os-domain", "EngineVersion": "OpenSearch_2.11"} + } +}, +"Outputs": { + "Ref": {"Value": {"Ref": "Domain"}} +} +}` + + outputs := createStackAndGetOutputs(t, client, "os-domain-stack", tmpl) + assert.Contains(t, outputs["Ref"], "unit-os-domain") + + _, err := backends.OpenSearch.Backend.DescribeDomain("unit-os-domain") + require.NoError(t, err) + + _, err = client.DeleteStack(t.Context(), &cfnsdk.DeleteStackInput{StackName: aws.String("os-domain-stack")}) + require.NoError(t, err) + + _, err = backends.OpenSearch.Backend.DescribeDomain("unit-os-domain") + require.Error(t, err) +} diff --git a/services/cloudformation/stack_instances.go b/services/cloudformation/stack_instances.go index f4944ab108..dcc8fa8ff1 100644 --- a/services/cloudformation/stack_instances.go +++ b/services/cloudformation/stack_instances.go @@ -19,19 +19,33 @@ type instanceTarget struct { ouID string } -// resolveInstanceTargets merges explicit accounts with OU-expanded accounts. +// Account filter type values (DeploymentTargets.AccountFilterType, +// cloudformation@v1.76.1 types/enums.go AccountFilterType). "" is the wire +// default, equivalent to accountFilterUnion (docs.aws.amazon.com/ +// AWSCloudFormation/latest/APIReference/API_DeploymentTargets.html: "This is +// the default value if AccountFilterType is not provided"). +const ( + accountFilterIntersection = "INTERSECTION" + accountFilterDifference = "DIFFERENCE" + accountFilterUnion = "UNION" +) + +// resolveInstanceTargets merges explicit accounts with OU-expanded accounts +// according to filterType (DeploymentTargets.AccountFilterType), matching +// API_DeploymentTargets.html: +// - "" / UNION: OU accounts plus the explicit accounts. +// - NONE: the OU accounts only (explicit accounts are ignored). +// - INTERSECTION: only explicit accounts that also belong to the OUs. +// - DIFFERENCE: OU accounts minus the explicit accounts. +// // ouIDs requires the StackSet's PermissionModel to be SERVICE_MANAGED, // matching real AWS, which rejects OU-based deployment targets on // self-managed StackSets. Must be called with b.mu held. func (b *InMemoryBackend) resolveInstanceTargets( - ss *StackSet, accounts, ouIDs []string, + ss *StackSet, accounts, ouIDs []string, filterType string, ) ([]instanceTarget, error) { - targets := make([]instanceTarget, 0, len(accounts)+len(ouIDs)) - for _, a := range accounts { - targets = append(targets, instanceTarget{account: a}) - } if len(ouIDs) == 0 { - return targets, nil + return combineAccountFilter(filterType, accounts, nil), nil } if ss.PermissionModel != stackSetPermissionServiceManaged { return nil, ErrServiceManagedRequired @@ -44,6 +58,7 @@ func (b *InMemoryBackend) resolveInstanceTargets( } seen := make(map[string]bool) + ouAccounts := make([]instanceTarget, 0, len(ouIDs)) for _, ou := range ouIDs { accts, err := b.orgDirectory.ResolveAccountIDsUnderParent(ou) if err != nil { @@ -54,17 +69,86 @@ func (b *InMemoryBackend) resolveInstanceTargets( continue } seen[a] = true - targets = append(targets, instanceTarget{account: a, ouID: ou}) + ouAccounts = append(ouAccounts, instanceTarget{account: a, ouID: ou}) + } + } + + return combineAccountFilter(filterType, accounts, ouAccounts), nil +} + +// combineAccountFilter applies filterType's documented set operation between +// the explicit account list and the OU-resolved accounts. ouAccounts' order +// is preserved for NONE/INTERSECTION/DIFFERENCE; explicit-then-OU order is +// preserved for UNION, matching the pre-existing union behavior. +func combineAccountFilter(filterType string, explicit []string, ouAccounts []instanceTarget) []instanceTarget { + ouByAccount := make(map[string]string, len(ouAccounts)) + ouOrder := make([]string, 0, len(ouAccounts)) + + for _, t := range ouAccounts { + if _, ok := ouByAccount[t.account]; !ok { + ouOrder = append(ouOrder, t.account) + } + + ouByAccount[t.account] = t.ouID + } + + explicitSet := make(map[string]bool, len(explicit)) + for _, a := range explicit { + explicitSet[a] = true + } + + switch filterType { + case valueNone: + return filterOUAccounts(ouOrder, ouByAccount, func(string) bool { return true }) + case accountFilterIntersection: + return filterOUAccounts(ouOrder, ouByAccount, func(a string) bool { return explicitSet[a] }) + case accountFilterDifference: + return filterOUAccounts(ouOrder, ouByAccount, func(a string) bool { return !explicitSet[a] }) + default: // "" or UNION + out := make([]instanceTarget, 0, len(explicit)+len(ouOrder)) + seen := make(map[string]bool, len(explicit)+len(ouOrder)) + + for _, a := range explicit { + if seen[a] { + continue + } + + seen[a] = true + out = append(out, instanceTarget{account: a, ouID: ouByAccount[a]}) + } + + for _, a := range ouOrder { + if seen[a] { + continue + } + + seen[a] = true + out = append(out, instanceTarget{account: a, ouID: ouByAccount[a]}) + } + + return out + } +} + +// filterOUAccounts returns the ouOrder accounts (in order) for which keep +// reports true, each carrying its resolved OU ID. +func filterOUAccounts(ouOrder []string, ouByAccount map[string]string, keep func(string) bool) []instanceTarget { + out := make([]instanceTarget, 0, len(ouOrder)) + + for _, a := range ouOrder { + if keep(a) { + out = append(out, instanceTarget{account: a, ouID: ouByAccount[a]}) } } - return targets, nil + return out } func (b *InMemoryBackend) CreateStackInstances( ctx context.Context, stackSetName string, accounts, ouIDs, regions []string, + filterType string, ) (string, error) { b.mu.Lock("CreateStackInstances") defer b.mu.Unlock() @@ -73,7 +157,7 @@ func (b *InMemoryBackend) CreateStackInstances( return "", ErrStackSetNotFound } - targets, err := b.resolveInstanceTargets(ss, accounts, ouIDs) + targets, err := b.resolveInstanceTargets(ss, accounts, ouIDs, filterType) if err != nil { return "", err } @@ -250,6 +334,7 @@ func (b *InMemoryBackend) DeleteStackInstances( stackSetName string, accounts, ouIDs, regions []string, retainStacks bool, + filterType string, ) (string, error) { b.mu.Lock("DeleteStackInstances") defer b.mu.Unlock() @@ -257,18 +342,16 @@ func (b *InMemoryBackend) DeleteStackInstances( if !ok { return "", ErrStackSetNotFound } - if len(ouIDs) > 0 { - targets, err := b.resolveInstanceTargets(ss, nil, ouIDs) - if err != nil { - return "", err - } - for _, t := range targets { - accounts = append(accounts, t.account) - } + + targets, err := b.resolveInstanceTargets(ss, accounts, ouIDs, filterType) + if err != nil { + return "", err } - failed := b.deleteMatchingStackInstances(ctx, stackSetName, accounts, regions, retainStacks) + + targetAccounts := instanceTargetAccounts(targets) + failed := b.deleteMatchingStackInstances(ctx, stackSetName, targetAccounts, regions, retainStacks) opID := b.recordStackSetOperation(stackSetName, "DELETE") - b.recordStackInstanceDeleteResults(stackSetName, opID, accounts, regions, failed) + b.recordStackInstanceDeleteResults(stackSetName, opID, targetAccounts, regions, failed) return opID, nil } @@ -276,6 +359,7 @@ func (b *InMemoryBackend) DeleteStackInstances( func (b *InMemoryBackend) UpdateStackInstances( stackSetName string, accounts, ouIDs, regions []string, + filterType string, ) (string, error) { b.mu.Lock("UpdateStackInstances") defer b.mu.Unlock() @@ -283,23 +367,31 @@ func (b *InMemoryBackend) UpdateStackInstances( if !ok { return "", ErrStackSetNotFound } - if len(ouIDs) > 0 { - targets, err := b.resolveInstanceTargets(ss, nil, ouIDs) - if err != nil { - return "", err - } - for _, t := range targets { - accounts = append(accounts, t.account) - } + + targets, err := b.resolveInstanceTargets(ss, accounts, ouIDs, filterType) + if err != nil { + return "", err } + + targetAccounts := instanceTargetAccounts(targets) opID := b.recordStackSetOperation(stackSetName, "UPDATE") - if len(accounts) > 0 && len(regions) > 0 { - b.recordOpResults(stackSetName, opID, accounts, regions, "SUCCEEDED") + if len(targetAccounts) > 0 && len(regions) > 0 { + b.recordOpResults(stackSetName, opID, targetAccounts, regions, "SUCCEEDED") } return opID, nil } +// instanceTargetAccounts extracts the account ID from each resolved target. +func instanceTargetAccounts(targets []instanceTarget) []string { + accounts := make([]string, 0, len(targets)) + for _, t := range targets { + accounts = append(accounts, t.account) + } + + return accounts +} + // ListStackInstancesFilter holds ListStackInstancesInput's optional // narrowing members (cloudformation@v1.76.1 api_op_ListStackInstances.go): // StackInstanceAccount/StackInstanceRegion match exactly, and Filters diff --git a/services/cloudformation/stack_instances_account_filter_test.go b/services/cloudformation/stack_instances_account_filter_test.go index dada391baa..8346671af7 100644 --- a/services/cloudformation/stack_instances_account_filter_test.go +++ b/services/cloudformation/stack_instances_account_filter_test.go @@ -1,120 +1,288 @@ package cloudformation_test import ( - "net/url" "testing" + "github.com/aws/aws-sdk-go-v2/aws" + cfnsdk "github.com/aws/aws-sdk-go-v2/service/cloudformation" + "github.com/aws/aws-sdk-go-v2/service/cloudformation/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/cloudformation" + "github.com/blackbirdworks/gopherstack/services/organizations" ) -// TestStackInstances_AccountFilterType covers DeploymentTargets.AccountFilterType -// across CreateStackInstances, UpdateStackInstances, and DeleteStackInstances: only -// the unset/NONE case (union of Accounts and OrganizationalUnitIds) is implemented, -// so INTERSECTION/DIFFERENCE/UNION must be rejected explicitly rather than silently -// computed as NONE. -func TestStackInstances_AccountFilterType(t *testing.T) { +// accountFilterFixture wires a SERVICE_MANAGED StackSet to a real +// Organizations backend with three accounts under one OU plus a fourth +// account outside it, so DeploymentTargets.AccountFilterType's four modes +// (NONE/INTERSECTION/DIFFERENCE/UNION, +// docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_DeploymentTargets.html) +// each produce a distinguishable target set. +type accountFilterFixture struct { + client *cfnsdk.Client + stackSetName string + ouID string + outsideAccount string + ouAccounts []string +} + +func newAccountFilterFixture(t *testing.T, stackSetName string) accountFilterFixture { + t.Helper() + + orgBackend := organizations.NewInMemoryBackend("000000000000", "us-east-1") + _, root, err := orgBackend.CreateOrganization("ALL") + require.NoError(t, err) + + ou, err := orgBackend.CreateOrganizationalUnit(root.ID, "Workloads", nil) + require.NoError(t, err) + + ouAccounts := make([]string, 0, 3) + for _, email := range []string{"a1@example.com", "a2@example.com", "a3@example.com"} { + status, acctErr := orgBackend.CreateAccount(email, email, "OrganizationAccountAccessRole", "ALLOW", nil) + require.NoError(t, acctErr) + require.NoError(t, orgBackend.MoveAccount(status.AccountID, root.ID, ou.ID)) + ouAccounts = append(ouAccounts, status.AccountID) + } + + outsideStatus, err := orgBackend.CreateAccount( + "outside@example.com", "outside@example.com", "OrganizationAccountAccessRole", "ALLOW", nil, + ) + require.NoError(t, err) + + cfnBackend := cloudformation.NewInMemoryBackendWithConfig( + "000000000000", "us-east-1", cloudformation.NewResourceCreator(nil), + ) + cfnBackend.SetOrganizationsDirectory(orgBackend) + + client := newTestClientForBackend(t, cfnBackend) + ctx := t.Context() + + _, err = client.ActivateOrganizationsAccess(ctx, &cfnsdk.ActivateOrganizationsAccessInput{}) + require.NoError(t, err) + + _, err = client.CreateStackSet(ctx, &cfnsdk.CreateStackSetInput{ + StackSetName: aws.String(stackSetName), + TemplateBody: aws.String(simpleTemplate), + PermissionModel: types.PermissionModelsServiceManaged, + }) + require.NoError(t, err) + + return accountFilterFixture{ + client: client, + stackSetName: stackSetName, + ouID: ou.ID, + ouAccounts: ouAccounts, + outsideAccount: outsideStatus.AccountID, + } +} + +func (f accountFilterFixture) listInstanceAccounts(t *testing.T) []string { + t.Helper() + + out, err := f.client.ListStackInstances(t.Context(), &cfnsdk.ListStackInstancesInput{ + StackSetName: aws.String(f.stackSetName), + }) + require.NoError(t, err) + + accounts := make([]string, 0, len(out.Summaries)) + for _, s := range out.Summaries { + accounts = append(accounts, aws.ToString(s.Account)) + } + + return accounts +} + +// TestStackInstances_AccountFilterType_Create drives CreateStackInstances +// through the real aws-sdk-go-v2 client with each AccountFilterType value +// CreateStackInstances supports and asserts ListStackInstances returns +// exactly the expected accounts. +func TestStackInstances_AccountFilterType_Create(t *testing.T) { t.Parallel() tests := []struct { name string - filterType string - wantReject bool + filterType types.AccountFilterType + wantIdx []int // indices into ouAccounts expected present }{ - {name: "unset", filterType: "", wantReject: false}, - {name: "none", filterType: "NONE", wantReject: false}, - {name: "intersection", filterType: "INTERSECTION", wantReject: true}, - {name: "difference", filterType: "DIFFERENCE", wantReject: true}, - {name: "union", filterType: "UNION", wantReject: true}, + {name: "none_ignores_explicit_accounts", filterType: types.AccountFilterTypeNone, wantIdx: []int{0, 1, 2}}, + {name: "intersection", filterType: types.AccountFilterTypeIntersection, wantIdx: []int{0, 1}}, + {name: "difference", filterType: types.AccountFilterTypeDifference, wantIdx: []int{2}}, } for _, tt := range tests { - t.Run("create_"+tt.name, func(t *testing.T) { + t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newHandler() - postForm(t, h, url.Values{ - "Action": {"CreateStackSet"}, - "StackSetName": {"filter-ss-" + tt.name}, - "TemplateBody": {simpleTemplate}, - }.Encode()) - - form := url.Values{ - "Action": {"CreateStackInstances"}, - "StackSetName": {"filter-ss-" + tt.name}, - "Accounts.member.1": {"111111111111"}, - "Regions.member.1": {"us-east-1"}, - } - if tt.filterType != "" { - form.Set("DeploymentTargets.AccountFilterType", tt.filterType) - } - rec := postForm(t, h, form.Encode()) + f := newAccountFilterFixture(t, "filter-create-"+tt.name) + ctx := t.Context() - if tt.wantReject { - assert.NotEqual(t, 200, rec.Code, "body: %s", rec.Body.String()) - assert.Contains(t, rec.Body.String(), "AccountFilterType") - } else { - require.Equal(t, 200, rec.Code, "body: %s", rec.Body.String()) - } - }) + // Explicit Accounts overlaps ouAccounts[0:2] and adds an outside + // account, so each filter type produces a distinguishable set. + explicit := []string{f.ouAccounts[0], f.ouAccounts[1], f.outsideAccount} - t.Run("update_"+tt.name, func(t *testing.T) { - t.Parallel() + _, err := f.client.CreateStackInstances(ctx, &cfnsdk.CreateStackInstancesInput{ + StackSetName: aws.String(f.stackSetName), + Regions: []string{"us-east-1"}, + DeploymentTargets: &types.DeploymentTargets{ + AccountFilterType: tt.filterType, + Accounts: explicit, + OrganizationalUnitIds: []string{f.ouID}, + }, + }) + require.NoError(t, err) - h := newHandler() - postForm(t, h, url.Values{ - "Action": {"CreateStackSet"}, - "StackSetName": {"filter-upd-ss-" + tt.name}, - "TemplateBody": {simpleTemplate}, - }.Encode()) - - form := url.Values{ - "Action": {"UpdateStackInstances"}, - "StackSetName": {"filter-upd-ss-" + tt.name}, - "Accounts.member.1": {"111111111111"}, - "Regions.member.1": {"us-east-1"}, - } - if tt.filterType != "" { - form.Set("DeploymentTargets.AccountFilterType", tt.filterType) + want := make([]string, 0, len(tt.wantIdx)) + for _, idx := range tt.wantIdx { + want = append(want, f.ouAccounts[idx]) } - rec := postForm(t, h, form.Encode()) - if tt.wantReject { - assert.NotEqual(t, 200, rec.Code, "body: %s", rec.Body.String()) - assert.Contains(t, rec.Body.String(), "AccountFilterType") - } else { - require.Equal(t, 200, rec.Code, "body: %s", rec.Body.String()) - } + assert.ElementsMatch(t, want, f.listInstanceAccounts(t)) }) + } +} - t.Run("delete_"+tt.name, func(t *testing.T) { - t.Parallel() +// TestStackInstances_AccountFilterType_UnionRejectedAtCreate verifies the +// documented restriction that UNION is not supported for CreateStackInstances +// operations. +func TestStackInstances_AccountFilterType_UnionRejectedAtCreate(t *testing.T) { + t.Parallel() - h := newHandler() - postForm(t, h, url.Values{ - "Action": {"CreateStackSet"}, - "StackSetName": {"filter-del-ss-" + tt.name}, - "TemplateBody": {simpleTemplate}, - }.Encode()) - - form := url.Values{ - "Action": {"DeleteStackInstances"}, - "StackSetName": {"filter-del-ss-" + tt.name}, - "Accounts.member.1": {"111111111111"}, - "Regions.member.1": {"us-east-1"}, - "RetainStacks": {"false"}, - } - if tt.filterType != "" { - form.Set("DeploymentTargets.AccountFilterType", tt.filterType) - } - rec := postForm(t, h, form.Encode()) + f := newAccountFilterFixture(t, "filter-create-union-rejected") - if tt.wantReject { - assert.NotEqual(t, 200, rec.Code, "body: %s", rec.Body.String()) - assert.Contains(t, rec.Body.String(), "AccountFilterType") - } else { - require.Equal(t, 200, rec.Code, "body: %s", rec.Body.String()) - } - }) + _, err := f.client.CreateStackInstances(t.Context(), &cfnsdk.CreateStackInstancesInput{ + StackSetName: aws.String(f.stackSetName), + Regions: []string{"us-east-1"}, + DeploymentTargets: &types.DeploymentTargets{ + AccountFilterType: types.AccountFilterTypeUnion, + Accounts: []string{f.outsideAccount}, + OrganizationalUnitIds: []string{f.ouID}, + }, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "UNION") +} + +// TestStackInstances_AccountFilterType_RequiredWhenBothGivenAtCreate verifies +// the documented rule that create operations specifying both +// OrganizationalUnitIds and Accounts must also specify AccountFilterType. +func TestStackInstances_AccountFilterType_RequiredWhenBothGivenAtCreate(t *testing.T) { + t.Parallel() + + f := newAccountFilterFixture(t, "filter-create-required") + + _, err := f.client.CreateStackInstances(t.Context(), &cfnsdk.CreateStackInstancesInput{ + StackSetName: aws.String(f.stackSetName), + Regions: []string{"us-east-1"}, + DeploymentTargets: &types.DeploymentTargets{ + Accounts: []string{f.outsideAccount}, + OrganizationalUnitIds: []string{f.ouID}, + }, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "AccountFilterType") +} + +// TestStackInstances_AccountFilterType_InvalidValue verifies an unsupported +// AccountFilterType enum value is rejected with ValidationError. +func TestStackInstances_AccountFilterType_InvalidValue(t *testing.T) { + t.Parallel() + + f := newAccountFilterFixture(t, "filter-create-invalid") + + _, err := f.client.CreateStackInstances(t.Context(), &cfnsdk.CreateStackInstancesInput{ + StackSetName: aws.String(f.stackSetName), + Regions: []string{"us-east-1"}, + DeploymentTargets: &types.DeploymentTargets{ + AccountFilterType: "BOGUS", + OrganizationalUnitIds: []string{f.ouID}, + }, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "AccountFilterType") +} + +// TestStackInstances_AccountFilterType_UpdateUnion verifies UNION (allowed on +// UpdateStackInstances, unlike Create): the touched accounts are the OU +// accounts plus the listed ones. UpdateStackInstances updates existing +// instances rather than provisioning new ones (gopherstack, like real AWS, +// requires an instance to already exist for an account/region pair to be +// updated), so the resolved target set is asserted via +// ListStackSetOperationResults rather than ListStackInstances. +func TestStackInstances_AccountFilterType_UpdateUnion(t *testing.T) { + t.Parallel() + + f := newAccountFilterFixture(t, "filter-update-union") + ctx := t.Context() + + _, err := f.client.CreateStackInstances(ctx, &cfnsdk.CreateStackInstancesInput{ + StackSetName: aws.String(f.stackSetName), + Regions: []string{"us-east-1"}, + DeploymentTargets: &types.DeploymentTargets{ + AccountFilterType: types.AccountFilterTypeNone, + OrganizationalUnitIds: []string{f.ouID}, + }, + }) + require.NoError(t, err) + require.ElementsMatch(t, f.ouAccounts, f.listInstanceAccounts(t)) + + updOut, err := f.client.UpdateStackInstances(ctx, &cfnsdk.UpdateStackInstancesInput{ + StackSetName: aws.String(f.stackSetName), + Regions: []string{"us-east-1"}, + DeploymentTargets: &types.DeploymentTargets{ + AccountFilterType: types.AccountFilterTypeUnion, + Accounts: []string{f.outsideAccount}, + OrganizationalUnitIds: []string{f.ouID}, + }, + }) + require.NoError(t, err) + + resultsOut, err := f.client.ListStackSetOperationResults(ctx, &cfnsdk.ListStackSetOperationResultsInput{ + StackSetName: aws.String(f.stackSetName), + OperationId: updOut.OperationId, + }) + require.NoError(t, err) + + touched := make([]string, 0, len(resultsOut.Summaries)) + for _, s := range resultsOut.Summaries { + touched = append(touched, aws.ToString(s.Account)) } + + want := append(append([]string{}, f.ouAccounts...), f.outsideAccount) + assert.ElementsMatch(t, want, touched) +} + +// TestStackInstances_AccountFilterType_DeleteDifference verifies DIFFERENCE +// on DeleteStackInstances: the accounts targeted for deletion are the OU +// accounts minus the listed ones, so the listed account's instance survives. +func TestStackInstances_AccountFilterType_DeleteDifference(t *testing.T) { + t.Parallel() + + f := newAccountFilterFixture(t, "filter-delete-difference") + ctx := t.Context() + + _, err := f.client.CreateStackInstances(ctx, &cfnsdk.CreateStackInstancesInput{ + StackSetName: aws.String(f.stackSetName), + Regions: []string{"us-east-1"}, + DeploymentTargets: &types.DeploymentTargets{ + AccountFilterType: types.AccountFilterTypeNone, + OrganizationalUnitIds: []string{f.ouID}, + }, + }) + require.NoError(t, err) + require.ElementsMatch(t, f.ouAccounts, f.listInstanceAccounts(t)) + + _, err = f.client.DeleteStackInstances(ctx, &cfnsdk.DeleteStackInstancesInput{ + StackSetName: aws.String(f.stackSetName), + Regions: []string{"us-east-1"}, + RetainStacks: aws.Bool(false), + DeploymentTargets: &types.DeploymentTargets{ + AccountFilterType: types.AccountFilterTypeDifference, + Accounts: []string{f.ouAccounts[0]}, + OrganizationalUnitIds: []string{f.ouID}, + }, + }) + require.NoError(t, err) + + assert.ElementsMatch(t, []string{f.ouAccounts[0]}, f.listInstanceAccounts(t)) } diff --git a/services/cloudformation/stack_instances_test.go b/services/cloudformation/stack_instances_test.go index 0e030328cd..5b15e044ab 100644 --- a/services/cloudformation/stack_instances_test.go +++ b/services/cloudformation/stack_instances_test.go @@ -24,6 +24,7 @@ func TestCreateStackInstances_ProvisionsChildStacks(t *testing.T) { []string{"111111111111", "222222222222"}, nil, []string{"us-east-1"}, + "", ) require.NoError(t, err) @@ -52,7 +53,7 @@ func TestDeleteStackInstances_TearsDownChildStacks(t *testing.T) { require.NoError(t, err) _, err = b.CreateStackInstances( - t.Context(), "teardown-ss", []string{"111111111111"}, nil, []string{"us-east-1"}, + t.Context(), "teardown-ss", []string{"111111111111"}, nil, []string{"us-east-1"}, "", ) require.NoError(t, err) @@ -62,7 +63,7 @@ func TestDeleteStackInstances_TearsDownChildStacks(t *testing.T) { childID := instances.Data[0].StackID _, err = b.DeleteStackInstances( - t.Context(), "teardown-ss", []string{"111111111111"}, nil, []string{"us-east-1"}, false, + t.Context(), "teardown-ss", []string{"111111111111"}, nil, []string{"us-east-1"}, false, "", ) require.NoError(t, err) @@ -85,7 +86,7 @@ func TestDeleteStackInstances_RetainStacksKeepsChildStack(t *testing.T) { require.NoError(t, err) _, err = b.CreateStackInstances( - t.Context(), "retain-ss", []string{"111111111111"}, nil, []string{"us-east-1"}, + t.Context(), "retain-ss", []string{"111111111111"}, nil, []string{"us-east-1"}, "", ) require.NoError(t, err) @@ -95,7 +96,7 @@ func TestDeleteStackInstances_RetainStacksKeepsChildStack(t *testing.T) { childID := instances.Data[0].StackID _, err = b.DeleteStackInstances( - t.Context(), "retain-ss", []string{"111111111111"}, nil, []string{"us-east-1"}, true, + t.Context(), "retain-ss", []string{"111111111111"}, nil, []string{"us-east-1"}, true, "", ) require.NoError(t, err) diff --git a/services/cloudformation/stack_lifecycle_test.go b/services/cloudformation/stack_lifecycle_test.go index a572cd3148..4f1fd37577 100644 --- a/services/cloudformation/stack_lifecycle_test.go +++ b/services/cloudformation/stack_lifecycle_test.go @@ -881,7 +881,7 @@ func TestStackSet_CreateUpdateDeleteWithInstances(t *testing.T) { // Create instances. accounts := []string{"111111111111", "222222222222"} regions := []string{"us-east-1", "us-west-2"} - _, err = b.CreateStackInstances(t.Context(), "my-ss", accounts, nil, regions) + _, err = b.CreateStackInstances(t.Context(), "my-ss", accounts, nil, regions, "") require.NoError(t, err) instances, err := b.ListStackInstances("my-ss", 0, "", cloudformation.ListStackInstancesFilter{}) @@ -899,7 +899,7 @@ func TestStackSet_CreateUpdateDeleteWithInstances(t *testing.T) { assert.Equal(t, "ACTIVE", updated.Status) // Delete instances. - _, err = b.DeleteStackInstances(t.Context(), "my-ss", accounts, nil, regions, false) + _, err = b.DeleteStackInstances(t.Context(), "my-ss", accounts, nil, regions, false, "") require.NoError(t, err) remaining, err := b.ListStackInstances("my-ss", 0, "", cloudformation.ListStackInstancesFilter{}) diff --git a/services/cloudformation/stackset_instance_feature_test.go b/services/cloudformation/stackset_instance_feature_test.go index bcb5e81008..c98bbba418 100644 --- a/services/cloudformation/stackset_instance_feature_test.go +++ b/services/cloudformation/stackset_instance_feature_test.go @@ -49,7 +49,7 @@ func TestStackInstance_StackIDAssigned(t *testing.T) { _, err := b.CreateStackSet("inst-test-ss", "test", simpleTemplate, cloudformation.StackSetOptions{}) require.NoError(t, err) - _, err = b.CreateStackInstances(t.Context(), "inst-test-ss", tc.accounts, nil, tc.regions) + _, err = b.CreateStackInstances(t.Context(), "inst-test-ss", tc.accounts, nil, tc.regions, "") require.NoError(t, err) instances, err := b.ListStackInstances("inst-test-ss", 0, "", cloudformation.ListStackInstancesFilter{}) @@ -76,11 +76,11 @@ func TestStackInstance_NoDuplicates(t *testing.T) { _, err := b.CreateStackSet("dedup-ss", "test", simpleTemplate, cloudformation.StackSetOptions{}) require.NoError(t, err) - _, err = b.CreateStackInstances(t.Context(), "dedup-ss", []string{"111111111111"}, nil, []string{"us-east-1"}) + _, err = b.CreateStackInstances(t.Context(), "dedup-ss", []string{"111111111111"}, nil, []string{"us-east-1"}, "") require.NoError(t, err) // Creating the same instance again should not duplicate it. - _, err = b.CreateStackInstances(t.Context(), "dedup-ss", []string{"111111111111"}, nil, []string{"us-east-1"}) + _, err = b.CreateStackInstances(t.Context(), "dedup-ss", []string{"111111111111"}, nil, []string{"us-east-1"}, "") require.NoError(t, err) instances, err := b.ListStackInstances("dedup-ss", 0, "", cloudformation.ListStackInstancesFilter{}) @@ -123,7 +123,7 @@ func TestStackSetOperationResults(t *testing.T) { _, err := b.CreateStackSet("op-results-ss", "test", simpleTemplate, cloudformation.StackSetOptions{}) require.NoError(t, err) - _, err = b.CreateStackInstances(t.Context(), "op-results-ss", tc.accounts, nil, tc.regions) + _, err = b.CreateStackInstances(t.Context(), "op-results-ss", tc.accounts, nil, tc.regions, "") require.NoError(t, err) // Get the operation ID from ListStackSetOperations. @@ -200,7 +200,7 @@ func TestDescribeStackInstance_Fields(t *testing.T) { b := newBackend() _, err := b.CreateStackSet("field-ss", "test", simpleTemplate, cloudformation.StackSetOptions{}) require.NoError(t, err) - _, err = b.CreateStackInstances(t.Context(), "field-ss", []string{"123456789012"}, nil, []string{"us-east-1"}) + _, err = b.CreateStackInstances(t.Context(), "field-ss", []string{"123456789012"}, nil, []string{"us-east-1"}, "") require.NoError(t, err) inst, err := b.DescribeStackInstance("field-ss", "123456789012", "us-east-1") @@ -225,9 +225,16 @@ func TestListStackSetOperations_SortedByCreationTime(t *testing.T) { require.NoError(t, err) // Create multiple operations by calling CreateStackInstances multiple times. - _, err = b.CreateStackInstances(t.Context(), "sort-ops-ss", []string{"111111111111"}, nil, []string{"us-east-1"}) + _, err = b.CreateStackInstances( + t.Context(), + "sort-ops-ss", + []string{"111111111111"}, + nil, + []string{"us-east-1"}, + "", + ) require.NoError(t, err) - _, err = b.UpdateStackInstances("sort-ops-ss", []string{"111111111111"}, nil, []string{"us-east-1"}) + _, err = b.UpdateStackInstances("sort-ops-ss", []string{"111111111111"}, nil, []string{"us-east-1"}, "") require.NoError(t, err) _, _, err = b.UpdateStackSet("sort-ops-ss", "", simpleTemplate, cloudformation.StackSetOptions{}) require.NoError(t, err) @@ -283,10 +290,18 @@ func TestDeleteStackInstances_Selective(t *testing.T) { _, err := b.CreateStackSet("del-sel-ss", "test", simpleTemplate, cloudformation.StackSetOptions{}) require.NoError(t, err) - _, err = b.CreateStackInstances(t.Context(), "del-sel-ss", tc.createAccounts, nil, tc.createRegions) + _, err = b.CreateStackInstances(t.Context(), "del-sel-ss", tc.createAccounts, nil, tc.createRegions, "") require.NoError(t, err) - _, err = b.DeleteStackInstances(t.Context(), "del-sel-ss", tc.deleteAccounts, nil, tc.deleteRegions, false) + _, err = b.DeleteStackInstances( + t.Context(), + "del-sel-ss", + tc.deleteAccounts, + nil, + tc.deleteRegions, + false, + "", + ) require.NoError(t, err) remaining, err := b.ListStackInstances("del-sel-ss", 0, "", cloudformation.ListStackInstancesFilter{}) diff --git a/services/cloudformation/store.go b/services/cloudformation/store.go index 8efbde833a..3e2069e233 100644 --- a/services/cloudformation/store.go +++ b/services/cloudformation/store.go @@ -73,14 +73,16 @@ type StorageBackend interface { ctx context.Context, stackSetName string, accounts, ouIDs, regions []string, + filterType string, ) (string, error) DeleteStackInstances( ctx context.Context, stackSetName string, accounts, ouIDs, regions []string, retainStacks bool, + filterType string, ) (string, error) - UpdateStackInstances(stackSetName string, accounts, ouIDs, regions []string) (string, error) + UpdateStackInstances(stackSetName string, accounts, ouIDs, regions []string, filterType string) (string, error) ListStackInstances( stackSetName string, maxResults int, nextToken string, filter ListStackInstancesFilter, ) (page.Page[StackInstance], error) diff --git a/services/cloudformation/store_direct_test.go b/services/cloudformation/store_direct_test.go index e1f13c3599..c7596500da 100644 --- a/services/cloudformation/store_direct_test.go +++ b/services/cloudformation/store_direct_test.go @@ -52,6 +52,7 @@ func TestStackSetDrift_UpdatesInstanceDriftStatus(t *testing.T) { []string{"111111111111"}, nil, []string{"us-east-1"}, + "", ) require.NoError(t, err) @@ -111,6 +112,7 @@ func TestStackSetOperationList(t *testing.T) { []string{"111"}, nil, []string{"us-east-1"}, + "", ) require.NoError(t, err) diff --git a/services/cloudformation/template.go b/services/cloudformation/template.go index 0721dc2baf..ad3f5f6f5d 100644 --- a/services/cloudformation/template.go +++ b/services/cloudformation/template.go @@ -1543,7 +1543,13 @@ func resolveGetAtt(logicalID, attrName string, ctx resolveCtx) string { resTypeAmplifyApp, resTypeAmplifyBranch, resTypeBatchSchedulingPolicy, resTypeBatchServiceEnvironment, resTypeEFSAccessPoint, - resTypeRedshiftClusterSubnetGroup: + resTypeRedshiftClusterSubnetGroup, + resTypeEC2PrefixList, resTypeEC2TGWPeeringAttachment, resTypeEC2TGWMulticastDomain, + resTypeEC2RouteServer, resTypeEC2RouteServerEndpoint, resTypeEC2RouteServerPeer, + resTypeEC2NetworkInsightsPath, resTypeElastiCacheUser, + resTypeKinesisVideoStream, resTypeKinesisVideoSignalingChannel, + resTypeECRPublicRepository, + resTypeKafkaConnectConnector, resTypeKafkaConnectCustomPlugin, resTypeKafkaConnectWorkerConfiguration: return v } } diff --git a/services/cloudtrail/events.go b/services/cloudtrail/events.go index f12a1b4152..d3d0d8568c 100644 --- a/services/cloudtrail/events.go +++ b/services/cloudtrail/events.go @@ -81,8 +81,11 @@ func (b *InMemoryBackend) trimEventsLocked() { b.events = kept + // In-place shift instead of reallocating: at steady state this runs every + // sweep, and the old alloc dominated allocator traffic (~25% of bytes). if excess := len(b.events) - maxStoredEvents; excess > 0 { - b.events = append([]Event(nil), b.events[excess:]...) + n := copy(b.events, b.events[excess:]) + b.events = b.events[:n] } } diff --git a/services/cloudtrail/events_bench_whitebox_test.go b/services/cloudtrail/events_bench_whitebox_test.go new file mode 100644 index 0000000000..bd5c4e6642 --- /dev/null +++ b/services/cloudtrail/events_bench_whitebox_test.go @@ -0,0 +1,33 @@ +package cloudtrail + +import ( + "testing" + "time" +) + +// BenchmarkTrimEventsLocked_AtCapacity models steady-state load: the store +// sits at maxStoredEvents and every sweep must shed the newest excess. +func BenchmarkTrimEventsLocked_AtCapacity(b *testing.B) { + be := NewInMemoryBackend("123456789012", "us-east-1") + + now := time.Now().UTC() + + be.events = make([]Event, maxStoredEvents, maxStoredEvents+trimEventsSweepEvery) + for i := range be.events { + be.events[i] = Event{EventTime: now, EventName: "PutObject"} + } + + extra := make([]Event, trimEventsSweepEvery) + for i := range extra { + extra[i] = Event{EventTime: now, EventName: "PutObject"} + } + + b.ReportAllocs() + + for b.Loop() { + be.mu.Lock("bench") + be.events = append(be.events, extra...) + be.trimEventsLocked() + be.mu.Unlock() + } +} diff --git a/services/cloudtrail/events_capacity_whitebox_test.go b/services/cloudtrail/events_capacity_whitebox_test.go new file mode 100644 index 0000000000..8500163948 --- /dev/null +++ b/services/cloudtrail/events_capacity_whitebox_test.go @@ -0,0 +1,34 @@ +package cloudtrail + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestTrimEventsLocked_CapsAtMaxStoredEvents pins trimEventsLocked's +// over-capacity branch: oldest events are dropped, newest are kept in order. +func TestTrimEventsLocked_CapsAtMaxStoredEvents(t *testing.T) { + t.Parallel() + + be := NewInMemoryBackend("000000000000", "us-east-1") + + const extra = 250 + + now := time.Now().UTC() + total := maxStoredEvents + extra + + be.events = make([]Event, total) + for i := range be.events { + be.events[i] = Event{EventID: string(rune(i)), EventTime: now, EventName: "FillerEvent"} + } + + be.trimEventsLocked() + + assert.Len(t, be.events, maxStoredEvents) + assert.Equal(t, string(rune(extra)), be.events[0].EventID, + "oldest surviving event should be the first non-dropped one") + assert.Equal(t, string(rune(total-1)), be.events[len(be.events)-1].EventID, + "newest event must survive") +} diff --git a/services/cloudwatchlogs/anomaly_detectors_test.go b/services/cloudwatchlogs/anomaly_detectors_test.go index bfa85a0543..c83cbe091a 100644 --- a/services/cloudwatchlogs/anomaly_detectors_test.go +++ b/services/cloudwatchlogs/anomaly_detectors_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -516,26 +517,28 @@ func TestCloudWatchLogsBackend_CreateLogAnomalyDetector_VisibilityTimeValidation func TestCloudWatchLogsBackend_UpdateLogAnomalyDetector_SetsLastModified(t *testing.T) { t.Parallel() - b := cloudwatchlogs.NewInMemoryBackend() - _, err := b.CreateLogGroup(context.Background(), "g", "", "") - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := cloudwatchlogs.NewInMemoryBackend() + _, err := b.CreateLogGroup(context.Background(), "g", "", "") + require.NoError(t, err) - groupARN := "arn:aws:logs:us-east-1:123456789012:log-group:g" - arn, err := b.CreateLogAnomalyDetector([]string{groupARN}, "d", "", "", "", 0) - require.NoError(t, err) + groupARN := "arn:aws:logs:us-east-1:123456789012:log-group:g" + arn, err := b.CreateLogAnomalyDetector([]string{groupARN}, "d", "", "", "", 0) + require.NoError(t, err) - before, err := b.GetLogAnomalyDetector(arn) - require.NoError(t, err) - createdAt := before.LastModifiedTimeStamp + before, err := b.GetLogAnomalyDetector(arn) + require.NoError(t, err) + createdAt := before.LastModifiedTimeStamp - time.Sleep(2 * time.Millisecond) + time.Sleep(2 * time.Millisecond) - err = b.UpdateLogAnomalyDetector(arn, "FIVE_MIN", 30, true) - require.NoError(t, err) + err = b.UpdateLogAnomalyDetector(arn, "FIVE_MIN", 30, true) + require.NoError(t, err) - after, err := b.GetLogAnomalyDetector(arn) - require.NoError(t, err) - assert.GreaterOrEqual(t, after.LastModifiedTimeStamp, createdAt) + after, err := b.GetLogAnomalyDetector(arn) + require.NoError(t, err) + assert.GreaterOrEqual(t, after.LastModifiedTimeStamp, createdAt) + }) } func TestCloudWatchLogsBackend_UpdateLogAnomalyDetector_VisibilityTimeValidation(t *testing.T) { diff --git a/services/cloudwatchlogs/queries_test.go b/services/cloudwatchlogs/queries_test.go index 415fa2cdc0..1f22c1853f 100644 --- a/services/cloudwatchlogs/queries_test.go +++ b/services/cloudwatchlogs/queries_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -400,7 +401,6 @@ func TestCloudWatchLogsBackend_QueryEviction_TTL(t *testing.T) { 0, 0, ) - // Sleep well beyond the TTL to avoid any scheduling jitter. time.Sleep(20 * time.Millisecond) // This new query triggers eviction; old-1 and old-2 should be removed. _, _ = b.StartQuery( @@ -442,14 +442,16 @@ func TestCloudWatchLogsBackend_QueryEviction_TTL(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := cloudwatchlogs.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - if tt.setup != nil { - tt.setup(t, b) - } + synctest.Test(t, func(t *testing.T) { + b := cloudwatchlogs.NewInMemoryBackendWithConfig("123456789012", "us-east-1") + if tt.setup != nil { + tt.setup(t, b) + } - queries, _, err := b.DescribeQueries("", "", "", "", 0) - require.NoError(t, err) - assert.Len(t, queries, tt.wantLen) + queries, _, err := b.DescribeQueries("", "", "", "", 0) + require.NoError(t, err) + assert.Len(t, queries, tt.wantLen) + }) }) } } diff --git a/services/codepipeline/webhooks.go b/services/codepipeline/webhooks.go index ca837fd2ea..4ef2d389f6 100644 --- a/services/codepipeline/webhooks.go +++ b/services/codepipeline/webhooks.go @@ -3,6 +3,7 @@ package codepipeline import ( "context" "fmt" + "maps" "sort" "github.com/google/uuid" @@ -55,6 +56,7 @@ func (b *InMemoryBackend) ListWebhooks(ctx context.Context) []*Webhook { result := make([]*Webhook, 0, len(entries)) for _, wh := range entries { cp := *wh + cp.Tags = maps.Clone(wh.Tags) result = append(result, &cp) } @@ -86,6 +88,7 @@ func (b *InMemoryBackend) PutWebhook(ctx context.Context, wh *Webhook) (*Webhook b.webhooks.Put(&cp) result := cp + result.Tags = maps.Clone(cp.Tags) return &result, nil } diff --git a/services/cognitoidp/auth_tokens.go b/services/cognitoidp/auth_tokens.go index 9509cfc2bd..94aec18092 100644 --- a/services/cognitoidp/auth_tokens.go +++ b/services/cognitoidp/auth_tokens.go @@ -473,6 +473,30 @@ func (b *InMemoryBackend) storeRefreshTokenLocked(token string, entry *refreshTo b.refreshTokensByUser[userKey] = make(map[string]struct{}) } b.refreshTokensByUser[userKey][token] = struct{}{} + + b.maybeEvictExpiredRefreshTokensLocked() +} + +// maybeEvictExpiredRefreshTokensLocked drops expired, never-refreshed tokens +// once the table is large (same pattern as sts). Caller holds b.mu. +func (b *InMemoryBackend) maybeEvictExpiredRefreshTokensLocked() { + if len(b.refreshTokens) < refreshTokenEvictThreshold { + return + } + + b.refreshTokenInsertsSinceSweep++ + if b.refreshTokenInsertsSinceSweep < refreshTokenEvictSweepInterval { + return + } + + b.refreshTokenInsertsSinceSweep = 0 + + now := time.Now().UTC() + for token, entry := range b.refreshTokens { + if !entry.ExpiresAt.IsZero() && !entry.ExpiresAt.After(now) { + b.deleteRefreshTokenLocked(token) + } + } } // tokenExpiryFor returns the configured token expiry duration for the given token type diff --git a/services/cognitoidp/persistence.go b/services/cognitoidp/persistence.go index 80a4b92f43..ec5643a6d9 100644 --- a/services/cognitoidp/persistence.go +++ b/services/cognitoidp/persistence.go @@ -398,6 +398,7 @@ func (b *InMemoryBackend) resetForIncompatibleSnapshotLocked() { b.tokenRevokedBeforeSeq = make(map[string]int64) b.tokenRevokedBefore = make(map[string]time.Time) b.tokenSeq = 0 + b.refreshTokenInsertsSinceSweep = 0 b.resourceTags = make(map[string]map[string]string) b.riskConfigurations = make(map[string]*RiskConfiguration) b.logDeliveryConfigs = make(map[string]*LogDeliveryConfig) diff --git a/services/cognitoidp/store.go b/services/cognitoidp/store.go index 097ea28a1e..eff8c0d6fe 100644 --- a/services/cognitoidp/store.go +++ b/services/cognitoidp/store.go @@ -45,6 +45,12 @@ const ( // defaultRefreshTokenTTL is the lifetime for refresh tokens. defaultRefreshTokenTTL = 30 * 24 * time.Hour + + // refreshTokenEvictThreshold: table size that arms the expired-token sweep. + refreshTokenEvictThreshold = 256 + + // refreshTokenEvictSweepInterval: inserts between sweeps once armed. + refreshTokenEvictSweepInterval = 64 ) // InMemoryBackend is the in-memory store for Cognito IDP resources. @@ -55,53 +61,54 @@ const ( // identity for a store.Table key; store_setup.go's registerAllTables doc // comment lists each one and why. type InMemoryBackend struct { - lambdaInvoker LambdaTriggerInvoker - domains *store.Table[UserPoolDomain] - resourceServers *store.Table[ResourceServer] - poolsByName *store.Index[UserPool] - clients *store.Table[UserPoolClient] - clientsByPool *store.Index[UserPoolClient] - users *store.Table[User] - usersByPool *store.Index[User] - usersBySub *store.Index[User] - refreshTokens map[string]*refreshTokenEntry - refreshTokensByClient map[string]map[string]struct{} - refreshTokensByUser map[string]map[string]struct{} - mfaSessions map[string]*mfaSessionEntry - groups *store.Table[Group] - logDeliveryConfigs map[string]*LogDeliveryConfig - groupMembers map[string]map[string]map[string]struct{} - riskConfigurations map[string]*RiskConfiguration - resourceServersByPool *store.Index[ResourceServer] - tokenRevokedBeforeSeq map[string]int64 - tokenRevokedBefore map[string]time.Time - registry *store.Registry - identityProviders *store.Table[IdentityProvider] - identityProvidersByPool *store.Index[IdentityProvider] - mu *lockmetrics.RWMutex - pools *store.Table[UserPool] - resourceTags map[string]map[string]string - groupsByPool *store.Index[Group] - uiCustomizations *store.Table[UICustomization] - managedLoginBrandings *store.Table[ManagedLoginBranding] - managedLoginBrandingsByPool *store.Index[ManagedLoginBranding] - terms *store.Table[Terms] - termsByPool *store.Index[Terms] - userImportJobs *store.Table[UserImportJob] - userImportJobsByPool *store.Index[UserImportJob] - poolMfaConfigs map[string]*UserPoolMfaFullConfig - attrVerificationCodes map[string]*attrVerificationEntry - typedRiskConfigurations *store.Table[TypedRiskConfiguration] - devices map[string]map[string]*Device - webauthnCredentials map[string]map[string]*WebAuthnCredential - authEvents map[string]map[string]*AuthEvent - userPoolReplicas *store.Table[UserPoolReplica] - userPoolReplicasByPool *store.Index[UserPoolReplica] - provisionedLimits map[string]int32 - accountID string - region string - endpoint string - tokenSeq int64 + lambdaInvoker LambdaTriggerInvoker + domains *store.Table[UserPoolDomain] + resourceServers *store.Table[ResourceServer] + poolsByName *store.Index[UserPool] + clients *store.Table[UserPoolClient] + clientsByPool *store.Index[UserPoolClient] + users *store.Table[User] + usersByPool *store.Index[User] + usersBySub *store.Index[User] + refreshTokens map[string]*refreshTokenEntry + refreshTokensByClient map[string]map[string]struct{} + refreshTokensByUser map[string]map[string]struct{} + mfaSessions map[string]*mfaSessionEntry + groups *store.Table[Group] + logDeliveryConfigs map[string]*LogDeliveryConfig + groupMembers map[string]map[string]map[string]struct{} + riskConfigurations map[string]*RiskConfiguration + resourceServersByPool *store.Index[ResourceServer] + tokenRevokedBeforeSeq map[string]int64 + tokenRevokedBefore map[string]time.Time + registry *store.Registry + identityProviders *store.Table[IdentityProvider] + identityProvidersByPool *store.Index[IdentityProvider] + mu *lockmetrics.RWMutex + pools *store.Table[UserPool] + resourceTags map[string]map[string]string + groupsByPool *store.Index[Group] + uiCustomizations *store.Table[UICustomization] + managedLoginBrandings *store.Table[ManagedLoginBranding] + managedLoginBrandingsByPool *store.Index[ManagedLoginBranding] + terms *store.Table[Terms] + termsByPool *store.Index[Terms] + userImportJobs *store.Table[UserImportJob] + userImportJobsByPool *store.Index[UserImportJob] + poolMfaConfigs map[string]*UserPoolMfaFullConfig + attrVerificationCodes map[string]*attrVerificationEntry + typedRiskConfigurations *store.Table[TypedRiskConfiguration] + devices map[string]map[string]*Device + webauthnCredentials map[string]map[string]*WebAuthnCredential + authEvents map[string]map[string]*AuthEvent + userPoolReplicas *store.Table[UserPoolReplica] + userPoolReplicasByPool *store.Index[UserPoolReplica] + provisionedLimits map[string]int32 + accountID string + region string + endpoint string + tokenSeq int64 + refreshTokenInsertsSinceSweep int } // NewInMemoryBackend creates a new InMemoryBackend. @@ -150,6 +157,7 @@ func (b *InMemoryBackend) Reset() { b.tokenRevokedBeforeSeq = make(map[string]int64) b.tokenRevokedBefore = make(map[string]time.Time) b.tokenSeq = 0 + b.refreshTokenInsertsSinceSweep = 0 b.resourceTags = make(map[string]map[string]string) b.riskConfigurations = make(map[string]*RiskConfiguration) b.logDeliveryConfigs = make(map[string]*LogDeliveryConfig) diff --git a/services/cognitoidp/users.go b/services/cognitoidp/users.go index 4f170f8f3f..cf703bb700 100644 --- a/services/cognitoidp/users.go +++ b/services/cognitoidp/users.go @@ -114,7 +114,8 @@ func (b *InMemoryBackend) AdminDeleteUser(userPoolID, username string) error { // deleteUserStateLocked removes the user record for poolID:username and every // piece of per-user state that would otherwise outlive it: refresh tokens, -// devices, auth events, WebAuthn credentials, and group memberships. Shared +// devices, auth events, WebAuthn credentials, sign-out revocation markers, +// and group memberships. Shared // by AdminDeleteUser, DeleteUser, and DeleteUserPool's cascade so a cleanup // added to one path can't drift from the others -- DeleteUserPool's cascade // was already fixed once to repeat this list by hand and missed groupMembers @@ -128,6 +129,8 @@ func (b *InMemoryBackend) deleteUserStateLocked(poolID, username string) { delete(b.devices, key) delete(b.authEvents, key) delete(b.webauthnCredentials, key) + delete(b.tokenRevokedBeforeSeq, key) + delete(b.tokenRevokedBefore, key) for _, members := range b.groupMembers[poolID] { delete(members, username) diff --git a/services/cognitoidp/whitebox_test.go b/services/cognitoidp/whitebox_test.go new file mode 100644 index 0000000000..8e04f0b898 --- /dev/null +++ b/services/cognitoidp/whitebox_test.go @@ -0,0 +1,94 @@ +package cognitoidp + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestCognitoIDP_RefreshTokenEviction proves storeRefreshTokenLocked +// opportunistically sweeps refresh tokens that expired naturally -- without +// ever being refreshed (InitiateAuthRefreshToken) or revoked (RevokeToken/ +// GlobalSignOut), the only two paths that otherwise delete an entry -- once +// the table grows past refreshTokenEvictThreshold. +func TestCognitoIDP_RefreshTokenEviction(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + seedExpired int + inserts int + wantSwept bool + }{ + {name: "below threshold keeps expired", seedExpired: 1, inserts: 1}, + { + name: "threshold and sweep interval evicts expired", + seedExpired: refreshTokenEvictThreshold + 16, + inserts: refreshTokenEvictSweepInterval, + wantSwept: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := NewInMemoryBackend("000000000000", "us-east-1", "") + past := time.Now().Add(-time.Hour) + live := time.Now().Add(time.Hour) + + b.mu.Lock("seed") + for i := range tt.seedExpired { + token := fmt.Sprintf("expired-%d", i) + b.refreshTokens[token] = &refreshTokenEntry{ + PoolID: "pool", ClientID: "client", Username: fmt.Sprintf("user-%d", i), ExpiresAt: past, + } + } + b.mu.Unlock() + + b.mu.Lock("insert") + for i := range tt.inserts { + token := fmt.Sprintf("live-%d", i) + b.storeRefreshTokenLocked(token, &refreshTokenEntry{ + PoolID: "pool", ClientID: "client", Username: fmt.Sprintf("liveuser-%d", i), ExpiresAt: live, + }) + } + b.mu.Unlock() + + b.mu.RLock("check") + _, stillPresent := b.refreshTokens["expired-0"] + b.mu.RUnlock() + + if tt.wantSwept { + assert.False(t, stillPresent, "expired refresh token should have been swept") + } else { + assert.True(t, stillPresent, "expired refresh token should remain below the eviction threshold") + } + }) + } +} + +// TestCognitoIDP_DeleteUserClearsRevocationMarkers proves deleteUserStateLocked +// (shared by AdminDeleteUser/DeleteUser/DeleteUserPool's cascade) removes the +// tokenRevokedBeforeSeq/tokenRevokedBefore sign-out markers for the deleted +// user, so they don't outlive the user they revoked tokens for. +func TestCognitoIDP_DeleteUserClearsRevocationMarkers(t *testing.T) { + t.Parallel() + + b := NewInMemoryBackend("000000000000", "us-east-1", "") + const poolID, username = "pool", "alice" + key := userStateKey(poolID, username) + + b.mu.Lock("seed") + b.tokenRevokedBeforeSeq[key] = 1 + b.tokenRevokedBefore[key] = time.Now() + b.deleteUserStateLocked(poolID, username) + _, seqPresent := b.tokenRevokedBeforeSeq[key] + _, timePresent := b.tokenRevokedBefore[key] + b.mu.Unlock() + + assert.False(t, seqPresent, "tokenRevokedBeforeSeq entry should be removed with the user") + assert.False(t, timePresent, "tokenRevokedBefore entry should be removed with the user") +} diff --git a/services/databrew/jobs_test.go b/services/databrew/jobs_test.go index a6366949d6..5374b60db3 100644 --- a/services/databrew/jobs_test.go +++ b/services/databrew/jobs_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/url" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -325,39 +326,45 @@ func TestStartJobRun_Success(t *testing.T) { func TestStartJobRun_TransitionsToSucceeded(t *testing.T) { t.Parallel() - b := newTestBackend() - _, err := b.CreateDataset( - context.Background(), - "ds", - "CSV", - s3Input("b", ""), - databrew.DatasetFormatOptions{}, - nil, - nil, - ) - require.NoError(t, err) - _, err = b.CreateJob( - context.Background(), - "run-j2", - "PROFILE", - "ds", - "", - "", - "", - nil, - nil, - databrew.JobExtras{}, - ) - require.NoError(t, err) - _, err = b.StartJobRun(context.Background(), "run-j2") - require.NoError(t, err) - // Poll for async state transition instead of fixed sleep. - require.Eventually(t, func() bool { - runs, _, listErr := b.ListJobRuns(context.Background(), "run-j2", 100, "") + synctest.Test(t, func(t *testing.T) { + b := newTestBackend() + _, err := b.CreateDataset( + context.Background(), + "ds", + "CSV", + s3Input("b", ""), + databrew.DatasetFormatOptions{}, + nil, + nil, + ) + require.NoError(t, err) + _, err = b.CreateJob( + context.Background(), + "run-j2", + "PROFILE", + "ds", + "", + "", + "", + nil, + nil, + databrew.JobExtras{}, + ) + require.NoError(t, err) + _, err = b.StartJobRun(context.Background(), "run-j2") + require.NoError(t, err) + + // jobRunTransitionDelay (unexported) is 100ms; cross it, then let the + // backend's transition goroutine run to completion. + time.Sleep(200 * time.Millisecond) + synctest.Wait() - return listErr == nil && len(runs) == 1 && runs[0].State == "SUCCEEDED" - }, 3*time.Second, 25*time.Millisecond) + runs, _, err := b.ListJobRuns(context.Background(), "run-j2", 100, "") + require.NoError(t, err) + require.Len(t, runs, 1) + assert.Equal(t, "SUCCEEDED", runs[0].State) + }) } func TestStartJobRun_JobNotFound(t *testing.T) { @@ -515,42 +522,43 @@ func TestStopJobRun_Success(t *testing.T) { func TestStopJobRun_AlreadySucceeded(t *testing.T) { t.Parallel() - b := newTestBackend() - _, err := b.CreateDataset( - context.Background(), - "ds", - "CSV", - s3Input("b", ""), - databrew.DatasetFormatOptions{}, - nil, - nil, - ) - require.NoError(t, err) - _, err = b.CreateJob( - context.Background(), - "stop-j2", - "PROFILE", - "ds", - "", - "", - "", - nil, - nil, - databrew.JobExtras{}, - ) - require.NoError(t, err) - run, err := b.StartJobRun(context.Background(), "stop-j2") - require.NoError(t, err) - // Wait for the async transition. - require.Eventually(t, func() bool { - runs, _, listErr := b.ListJobRuns(context.Background(), "stop-j2", 100, "") - - return listErr == nil && len(runs) == 1 && runs[0].State == "SUCCEEDED" - }, 3*time.Second, 25*time.Millisecond) - // Stopping a SUCCEEDED run should be a no-op (returns the run). - stopped, err := b.StopJobRun(context.Background(), "stop-j2", run.RunID) - require.NoError(t, err) - assert.Equal(t, "SUCCEEDED", stopped.State) + + synctest.Test(t, func(t *testing.T) { + b := newTestBackend() + _, err := b.CreateDataset( + context.Background(), + "ds", + "CSV", + s3Input("b", ""), + databrew.DatasetFormatOptions{}, + nil, + nil, + ) + require.NoError(t, err) + _, err = b.CreateJob( + context.Background(), + "stop-j2", + "PROFILE", + "ds", + "", + "", + "", + nil, + nil, + databrew.JobExtras{}, + ) + require.NoError(t, err) + run, err := b.StartJobRun(context.Background(), "stop-j2") + require.NoError(t, err) + + time.Sleep(200 * time.Millisecond) + synctest.Wait() + + // Stopping a SUCCEEDED run should be a no-op (returns the run). + stopped, err := b.StopJobRun(context.Background(), "stop-j2", run.RunID) + require.NoError(t, err) + assert.Equal(t, "SUCCEEDED", stopped.State) + }) } func TestStopJobRun_NotFound_NoRuns(t *testing.T) { @@ -938,33 +946,36 @@ func TestListJobs_Filters(t *testing.T) { func TestJobRunIdField_RoundTrip(t *testing.T) { t.Parallel() - h := newTestHandler() - databrewReq(t, h, http.MethodPost, "/databrew/v1/profileJobs", - map[string]any{"Name": "rt-job"}) - - startRec := databrewReq(t, h, http.MethodPost, "/databrew/v1/jobs/rt-job/startJobRun", nil) - require.Equal(t, http.StatusOK, startRec.Code) - - var startResp map[string]any - require.NoError(t, json.Unmarshal(startRec.Body.Bytes(), &startResp)) - runID, ok := startResp["RunId"].(string) - require.True(t, ok) - require.NotEmpty(t, runID) - - // Wait for transition so DescribeJobRun is non-empty. - time.Sleep(200 * time.Millisecond) - - descRec := databrewReq(t, h, http.MethodGet, "/databrew/v1/jobs/rt-job/jobRun/"+runID, nil) - require.Equal(t, http.StatusOK, descRec.Code) - var descResp map[string]any - require.NoError(t, json.Unmarshal(descRec.Body.Bytes(), &descResp)) - assert.Equal(t, runID, descResp["RunId"]) - - stopRec := databrewReq(t, h, http.MethodPost, "/databrew/v1/jobs/rt-job/jobRun/"+runID, nil) - require.Equal(t, http.StatusOK, stopRec.Code) - var stopResp map[string]any - require.NoError(t, json.Unmarshal(stopRec.Body.Bytes(), &stopResp)) - assert.Equal(t, runID, stopResp["RunId"]) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler() + databrewReq(t, h, http.MethodPost, "/databrew/v1/profileJobs", + map[string]any{"Name": "rt-job"}) + + startRec := databrewReq(t, h, http.MethodPost, "/databrew/v1/jobs/rt-job/startJobRun", nil) + require.Equal(t, http.StatusOK, startRec.Code) + + var startResp map[string]any + require.NoError(t, json.Unmarshal(startRec.Body.Bytes(), &startResp)) + runID, ok := startResp["RunId"].(string) + require.True(t, ok) + require.NotEmpty(t, runID) + + // jobRunTransitionDelay (unexported) is 100ms; cross it so DescribeJobRun is non-empty. + time.Sleep(200 * time.Millisecond) + synctest.Wait() + + descRec := databrewReq(t, h, http.MethodGet, "/databrew/v1/jobs/rt-job/jobRun/"+runID, nil) + require.Equal(t, http.StatusOK, descRec.Code) + var descResp map[string]any + require.NoError(t, json.Unmarshal(descRec.Body.Bytes(), &descResp)) + assert.Equal(t, runID, descResp["RunId"]) + + stopRec := databrewReq(t, h, http.MethodPost, "/databrew/v1/jobs/rt-job/jobRun/"+runID, nil) + require.Equal(t, http.StatusOK, stopRec.Code) + var stopResp map[string]any + require.NoError(t, json.Unmarshal(stopRec.Body.Bytes(), &stopResp)) + assert.Equal(t, runID, stopResp["RunId"]) + }) } // ---- Job extras: ProfileConfiguration/JobSample/ValidationConfigurations, diff --git a/services/databrew/shutdown_test.go b/services/databrew/shutdown_test.go index 527e4e81b9..8570dc36d4 100644 --- a/services/databrew/shutdown_test.go +++ b/services/databrew/shutdown_test.go @@ -3,8 +3,10 @@ package databrew_test import ( "context" "testing" + "testing/synctest" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/blackbirdworks/gopherstack/services/databrew" @@ -135,18 +137,23 @@ func TestBackendShutdown(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b, job := tt.build(t) - if job == "" { - return - } - - // Give any (incorrectly) leaked goroutine time to fire so a - // false negative would surface. - require.Never(t, func() bool { - runs, _, err := b.ListJobRuns(context.Background(), job, 100, "") - return err == nil && len(runs) == 1 && runs[0].State == "SUCCEEDED" - }, 250*time.Millisecond, 25*time.Millisecond) + synctest.Test(t, func(t *testing.T) { + b, job := tt.build(t) + if job == "" { + return + } + + // Cross the 100ms transition delay: a leaked goroutine would + // have fired by now. + time.Sleep(250 * time.Millisecond) + synctest.Wait() + + runs, _, err := b.ListJobRuns(context.Background(), job, 100, "") + require.NoError(t, err) + require.Len(t, runs, 1) + assert.NotEqual(t, "SUCCEEDED", runs[0].State) + }) }) } } @@ -156,38 +163,42 @@ func TestBackendShutdown(t *testing.T) { func TestResetDoesNotStopTransitions(t *testing.T) { t.Parallel() - b := databrew.NewInMemoryBackendWithContext(t.Context(), "123456789012", "us-east-1") - b.Reset() - - _, err := b.CreateDataset( - context.Background(), - "ds", - "CSV", - s3Input("b", ""), - databrew.DatasetFormatOptions{}, - nil, - nil, - ) - require.NoError(t, err) - _, err = b.CreateJob( - context.Background(), - "post-reset", - "PROFILE", - "ds", - "", - "", - "", - nil, - nil, - databrew.JobExtras{}, - ) - require.NoError(t, err) - _, err = b.StartJobRun(context.Background(), "post-reset") - require.NoError(t, err) - - require.Eventually(t, func() bool { - runs, _, listErr := b.ListJobRuns(context.Background(), "post-reset", 100, "") - - return listErr == nil && len(runs) == 1 && runs[0].State == "SUCCEEDED" - }, 3*time.Second, 25*time.Millisecond) + synctest.Test(t, func(t *testing.T) { + b := databrew.NewInMemoryBackendWithContext(t.Context(), "123456789012", "us-east-1") + b.Reset() + + _, err := b.CreateDataset( + context.Background(), + "ds", + "CSV", + s3Input("b", ""), + databrew.DatasetFormatOptions{}, + nil, + nil, + ) + require.NoError(t, err) + _, err = b.CreateJob( + context.Background(), + "post-reset", + "PROFILE", + "ds", + "", + "", + "", + nil, + nil, + databrew.JobExtras{}, + ) + require.NoError(t, err) + _, err = b.StartJobRun(context.Background(), "post-reset") + require.NoError(t, err) + + time.Sleep(200 * time.Millisecond) + synctest.Wait() + + runs, _, err := b.ListJobRuns(context.Background(), "post-reset", 100, "") + require.NoError(t, err) + require.Len(t, runs, 1) + assert.Equal(t, "SUCCEEDED", runs[0].State) + }) } diff --git a/services/datasync/models.go b/services/datasync/models.go index 2258d8da5c..6061bd646f 100644 --- a/services/datasync/models.go +++ b/services/datasync/models.go @@ -23,7 +23,7 @@ func (a *storedAgent) toAgent() Agent { Status: a.Status, EndpointType: a.EndpointType, CreationTime: a.CreationTime, - Tags: a.Tags, + Tags: maps.Clone(a.Tags), } } @@ -332,7 +332,7 @@ func (t *storedTask) toTask() Task { CloudWatchLogGroupArn: t.CloudWatchLogGroupArn, CurrentTaskExecutionArn: t.CurrentTaskExecutionArn, CreationTime: t.CreationTime, - Tags: t.Tags, + Tags: maps.Clone(t.Tags), Options: maps.Clone(t.Options), ManifestConfig: maps.Clone(t.ManifestConfig), TaskReportConfig: maps.Clone(t.TaskReportConfig), diff --git a/services/dsql/PARITY.md b/services/dsql/PARITY.md new file mode 100644 index 0000000000..fa8993b5ac --- /dev/null +++ b/services/dsql/PARITY.md @@ -0,0 +1,100 @@ +--- +service: dsql +sdk_module: aws-sdk-go-v2/service/dsql@v1.22.1 +last_audit_commit: 5a0bc403e # HEAD at audit time, pre-commit +last_audit_date: 2026-09-26 +overall: B # new service, control plane only, unit-tested against the real SDK client +ops: + CreateCluster: {wire: ok, errors: ok, state: ok, persist: ok, note: "CREATING, lazily flips to ACTIVE on next read after a short deadline -- see items_still_open"} + GetCluster: {wire: ok, errors: ok, state: ok, persist: ok} + ListClusters: {wire: ok, errors: ok, state: ok, persist: ok, note: "opaque nextToken via pkgs/page"} + UpdateCluster: {wire: ok, errors: ok, state: ok, persist: ok, note: "UPDATING, lazily flips back to ACTIVE; KmsEncryptionKey=AWS_OWNED_KMS_KEY reverts to the AWS-owned key per SDK doc comment"} + DeleteCluster: {wire: ok, errors: ok, state: ok, persist: ok, note: "DeletionProtectionEnabled blocks delete (ValidationException, reason=deletionProtectionEnabled); otherwise DELETING, lazily removed on next read"} + GetClusterPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + PutClusterPolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "expectedPolicyVersion optimistic lock; bypassPolicyLockoutSafetyCheck accepted but not evaluated -- see items_still_open"} + DeleteClusterPolicy: {wire: ok, errors: ok, state: ok, persist: ok, note: "expectedPolicyVersion optimistic lock"} + GetVpcEndpointServiceName: {wire: ok, errors: ok, state: ok, persist: ok, note: "wire-shaped names only; no real PrivateLink plane -- see items_still_open"} + CreateStream: {wire: ok, errors: ok, state: ok, persist: ok, note: "CREATING, lazily flips to ACTIVE on next read"} + GetStream: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteStream: {wire: ok, errors: ok, state: ok, persist: ok, note: "removed immediately -- see items_still_open"} + ListStreams: {wire: ok, errors: ok, state: ok, persist: ok, note: "opaque nextToken via pkgs/page"} + TagResource: {wire: ok, errors: ok, state: ok, persist: ok, note: "cluster ARN only, per SDK doc comment"} + UntagResource: {wire: ok, errors: ok, state: ok, persist: ok} + ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok} +families: + Cluster: {status: ok, note: "Create/Get/List/Update/Delete verified end-to-end against the real aws-sdk-go-v2 client over an httptest server -- wire shapes (lowerCamelCase JSON, unlike most restjson1 services in this repo), epoch creationTime, ARN format (arn:aws:dsql:region:account:cluster/id), the .dsql..on.aws endpoint format, multiRegionProperties/witnessRegion round-trip, deletionProtectionEnabled enforcement, and error deserialization (ResourceNotFoundException/ValidationException/ServiceQuotaExceededException) all round-trip cleanly."} + ClusterPolicy: {status: ok, note: "Get/Put/Delete round-trip the policy document and an opaque policyVersion token; PutClusterPolicy/DeleteClusterPolicy both honor expectedPolicyVersion (ConflictException on mismatch), matching the SDK's optimistic-concurrency doc comments."} + Stream: {status: ok, note: "Create/Get/Delete/List verified against the real client; streamIdentifier is scoped to its owning cluster (composite table key), matching the /stream/{clusterId}/{streamId} wire path. Only the Kinesis targetDefinition variant exists in the pinned SDK, so that's the only one implemented."} + Tags: {status: ok, note: "One generic tag family keyed by cluster ARN, matching real AWS (TagResource/UntagResource/ListTagsForResource operate on dsql:cluster resources only)."} +gaps: [] +items_still_open: + - "CREATING/UPDATING/DELETING transient cluster and stream states use a short, fixed lazy + deadline (750ms for clusters, 500ms for streams) rather than a background reconciler or an + AWS-realistic multi-second/multi-minute provisioning time: a client that reads twice in a + row observes the terminal state almost immediately. This is a deliberate simplification + (explicitly authorized as either an honest immediate-ACTIVE or a lazy deadline) chosen so + terraform-provider-aws's ClusterActiveWaiter/ClusterNotExistsWaiter (2s minimum poll + interval) always observes the terminal state on their very first poll." + - "PutClusterPolicy's bypassPolicyLockoutSafetyCheck is accepted and stored on the wire + request but never evaluated: real AWS parses the policy document and refuses to apply one + that would lock the caller out of the cluster unless this flag is set. This backend has no + IAM policy evaluation engine (no service in this repo does), so every PutClusterPolicy call + succeeds regardless of the flag -- structural, out of scope for this pass." + - "GetVpcEndpointServiceName returns wire-shaped serviceName/clusterVpcEndpoint values but + there is no real VPC/PrivateLink plane behind them -- structural, matches how every other + VPC-endpoint-service-name-style operation in this repo (e.g. services/rds) is handled." + - "DeleteStream removes the stream synchronously rather than lingering through a DELETING + state first: real AWS's StreamStatus enum includes DELETING, but nothing else in this + backend or in terraform-provider-aws observes a stream's intermediate delete state, so this + is behaviorally equivalent for any client that only checks for ResourceNotFoundException + afterward." + - "Multi-Region peering (multiRegionProperties.clusters) is stored and echoed back exactly as + given but not enforced: creating/updating a cluster with peer cluster ARNs does not + validate that those peers exist or reciprocally link back to this cluster. Each dsql + backend instance is a single account/region process, matching how every other + multi-region-aware service in this repo (e.g. services/dynamodbstreams's global tables) + treats cross-region state as opaque input." +--- + +## Notes + +Initial implementation (2026-09-26, bd issue gopherstack-7r6bz): control-plane +REST-JSON API modeled after services/kinesisvideo (package layout, +lockmetrics, persistence) and services/kafka (manual method+path routing, +since DSQL is a real path-parameter REST API: /cluster/{id}, +/cluster/{id}/policy, /stream/{clusterId}/{streamId}, unlike kinesisvideo's +flat action-named paths). Every operation mutates/reads real in-memory state +via pkgs/store.Table + pkgs/lockmetrics.RWMutex, with JSON snapshot/restore +wired into pkgs/persistence (additive inventory row in +pkgs/persistence/testdata/snapshot_inventory.json). + +Wire shapes, HTTP methods/paths, and error codes (ConflictException 409, +ResourceNotFoundException 404, ValidationException 400, +ServiceQuotaExceededException 402) were verified against the pinned +aws-sdk-go-v2/service/dsql@v1.22.1 request_snapshot/*.snap and +response_snapshot/*.snap fixtures (the module's own smithy-generated +request/response byte fixtures), not just the Go struct definitions. DSQL is +unusual among this repo's restjson1 services in emitting lowerCamelCase JSON +field names (clientToken, deletionProtectionEnabled, ...) rather than +PascalCase. + +Two pre-existing, unrelated services' RouteMatchers over-claimed a path +prefix DSQL's real wire shape also needs, both fixed by guarding the other +service's claim rather than raising DSQL's MatchPriority (per +.claude/memories -- route-matcher-prefix-collision): + +- Inspector2 unconditionally claimed the "/cluster/" prefix for its one real + operation (POST /cluster/get); added to its existing + ambiguousRouteMatchPrefixes map (the same mechanism it already uses for + /findings/, /members/, /configuration/), gated by its existing + isInspector2Request helper. +- EKS unconditionally claimed the "/clusters/" prefix; DSQL's + GetVpcEndpointServiceName lives at the real wire path + /clusters/{id}/vpc-endpoint-service-name (plural "clusters", unlike every + other DSQL cluster operation's singular "/cluster/{id}"), confirmed against + request_snapshot/GetVpcEndpointServiceName.request.snap. EKS has no such + operation, so that exact suffix is carved out via a new + isDSQLVpcEndpointServiceNamePath helper. + +`go run ./cmd/routecollisions` before/after: both new collisions are +(guarded/guarded); no new unguarded collisions. diff --git a/services/dsql/README.md b/services/dsql/README.md new file mode 100644 index 0000000000..a96babdd32 --- /dev/null +++ b/services/dsql/README.md @@ -0,0 +1,27 @@ + +# Dsql + +**Parity grade: B** · SDK `aws-sdk-go-v2/service/dsql@v1.22.1` · last audited 2026-09-26 (`5a0bc403e`) + +## Coverage + +| Metric | Value | +| --- | --- | +| PARITY entries audited | 16 (16 ok) | +| Feature families | 4 (4 ok) | +| Known gaps | 5 | +| Deferred items | 0 | +| Resource leaks | unknown | + +### Known gaps + +- "CREATING/UPDATING/DELETING transient cluster and stream states use a short, fixed lazy deadline (750ms for clusters, 500ms for streams) rather than a background reconciler or an AWS-realistic multi-second/multi-minute provisioning time: a client that reads twice in a row observes the terminal state almost immediately. This is a deliberate simplification (explicitly authorized as either an honest immediate-ACTIVE or a lazy deadline) chosen so terraform-provider-aws's ClusterActiveWaiter/ClusterNotExistsWaiter (2s minimum poll interval) always observes the terminal state on their very first poll." +- "PutClusterPolicy's bypassPolicyLockoutSafetyCheck is accepted and stored on the wire request but never evaluated: real AWS parses the policy document and refuses to apply one that would lock the caller out of the cluster unless this flag is set. This backend has no IAM policy evaluation engine (no service in this repo does), so every PutClusterPolicy call succeeds regardless of the flag -- structural, out of scope for this pass." +- "GetVpcEndpointServiceName returns wire-shaped serviceName/clusterVpcEndpoint values but there is no real VPC/PrivateLink plane behind them -- structural, matches how every other VPC-endpoint-service-name-style operation in this repo (e.g. services/rds) is handled." +- "DeleteStream removes the stream synchronously rather than lingering through a DELETING state first: real AWS's StreamStatus enum includes DELETING, but nothing else in this backend or in terraform-provider-aws observes a stream's intermediate delete state, so this is behaviorally equivalent for any client that only checks for ResourceNotFoundException afterward." +- "Multi-Region peering (multiRegionProperties.clusters) is stored and echoed back exactly as given but not enforced: creating/updating a cluster with peer cluster ARNs does not validate that those peers exist or reciprocally link back to this cluster. Each dsql backend instance is a single account/region process, matching how every other multi-region-aware service in this repo (e.g. services/dynamodbstreams's global tables) treats cross-region state as opaque input." + +## More + +- [Full parity audit](PARITY.md) +- [All services](../../README.md#services) diff --git a/services/dsql/clusters.go b/services/dsql/clusters.go new file mode 100644 index 0000000000..3ade99859a --- /dev/null +++ b/services/dsql/clusters.go @@ -0,0 +1,187 @@ +package dsql + +import ( + "maps" + "sort" + "time" + + "github.com/blackbirdworks/gopherstack/pkgs/page" +) + +const defaultListLimit = 100 + +// CreateCluster creates a new Aurora DSQL cluster. New clusters start +// CREATING and lazily transition to ACTIVE on the next read once +// clusterActivationDelay elapses -- see PARITY.md for why this is a lazy +// deadline rather than a background reconciler. +func (b *InMemoryBackend) CreateCluster(accountID, region string, in CreateClusterInput) (*Cluster, error) { + if err := validateTags(in.Tags); err != nil { + return nil, err + } + + if err := validateMultiRegion(in.MultiRegion, region); err != nil { + return nil, err + } + + b.mu.Lock("CreateCluster") + defer b.mu.Unlock() + + if b.countClustersLocked(accountID, region) >= maxClustersPerAccountRegion { + return nil, ErrClusterQuotaExceeded + } + + identifier := newIdentifier() + now := time.Now().UTC() + + tags := make(map[string]string, len(in.Tags)) + maps.Copy(tags, in.Tags) + + c := &Cluster{ + Identifier: identifier, + ARN: clusterARN(region, accountID, identifier), + Endpoint: clusterEndpoint(identifier, region), + AccountID: accountID, + Region: region, + Status: statusCreating, + CreationTime: now, + PendingUntil: now.Add(clusterActivationDelay), + KmsEncryptionKey: in.KmsEncryptionKey, + DeletionProtectionEnabled: in.DeletionProtectionEnabled, + MultiRegion: in.MultiRegion.clone(), + Tags: tags, + } + + if in.Policy != "" { + c.Policy = &ClusterPolicy{Policy: in.Policy, Version: newVersionToken()} + } + + b.clusters.Put(c) + + return c.clone(), nil +} + +func (b *InMemoryBackend) countClustersLocked(accountID, region string) int { + n := 0 + + for _, c := range b.clusters.All() { + if c.AccountID == accountID && c.Region == region { + n++ + } + } + + return n +} + +// GetCluster returns the current information about a cluster. +func (b *InMemoryBackend) GetCluster(identifier string) (*Cluster, error) { + b.mu.Lock("GetCluster") + defer b.mu.Unlock() + + c, err := b.resolveClusterLocked(identifier) + if err != nil { + return nil, err + } + + return c.clone(), nil +} + +// ListClusters returns clusters ordered by identifier, paginated by nextToken/maxResults. +func (b *InMemoryBackend) ListClusters(nextToken string, maxResults int) ([]*Cluster, string, error) { + b.mu.Lock("ListClusters") + defer b.mu.Unlock() + + all := b.clusters.All() + + live := make([]*Cluster, 0, len(all)) + + for _, c := range all { + b.advanceClusterLocked(c) + + if c.Status == statusDeleting && time.Now().After(c.PendingUntil) { + b.clusters.Delete(c.Identifier) + + continue + } + + live = append(live, c.clone()) + } + + sort.Slice(live, func(i, j int) bool { return live[i].Identifier < live[j].Identifier }) + + p := page.New(live, nextToken, maxResults, defaultListLimit) + + return p.Data, p.Next, nil +} + +// UpdateCluster updates a cluster's mutable configuration and transitions it +// through UPDATING back to ACTIVE on the next read. +func (b *InMemoryBackend) UpdateCluster(identifier string, in UpdateClusterInput) (*Cluster, error) { + b.mu.Lock("UpdateCluster") + defer b.mu.Unlock() + + c, err := b.resolveClusterLocked(identifier) + if err != nil { + return nil, err + } + + if in.MultiRegion != nil { + if validateErr := validateMultiRegion(in.MultiRegion, c.Region); validateErr != nil { + return nil, validateErr + } + + c.MultiRegion = in.MultiRegion.clone() + } + + if in.DeletionProtectionEnabled != nil { + c.DeletionProtectionEnabled = *in.DeletionProtectionEnabled + } + + switch in.KmsEncryptionKey { + case "": + case encryptionTypeAWSOwned: + c.KmsEncryptionKey = "" + default: + c.KmsEncryptionKey = in.KmsEncryptionKey + } + + now := time.Now().UTC() + c.Status = statusUpdating + c.PendingUntil = now.Add(clusterActivationDelay) + + return c.clone(), nil +} + +// DeleteCluster marks a cluster DELETING; it is lazily removed from the +// table clusterDeletionDelay after this call, on the next resolve. A cluster +// with deletion protection enabled cannot be deleted. +func (b *InMemoryBackend) DeleteCluster(identifier string) (*Cluster, error) { + b.mu.Lock("DeleteCluster") + defer b.mu.Unlock() + + c, err := b.resolveClusterLocked(identifier) + if err != nil { + return nil, err + } + + if c.DeletionProtectionEnabled { + return nil, ErrDeletionProtected + } + + now := time.Now().UTC() + c.Status = statusDeleting + c.PendingUntil = now.Add(clusterDeletionDelay) + + return c.clone(), nil +} + +func validateMultiRegion(m *MultiRegionProperties, region string) error { + if m == nil || m.WitnessRegion == "" || region == "" { + return nil + } + + if m.WitnessRegion == region { + return ErrValidation + } + + return nil +} diff --git a/services/dsql/clusters_test.go b/services/dsql/clusters_test.go new file mode 100644 index 0000000000..b568af7bb6 --- /dev/null +++ b/services/dsql/clusters_test.go @@ -0,0 +1,232 @@ +package dsql_test + +import ( + "errors" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + dsqlsdk "github.com/aws/aws-sdk-go-v2/service/dsql" + "github.com/aws/aws-sdk-go-v2/service/dsql/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateCluster(t *testing.T) { + t.Parallel() + + tests := []struct { + input dsqlsdk.CreateClusterInput + name string + }{ + {name: "minimal", input: dsqlsdk.CreateClusterInput{}}, + { + name: "with deletion protection and tags", + input: dsqlsdk.CreateClusterInput{ + DeletionProtectionEnabled: aws.Bool(true), + Tags: map[string]string{"env": "test"}, + }, + }, + { + name: "with multi-region properties", + input: dsqlsdk.CreateClusterInput{ + MultiRegionProperties: &types.MultiRegionProperties{ + WitnessRegion: aws.String("us-west-2"), + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + out, err := client.CreateCluster(t.Context(), &tt.input) + require.NoError(t, err) + assert.Contains(t, aws.ToString(out.Arn), "arn:aws:dsql:"+testRegion+":"+testAccountID+":cluster/") + assert.Equal(t, types.ClusterStatusCreating, out.Status) + assert.Contains(t, aws.ToString(out.Endpoint), ".dsql."+testRegion+".on.aws") + assert.NotEmpty(t, aws.ToString(out.Identifier)) + }) + } +} + +func TestCreateCluster_WitnessRegionEqualsClusterRegion(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.CreateCluster(t.Context(), &dsqlsdk.CreateClusterInput{ + MultiRegionProperties: &types.MultiRegionProperties{WitnessRegion: aws.String(testRegion)}, + }) + require.Error(t, err) + assertAPIErrorCode(t, err, "ValidationException") +} + +func TestCreateCluster_QuotaExceeded(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + const quota = 20 + + for range quota { + _, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + } + + _, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.Error(t, err) + assertAPIErrorCode(t, err, "ServiceQuotaExceededException") +} + +func TestGetCluster(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{ + Tags: map[string]string{"env": "test"}, + }) + require.NoError(t, err) + + out, err := client.GetCluster(ctx, &dsqlsdk.GetClusterInput{Identifier: created.Identifier}) + require.NoError(t, err) + assert.Equal(t, aws.ToString(created.Identifier), aws.ToString(out.Identifier)) + assert.Equal(t, aws.ToString(created.Arn), aws.ToString(out.Arn)) + assert.Equal(t, map[string]string{"env": "test"}, out.Tags) + assert.NotZero(t, aws.ToTime(out.CreationTime)) + require.NotNil(t, out.EncryptionDetails) + assert.Equal(t, types.EncryptionTypeAwsOwnedKmsKey, out.EncryptionDetails.EncryptionType) +} + +func TestGetCluster_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.GetCluster(t.Context(), &dsqlsdk.GetClusterInput{Identifier: aws.String("does-not-exist")}) + require.Error(t, err) + assertAPIErrorCode(t, err, "ResourceNotFoundException") +} + +func TestListClusters(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + const n = 3 + + for range n { + _, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + } + + out, err := client.ListClusters(ctx, &dsqlsdk.ListClustersInput{}) + require.NoError(t, err) + assert.Len(t, out.Clusters, n) +} + +func TestUpdateCluster(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + + out, err := client.UpdateCluster(ctx, &dsqlsdk.UpdateClusterInput{ + Identifier: created.Identifier, + DeletionProtectionEnabled: aws.Bool(true), + }) + require.NoError(t, err) + assert.Equal(t, aws.ToString(created.Identifier), aws.ToString(out.Identifier)) + assert.Equal(t, types.ClusterStatusUpdating, out.Status) + + got, err := client.GetCluster(ctx, &dsqlsdk.GetClusterInput{Identifier: created.Identifier}) + require.NoError(t, err) + assert.True(t, aws.ToBool(got.DeletionProtectionEnabled)) +} + +func TestUpdateCluster_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.UpdateCluster(t.Context(), &dsqlsdk.UpdateClusterInput{Identifier: aws.String("nope")}) + require.Error(t, err) + assertAPIErrorCode(t, err, "ResourceNotFoundException") +} + +func TestDeleteCluster(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + + out, err := client.DeleteCluster(ctx, &dsqlsdk.DeleteClusterInput{Identifier: created.Identifier}) + require.NoError(t, err) + assert.Equal(t, types.ClusterStatusDeleting, out.Status) + + require.Eventually(t, func() bool { + _, getErr := client.GetCluster(ctx, &dsqlsdk.GetClusterInput{Identifier: created.Identifier}) + + var apiErr smithy.APIError + + return getErr != nil && errors.As(getErr, &apiErr) && apiErr.ErrorCode() == "ResourceNotFoundException" + }, waitTimeout, pollInterval) +} + +func TestDeleteCluster_DeletionProtectionBlocks(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{ + DeletionProtectionEnabled: aws.Bool(true), + }) + require.NoError(t, err) + + _, err = client.DeleteCluster(ctx, &dsqlsdk.DeleteClusterInput{Identifier: created.Identifier}) + require.Error(t, err) + assertAPIErrorCode(t, err, "ValidationException") +} + +func TestGetVpcEndpointServiceName(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + + out, err := client.GetVpcEndpointServiceName(ctx, &dsqlsdk.GetVpcEndpointServiceNameInput{ + Identifier: created.Identifier, + }) + require.NoError(t, err) + assert.NotEmpty(t, aws.ToString(out.ServiceName)) + assert.NotEmpty(t, aws.ToString(out.ClusterVpcEndpoint)) +} + +func TestGetVpcEndpointServiceName_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.GetVpcEndpointServiceName(t.Context(), &dsqlsdk.GetVpcEndpointServiceNameInput{ + Identifier: aws.String("nope"), + }) + require.Error(t, err) + assertAPIErrorCode(t, err, "ResourceNotFoundException") +} diff --git a/services/dsql/errors.go b/services/dsql/errors.go new file mode 100644 index 0000000000..86dd6c3e88 --- /dev/null +++ b/services/dsql/errors.go @@ -0,0 +1,30 @@ +package dsql + +import ( + "errors" + + "github.com/blackbirdworks/gopherstack/pkgs/awserr" +) + +var ( + // ErrClusterNotFound is returned when a cluster identifier does not exist. + ErrClusterNotFound = awserr.New("cluster not found", awserr.ErrNotFound) + // ErrStreamNotFound is returned when a stream identifier does not exist. + ErrStreamNotFound = awserr.New("stream not found", awserr.ErrNotFound) + // ErrPolicyNotFound is returned when a cluster has no resource policy set. + ErrPolicyNotFound = awserr.New("cluster policy not found", awserr.ErrNotFound) + // ErrDeletionProtected is returned when DeleteCluster is called on a + // cluster with deletionProtectionEnabled set. + ErrDeletionProtected = awserr.New("cluster has deletion protection enabled", awserr.ErrInvalidParameter) + // ErrValidation is returned when request input fails validation. + ErrValidation = awserr.New("invalid argument", awserr.ErrInvalidParameter) + // ErrPolicyVersionMismatch is returned when an expectedPolicyVersion does + // not match the cluster's current policy version. + ErrPolicyVersionMismatch = awserr.New("policy version mismatch", awserr.ErrConflict) + // ErrClusterQuotaExceeded is returned when an account/region would exceed + // the emulated per-region cluster quota. + ErrClusterQuotaExceeded = errors.New("cluster quota exceeded") + // ErrStreamQuotaExceeded is returned when a cluster would exceed the + // emulated per-cluster stream quota. + ErrStreamQuotaExceeded = errors.New("stream quota exceeded") +) diff --git a/services/dsql/handler.go b/services/dsql/handler.go new file mode 100644 index 0000000000..a5402ce880 --- /dev/null +++ b/services/dsql/handler.go @@ -0,0 +1,243 @@ +package dsql + +import ( + "context" + "errors" + "net/http" + "strings" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/pkgs/awserr" + "github.com/blackbirdworks/gopherstack/pkgs/httputils" + "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/service" +) + +const dsqlMatchPriority = service.PriorityPathVersioned + +// Handler is the HTTP handler for the Aurora DSQL REST-JSON control-plane API. +type Handler struct { + Backend StorageBackend + AccountID string + DefaultRegion string +} + +// NewHandler creates a new Aurora DSQL handler backed by backend. +func NewHandler(backend StorageBackend) *Handler { + return &Handler{Backend: backend} +} + +// Name returns the service name. +func (h *Handler) Name() string { return "DSQL" } + +// Reset clears all backend state. +func (h *Handler) Reset() { h.Backend.Reset() } + +// GetSupportedOperations returns the list of supported operations. +func (h *Handler) GetSupportedOperations() []string { + return []string{ + opCreateCluster, + opGetCluster, + opListClusters, + opUpdateCluster, + opDeleteCluster, + opGetClusterPolicy, + opPutClusterPolicy, + opDeleteClusterPolicy, + opGetVpcEndpointServiceName, + opTagResource, + opUntagResource, + opListTagsForResource, + opCreateStream, + opGetStream, + opDeleteStream, + opListStreams, + } +} + +// ChaosServiceName returns the lowercase AWS service name for fault rule matching. +func (h *Handler) ChaosServiceName() string { return dsqlServiceName } + +// ChaosOperations returns all operations that can be fault-injected. +func (h *Handler) ChaosOperations() []string { return h.GetSupportedOperations() } + +// ChaosRegions returns all regions this handler handles. +func (h *Handler) ChaosRegions() []string { return []string{h.DefaultRegion} } + +// RouteMatcher returns a function that matches Aurora DSQL REST API requests. +// +// /tags/{arn} is SigV4-scoped rather than claimed unconditionally: it is a +// generic-tagging path many restjson1 services reuse verbatim (see +// .claude/memories/route-matcher-prefix-collision.md), so an unscoped claim +// here would swallow another service's own TagResource/UntagResource/ +// ListTagsForResource requests. +func (h *Handler) RouteMatcher() service.Matcher { + return func(c *echo.Context) bool { + path := effectivePath(c.Request()) + + if path == pathClusterRoot || path == pathClusterRoot+"/" { + return true + } + + if isDSQLClusterPolicyOrResourcePath(path) { + return true + } + + if strings.HasPrefix(path, pathClustersPrefix) && strings.HasSuffix(path, vpcEndpointServiceNameSuffix) { + return true + } + + if strings.HasPrefix(path, pathStreamPrefix) { + return true + } + + if strings.HasPrefix(path, pathTagsPrefix) { + svc := httputils.ExtractServiceFromRequest(c.Request()) + + return svc == "" || svc == dsqlServiceName + } + + return false + } +} + +// isDSQLClusterPolicyOrResourcePath reports whether path is a DSQL cluster +// resource path (/cluster/{id} or /cluster/{id}/policy). It is a named +// helper (rather than an inline strings.HasPrefix) because Inspector2's own +// RouteMatcher over-claims the same "/cluster/" prefix for its unrelated +// "/cluster/get" operation (gopherstack-7r6bz); Inspector2 guards its claim +// against this path family via its own ambiguousRouteMatchPrefixes map. +func isDSQLClusterPolicyOrResourcePath(path string) bool { + return strings.HasPrefix(path, pathClusterPrefix) +} + +// MatchPriority returns the routing priority. +func (h *Handler) MatchPriority() int { return dsqlMatchPriority } + +// ExtractOperation extracts the operation name from the request. +func (h *Handler) ExtractOperation(c *echo.Context) string { + op, _ := parseDSQLPath(c.Request().Method, effectivePath(c.Request())) + + return op +} + +// ExtractResource extracts the resource identifier from the request. +func (h *Handler) ExtractResource(c *echo.Context) string { + _, resource := parseDSQLPath(c.Request().Method, effectivePath(c.Request())) + + return resource +} + +// effectivePath returns the raw (percent-encoded) path if available, otherwise the decoded path. +func effectivePath(r *http.Request) string { + if r.URL.RawPath != "" { + return r.URL.RawPath + } + + return r.URL.Path +} + +// contextWithRegion returns the request context with the resolved AWS region attached. +func (h *Handler) contextWithRegion(c *echo.Context) context.Context { + region := httputils.ExtractRegionFromRequest(c.Request(), h.DefaultRegion) + + return context.WithValue(c.Request().Context(), regionContextKey{}, region) +} + +type regionContextKey struct{} + +func regionFromContext(ctx context.Context, defaultRegion string) string { + if r, ok := ctx.Value(regionContextKey{}).(string); ok && r != "" { + return r + } + + return defaultRegion +} + +// Handler returns the Echo handler function for Aurora DSQL requests. +func (h *Handler) Handler() echo.HandlerFunc { + return func(c *echo.Context) error { + ctx := h.contextWithRegion(c) + log := logger.Load(ctx) + + method := c.Request().Method + path := effectivePath(c.Request()) + + op, resource := parseDSQLPath(method, path) + if op == "" { + return h.writeError( + c, http.StatusBadRequest, "ValidationException", "unknown operation", validationReasonOther, + ) + } + + body, err := httputils.ReadBody(c.Request()) + if err != nil { + log.ErrorContext(ctx, "dsql: failed to read request body", "error", err) + + return h.writeError( + c, http.StatusInternalServerError, "InternalServerException", "failed to read request body", "", + ) + } + + log.DebugContext(ctx, "dsql request", "op", op, "resource", resource) + + return h.dispatch(ctx, c, op, resource, body) + } +} + +// dispatch routes a parsed operation to the appropriate handler. +func (h *Handler) dispatch(ctx context.Context, c *echo.Context, op, resource string, body []byte) error { + if ok, err := h.dispatchClusterOps(ctx, c, op, resource, body); ok { + return err + } + + if ok, err := h.dispatchPolicyOps(c, op, resource, body); ok { + return err + } + + if ok, err := h.dispatchTagOps(c, op, resource, body); ok { + return err + } + + if ok, err := h.dispatchStreamOps(c, op, resource, body); ok { + return err + } + + return h.writeError( + c, http.StatusBadRequest, "ValidationException", "unknown operation: "+op, validationReasonOther, + ) +} + +// writeError writes a DSQL restJson1 error response. +func (h *Handler) writeError(c *echo.Context, status int, errType, message, reason string) error { + return c.JSON(status, errorResponse{Type: errType, Message: message, Reason: reason}) +} + +// writeInvalidBody writes the common ValidationException response for an +// unparseable request body. +func (h *Handler) writeInvalidBody(c *echo.Context) error { + return h.writeError( + c, http.StatusBadRequest, "ValidationException", "invalid request body", validationReasonFieldError, + ) +} + +// writeBackendError maps a backend error to the matching AWS error response. +func (h *Handler) writeBackendError(c *echo.Context, err error) error { + switch { + case errors.Is(err, ErrDeletionProtected): + return h.writeError(c, http.StatusBadRequest, "ValidationException", err.Error(), validationReasonLockedOut) + case errors.Is(err, ErrClusterQuotaExceeded), errors.Is(err, ErrStreamQuotaExceeded): + return h.writeError(c, http.StatusPaymentRequired, "ServiceQuotaExceededException", err.Error(), "") + case errors.Is(err, ErrPolicyVersionMismatch): + return h.writeError(c, http.StatusConflict, "ConflictException", err.Error(), "") + case errors.Is(err, awserr.ErrNotFound): + return h.writeError(c, http.StatusNotFound, "ResourceNotFoundException", err.Error(), "") + case errors.Is(err, awserr.ErrConflict): + return h.writeError(c, http.StatusConflict, "ConflictException", err.Error(), "") + case errors.Is(err, awserr.ErrInvalidParameter): + return h.writeError(c, http.StatusBadRequest, "ValidationException", err.Error(), validationReasonFieldError) + default: + return h.writeError(c, http.StatusInternalServerError, "InternalServerException", err.Error(), "") + } +} diff --git a/services/dsql/handler_clusters.go b/services/dsql/handler_clusters.go new file mode 100644 index 0000000000..a3ae814baf --- /dev/null +++ b/services/dsql/handler_clusters.go @@ -0,0 +1,138 @@ +package dsql + +import ( + "context" + "encoding/json" + "net/http" + "strconv" + + "github.com/labstack/echo/v5" +) + +// dispatchClusterOps handles cluster CRUD and vpc-endpoint-service-name operations. +func (h *Handler) dispatchClusterOps( + ctx context.Context, + c *echo.Context, + op, resource string, + body []byte, +) (bool, error) { + switch op { + case opCreateCluster: + return true, h.handleCreateCluster(ctx, c, body) + case opGetCluster: + return true, h.handleGetCluster(c, resource) + case opListClusters: + return true, h.handleListClusters(c) + case opUpdateCluster: + return true, h.handleUpdateCluster(c, resource, body) + case opDeleteCluster: + return true, h.handleDeleteCluster(c, resource) + case opGetVpcEndpointServiceName: + return true, h.handleGetVpcEndpointServiceName(ctx, c, resource) + } + + return false, nil +} + +func (h *Handler) handleCreateCluster(ctx context.Context, c *echo.Context, body []byte) error { + var req createClusterRequest + if len(body) > 0 { + if err := json.Unmarshal(body, &req); err != nil { + return h.writeInvalidBody(c) + } + } + + region := regionFromContext(ctx, h.DefaultRegion) + + in := CreateClusterInput{ + DeletionProtectionEnabled: req.DeletionProtectionEnabled, + KmsEncryptionKey: req.KmsEncryptionKey, + MultiRegion: multiRegionFromDTO(req.MultiRegionProperties), + Policy: req.Policy, + Tags: req.Tags, + } + + cluster, err := h.Backend.CreateCluster(h.AccountID, region, in) + if err != nil { + return h.writeBackendError(c, err) + } + + return c.JSON(http.StatusOK, clusterResponseFromCluster(cluster, false)) +} + +func (h *Handler) handleGetCluster(c *echo.Context, identifier string) error { + cluster, err := h.Backend.GetCluster(identifier) + if err != nil { + return h.writeBackendError(c, err) + } + + return c.JSON(http.StatusOK, clusterResponseFromCluster(cluster, true)) +} + +func (h *Handler) handleListClusters(c *echo.Context) error { + q := c.Request().URL.Query() + + maxResults := 0 + if v := q.Get("max-results"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + maxResults = n + } + } + + clusters, next, err := h.Backend.ListClusters(q.Get("next-token"), maxResults) + if err != nil { + return h.writeBackendError(c, err) + } + + summaries := make([]clusterSummaryDTO, 0, len(clusters)) + for _, cl := range clusters { + summaries = append(summaries, clusterSummaryDTO{Arn: cl.ARN, Identifier: cl.Identifier}) + } + + return c.JSON(http.StatusOK, listClustersResponse{Clusters: summaries, NextToken: next}) +} + +func (h *Handler) handleUpdateCluster(c *echo.Context, identifier string, body []byte) error { + var req updateClusterRequest + if len(body) > 0 { + if err := json.Unmarshal(body, &req); err != nil { + return h.writeInvalidBody(c) + } + } + + in := UpdateClusterInput{ + DeletionProtectionEnabled: req.DeletionProtectionEnabled, + KmsEncryptionKey: req.KmsEncryptionKey, + MultiRegion: multiRegionFromDTO(req.MultiRegionProperties), + } + + cluster, err := h.Backend.UpdateCluster(identifier, in) + if err != nil { + return h.writeBackendError(c, err) + } + + return c.JSON(http.StatusOK, updateOrDeleteResponseFromCluster(cluster)) +} + +func (h *Handler) handleDeleteCluster(c *echo.Context, identifier string) error { + cluster, err := h.Backend.DeleteCluster(identifier) + if err != nil { + return h.writeBackendError(c, err) + } + + return c.JSON(http.StatusOK, updateOrDeleteResponseFromCluster(cluster)) +} + +func (h *Handler) handleGetVpcEndpointServiceName(ctx context.Context, c *echo.Context, identifier string) error { + region := regionFromContext(ctx, h.DefaultRegion) + + serviceName, clusterVpcEndpoint, err := h.Backend.GetVpcEndpointServiceName(identifier, region) + if err != nil { + return h.writeBackendError(c, err) + } + + return c.JSON(http.StatusOK, getVpcEndpointServiceNameResponse{ + ClusterVpcEndpoint: clusterVpcEndpoint, + ServiceName: serviceName, + }) +} diff --git a/services/dsql/handler_policy.go b/services/dsql/handler_policy.go new file mode 100644 index 0000000000..8743d03dce --- /dev/null +++ b/services/dsql/handler_policy.go @@ -0,0 +1,56 @@ +package dsql + +import ( + "encoding/json" + "net/http" + + "github.com/labstack/echo/v5" +) + +// dispatchPolicyOps handles cluster resource-policy operations. +func (h *Handler) dispatchPolicyOps(c *echo.Context, op, resource string, body []byte) (bool, error) { + switch op { + case opGetClusterPolicy: + return true, h.handleGetClusterPolicy(c, resource) + case opPutClusterPolicy: + return true, h.handlePutClusterPolicy(c, resource, body) + case opDeleteClusterPolicy: + return true, h.handleDeleteClusterPolicy(c, resource) + } + + return false, nil +} + +func (h *Handler) handleGetClusterPolicy(c *echo.Context, identifier string) error { + policy, err := h.Backend.GetClusterPolicy(identifier) + if err != nil { + return h.writeBackendError(c, err) + } + + return c.JSON(http.StatusOK, policyResponse(policy)) +} + +func (h *Handler) handlePutClusterPolicy(c *echo.Context, identifier string, body []byte) error { + var req putClusterPolicyRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeInvalidBody(c) + } + + policy, err := h.Backend.PutClusterPolicy(identifier, req.Policy, req.ExpectedPolicyVersion) + if err != nil { + return h.writeBackendError(c, err) + } + + return c.JSON(http.StatusOK, policyVersionResponse{PolicyVersion: policy.Version}) +} + +func (h *Handler) handleDeleteClusterPolicy(c *echo.Context, identifier string) error { + q := c.Request().URL.Query() + + policy, err := h.Backend.DeleteClusterPolicy(identifier, q.Get("expected-policy-version")) + if err != nil { + return h.writeBackendError(c, err) + } + + return c.JSON(http.StatusOK, policyVersionResponse{PolicyVersion: policy.Version}) +} diff --git a/services/dsql/handler_streams.go b/services/dsql/handler_streams.go new file mode 100644 index 0000000000..ea7031714b --- /dev/null +++ b/services/dsql/handler_streams.go @@ -0,0 +1,91 @@ +package dsql + +import ( + "encoding/json" + "net/http" + "strconv" + + "github.com/labstack/echo/v5" +) + +// dispatchStreamOps handles cluster change-data-capture stream operations. +func (h *Handler) dispatchStreamOps(c *echo.Context, op, resource string, body []byte) (bool, error) { + switch op { + case opCreateStream: + return true, h.handleCreateStream(c, resource, body) + case opGetStream: + return true, h.handleGetStream(c, resource) + case opDeleteStream: + return true, h.handleDeleteStream(c, resource) + case opListStreams: + return true, h.handleListStreams(c, resource) + } + + return false, nil +} + +func (h *Handler) handleCreateStream(c *echo.Context, clusterIdentifier string, body []byte) error { + var req createStreamRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeInvalidBody(c) + } + + in := CreateStreamInput{ + Format: req.Format, + Ordering: req.Ordering, + Tags: req.Tags, + Target: targetFromDTO(req.TargetDefinition), + } + + stream, err := h.Backend.CreateStream(clusterIdentifier, in) + if err != nil { + return h.writeBackendError(c, err) + } + + return c.JSON(http.StatusOK, streamResponseFromStream(stream, false)) +} + +func (h *Handler) handleGetStream(c *echo.Context, key string) error { + clusterIdentifier, streamIdentifier := splitStreamKey(key) + + stream, err := h.Backend.GetStream(clusterIdentifier, streamIdentifier) + if err != nil { + return h.writeBackendError(c, err) + } + + return c.JSON(http.StatusOK, streamResponseFromStream(stream, true)) +} + +func (h *Handler) handleDeleteStream(c *echo.Context, key string) error { + clusterIdentifier, streamIdentifier := splitStreamKey(key) + + stream, err := h.Backend.DeleteStream(clusterIdentifier, streamIdentifier) + if err != nil { + return h.writeBackendError(c, err) + } + + return c.JSON(http.StatusOK, deleteStreamResponseFromStream(stream)) +} + +func (h *Handler) handleListStreams(c *echo.Context, clusterIdentifier string) error { + q := c.Request().URL.Query() + + maxResults := 0 + if v := q.Get("max-results"); v != "" { + if n, err := strconv.Atoi(v); err == nil { + maxResults = n + } + } + + streams, next, err := h.Backend.ListStreams(clusterIdentifier, q.Get("next-token"), maxResults) + if err != nil { + return h.writeBackendError(c, err) + } + + summaries := make([]streamSummaryDTO, 0, len(streams)) + for _, s := range streams { + summaries = append(summaries, streamSummaryFromStream(s)) + } + + return c.JSON(http.StatusOK, listStreamsResponse{NextToken: next, Streams: summaries}) +} diff --git a/services/dsql/handler_tags.go b/services/dsql/handler_tags.go new file mode 100644 index 0000000000..16c71a726c --- /dev/null +++ b/services/dsql/handler_tags.go @@ -0,0 +1,54 @@ +package dsql + +import ( + "encoding/json" + "net/http" + + "github.com/labstack/echo/v5" +) + +// dispatchTagOps handles the generic ARN-scoped tagging trio. +func (h *Handler) dispatchTagOps(c *echo.Context, op, resource string, body []byte) (bool, error) { + switch op { + case opTagResource: + return true, h.handleTagResource(c, resource, body) + case opUntagResource: + return true, h.handleUntagResource(c, resource) + case opListTagsForResource: + return true, h.handleListTagsForResource(c, resource) + } + + return false, nil +} + +func (h *Handler) handleTagResource(c *echo.Context, resourceARN string, body []byte) error { + var req tagResourceRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeInvalidBody(c) + } + + if err := h.Backend.TagResource(resourceARN, req.Tags); err != nil { + return h.writeBackendError(c, err) + } + + return c.JSON(http.StatusOK, struct{}{}) +} + +func (h *Handler) handleUntagResource(c *echo.Context, resourceARN string) error { + tagKeys := c.Request().URL.Query()["tagKeys"] + + if err := h.Backend.UntagResource(resourceARN, tagKeys); err != nil { + return h.writeBackendError(c, err) + } + + return c.JSON(http.StatusOK, struct{}{}) +} + +func (h *Handler) handleListTagsForResource(c *echo.Context, resourceARN string) error { + tags, err := h.Backend.ListTagsForResource(resourceARN) + if err != nil { + return h.writeBackendError(c, err) + } + + return c.JSON(http.StatusOK, listTagsForResourceResponse{Tags: tags}) +} diff --git a/services/dsql/handler_test.go b/services/dsql/handler_test.go new file mode 100644 index 0000000000..8bbd5d8ec0 --- /dev/null +++ b/services/dsql/handler_test.go @@ -0,0 +1,73 @@ +package dsql_test + +import ( + "net/http/httptest" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awscfg "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + dsqlsdk "github.com/aws/aws-sdk-go-v2/service/dsql" + "github.com/aws/smithy-go" + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/dsql" +) + +const ( + testRegion = "us-east-1" + testAccountID = "123456789012" + + waitTimeout = 5 * time.Second + pollInterval = 50 * time.Millisecond +) + +// assertAPIErrorCode fails the test unless err is a smithy API error with the given code. +func assertAPIErrorCode(t *testing.T, err error, code string) { + t.Helper() + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, code, apiErr.ErrorCode()) +} + +// newTestClient stands up the real aws-sdk-go-v2 dsql client against an +// httptest server running this package's Handler, wired through the same +// pkgs/service registry/router used in production. +func newTestClient(t *testing.T, h *dsql.Handler) *dsqlsdk.Client { + t.Helper() + + e := echo.New() + registry := service.NewRegistry() + require.NoError(t, registry.Register(h)) + e.Use(service.NewServiceRouter(registry).RouteHandler()) + + srv := httptest.NewServer(e) + t.Cleanup(srv.Close) + + cfg, err := awscfg.LoadDefaultConfig( + t.Context(), + awscfg.WithRegion(testRegion), + awscfg.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider("test", "test", ""), + ), + ) + require.NoError(t, err) + + return dsqlsdk.NewFromConfig(cfg, func(o *dsqlsdk.Options) { + o.BaseEndpoint = aws.String(srv.URL) + }) +} + +func newTestHandler() *dsql.Handler { + backend := dsql.NewInMemoryBackend() + h := dsql.NewHandler(backend) + h.AccountID = testAccountID + h.DefaultRegion = testRegion + + return h +} diff --git a/services/dsql/interfaces.go b/services/dsql/interfaces.go new file mode 100644 index 0000000000..6a747ab270 --- /dev/null +++ b/services/dsql/interfaces.go @@ -0,0 +1,57 @@ +package dsql + +// StorageBackend is the interface for the Aurora DSQL backend. +type StorageBackend interface { + CreateCluster(accountID, region string, in CreateClusterInput) (*Cluster, error) + GetCluster(identifier string) (*Cluster, error) + ListClusters(nextToken string, maxResults int) ([]*Cluster, string, error) + UpdateCluster(identifier string, in UpdateClusterInput) (*Cluster, error) + DeleteCluster(identifier string) (*Cluster, error) + + GetClusterPolicy(identifier string) (*ClusterPolicy, error) + PutClusterPolicy(identifier, policy, expectedVersion string) (*ClusterPolicy, error) + DeleteClusterPolicy(identifier, expectedVersion string) (*ClusterPolicy, error) + + GetVpcEndpointServiceName(identifier, region string) (serviceName, clusterVpcEndpoint string, err error) + + CreateStream(clusterIdentifier string, in CreateStreamInput) (*Stream, error) + GetStream(clusterIdentifier, streamIdentifier string) (*Stream, error) + DeleteStream(clusterIdentifier, streamIdentifier string) (*Stream, error) + ListStreams(clusterIdentifier, nextToken string, maxResults int) ([]*Stream, string, error) + + TagResource(resourceARN string, tags map[string]string) error + UntagResource(resourceARN string, tagKeys []string) error + ListTagsForResource(resourceARN string) (map[string]string, error) + + Reset() +} + +// CreateClusterInput carries CreateCluster's optional fields. +type CreateClusterInput struct { + MultiRegion *MultiRegionProperties + Tags map[string]string + Policy string + KmsEncryptionKey string + DeletionProtectionEnabled bool +} + +// UpdateClusterInput carries UpdateCluster's optional fields. A nil pointer +// means "leave unchanged"; KmsEncryptionKey uses the empty string to mean +// "unchanged" and the reserved value "AWS_OWNED_KMS_KEY" to mean "revert to +// the AWS owned key", matching the real API's documented semantics. +type UpdateClusterInput struct { + MultiRegion *MultiRegionProperties + DeletionProtectionEnabled *bool + KmsEncryptionKey string +} + +// CreateStreamInput carries CreateStream's fields. +type CreateStreamInput struct { + Target *StreamTarget + Tags map[string]string + Format string + Ordering string +} + +// Compile-time assertion that InMemoryBackend implements StorageBackend. +var _ StorageBackend = (*InMemoryBackend)(nil) diff --git a/services/dsql/models.go b/services/dsql/models.go new file mode 100644 index 0000000000..2966a220d5 --- /dev/null +++ b/services/dsql/models.go @@ -0,0 +1,141 @@ +package dsql + +import ( + "maps" + "slices" + "time" +) + +// Cluster status values, matching aws-sdk-go-v2/service/dsql/types.ClusterStatus. +const ( + statusCreating = "CREATING" + statusActive = "ACTIVE" + statusUpdating = "UPDATING" + statusDeleting = "DELETING" +) + +// Stream status values, matching aws-sdk-go-v2/service/dsql/types.StreamStatus. +const ( + streamStatusCreating = "CREATING" + streamStatusActive = "ACTIVE" +) + +const ( + encryptionTypeAWSOwned = "AWS_OWNED_KMS_KEY" + encryptionTypeCustomer = "CUSTOMER_MANAGED_KMS_KEY" + encryptionStatusEnabled = "ENABLED" + validationReasonLockedOut = "deletionProtectionEnabled" + validationReasonFieldError = "fieldValidationFailed" + validationReasonOther = "other" +) + +// MultiRegionProperties mirrors types.MultiRegionProperties. +type MultiRegionProperties struct { + WitnessRegion string + Clusters []string +} + +func (m *MultiRegionProperties) clone() *MultiRegionProperties { + if m == nil { + return nil + } + + cp := *m + cp.Clusters = slices.Clone(m.Clusters) + + return &cp +} + +// ClusterPolicy is a resource-based policy attached to a cluster. +type ClusterPolicy struct { + Policy string + Version string +} + +// Cluster is the persisted representation of an Aurora DSQL cluster. +type Cluster struct { + CreationTime time.Time + PendingUntil time.Time + Policy *ClusterPolicy + MultiRegion *MultiRegionProperties + Tags map[string]string + Identifier string + ARN string + Endpoint string + Status string + KmsEncryptionKey string + AccountID string + Region string + DeletionProtectionEnabled bool +} + +func (c *Cluster) clone() *Cluster { + if c == nil { + return nil + } + + cp := *c + cp.Tags = make(map[string]string, len(c.Tags)) + maps.Copy(cp.Tags, c.Tags) + cp.MultiRegion = c.MultiRegion.clone() + + if c.Policy != nil { + p := *c.Policy + cp.Policy = &p + } + + return &cp +} + +func (c *Cluster) encryptionType() string { + if c.KmsEncryptionKey == "" { + return encryptionTypeAWSOwned + } + + return encryptionTypeCustomer +} + +func (c *Cluster) kmsKeyARN() string { + if c.KmsEncryptionKey == "" { + return "" + } + + return c.KmsEncryptionKey +} + +// StreamTarget mirrors types.TargetDefinitionMemberKinesis. +type StreamTarget struct { + RoleArn string + StreamArn string +} + +// Stream is the persisted representation of an Aurora DSQL change-data-capture stream. +type Stream struct { + CreationTime time.Time + PendingUntil time.Time + Target *StreamTarget + Tags map[string]string + ClusterIdentifier string + StreamIdentifier string + ARN string + Status string + Format string + Ordering string +} + +func (s *Stream) clone() *Stream { + if s == nil { + return nil + } + + cp := *s + cp.Tags = make(map[string]string, len(s.Tags)) + maps.Copy(cp.Tags, s.Tags) + + if s.Target != nil { + t := *s.Target + cp.Target = &t + } + + return &cp +} diff --git a/services/dsql/persistence.go b/services/dsql/persistence.go new file mode 100644 index 0000000000..f5c26dac25 --- /dev/null +++ b/services/dsql/persistence.go @@ -0,0 +1,99 @@ +package dsql + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/persistence" +) + +// ErrNoSnapshot is returned when a backend does not support snapshot/restore. +var ErrNoSnapshot = errors.New("backend does not support restore") + +// Snapshottable is an optional interface a StorageBackend may implement to +// support snapshot/restore for persistence or test isolation. +type Snapshottable interface { + Snapshot(ctx context.Context) []byte + Restore(context.Context, []byte) error +} + +// dsqlSnapshotVersion identifies the shape of [backendSnapshot]. Bump it +// whenever a change would make an older snapshot unsafe to decode as the +// current shape; Restore discards (rather than partially decodes) any mismatch. +const dsqlSnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the backend. Tables +// holds one JSON-encoded array per registered table name (clusters, streams +// -- see store_setup.go), produced by b.registry.SnapshotAll(). +type backendSnapshot struct { + Tables map[string]json.RawMessage `json:"tables"` + Version int `json:"version"` +} + +// Snapshot serializes backend state to JSON. +func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { + b.mu.RLock("Snapshot") + defer b.mu.RUnlock() + + tables, err := b.registry.SnapshotAll() + if err != nil { + logger.Load(ctx).WarnContext(ctx, "dsql: snapshot table marshal failed", "error", err) + + return nil + } + + snap := backendSnapshot{Version: dsqlSnapshotVersion, Tables: tables} + + return persistence.MarshalSnapshot(ctx, dsqlServiceName, &snap) +} + +// Restore deserializes backend state from a JSON snapshot. +func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { + var snap backendSnapshot + + if err := persistence.UnmarshalSnapshot(ctx, dsqlServiceName, data, &snap); err != nil { + return err + } + + b.mu.Lock("Restore") + defer b.mu.Unlock() + + if snap.Version != dsqlSnapshotVersion { + logger.Load(ctx).WarnContext(ctx, + "dsql: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", dsqlSnapshotVersion) + + b.registry.ResetAll() + + return nil + } + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("dsql: restore snapshot tables: %w", err) + } + + return nil +} + +// Snapshot implements persistence by delegating to the backend if it supports it. +func (h *Handler) Snapshot(ctx context.Context) []byte { + s, ok := h.Backend.(Snapshottable) + if !ok { + return nil + } + + return s.Snapshot(ctx) +} + +// Restore implements persistence by delegating to the backend if it supports it. +func (h *Handler) Restore(ctx context.Context, data []byte) error { + s, ok := h.Backend.(Snapshottable) + if !ok { + return ErrNoSnapshot + } + + return s.Restore(ctx, data) +} diff --git a/services/dsql/policy.go b/services/dsql/policy.go new file mode 100644 index 0000000000..9840154454 --- /dev/null +++ b/services/dsql/policy.go @@ -0,0 +1,73 @@ +package dsql + +// GetClusterPolicy returns a cluster's resource-based policy. +func (b *InMemoryBackend) GetClusterPolicy(identifier string) (*ClusterPolicy, error) { + b.mu.Lock("GetClusterPolicy") + defer b.mu.Unlock() + + c, err := b.resolveClusterLocked(identifier) + if err != nil { + return nil, err + } + + if c.Policy == nil { + return nil, ErrPolicyNotFound + } + + p := *c.Policy + + return &p, nil +} + +// PutClusterPolicy creates or replaces a cluster's resource-based policy. If +// expectedVersion is non-empty it must match the current policy version +// (optimistic concurrency), matching PutClusterPolicyInput's +// expectedPolicyVersion semantics. +func (b *InMemoryBackend) PutClusterPolicy(identifier, policy, expectedVersion string) (*ClusterPolicy, error) { + if policy == "" { + return nil, ErrValidation + } + + b.mu.Lock("PutClusterPolicy") + defer b.mu.Unlock() + + c, err := b.resolveClusterLocked(identifier) + if err != nil { + return nil, err + } + + if expectedVersion != "" && (c.Policy == nil || c.Policy.Version != expectedVersion) { + return nil, ErrPolicyVersionMismatch + } + + c.Policy = &ClusterPolicy{Policy: policy, Version: newVersionToken()} + + p := *c.Policy + + return &p, nil +} + +// DeleteClusterPolicy removes a cluster's resource-based policy. If +// expectedVersion is non-empty it must match the current policy version. +func (b *InMemoryBackend) DeleteClusterPolicy(identifier, expectedVersion string) (*ClusterPolicy, error) { + b.mu.Lock("DeleteClusterPolicy") + defer b.mu.Unlock() + + c, err := b.resolveClusterLocked(identifier) + if err != nil { + return nil, err + } + + if c.Policy == nil { + return nil, ErrPolicyNotFound + } + + if expectedVersion != "" && c.Policy.Version != expectedVersion { + return nil, ErrPolicyVersionMismatch + } + + p := *c.Policy + c.Policy = nil + + return &p, nil +} diff --git a/services/dsql/policy_test.go b/services/dsql/policy_test.go new file mode 100644 index 0000000000..6c5fb34af8 --- /dev/null +++ b/services/dsql/policy_test.go @@ -0,0 +1,122 @@ +package dsql_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + dsqlsdk "github.com/aws/aws-sdk-go-v2/service/dsql" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testPolicyDoc = `{"Version":"2012-10-17","Statement":[` + + `{"Effect":"Allow","Principal":"*","Action":"dsql:DbConnect","Resource":"*"}]}` + +func TestGetClusterPolicy_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + + _, err = client.GetClusterPolicy(ctx, &dsqlsdk.GetClusterPolicyInput{Identifier: created.Identifier}) + require.Error(t, err) + assertAPIErrorCode(t, err, "ResourceNotFoundException") +} + +func TestPutAndGetClusterPolicy(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + + putOut, err := client.PutClusterPolicy(ctx, &dsqlsdk.PutClusterPolicyInput{ + Identifier: created.Identifier, + Policy: aws.String(testPolicyDoc), + }) + require.NoError(t, err) + assert.NotEmpty(t, aws.ToString(putOut.PolicyVersion)) + + getOut, err := client.GetClusterPolicy(ctx, &dsqlsdk.GetClusterPolicyInput{Identifier: created.Identifier}) + require.NoError(t, err) + assert.JSONEq(t, testPolicyDoc, aws.ToString(getOut.Policy)) + assert.Equal(t, aws.ToString(putOut.PolicyVersion), aws.ToString(getOut.PolicyVersion)) +} + +func TestPutClusterPolicy_ExpectedVersionMismatch(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + + _, err = client.PutClusterPolicy(ctx, &dsqlsdk.PutClusterPolicyInput{ + Identifier: created.Identifier, + Policy: aws.String(testPolicyDoc), + }) + require.NoError(t, err) + + _, err = client.PutClusterPolicy(ctx, &dsqlsdk.PutClusterPolicyInput{ + Identifier: created.Identifier, + Policy: aws.String(testPolicyDoc), + ExpectedPolicyVersion: aws.String("stale-version"), + }) + require.Error(t, err) + assertAPIErrorCode(t, err, "ConflictException") +} + +func TestDeleteClusterPolicy(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + + putOut, err := client.PutClusterPolicy(ctx, &dsqlsdk.PutClusterPolicyInput{ + Identifier: created.Identifier, + Policy: aws.String(testPolicyDoc), + }) + require.NoError(t, err) + + _, err = client.DeleteClusterPolicy(ctx, &dsqlsdk.DeleteClusterPolicyInput{ + Identifier: created.Identifier, + ExpectedPolicyVersion: putOut.PolicyVersion, + }) + require.NoError(t, err) + + _, err = client.GetClusterPolicy(ctx, &dsqlsdk.GetClusterPolicyInput{Identifier: created.Identifier}) + require.Error(t, err) + assertAPIErrorCode(t, err, "ResourceNotFoundException") +} + +func TestDeleteClusterPolicy_ExpectedVersionMismatch(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + + _, err = client.PutClusterPolicy(ctx, &dsqlsdk.PutClusterPolicyInput{ + Identifier: created.Identifier, + Policy: aws.String(testPolicyDoc), + }) + require.NoError(t, err) + + _, err = client.DeleteClusterPolicy(ctx, &dsqlsdk.DeleteClusterPolicyInput{ + Identifier: created.Identifier, + ExpectedPolicyVersion: aws.String("stale-version"), + }) + require.Error(t, err) + assertAPIErrorCode(t, err, "ConflictException") +} diff --git a/services/dsql/provider.go b/services/dsql/provider.go new file mode 100644 index 0000000000..289df6c3db --- /dev/null +++ b/services/dsql/provider.go @@ -0,0 +1,23 @@ +package dsql + +import "github.com/blackbirdworks/gopherstack/pkgs/service" + +// Provider implements service.Provider for the Aurora DSQL service. +type Provider struct{} + +// Name returns the provider name. +func (p *Provider) Name() string { return "DSQL" } + +// Init initializes the Aurora DSQL service backend and handler. +// +//nolint:ireturn,nolintlint // architecturally required to return interface +func (p *Provider) Init(ctx *service.AppContext) (service.Registerable, error) { + accountID, region := service.AccountRegionOrDefault(ctx) + + backend := NewInMemoryBackend() + handler := NewHandler(backend) + handler.AccountID = accountID + handler.DefaultRegion = region + + return handler, nil +} diff --git a/services/dsql/routes.go b/services/dsql/routes.go new file mode 100644 index 0000000000..213ec90bf8 --- /dev/null +++ b/services/dsql/routes.go @@ -0,0 +1,150 @@ +package dsql + +import ( + "net/http" + "net/url" + "strings" +) + +const ( + pathClusterRoot = "/cluster" + pathClusterPrefix = "/cluster/" + pathClustersPrefix = "/clusters/" // plural: GetVpcEndpointServiceName only + pathTagsPrefix = "/tags/" + pathStreamPrefix = "/stream/" + + policySuffix = "/policy" + vpcEndpointServiceNameSuffix = "/vpc-endpoint-service-name" +) + +// Operation names, matching the AWS API exactly. +const ( + opCreateCluster = "CreateCluster" + opGetCluster = "GetCluster" + opListClusters = "ListClusters" + opUpdateCluster = "UpdateCluster" + opDeleteCluster = "DeleteCluster" + + opGetClusterPolicy = "GetClusterPolicy" + opPutClusterPolicy = "PutClusterPolicy" + opDeleteClusterPolicy = "DeleteClusterPolicy" + + opGetVpcEndpointServiceName = "GetVpcEndpointServiceName" + + opTagResource = "TagResource" + opUntagResource = "UntagResource" + opListTagsForResource = "ListTagsForResource" + + opCreateStream = "CreateStream" + opGetStream = "GetStream" + opDeleteStream = "DeleteStream" + opListStreams = "ListStreams" +) + +// parseDSQLPath parses an HTTP method + path into an operation name and a +// resource identifier: a cluster identifier, a "clusterId/streamId" +// composite (see streamKey), or a resource ARN for the /tags/ family. +func parseDSQLPath(method, path string) (string, string) { + switch { + case path == pathClusterRoot || path == pathClusterRoot+"/": + return parseClusterRoot(method) + case strings.HasPrefix(path, pathClusterPrefix): + return parseClusterResource(method, path[len(pathClusterPrefix):]) + case strings.HasPrefix(path, pathClustersPrefix): + return parseVpcEndpointServiceName(method, path[len(pathClustersPrefix):]) + case strings.HasPrefix(path, pathTagsPrefix): + return parseTagsResource(method, path[len(pathTagsPrefix):]) + case strings.HasPrefix(path, pathStreamPrefix): + return parseStreamResource(method, path[len(pathStreamPrefix):]) + } + + return "", "" +} + +func parseClusterRoot(method string) (string, string) { + switch method { + case http.MethodGet: + return opListClusters, "" + case http.MethodPost: + return opCreateCluster, "" + } + + return "", "" +} + +// parseClusterResource routes /cluster/{id} and /cluster/{id}/policy paths. +func parseClusterResource(method, remainder string) (string, string) { + if id, ok := strings.CutSuffix(remainder, policySuffix); ok { + switch method { + case http.MethodGet: + return opGetClusterPolicy, id + case http.MethodPost: + return opPutClusterPolicy, id + case http.MethodDelete: + return opDeleteClusterPolicy, id + } + + return "", "" + } + + switch method { + case http.MethodGet: + return opGetCluster, remainder + case http.MethodPost: + return opUpdateCluster, remainder + case http.MethodDelete: + return opDeleteCluster, remainder + } + + return "", "" +} + +func parseVpcEndpointServiceName(method, remainder string) (string, string) { + id, ok := strings.CutSuffix(remainder, vpcEndpointServiceNameSuffix) + if !ok || method != http.MethodGet { + return "", "" + } + + return opGetVpcEndpointServiceName, id +} + +func parseTagsResource(method, remainder string) (string, string) { + decoded, _ := url.PathUnescape(remainder) + + switch method { + case http.MethodGet: + return opListTagsForResource, decoded + case http.MethodPost: + return opTagResource, decoded + case http.MethodDelete: + return opUntagResource, decoded + } + + return "", "" +} + +// parseStreamResource routes /stream/{clusterId} (list/create) and +// /stream/{clusterId}/{streamId} (get/delete) paths. +func parseStreamResource(method, remainder string) (string, string) { + clusterID, streamID, hasStream := strings.Cut(remainder, "/") + + if !hasStream { + switch method { + case http.MethodGet: + return opListStreams, clusterID + case http.MethodPost: + return opCreateStream, clusterID + } + + return "", "" + } + + switch method { + case http.MethodGet: + return opGetStream, streamKey(clusterID, streamID) + case http.MethodDelete: + return opDeleteStream, streamKey(clusterID, streamID) + } + + return "", "" +} diff --git a/services/dsql/store.go b/services/dsql/store.go new file mode 100644 index 0000000000..4bec44463b --- /dev/null +++ b/services/dsql/store.go @@ -0,0 +1,214 @@ +package dsql + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "strings" + "time" + + "github.com/blackbirdworks/gopherstack/pkgs/arn" + "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +const ( + dsqlServiceName = "dsql" + + // clusterActivationDelay/clusterDeletionDelay bound how long a cluster + // stays CREATING/UPDATING or DELETING before this backend lazily + // advances it to ACTIVE or removes it on the next read -- see + // PARITY.md's items_still_open for why this is a lazy deadline rather + // than a background reconciler. + clusterActivationDelay = 750 * time.Millisecond + clusterDeletionDelay = 750 * time.Millisecond + streamActivationDelay = 500 * time.Millisecond + + maxClustersPerAccountRegion = 20 + maxStreamsPerCluster = 20 + maxTagsPerResource = 50 + + identifierLength = 26 + identifierAlpha = "abcdefghijklmnopqrstuvwxyz0123456789" +) + +// InMemoryBackend is the in-memory implementation of StorageBackend. +type InMemoryBackend struct { + clusters *store.Table[Cluster] + streams *store.Table[Stream] + registry *store.Registry + mu *lockmetrics.RWMutex +} + +// NewInMemoryBackend creates a new in-memory Aurora DSQL backend. +func NewInMemoryBackend() *InMemoryBackend { + b := &InMemoryBackend{ + registry: store.NewRegistry(), + mu: lockmetrics.New(dsqlServiceName), + } + + registerAllTables(b) + + return b +} + +// Reset clears all backend state. +func (b *InMemoryBackend) Reset() { + b.mu.Lock("Reset") + defer b.mu.Unlock() + + b.registry.ResetAll() +} + +// newIdentifier returns a random lowercase-alphanumeric identifier matching +// the shape of a real DSQL cluster/stream identifier (documented as an +// opaque generated ID, not caller-supplied). +func newIdentifier() string { + buf := make([]byte, identifierLength) + if _, err := rand.Read(buf); err != nil { + // crypto/rand.Read failing is not something callers can recover from + // meaningfully; fall back to a hex timestamp so the backend never + // panics on a degraded entropy source. + return hex.EncodeToString([]byte(fmt.Sprintf("%x", time.Now().UnixNano())))[:identifierLength] + } + + out := make([]byte, identifierLength) + for i, v := range buf { + out[i] = identifierAlpha[int(v)%len(identifierAlpha)] + } + + return string(out) +} + +// newVersionToken returns a short random hex token used for cluster policy +// optimistic-concurrency versions. +func newVersionToken() string { + buf := make([]byte, versionTokenBytes) + if _, err := rand.Read(buf); err != nil { + return hex.EncodeToString([]byte(fmt.Sprintf("%x", time.Now().UnixNano()))) + } + + return hex.EncodeToString(buf) +} + +const versionTokenBytes = 8 + +func clusterARN(region, accountID, identifier string) string { + return arn.Build(dsqlServiceName, region, accountID, "cluster/"+identifier) +} + +func clusterEndpoint(identifier, region string) string { + return fmt.Sprintf("%s.dsql.%s.on.aws", identifier, region) +} + +func streamARN(region, accountID, clusterIdentifier, streamIdentifier string) string { + return arn.Build(dsqlServiceName, region, accountID, + fmt.Sprintf("cluster/%s/stream/%s", clusterIdentifier, streamIdentifier)) +} + +// streamKey returns the composite primary key for the streams table: stream +// identifiers are only unique within their owning cluster. +func streamKey(clusterIdentifier, streamIdentifier string) string { + return clusterIdentifier + "/" + streamIdentifier +} + +// splitStreamKey is the inverse of streamKey. +func splitStreamKey(key string) (string, string) { + clusterIdentifier, streamIdentifier, _ := strings.Cut(key, "/") + + return clusterIdentifier, streamIdentifier +} + +func validateTags(tags map[string]string) error { + if len(tags) > maxTagsPerResource { + return ErrValidation + } + + for k := range tags { + if k == "" { + return ErrValidation + } + } + + return nil +} + +// resolveClusterLocked returns the live (mutable) cluster for identifier +// after applying any due lazy status transition. Callers must hold b.mu for +// writing. A cluster whose DELETING deadline has passed is removed from the +// table and reported as not found, matching real AWS once deletion completes. +func (b *InMemoryBackend) resolveClusterLocked(identifier string) (*Cluster, error) { + c, ok := b.clusters.Get(identifier) + if !ok { + return nil, ErrClusterNotFound + } + + b.advanceClusterLocked(c) + + if c.Status == statusDeleting && time.Now().After(c.PendingUntil) { + b.clusters.Delete(identifier) + + return nil, ErrClusterNotFound + } + + return c, nil +} + +// advanceClusterLocked flips a CREATING/UPDATING cluster to ACTIVE once its +// PendingUntil deadline has passed. Callers must hold b.mu for writing. +func (b *InMemoryBackend) advanceClusterLocked(c *Cluster) { + if c.PendingUntil.IsZero() || time.Now().Before(c.PendingUntil) { + return + } + + switch c.Status { + case statusCreating, statusUpdating: + c.Status = statusActive + c.PendingUntil = time.Time{} + } +} + +// resolveStreamLocked returns the live (mutable) stream, applying any due +// lazy activation. Callers must hold b.mu for writing. +func (b *InMemoryBackend) resolveStreamLocked(clusterIdentifier, streamIdentifier string) (*Stream, error) { + s, ok := b.streams.Get(streamKey(clusterIdentifier, streamIdentifier)) + if !ok { + return nil, ErrStreamNotFound + } + + b.advanceStreamLocked(s) + + return s, nil +} + +func (b *InMemoryBackend) advanceStreamLocked(s *Stream) { + if s.PendingUntil.IsZero() || time.Now().Before(s.PendingUntil) { + return + } + + if s.Status == streamStatusCreating { + s.Status = streamStatusActive + s.PendingUntil = time.Time{} + } +} + +// clusterIdentifierFromResourceARN extracts the cluster identifier from a +// DSQL cluster ARN (arn:{partition}:dsql:{region}:{account}:cluster/{id}), +// used by TagResource/UntagResource/ListTagsForResource, which key off an +// ARN rather than a bare identifier. +func clusterIdentifierFromResourceARN(resourceARN string) (string, bool) { + // arn:partition:service:region:account:resource + parts := strings.SplitN(resourceARN, ":", arnPartsCount) + if len(parts) != arnPartsCount || parts[2] != dsqlServiceName { + return "", false + } + + name, ok := strings.CutPrefix(parts[5], "cluster/") + if !ok { + return "", false + } + + return name, true +} + +const arnPartsCount = 6 diff --git a/services/dsql/store_setup.go b/services/dsql/store_setup.go new file mode 100644 index 0000000000..fea93ab2ba --- /dev/null +++ b/services/dsql/store_setup.go @@ -0,0 +1,15 @@ +package dsql + +import "github.com/blackbirdworks/gopherstack/pkgs/store" + +func clusterKeyFn(v *Cluster) string { return v.Identifier } + +func streamKeyFn(v *Stream) string { return streamKey(v.ClusterIdentifier, v.StreamIdentifier) } + +// registerAllTables registers every backend resource table exactly once. +// Must be called during construction only -- store.Register panics on a +// duplicate name. +func registerAllTables(b *InMemoryBackend) { + b.clusters = store.Register(b.registry, "clusters", store.New(clusterKeyFn)) + b.streams = store.Register(b.registry, "streams", store.New(streamKeyFn)) +} diff --git a/services/dsql/streams.go b/services/dsql/streams.go new file mode 100644 index 0000000000..c76df54772 --- /dev/null +++ b/services/dsql/streams.go @@ -0,0 +1,130 @@ +package dsql + +import ( + "maps" + "sort" + "time" + + "github.com/blackbirdworks/gopherstack/pkgs/page" +) + +// CreateStream creates a Kinesis change-data-capture stream on a cluster. +// New streams start CREATING and lazily transition to ACTIVE on the next +// read once streamActivationDelay elapses. +func (b *InMemoryBackend) CreateStream(clusterIdentifier string, in CreateStreamInput) (*Stream, error) { + if err := validateTags(in.Tags); err != nil { + return nil, err + } + + if in.Target == nil || in.Target.RoleArn == "" || in.Target.StreamArn == "" { + return nil, ErrValidation + } + + b.mu.Lock("CreateStream") + defer b.mu.Unlock() + + c, err := b.resolveClusterLocked(clusterIdentifier) + if err != nil { + return nil, err + } + + if b.countStreamsLocked(clusterIdentifier) >= maxStreamsPerCluster { + return nil, ErrStreamQuotaExceeded + } + + streamIdentifier := newIdentifier() + now := time.Now().UTC() + + tags := make(map[string]string, len(in.Tags)) + maps.Copy(tags, in.Tags) + + target := *in.Target + + s := &Stream{ + ClusterIdentifier: clusterIdentifier, + StreamIdentifier: streamIdentifier, + ARN: streamARN(c.Region, c.AccountID, clusterIdentifier, streamIdentifier), + Status: streamStatusCreating, + Format: in.Format, + Ordering: in.Ordering, + CreationTime: now, + PendingUntil: now.Add(streamActivationDelay), + Target: &target, + Tags: tags, + } + + b.streams.Put(s) + + return s.clone(), nil +} + +func (b *InMemoryBackend) countStreamsLocked(clusterIdentifier string) int { + n := 0 + + for _, s := range b.streams.All() { + if s.ClusterIdentifier == clusterIdentifier { + n++ + } + } + + return n +} + +// GetStream returns the current information about a stream. +func (b *InMemoryBackend) GetStream(clusterIdentifier, streamIdentifier string) (*Stream, error) { + b.mu.Lock("GetStream") + defer b.mu.Unlock() + + s, err := b.resolveStreamLocked(clusterIdentifier, streamIdentifier) + if err != nil { + return nil, err + } + + return s.clone(), nil +} + +// DeleteStream removes a stream immediately (real AWS transitions through +// DELETING, but nothing else in this backend observes a stream's +// intermediate delete state, so removing it synchronously here is +// behaviorally equivalent to any client that only checks for +// ResourceNotFoundException afterward). +func (b *InMemoryBackend) DeleteStream(clusterIdentifier, streamIdentifier string) (*Stream, error) { + b.mu.Lock("DeleteStream") + defer b.mu.Unlock() + + s, err := b.resolveStreamLocked(clusterIdentifier, streamIdentifier) + if err != nil { + return nil, err + } + + b.streams.Delete(streamKey(clusterIdentifier, streamIdentifier)) + + return s.clone(), nil +} + +// ListStreams returns a cluster's streams ordered by stream identifier, paginated by nextToken/maxResults. +func (b *InMemoryBackend) ListStreams(clusterIdentifier, nextToken string, maxResults int) ([]*Stream, string, error) { + b.mu.Lock("ListStreams") + defer b.mu.Unlock() + + if _, err := b.resolveClusterLocked(clusterIdentifier); err != nil { + return nil, "", err + } + + matched := make([]*Stream, 0) + + for _, s := range b.streams.All() { + if s.ClusterIdentifier != clusterIdentifier { + continue + } + + b.advanceStreamLocked(s) + matched = append(matched, s.clone()) + } + + sort.Slice(matched, func(i, j int) bool { return matched[i].StreamIdentifier < matched[j].StreamIdentifier }) + + p := page.New(matched, nextToken, maxResults, defaultListLimit) + + return p.Data, p.Next, nil +} diff --git a/services/dsql/streams_test.go b/services/dsql/streams_test.go new file mode 100644 index 0000000000..ca053d9b21 --- /dev/null +++ b/services/dsql/streams_test.go @@ -0,0 +1,202 @@ +package dsql_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + dsqlsdk "github.com/aws/aws-sdk-go-v2/service/dsql" + "github.com/aws/aws-sdk-go-v2/service/dsql/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testKinesisTarget() types.TargetDefinition { + return &types.TargetDefinitionMemberKinesis{ + Value: types.KinesisTargetDefinition{ + RoleArn: aws.String("arn:aws:iam::123456789012:role/dsql-stream-role"), + StreamArn: aws.String("arn:aws:kinesis:us-east-1:123456789012:stream/dsql-cdc"), + }, + } +} + +func TestCreateStream(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + cluster, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + + out, err := client.CreateStream(ctx, &dsqlsdk.CreateStreamInput{ + ClusterIdentifier: cluster.Identifier, + TargetDefinition: testKinesisTarget(), + Format: types.StreamFormatJson, + Ordering: types.StreamOrderingUnordered, + }) + require.NoError(t, err) + assert.Equal(t, aws.ToString(cluster.Identifier), aws.ToString(out.ClusterIdentifier)) + assert.Equal(t, types.StreamStatusCreating, out.Status) + assert.NotEmpty(t, aws.ToString(out.StreamIdentifier)) +} + +func TestCreateStream_ClusterNotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.CreateStream(t.Context(), &dsqlsdk.CreateStreamInput{ + ClusterIdentifier: aws.String("nope"), + TargetDefinition: testKinesisTarget(), + Format: types.StreamFormatJson, + Ordering: types.StreamOrderingUnordered, + }) + require.Error(t, err) + assertAPIErrorCode(t, err, "ResourceNotFoundException") +} + +func TestCreateStream_MissingTargetIsValidationError(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + cluster, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + + _, err = client.CreateStream(ctx, &dsqlsdk.CreateStreamInput{ClusterIdentifier: cluster.Identifier}) + require.Error(t, err) +} + +func TestGetStream(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + cluster, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + + created, err := client.CreateStream(ctx, &dsqlsdk.CreateStreamInput{ + ClusterIdentifier: cluster.Identifier, + TargetDefinition: testKinesisTarget(), + Format: types.StreamFormatJson, + Ordering: types.StreamOrderingUnordered, + Tags: map[string]string{"env": "test"}, + }) + require.NoError(t, err) + + out, err := client.GetStream(ctx, &dsqlsdk.GetStreamInput{ + ClusterIdentifier: cluster.Identifier, + StreamIdentifier: created.StreamIdentifier, + }) + require.NoError(t, err) + assert.Equal(t, aws.ToString(created.StreamIdentifier), aws.ToString(out.StreamIdentifier)) + assert.Equal(t, map[string]string{"env": "test"}, out.Tags) + require.NotNil(t, out.TargetDefinition) +} + +func TestGetStream_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + cluster, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + + _, err = client.GetStream(ctx, &dsqlsdk.GetStreamInput{ + ClusterIdentifier: cluster.Identifier, + StreamIdentifier: aws.String("nope"), + }) + require.Error(t, err) + assertAPIErrorCode(t, err, "ResourceNotFoundException") +} + +func TestListStreams(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + cluster, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + + const n = 3 + + for range n { + _, streamErr := client.CreateStream(ctx, &dsqlsdk.CreateStreamInput{ + ClusterIdentifier: cluster.Identifier, + TargetDefinition: testKinesisTarget(), + Format: types.StreamFormatJson, + Ordering: types.StreamOrderingUnordered, + }) + require.NoError(t, streamErr) + } + + out, err := client.ListStreams(ctx, &dsqlsdk.ListStreamsInput{ClusterIdentifier: cluster.Identifier}) + require.NoError(t, err) + assert.Len(t, out.Streams, n) +} + +func TestDeleteStream(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + cluster, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + + created, err := client.CreateStream(ctx, &dsqlsdk.CreateStreamInput{ + ClusterIdentifier: cluster.Identifier, + TargetDefinition: testKinesisTarget(), + Format: types.StreamFormatJson, + Ordering: types.StreamOrderingUnordered, + }) + require.NoError(t, err) + + _, err = client.DeleteStream(ctx, &dsqlsdk.DeleteStreamInput{ + ClusterIdentifier: cluster.Identifier, + StreamIdentifier: created.StreamIdentifier, + }) + require.NoError(t, err) + + _, err = client.GetStream(ctx, &dsqlsdk.GetStreamInput{ + ClusterIdentifier: cluster.Identifier, + StreamIdentifier: created.StreamIdentifier, + }) + require.Error(t, err) + assertAPIErrorCode(t, err, "ResourceNotFoundException") +} + +func TestCreateStream_QuotaExceeded(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + cluster, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{}) + require.NoError(t, err) + + const quota = 20 + + for range quota { + _, streamErr := client.CreateStream(ctx, &dsqlsdk.CreateStreamInput{ + ClusterIdentifier: cluster.Identifier, + TargetDefinition: testKinesisTarget(), + Format: types.StreamFormatJson, + Ordering: types.StreamOrderingUnordered, + }) + require.NoError(t, streamErr) + } + + _, err = client.CreateStream(ctx, &dsqlsdk.CreateStreamInput{ + ClusterIdentifier: cluster.Identifier, + TargetDefinition: testKinesisTarget(), + Format: types.StreamFormatJson, + Ordering: types.StreamOrderingUnordered, + }) + require.Error(t, err) + assertAPIErrorCode(t, err, "ServiceQuotaExceededException") +} diff --git a/services/dsql/tags.go b/services/dsql/tags.go new file mode 100644 index 0000000000..103246d100 --- /dev/null +++ b/services/dsql/tags.go @@ -0,0 +1,74 @@ +package dsql + +import "maps" + +// TagResource adds or replaces tags on a cluster identified by its ARN. +func (b *InMemoryBackend) TagResource(resourceARN string, tags map[string]string) error { + if err := validateTags(tags); err != nil { + return err + } + + identifier, ok := clusterIdentifierFromResourceARN(resourceARN) + if !ok { + return ErrValidation + } + + b.mu.Lock("TagResource") + defer b.mu.Unlock() + + c, err := b.resolveClusterLocked(identifier) + if err != nil { + return err + } + + if len(c.Tags)+len(tags) > maxTagsPerResource { + return ErrValidation + } + + maps.Copy(c.Tags, tags) + + return nil +} + +// UntagResource removes tags from a cluster by key. +func (b *InMemoryBackend) UntagResource(resourceARN string, tagKeys []string) error { + identifier, ok := clusterIdentifierFromResourceARN(resourceARN) + if !ok { + return ErrValidation + } + + b.mu.Lock("UntagResource") + defer b.mu.Unlock() + + c, err := b.resolveClusterLocked(identifier) + if err != nil { + return err + } + + for _, k := range tagKeys { + delete(c.Tags, k) + } + + return nil +} + +// ListTagsForResource returns all tags on a cluster. +func (b *InMemoryBackend) ListTagsForResource(resourceARN string) (map[string]string, error) { + identifier, ok := clusterIdentifierFromResourceARN(resourceARN) + if !ok { + return nil, ErrValidation + } + + b.mu.Lock("ListTagsForResource") + defer b.mu.Unlock() + + c, err := b.resolveClusterLocked(identifier) + if err != nil { + return nil, err + } + + out := make(map[string]string, len(c.Tags)) + maps.Copy(out, c.Tags) + + return out, nil +} diff --git a/services/dsql/tags_test.go b/services/dsql/tags_test.go new file mode 100644 index 0000000000..4b0d32a63a --- /dev/null +++ b/services/dsql/tags_test.go @@ -0,0 +1,65 @@ +package dsql_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + dsqlsdk "github.com/aws/aws-sdk-go-v2/service/dsql" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTagResourceListUntag(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + cluster, err := client.CreateCluster(ctx, &dsqlsdk.CreateClusterInput{Tags: map[string]string{"a": "1"}}) + require.NoError(t, err) + + _, err = client.TagResource(ctx, &dsqlsdk.TagResourceInput{ + ResourceArn: cluster.Arn, + Tags: map[string]string{"b": "2"}, + }) + require.NoError(t, err) + + listOut, err := client.ListTagsForResource(ctx, &dsqlsdk.ListTagsForResourceInput{ResourceArn: cluster.Arn}) + require.NoError(t, err) + assert.Equal(t, map[string]string{"a": "1", "b": "2"}, listOut.Tags) + + _, err = client.UntagResource(ctx, &dsqlsdk.UntagResourceInput{ + ResourceArn: cluster.Arn, + TagKeys: []string{"a"}, + }) + require.NoError(t, err) + + listOut, err = client.ListTagsForResource(ctx, &dsqlsdk.ListTagsForResourceInput{ResourceArn: cluster.Arn}) + require.NoError(t, err) + assert.Equal(t, map[string]string{"b": "2"}, listOut.Tags) +} + +func TestTagResource_ClusterNotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.TagResource(t.Context(), &dsqlsdk.TagResourceInput{ + ResourceArn: aws.String("arn:aws:dsql:us-east-1:123456789012:cluster/does-not-exist"), + Tags: map[string]string{"a": "1"}, + }) + require.Error(t, err) + assertAPIErrorCode(t, err, "ResourceNotFoundException") +} + +func TestListTagsForResource_InvalidARN(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.ListTagsForResource(t.Context(), &dsqlsdk.ListTagsForResourceInput{ + ResourceArn: aws.String("not-an-arn"), + }) + require.Error(t, err) + assertAPIErrorCode(t, err, "ValidationException") +} diff --git a/services/dsql/vpcendpoint.go b/services/dsql/vpcendpoint.go new file mode 100644 index 0000000000..781227b0f7 --- /dev/null +++ b/services/dsql/vpcendpoint.go @@ -0,0 +1,22 @@ +package dsql + +import "fmt" + +// GetVpcEndpointServiceName returns the VPC endpoint service name and +// cluster-specific VPC endpoint DNS name a client would use to reach the +// cluster privately. There is no real VPC/PrivateLink plane behind this +// emulator, so both values are wire-shaped but not backed by a functioning +// endpoint -- see PARITY.md. +func (b *InMemoryBackend) GetVpcEndpointServiceName(identifier, region string) (string, string, error) { + b.mu.Lock("GetVpcEndpointServiceName") + defer b.mu.Unlock() + + if _, err := b.resolveClusterLocked(identifier); err != nil { + return "", "", err + } + + serviceName := fmt.Sprintf("com.amazonaws.%s.dsql", region) + clusterVpcEndpoint := fmt.Sprintf("%s.vpce.dsql.%s.on.aws", identifier, region) + + return serviceName, clusterVpcEndpoint, nil +} diff --git a/services/dsql/wire.go b/services/dsql/wire.go new file mode 100644 index 0000000000..1ee38e6f4a --- /dev/null +++ b/services/dsql/wire.go @@ -0,0 +1,273 @@ +package dsql + +import "github.com/blackbirdworks/gopherstack/pkgs/awstime" + +// Wire DTOs for the Aurora DSQL REST-JSON control plane. Field names match +// the pinned aws-sdk-go-v2/service/dsql@v1.22.1 request/response snapshots +// exactly: unlike most restJson1 services in this repo, DSQL emits +// lowerCamelCase JSON field names (bypassPolicyLockoutSafetyCheck, +// clientToken, deletionProtectionEnabled, ...), confirmed against +// request_snapshot/*.snap and response_snapshot/*.snap in the module. + +type multiRegionPropertiesDTO struct { + WitnessRegion string `json:"witnessRegion,omitempty"` + Clusters []string `json:"clusters,omitempty"` +} + +func multiRegionToDTO(m *MultiRegionProperties) *multiRegionPropertiesDTO { + if m == nil { + return nil + } + + return &multiRegionPropertiesDTO{WitnessRegion: m.WitnessRegion, Clusters: m.Clusters} +} + +func multiRegionFromDTO(dto *multiRegionPropertiesDTO) *MultiRegionProperties { + if dto == nil { + return nil + } + + return &MultiRegionProperties{WitnessRegion: dto.WitnessRegion, Clusters: dto.Clusters} +} + +type encryptionDetailsDTO struct { + EncryptionStatus string `json:"encryptionStatus"` + EncryptionType string `json:"encryptionType"` + KmsKeyArn string `json:"kmsKeyArn,omitempty"` +} + +func encryptionDetailsFromCluster(c *Cluster) *encryptionDetailsDTO { + return &encryptionDetailsDTO{ + EncryptionStatus: encryptionStatusEnabled, + EncryptionType: c.encryptionType(), + KmsKeyArn: c.kmsKeyARN(), + } +} + +type createClusterRequest struct { + MultiRegionProperties *multiRegionPropertiesDTO `json:"multiRegionProperties,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + ClientToken string `json:"clientToken,omitempty"` + KmsEncryptionKey string `json:"kmsEncryptionKey,omitempty"` + Policy string `json:"policy,omitempty"` + BypassPolicyLockoutSafetyCheck bool `json:"bypassPolicyLockoutSafetyCheck,omitempty"` + DeletionProtectionEnabled bool `json:"deletionProtectionEnabled,omitempty"` +} + +type clusterResponse struct { + MultiRegionProperties *multiRegionPropertiesDTO `json:"multiRegionProperties,omitempty"` + EncryptionDetails *encryptionDetailsDTO `json:"encryptionDetails,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Arn string `json:"arn"` + Identifier string `json:"identifier"` + Status string `json:"status"` + Endpoint string `json:"endpoint,omitempty"` + CreationTime float64 `json:"creationTime"` + DeletionProtectionEnabled bool `json:"deletionProtectionEnabled"` +} + +func clusterResponseFromCluster(c *Cluster, includeTags bool) clusterResponse { + resp := clusterResponse{ + Arn: c.ARN, + CreationTime: awstime.Epoch(c.CreationTime), + DeletionProtectionEnabled: c.DeletionProtectionEnabled, + EncryptionDetails: encryptionDetailsFromCluster(c), + Endpoint: c.Endpoint, + Identifier: c.Identifier, + MultiRegionProperties: multiRegionToDTO(c.MultiRegion), + Status: c.Status, + } + + if includeTags { + resp.Tags = c.Tags + } + + return resp +} + +type updateClusterRequest struct { + MultiRegionProperties *multiRegionPropertiesDTO `json:"multiRegionProperties,omitempty"` + DeletionProtectionEnabled *bool `json:"deletionProtectionEnabled,omitempty"` + ClientToken string `json:"clientToken,omitempty"` + KmsEncryptionKey string `json:"kmsEncryptionKey,omitempty"` +} + +type updateOrDeleteClusterResponse struct { + Arn string `json:"arn"` + Identifier string `json:"identifier"` + Status string `json:"status"` + CreationTime float64 `json:"creationTime"` +} + +func updateOrDeleteResponseFromCluster(c *Cluster) updateOrDeleteClusterResponse { + return updateOrDeleteClusterResponse{ + Arn: c.ARN, + CreationTime: awstime.Epoch(c.CreationTime), + Identifier: c.Identifier, + Status: c.Status, + } +} + +type clusterSummaryDTO struct { + Arn string `json:"arn"` + Identifier string `json:"identifier"` +} + +type listClustersResponse struct { + NextToken string `json:"nextToken,omitempty"` + Clusters []clusterSummaryDTO `json:"clusters"` +} + +type getClusterPolicyResponse struct { + Policy string `json:"policy,omitempty"` + PolicyVersion string `json:"policyVersion,omitempty"` +} + +type putClusterPolicyRequest struct { + ClientToken string `json:"clientToken,omitempty"` + ExpectedPolicyVersion string `json:"expectedPolicyVersion,omitempty"` + Policy string `json:"policy"` + BypassPolicyLockoutSafetyCheck bool `json:"bypassPolicyLockoutSafetyCheck,omitempty"` +} + +type policyVersionResponse struct { + PolicyVersion string `json:"policyVersion,omitempty"` +} + +func policyResponse(p *ClusterPolicy) getClusterPolicyResponse { + return getClusterPolicyResponse{Policy: p.Policy, PolicyVersion: p.Version} +} + +type getVpcEndpointServiceNameResponse struct { + ClusterVpcEndpoint string `json:"clusterVpcEndpoint,omitempty"` + ServiceName string `json:"serviceName,omitempty"` +} + +type tagResourceRequest struct { + Tags map[string]string `json:"tags"` +} + +type listTagsForResourceResponse struct { + Tags map[string]string `json:"tags"` +} + +type kinesisTargetDefinitionDTO struct { + RoleArn string `json:"roleArn"` + StreamArn string `json:"streamArn"` +} + +type targetDefinitionDTO struct { + Kinesis *kinesisTargetDefinitionDTO `json:"kinesis,omitempty"` +} + +func targetToDTO(t *StreamTarget) *targetDefinitionDTO { + if t == nil { + return nil + } + + return &targetDefinitionDTO{Kinesis: &kinesisTargetDefinitionDTO{RoleArn: t.RoleArn, StreamArn: t.StreamArn}} +} + +func targetFromDTO(dto *targetDefinitionDTO) *StreamTarget { + if dto == nil || dto.Kinesis == nil { + return nil + } + + return &StreamTarget{RoleArn: dto.Kinesis.RoleArn, StreamArn: dto.Kinesis.StreamArn} +} + +type createStreamRequest struct { + TargetDefinition *targetDefinitionDTO `json:"targetDefinition"` + Tags map[string]string `json:"tags,omitempty"` + ClientToken string `json:"clientToken,omitempty"` + Format string `json:"format,omitempty"` + Ordering string `json:"ordering,omitempty"` +} + +type streamResponse struct { + TargetDefinition *targetDefinitionDTO `json:"targetDefinition,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Arn string `json:"arn"` + ClusterIdentifier string `json:"clusterIdentifier"` + StreamIdentifier string `json:"streamIdentifier"` + Status string `json:"status"` + Format string `json:"format,omitempty"` + Ordering string `json:"ordering,omitempty"` + CreationTime float64 `json:"creationTime"` +} + +func streamResponseFromStream(s *Stream, includeExtras bool) streamResponse { + resp := streamResponse{ + Arn: s.ARN, + ClusterIdentifier: s.ClusterIdentifier, + CreationTime: awstime.Epoch(s.CreationTime), + Format: s.Format, + Ordering: s.Ordering, + Status: s.Status, + StreamIdentifier: s.StreamIdentifier, + } + + if includeExtras { + resp.Tags = s.Tags + resp.TargetDefinition = targetToDTO(s.Target) + } + + return resp +} + +type updateOrDeleteStreamResponse struct { + Arn string `json:"arn"` + ClusterIdentifier string `json:"clusterIdentifier"` + StreamIdentifier string `json:"streamIdentifier"` + Status string `json:"status"` + CreationTime float64 `json:"creationTime"` +} + +func deleteStreamResponseFromStream(s *Stream) updateOrDeleteStreamResponse { + return updateOrDeleteStreamResponse{ + Arn: s.ARN, + ClusterIdentifier: s.ClusterIdentifier, + CreationTime: awstime.Epoch(s.CreationTime), + Status: s.Status, + StreamIdentifier: s.StreamIdentifier, + } +} + +type streamSummaryDTO struct { + Arn string `json:"arn"` + ClusterIdentifier string `json:"clusterIdentifier"` + StreamIdentifier string `json:"streamIdentifier"` + Status string `json:"status"` + CreationTime float64 `json:"creationTime"` +} + +func streamSummaryFromStream(s *Stream) streamSummaryDTO { + return streamSummaryDTO{ + Arn: s.ARN, + ClusterIdentifier: s.ClusterIdentifier, + CreationTime: awstime.Epoch(s.CreationTime), + Status: s.Status, + StreamIdentifier: s.StreamIdentifier, + } +} + +type listStreamsResponse struct { + NextToken string `json:"nextToken,omitempty"` + Streams []streamSummaryDTO `json:"streams"` +} + +type validationFieldDTO struct { + Message string `json:"message,omitempty"` + Name string `json:"name,omitempty"` +} + +type errorResponse struct { + Type string `json:"__type"` + Message string `json:"message,omitempty"` + Reason string `json:"reason,omitempty"` + ResourceID string `json:"resourceId,omitempty"` + ResourceType string `json:"resourceType,omitempty"` + ServiceCode string `json:"serviceCode,omitempty"` + QuotaCode string `json:"quotaCode,omitempty"` + FieldList []validationFieldDTO `json:"fieldList,omitempty"` +} diff --git a/services/dynamodb/PARITY.md b/services/dynamodb/PARITY.md index d3c8823b22..d5b5453eea 100644 --- a/services/dynamodb/PARITY.md +++ b/services/dynamodb/PARITY.md @@ -1,14 +1,21 @@ --- service: dynamodb sdk_module: aws-sdk-go-v2/service/dynamodb@v1.67.0 # version audited against (go.mod pin) -last_audit_commit: cd027034c # 2026-09-20 autoscaling-dynamodb-kms-and-cloudwatch terraform sweep: DisableKinesisStreamingDestination DISABLED-not-removed fix; prior: 176ddc764 -last_audit_date: 2026-09-20 # prior: 2026-09-19 -- manifest-harvest pass: fixed UpdateGlobalTableSettings autoscaling - # accept-and-drop gap and DisableKinesisStreamingDestination's never-echoed - # EnableKinesisStreamingConfiguration -- see global_table_settings_autoscaling/ - # kinesis_streaming_disable_echo families below. Did not re-litigate - # ConfirmRemoveSelfResourceAccess (no IAM evaluator, gopherstack-cu4g) or - # UpdateTableReplicaAutoScaling's ReplicaUpdates (no per-replica field to - # route into) -- both re-verified genuine in a prior pass. +last_audit_commit: e1e3f187f # 2026-09-26 global-tables-v2-autoscaling pass: ReplicaUpdates + AutoScalingRoleArn/ScalingPolicies; prior: cd027034c +last_audit_date: 2026-09-26 # prior: 2026-09-20 -- autoscaling-dynamodb-kms-and-cloudwatch terraform sweep: DisableKinesisStreamingDestination DISABLED-not-removed fix + # 2026-09-26 (this audit): UpdateTableReplicaAutoScaling's ReplicaUpdates + # (per-replica read-capacity + per-replica-per-GSI read-capacity) is now + # wired end to end (wire, backend, Describe echo) -- previously accepted + # nowhere on the wire, the exact "no per-replica field to route into" gap + # the 2026-09-20 audit re-verified genuine. AutoScalingRoleArn and + # ScalingPolicies (TargetTrackingScalingPolicyConfiguration) are now + # accepted, stored, and echoed back exactly as the caller supplied them + # (not fabricated -- no IAM/policy engine backs them). Also found and fixed + # PARITY.md staleness: the gopherstack-1vv2 items_still_open entry claiming + # ReplicaAutoScalingDescription.GlobalSecondaryIndexes was "never populated" + # was already false by the time of this audit -- a prior commit had already + # wired the per-GSI write-capacity echo into replicaAutoScalingDescriptionsRLocked + # without updating items_still_open (see autoscaling family below). overall: A # gopherstack-rkmp deep pass (this audit, 2026-08-14): struct-field-diffed every wire model against the pinned SDK (see Notes) and fixed 3 more wire drops -- Query/Scan AttributesToGet (undeclared, and even where declared elsewhere the projection resolver never consulted it for these two ops), GSI/LSI IndexArn (+GSI IndexSizeBytes/Backfilling), ListBackups BackupSummary.BackupSizeBytes. PARITY.md itself was stale by 6 commits (7a2189b06..bc2e6285a) before this update -- see Notes. CONFIRMED FIXED, previously an open gap here: GSI/LSI Query full-scan (17c0ac7a7 added real per-GSI/LSI indexes; gopherstack-anlc verified 4.8-5.0us flat vs 1.82-28.0ms before). gopherstack-lze5 (2026-08-14, follow-up pass): PutItem/UpdateItem/DeleteItem's legacy Expected/ConditionalOperator/AttributeUpdates parameters -- the conditional-check-bypass and no-op-write bugs -- are now FIXED by translation into the existing expr evaluator. gopherstack-yvs8 (2026-08-14, follow-up to lze5): Query/Scan's legacy KeyConditions/QueryFilter/ScanFilter -- the "ScanFilter/QueryFilter silently returns every item" and "KeyConditions silently dropped" failure modes -- are now FIXED the same way (translation into KeyConditionExpression/FilterExpression, reusing the existing evaluator paths); see gaps for the KeySchema-reordering writeup. ReturnConsumedCapacity=INDEXES dead code (gopherstack-glfv) also still open -- see gaps. protocol: json-1.0 (DynamoDB_20120810 targets) families: @@ -20,7 +27,7 @@ families: janitor_ttl: {status: ok, note: PROVEN batched-lock, ctx-cancel, quickselect eviction, ring-buffer compaction} datalayer: {status: ok, note: RE-AUDITED — ce30166a converted db.Tables/Backups/GlobalTables/exports/imports/streamARNIndex from raw maps to pkgs/store.Table+Index (composite key tableKey(region,name), region derived by parsing TableArn via tableRegion()). Verified every insertion site (CreateTable, RestoreTable, CreateGlobalTable replicas, cloneTableSchema, applyOneReplicaTableEntry) builds TableArn with the same region string used as the store key *before* Put, so tableRegion(t) round-trips correctly; TableArn is never mutated post-insert. No stale map-key leaks (tablesByRegion Index auto-empties groups on last delete, unlike the old per-region submap). Persistence snapshot reshaped map->sorted slice + added a schema version gate (old snapshots discarded cleanly on upgrade, matching the sqs/ec2 precedent) — intentional, not a parity bug.} admin_lists: {status: ok, note: gopherstack-6flj (2026-08-15) wrapper-key sweep of all 22 List+Describe+Get ops (ListBackups/ListContributorInsights/ListExports/ListGlobalTables/ListImports/ListTables/ListTagsOfResource, the 13 Describe* ops, GetItem, GetResourcePolicy) — every top-level wrapper key diffed field-by-field against its own api_op_*.go Output struct in the pinned aws-sdk-go-v2/service/dynamodb@v1.63.1 module cache; all correct, no wrong/silent-empty key found, no shared-converter cross-op mismatch (exportTableToPointInTimeOutput is legitimately shared by ExportTableToPointInTime/DescribeExport — both real Outputs are ExportDescription-only). One real gap found and fixed: DescribeContributorInsightsOutput.LastUpdateDateTime (deserializers.go:18441, epoch-seconds) was entirely unmodeled — the backend never tracked when contributor insights was last toggled. Fixed by adding Table.ContributorInsightsLastUpdate (set in setContributorInsightsLocked on every UpdateContributorInsights call) and emitting it only once non-zero (a never-toggled table reports it absent, matching AWS's own "populated once an action has occurred" behavior, not a fabricated zero time). See gaps for FailureException (same struct, correctly left unmodeled). Re-verified 2026-09-19 (over-wide-response sweep, gopherstack): this prior pass only diffed top-level wrapper keys; this pass diffed each List op's item shape member-by-member against dynamodb@v1.67.0 and confirmed all four already exact -- ListBackups' BackupSummary (backup_ops.go:187-198; BackupExpiryDateTime correctly absent, genuinely inapplicable since CreateBackup only ever produces BackupTypeUser backups, per the real API's own "applicable ... for backups created by AWS Backup" doc), ListContributorInsights' ContributorInsightsSummary (contributor_insights.go:164-168; IndexName correctly absent -- table-level summaries only, matching this backend's documented GSI-mirrors-table design), ListExports' ExportSummary (import_export_s3.go:995-1000), ListImports' ImportSummary (import_export_s3.go:700-709). Proven via TestListSummaryShapes (list_summary_shapes_test.go, real client, all four ops).} - autoscaling: {status: fixed, note: "2026-08-21 (gopherstack-1vv2, InMemoryDB receiver-scope sweep): UpdateTableReplicaAutoScaling built a brand-new autoScalingSettings from only the current call's fields and assigned it wholesale over table.AutoScaling. GlobalSecondaryIndexUpdates and ProvisionedWriteCapacityAutoScalingUpdate are independently optional on the real input (api_op_UpdateTableReplicaAutoScaling.go) -- a call updating only one GSI's auto scaling settings silently wiped a previously-set table-level write-capacity autoscaling config, and vice versa. Fixed: autoScalingSettingsFromInput -> mergeAutoScalingSettingsFromInput, which merges into the existing table.AutoScaling (creating one only if nil) instead of replacing it. TestUpdateTableReplicaAutoScaling_WriteAndGSIUpdatesDontClobberEachOther (autoscaling_status_agreement_internal_test.go), hand-verified to fail against unfixed code. Other InMemoryDB Update* methods checked in the same sweep (UpdateContinuousBackups/UpdateContributorInsights/UpdateGlobalTable/UpdateGlobalTableSettings/UpdateItem/UpdateKinesisStreamingDestination/UpdateTable/UpdateTimeToLive) already merge field-by-field or are single-scalar toggles -- no further bugs of this shape found. See gaps: GlobalSecondaryIndexes autoscaling settings are stored but never echoed back on ReplicaAutoScalingDescription (a separate, pre-existing accept-and-drop gap, not touched by this fix)."} + autoscaling: {status: fixed, note: "2026-08-21 (gopherstack-1vv2, InMemoryDB receiver-scope sweep): UpdateTableReplicaAutoScaling built a brand-new autoScalingSettings from only the current call's fields and assigned it wholesale over table.AutoScaling. GlobalSecondaryIndexUpdates and ProvisionedWriteCapacityAutoScalingUpdate are independently optional on the real input (api_op_UpdateTableReplicaAutoScaling.go) -- a call updating only one GSI's auto scaling settings silently wiped a previously-set table-level write-capacity autoscaling config, and vice versa. Fixed: autoScalingSettingsFromInput -> mergeAutoScalingSettingsFromInput, which merges into the existing table.AutoScaling (creating one only if nil) instead of replacing it. TestUpdateTableReplicaAutoScaling_WriteAndGSIUpdatesDontClobberEachOther (autoscaling_status_agreement_internal_test.go), hand-verified to fail against unfixed code. Other InMemoryDB Update* methods checked in the same sweep (UpdateContinuousBackups/UpdateContributorInsights/UpdateGlobalTable/UpdateGlobalTableSettings/UpdateItem/UpdateKinesisStreamingDestination/UpdateTable/UpdateTimeToLive) already merge field-by-field or are single-scalar toggles -- no further bugs of this shape found. GlobalSecondaryIndexes autoscaling settings being stored but never echoed on ReplicaAutoScalingDescription was fixed in a later, undated commit (confirmed by reading replicaAutoScalingDescriptionsRLocked, which already builds gsiDescriptions from table.AutoScaling.GlobalSecondaryIndexes) -- items_still_open still listed it as open until this audit corrected the staleness. 2026-09-26 (this pass, global-tables-v2-autoscaling): (1) ReplicaUpdates ([]types.ReplicaAutoScalingUpdate, RegionName + ReplicaProvisionedReadCapacityAutoScalingUpdate + ReplicaGlobalSecondaryIndexUpdates) is now accepted on the wire (handler_autoscaling.go's replicaAutoScalingUpdateWire), merged per-replica without clobbering other replicas (mergeReplicaAutoScalingFromUpdates, store.go's new Table.ReplicaAutoScaling map keyed by RegionName), validated (ResourceNotFoundException if the named region isn't one of the table's replicas, ValidationException if MinimumUnits>MaximumUnits), and echoed back as ReplicaProvisionedReadCapacityAutoScalingSettings + per-GSI ProvisionedReadCapacityAutoScalingSettings on both Update and Describe. (2) AutoScalingRoleArn and ScalingPolicyUpdate/ScalingPolicies (TargetTrackingScalingPolicyConfiguration: TargetValue/DisableScaleIn/ScaleInCooldown/ScaleOutCooldown) are now accepted, stored on autoScalingThroughput, and echoed back exactly as the caller supplied them on every AutoScalingSettingsDescription this package emits (table-level write, per-GSI write, per-replica read, per-replica-per-GSI read) -- an honest echo of the caller's own input, not fabrication: this backend still has no IAM-role or scaling-policy evaluation engine behind these values. (3) UpdateTableReplicaAutoScaling now validates BillingMode==PROVISIONED before accepting any actual settings change (ValidationException on a PAY_PER_REQUEST table; a bare TableName-only call, as used to refresh replica status, is exempt) -- own wording, disclosed: no verbatim AWS rejection string was found (see AutoScalingSettingsUpdate docs + Terraform/CDK issue reports establishing the underlying PROVISIONED-only constraint). Tests: autoscaling_replica_updates_test.go (real aws-sdk-go-v2 client, table-driven validation cases, ReplicaUpdates round-trip, unknown-region ResourceNotFoundException). 2026-09-26 (gopherstack-101r, applicationautoscaling cross-service wiring): the deferred cross-service decision above is now wired. services/applicationautoscaling (source: itself, not this package -- avoids the dynamodb<->applicationautoscaling import cycle) pushes RegisterScalableTarget/PutScalingPolicy(ServiceNamespace=dynamodb) straight into this table's own AutoScaling/ReplicaAutoScaling state via this package's own UpdateTableReplicaAutoScaling/DescribeTableReplicaAutoScaling (dynamodb remains the single source of truth; applicationautoscaling keeps no separate copy for the dynamodb namespace), so a target/policy registered through either API is immediately visible through the other. Also required here, found while making that wiring actually usable for the common case: DescribeTableReplicaAutoScaling/UpdateTableReplicaAutoScaling's Replicas list was empty for a plain (non-global-table) table -- table.Replicas (see buildReplicasExcluding in global_tables.go) intentionally excludes a table's own home region once real Global Tables replicas exist elsewhere, but was never populated with anything for a table that has no Global Tables replication at all, even though real DynamoDB reports exactly one Replicas entry (its own region) for a single-region table -- the dominant real-world case for Application Auto Scaling against DynamoDB, per Terraform's aws_appautoscaling_target/aws_appautoscaling_policy needing zero Global Tables involvement. Fixed with autoScalingReplicaEntries (autoscaling.go): synthesizes one virtual replica entry for the table's own home region (tableRegion(table)) when table.Replicas is empty, used by both replicaAutoScalingDescriptionsRLocked (Describe/Update's response) and tableHasReplicaRegion (ReplicaUpdates' region validation) -- confined to this file, does not touch table.Replicas itself or any other consumer (DescribeTable, ListGlobalTables, etc.). TestDescribeTableReplicaAutoScaling/DescribeTableReplicaAutoScaling_NoReplicas (backup_replica_test.go), which had asserted the old empty-Replicas behavior as correct, was updated to assert the single home-region entry instead. Not wired, disclosed (matches real AWS, confirmed via Application Auto Scaling's own docs + a real-account error transcript, not guessed): DeregisterScalableTarget does not clear this table's AutoScaling/ReplicaAutoScaling state, and deleting a table does not deregister its Application Auto Scaling scalable targets -- real AWS leaves both orphaned the same way (Application Auto Scaling's RegisterScalableTarget/DeregisterScalableTarget docs describe cleanup as the caller's own responsibility; see services/applicationautoscaling/PARITY.md for the full writeup and citations)."} global_table_settings_autoscaling: {status: fixed, note: "2026-08-23 (manifest-harvest pass): UpdateGlobalTableSettingsInput's GlobalTableProvisionedWriteCapacityAutoScalingSettingsUpdate, GlobalTableGlobalSecondaryIndexSettingsUpdate (global, not per-replica, per-GSI write autoscaling), ReplicaSettingsUpdate[].ReplicaProvisionedReadCapacityAutoScalingSettingsUpdate, and ReplicaGlobalSecondaryIndexSettingsUpdate[].ProvisionedReadCapacityAutoScalingSettingsUpdate (api_op_UpdateGlobalTableSettings.go, types.go:2891/2962/1881) were all accepted on the wire (handler_global_tables.go's updateGlobalTableSettingsInput had no struct fields for any of them) then silently dropped -- an accept-and-drop wire gap, same class as UpdateTableReplicaAutoScaling's pre-1vv2-fix clobber bug but never wired at all rather than clobbered. Fixed: StoredGlobalTable gained WriteCapacityAutoScaling/GSIWriteCapacityAutoScaling, StoredReplicaSettings/StoredReplicaGSISettings gained ReadCapacityAutoScaling, all reusing the existing autoScalingThroughput persisted shape and throughputFromUpdate/sdkAutoScalingSettingsDescription converters UpdateTableReplicaAutoScaling already has (autoscaling.go) -- no new evaluator. Both UpdateGlobalTableSettings and DescribeGlobalTableSettings now echo the same stored settings (global write-capacity autoscaling applies uniformly across replicas, matching how WriteCapacityUnits already does, since it is a global-table-level setting in the v1 API, not per-replica). Verified via TestGlobalTableSettings_AutoScaling, driven through the real aws-sdk-go-v2 client, hand-reverted (services/dynamodb/{global_tables,handler_global_tables,store}.go) to confirm it fails against unfixed code (nil ReplicaProvisionedWriteCapacityAutoScalingSettings), restored, md5sum identical. Additive-only struct fields; pkgs/persistence snapshot-version guard confirmed no bump needed."} kinesis_streaming_disable_echo: {status: fixed, note: "2026-08-23 (manifest-harvest pass): DisableKinesisStreamingDestinationOutput.EnableKinesisStreamingConfiguration (deserializers.go:18931 -- a real modeled response member on Disable despite its SDK doc comment reading 'the destination for the Kinesis streaming information that is being enabled', a codegen doc-comment artifact shared with Enable/Update, not evidence the field is request-only) was never populated; DisableKinesisStreamingDestination always returned it as nil/absent even though the backend already tracked the destination's precision (KinesisDestinationEntry.Precision) right up until deleting it. Fixed: removeKinesisDestinationLocked now returns the removed entry's precision, echoed back as EnableKinesisStreamingConfiguration (defaulting to MILLISECOND, matching Enable/Describe's existing default). Verified via TestDisableKinesisStreamingDestination_EchoesConfig, hand-reverted (kinesis_streaming.go, handler_kinesis_streaming.go) to confirm nil response before the fix, restored, md5sum identical."} pagination_sweep: {status: fixed, note: "2026-08-28/29 (wrapper-key-sweep-rds-cloudwatch-sqs-sns pagination pass): audited every List/Describe/Query/Scan op with a page-size + continuation member against the pinned SDK. ListGlobalTables' applyGlobalTableLimit only capped the page when the caller supplied an explicit Limit; an omitted Limit (ListGlobalTablesInput.Limit doc, api_op_ListGlobalTables.go:35, 'if the parameter is not specified, DynamoDB defaults to 100') returned every global table uncapped with no LastEvaluatedGlobalTableName. Fixed: applyGlobalTableLimit now falls back to defaultListGlobalTablesLimit=100. TestListGlobalTables_DefaultLimitPagination (wire_field_fixes_test.go) creates 105 global tables, drives the real SDK client through the full pagination loop with no Limit set, and asserts each page is <=100 and the union is exactly the 105 names with no duplicates; hand-reverted to confirm it fails against unfixed code (page of 105), restored. Everything else audited CORRECT: Query/Scan's Limit-as-items-examined + post-limit-filter + ExclusiveStartKey/LastEvaluatedKey semantics (item_ops_query.go/item_ops_scan.go) match AWS's own documented 'LastEvaluatedKey may be non-nil with nothing left to return' behavior -- collectQueryPage emits LastEvaluatedKey whenever the Limit boundary is hit, including on the true last item (no i # DynamoDB -**Parity grade: A** · SDK `aws-sdk-go-v2/service/dynamodb@v1.67.0` · last audited 2026-09-20 (`cd027034c`) · protocol json-1.0 (DynamoDB_20120810 targets) +**Parity grade: A** · SDK `aws-sdk-go-v2/service/dynamodb@v1.67.0` · last audited 2026-09-26 (`e1e3f187f`) · protocol json-1.0 (DynamoDB_20120810 targets) ## Coverage | Metric | Value | | --- | --- | | Feature families | 15 (15 ok) | -| Known gaps | 8 | +| Known gaps | 6 | | Deferred items | 2 | | Resource leaks | clean | ### Known gaps -- "2026-09-11 (gopherstack-l3vv part c, disclosed, not modeled): ReplicaProvisionedReadCapacityAutoScalingSettings/ ReplicaProvisionedWriteCapacityAutoScalingSettings (both top-level, via GlobalTableProvisionedWriteCapacityAutoScalingSettingsUpdate/ ReplicaProvisionedReadCapacityAutoScalingSettingsUpdate, and per-GSI) DO echo real MinimumUnits/MaximumUnits/AutoScalingDisabled (fixed 2026-08-23, see global_table_settings_autoscaling above, reusing autoscaling.go's autoScalingThroughput/sdkAutoScalingSettingsDescription), but the real AutoScalingSettingsDescription (dynamodb@v1.67.0 types.go) also carries AutoScalingRoleArn *string and ScalingPolicies []AutoScalingPolicyDescription (each a TargetTrackingScalingPolicyConfiguration with PredefinedMetricSpecification/TargetValue/Scale{In,Out}Cooldown/ DisableScaleIn) -- a real IAM-role-backed autoscaling policy object, not a throughput range. This backend tracks no such policy state anywhere for legacy v1 global tables (nor does the separate v2 UpdateTableReplicaAutoScaling path on Table.AutoScaling): AutoScalingRoleArn and ScalingPolicies are always left nil/empty on every AutoScalingSettingsDescription this package emits. Fabricating a role ARN or a policy list with no real policy engine behind it would violate the no-fabricated-data rule; left honestly absent, same category as the already-documented incremental-export and per-replica-autoscaling-via- ReplicaUpdates gaps -- a genuine feature gap, not a wire drop." -- "2026-08-21 (gopherstack-1vv2): ReplicaAutoScalingDescription.GlobalSecondaryIndexes (types.go:2642) is never populated by UpdateTableReplicaAutoScaling or DescribeTableReplicaAutoScaling -- replicaAutoScalingDescriptionsRLocked only ever echoes table-level Write settings per replica. Per-GSI autoscaling settings ARE stored (autoScalingSettings.GlobalSecondaryIndexes, now correctly merged rather than clobbered -- see autoscaling family) but a real client reading them back via Update or Describe always sees an empty list regardless of what was configured. Pre-existing, found while fixing the clobber bug above; not fixed here since it's an accept-and-drop wire gap, a different bug class from this pass's scope." - "2026-08-15 (gopherstack-6flj, disclosed, not fixed): DescribeContributorInsightsOutput.FailureException (types.FailureException{ExceptionName, ExceptionDescription}, api_op_DescribeContributorInsights.go) remains unmodeled. This backend's UpdateContributorInsights/DescribeContributorInsights never fail to enable/disable contributor insights (no IAM/service-limit failure model exists anywhere in this service), so there is no honest non-nil value to populate this field with -- always leaving it nil is the accurate representation, not a gap being papered over. LastUpdateDateTime (same struct) was the real, fixable gap and is now fixed -- see admin_lists family above." - "2026-08-14 (gopherstack-lze5, CORRECTNESS, PARTIALLY FIXED): Expected, ConditionalOperator, and AttributeUpdates (PutItem/UpdateItem/DeleteItem's legacy pre-expression parameters) are now implemented -- the conditional-check-bypass and no-op-write failure modes this issue was filed for. Fixed by translation, not a second evaluator: legacy_conditions.go converts each legacy Expected/Condition into an equivalent ConditionExpression fragment (aliased #name/:value placeholders synthesized per attribute, joined by ConditionalOperator's AND/OR, default AND -- see legacyConditionalJoiner) and each AttributeUpdates entry into an equivalent UpdateExpression fragment (PUT -> SET, DELETE w/o Value -> REMOVE, DELETE w/ a set Value -> DELETE, ADD -> ADD; action-semantics citations: types/types.go:197-269 AttributeValueUpdate doc), then hands the rewritten request to the SAME evaluator (services/dynamodb/expr, via the existing checkPutCondition/checkUpdateCondition/checkDeleteCondition/doUpdate) real PutItem/UpdateItem/DeleteItem already used for ConditionExpression/ UpdateExpression. ComparisonOperator set: EQ/NE/LE/LT/GE/GT/NOT_NULL/NULL/ CONTAINS/NOT_CONTAINS/BEGINS_WITH/IN/BETWEEN, all implemented (renderComparison, citing types/types.go:1279-1391 for operator semantics and arg counts). Expected's old Value/Exists style and its Value/Exists-vs-ComparisonOperator mutual exclusion cite types/types.go:1240-1256 verbatim. Mutual exclusion between legacy and expression parameters is enforced per-operation (any of Expected/ConditionalOperator/AttributeUpdates set alongside any of ConditionExpression/UpdateExpression -> ValidationException) -- this specific rejection is well-established real DynamoDB behavior but has no client-side SDK validation to cite a line number against, so the error wording is our own, not a verified verbatim AWS string. Tested driving the real aws-sdk-go-v2 client and asserting behaviour (ConditionalCheckFailedException + item unchanged on a failing Expected, ADD-on-number increments, ADD-on-set unions, DELETE-with-set-value subtracts, DELETE-without-value removes), not just call success -- legacy_conditional_params_test.go; each covered case was hand-verified to fail with unfixed code (e.g. 'An error is expected but got nil... expected: *types.ConditionalCheckFailedException'). - "2026-08-14 (gopherstack-rkmp/gopherstack-glfv, CORRECTNESS, flagged not fixed): ReturnConsumedCapacity=INDEXES never returns a per-index breakdown on any operation. capacity.go's buildConsumedCapacityWithIndexes/applyIndexBreakdowns correctly build types.ConsumedCapacity.Table/GlobalSecondaryIndexes/ LocalSecondaryIndexes and are unit-tested in isolation, but grep confirms they are called from nowhere except export_test.go -- every real operation (PutItem/UpdateItem/DeleteItem/Query/Scan/BatchGetItem/BatchWriteItem/ TransactGetItems/TransactWriteItems) builds a bare ConsumedCapacity{TableName, CapacityUnits, Read/WriteCapacityUnits} literal directly, so INDEXES and TOTAL produce byte-identical output everywhere. TestConsumedCapacityIndexes_PutItem is misleadingly named: despite the name and a GSI fixture, it actually requests TOTAL and never exercises the INDEXES path -- the same 'test looked like coverage and wasn't' pattern noted below for the pre-53cfd590b tests. Read-side fix (100% of RCU to the queried index) is straightforward; write-side fix (attributing WCU across every GSI/LSI a written item's key populates) needs AWS billing semantics not verified against a real account this pass, so it's flagged rather than guessed, per the no-fabrication rule." diff --git a/services/dynamodb/autoscaling.go b/services/dynamodb/autoscaling.go index bc7d46c8a7..54c009c75b 100644 --- a/services/dynamodb/autoscaling.go +++ b/services/dynamodb/autoscaling.go @@ -6,6 +6,7 @@ package dynamodb import ( "context" + "fmt" "sort" "github.com/aws/aws-sdk-go-v2/aws" @@ -58,9 +59,87 @@ func mergeAutoScalingSettingsFromInput( return s } +// mergeReplicaAutoScalingFromUpdates merges UpdateTableReplicaAutoScalingInput's +// ReplicaUpdates (types.ReplicaAutoScalingUpdate, keyed by RegionName) into +// existing, the same merge-not-replace treatment mergeAutoScalingSettingsFromInput +// gives table-level settings: a call updating one replica's read capacity must +// not disturb another replica's, or that replica's own GSI settings. +func mergeReplicaAutoScalingFromUpdates( + existing map[string]*replicaAutoScalingSettings, + updates []types.ReplicaAutoScalingUpdate, +) map[string]*replicaAutoScalingSettings { + if len(updates) == 0 { + return existing + } + + out := existing + if out == nil { + out = make(map[string]*replicaAutoScalingSettings, len(updates)) + } + + for _, u := range updates { + region := aws.ToString(u.RegionName) + if region == "" { + continue + } + + out[region] = mergeOneReplicaAutoScalingUpdate(out[region], u) + } + + return out +} + +// mergeOneReplicaAutoScalingUpdate merges a single ReplicaAutoScalingUpdate +// into rs (nil if this replica has never been updated before). +func mergeOneReplicaAutoScalingUpdate( + rs *replicaAutoScalingSettings, + u types.ReplicaAutoScalingUpdate, +) *replicaAutoScalingSettings { + if rs == nil { + rs = &replicaAutoScalingSettings{} + } + + if u.ReplicaProvisionedReadCapacityAutoScalingUpdate != nil { + rs.Read = throughputFromUpdate(u.ReplicaProvisionedReadCapacityAutoScalingUpdate) + } + + if len(u.ReplicaGlobalSecondaryIndexUpdates) > 0 { + rs.GlobalSecondaryIndexes = mergeReplicaGSIAutoScalingUpdates( + rs.GlobalSecondaryIndexes, + u.ReplicaGlobalSecondaryIndexUpdates, + ) + } + + return rs +} + +// mergeReplicaGSIAutoScalingUpdates merges a replica's per-GSI read-capacity +// updates into existing (nil if none stored yet). +func mergeReplicaGSIAutoScalingUpdates( + existing map[string]*autoScalingThroughput, + updates []types.ReplicaGlobalSecondaryIndexAutoScalingUpdate, +) map[string]*autoScalingThroughput { + out := existing + if out == nil { + out = make(map[string]*autoScalingThroughput, len(updates)) + } + + for _, g := range updates { + if g.IndexName == nil { + continue + } + out[*g.IndexName] = throughputFromUpdate(g.ProvisionedReadCapacityAutoScalingUpdate) + } + + return out +} + // throughputFromUpdate translates the SDK AutoScalingSettingsUpdate struct // into the persisted shape. Returns nil when no fields were supplied so the -// caller can distinguish "explicitly cleared" from "untouched". +// caller can distinguish "explicitly cleared" from "untouched". Echoes +// AutoScalingRoleArn/ScalingPolicyUpdate back exactly as supplied -- this +// emulator has no IAM-role or scaling-policy engine, so it is not fabricating +// a value, only round-tripping the caller's own input. func throughputFromUpdate(u *types.AutoScalingSettingsUpdate) *autoScalingThroughput { if u == nil { return nil @@ -69,30 +148,148 @@ func throughputFromUpdate(u *types.AutoScalingSettingsUpdate) *autoScalingThroug out := &autoScalingThroughput{ MinCapacity: u.MinimumUnits, MaxCapacity: u.MaximumUnits, + RoleArn: u.AutoScalingRoleArn, } if u.AutoScalingDisabled != nil { out.Disabled = *u.AutoScalingDisabled } - if u.ScalingPolicyUpdate != nil && - u.ScalingPolicyUpdate.TargetTrackingScalingPolicyConfiguration != nil { - out.TargetUtilizPct = u.ScalingPolicyUpdate.TargetTrackingScalingPolicyConfiguration.TargetValue + if u.ScalingPolicyUpdate != nil { + out.PolicyName = u.ScalingPolicyUpdate.PolicyName + if tt := u.ScalingPolicyUpdate.TargetTrackingScalingPolicyConfiguration; tt != nil { + out.TargetUtilizPct = tt.TargetValue + out.DisableScaleIn = tt.DisableScaleIn + out.ScaleInCooldown = tt.ScaleInCooldown + out.ScaleOutCooldown = tt.ScaleOutCooldown + } } return out } -// applyAutoScalingSettingsLocked sets table.AutoScaling from input and -// returns the table's name and status under a single defer-protected table.mu.Lock. +// validateAutoScalingSettingsUpdate rejects MinimumUnits > MaximumUnits when +// both are supplied. Real DynamoDB documents the two as independent bounds +// (API_AutoScalingSettingsUpdate.html) but publishes no verbatim rejection +// string for an inverted range; this wording is our own, disclosed the same +// way as this file's other undocumented-error-text validations. +func validateAutoScalingSettingsUpdate(u *types.AutoScalingSettingsUpdate) error { + if u == nil { + return nil + } + if u.MinimumUnits != nil && u.MaximumUnits != nil && *u.MinimumUnits > *u.MaximumUnits { + return NewValidationException("MinimumUnits must be less than or equal to MaximumUnits") + } + + return nil +} + +// autoScalingUpdateRequestsSettings reports whether input asks to change any +// autoscaling configuration at all. A bare TableName (as a real client sends +// to refresh replica status) must not trip the PROVISIONED-only gate below. +func autoScalingUpdateRequestsSettings(input *dynamodb.UpdateTableReplicaAutoScalingInput) bool { + return input.ProvisionedWriteCapacityAutoScalingUpdate != nil || + len(input.GlobalSecondaryIndexUpdates) > 0 || + len(input.ReplicaUpdates) > 0 +} + +// validateAutoScalingUpdateInput checks every AutoScalingSettingsUpdate the +// input carries -- table-level, per-GSI, per-replica, and per-replica-per-GSI. +func validateAutoScalingUpdateInput(input *dynamodb.UpdateTableReplicaAutoScalingInput) error { + if err := validateAutoScalingSettingsUpdate(input.ProvisionedWriteCapacityAutoScalingUpdate); err != nil { + return err + } + + for _, g := range input.GlobalSecondaryIndexUpdates { + if err := validateAutoScalingSettingsUpdate(g.ProvisionedWriteCapacityAutoScalingUpdate); err != nil { + return err + } + } + + for _, r := range input.ReplicaUpdates { + if err := validateAutoScalingSettingsUpdate(r.ReplicaProvisionedReadCapacityAutoScalingUpdate); err != nil { + return err + } + for _, g := range r.ReplicaGlobalSecondaryIndexUpdates { + if err := validateAutoScalingSettingsUpdate(g.ProvisionedReadCapacityAutoScalingUpdate); err != nil { + return err + } + } + } + + return nil +} + +// tableHasReplicaRegion also accepts the home region when table has no +// explicit replicas (see autoScalingReplicaEntries). Callers must hold table.mu. +func tableHasReplicaRegion(table *Table, region string) bool { + if len(table.Replicas) == 0 { + return region == tableRegion(table) + } + + for _, r := range table.Replicas { + if r.RegionName == region { + return true + } + } + + return false +} + +// replicaStatusEntry is one (region, status) row for autoscaling reporting. +type replicaStatusEntry struct { + region string + status string +} + +// table.Replicas excludes the home region once real replicas exist but is +// empty for a plain table; real AWS still reports that region, so synthesize it. +func autoScalingReplicaEntries(table *Table) []replicaStatusEntry { + if len(table.Replicas) == 0 { + return []replicaStatusEntry{{region: tableRegion(table), status: table.Status}} + } + + entries := make([]replicaStatusEntry, len(table.Replicas)) + for i, r := range table.Replicas { + entries[i] = replicaStatusEntry{region: r.RegionName, status: r.ReplicaStatus} + } + + return entries +} + +// applyAutoScalingSettingsLocked validates and applies input under a single +// defer-protected table.mu.Lock, returning the table's name and status. func applyAutoScalingSettingsLocked( table *Table, input *dynamodb.UpdateTableReplicaAutoScalingInput, -) (string, string) { +) (string, string, error) { table.mu.Lock("UpdateTableReplicaAutoScaling") defer table.mu.Unlock() + if autoScalingUpdateRequestsSettings(input) && isOnDemandTable(table.BillingMode) { + return "", "", NewValidationException( + "AutoScaling is not available for tables with PAY_PER_REQUEST billing mode", + ) + } + + if err := validateAutoScalingUpdateInput(input); err != nil { + return "", "", err + } + + for _, r := range input.ReplicaUpdates { + region := aws.ToString(r.RegionName) + if region != "" && !tableHasReplicaRegion(table, region) { + return "", "", NewResourceNotFoundException( + fmt.Sprintf("Replica not found for region: %s", region), + ) + } + } + table.AutoScaling = mergeAutoScalingSettingsFromInput(table.AutoScaling, input) + table.ReplicaAutoScaling = mergeReplicaAutoScalingFromUpdates( + table.ReplicaAutoScaling, + input.ReplicaUpdates, + ) - return table.Name, table.Status + return table.Name, table.Status, nil } // --- UpdateTableReplicaAutoScaling --- @@ -112,7 +309,11 @@ func (db *InMemoryDB) UpdateTableReplicaAutoScaling( return nil, err } - tableName, _ := applyAutoScalingSettingsLocked(table, input) + tableName, _, applyErr := applyAutoScalingSettingsLocked(table, input) + if applyErr != nil { + return nil, applyErr + } + tableStatus, replicaDescs := replicaAutoScalingDescriptionsRLocked(table) return &dynamodb.UpdateTableReplicaAutoScalingOutput{ @@ -126,65 +327,119 @@ func (db *InMemoryDB) UpdateTableReplicaAutoScaling( // sdkAutoScalingSettingsDescription converts a persisted autoScalingThroughput // into the SDK description type, or nil if t is nil (no settings configured). -func sdkAutoScalingSettingsDescription(t *autoScalingThroughput) *types.AutoScalingSettingsDescription { +// AutoScalingRoleArn and ScalingPolicies are echoed back exactly as the +// client supplied them on the matching Update call. +func sdkAutoScalingSettingsDescription( + t *autoScalingThroughput, +) *types.AutoScalingSettingsDescription { if t == nil { return nil } disabled := t.Disabled - return &types.AutoScalingSettingsDescription{ + desc := &types.AutoScalingSettingsDescription{ MinimumUnits: t.MinCapacity, MaximumUnits: t.MaxCapacity, AutoScalingDisabled: &disabled, + AutoScalingRoleArn: t.RoleArn, } + + if t.TargetUtilizPct != nil { + desc.ScalingPolicies = []types.AutoScalingPolicyDescription{ + { + PolicyName: t.PolicyName, + TargetTrackingScalingPolicyConfiguration: &types.AutoScalingTargetTrackingScalingPolicyConfigurationDescription{ + TargetValue: t.TargetUtilizPct, + DisableScaleIn: t.DisableScaleIn, + ScaleInCooldown: t.ScaleInCooldown, + ScaleOutCooldown: t.ScaleOutCooldown, + }, + }, + } + } + + return desc } // --- DescribeTableReplicaAutoScaling --- -// replicaAutoScalingDescriptionsRLocked copies table.Status and table.Replicas, -// along with the table's write-capacity autoscaling settings (applied -// uniformly to every replica -- this emulator doesn't model per-replica -// overrides), into the SDK description type under a defer-protected -// table.mu.RLock. -func replicaAutoScalingDescriptionsRLocked(table *Table) (string, []types.ReplicaAutoScalingDescription) { +// buildReplicaGSIAutoScalingDescriptions merges per-index write settings +// (table-wide, from table.AutoScaling.GlobalSecondaryIndexes) with per-index +// read settings (per-replica, from table.ReplicaAutoScaling[region]) into one +// sorted ReplicaGlobalSecondaryIndexAutoScalingDescription list. +func buildReplicaGSIAutoScalingDescriptions( + write, read map[string]*types.AutoScalingSettingsDescription, +) []types.ReplicaGlobalSecondaryIndexAutoScalingDescription { + if len(write) == 0 && len(read) == 0 { + return nil + } + + names := make(map[string]struct{}, len(write)+len(read)) + for name := range write { + names[name] = struct{}{} + } + for name := range read { + names[name] = struct{}{} + } + + out := make([]types.ReplicaGlobalSecondaryIndexAutoScalingDescription, 0, len(names)) + for name := range names { + idxName := name + out = append(out, types.ReplicaGlobalSecondaryIndexAutoScalingDescription{ + IndexName: &idxName, + IndexStatus: types.IndexStatusActive, + ProvisionedWriteCapacityAutoScalingSettings: write[name], + ProvisionedReadCapacityAutoScalingSettings: read[name], + }) + } + sort.Slice(out, func(i, j int) bool { return *out[i].IndexName < *out[j].IndexName }) + + return out +} + +// replicaAutoScalingDescriptionsRLocked copies table.Status, its replica +// regions (see autoScalingReplicaEntries), and their write/read settings. +func replicaAutoScalingDescriptionsRLocked( + table *Table, +) (string, []types.ReplicaAutoScalingDescription) { table.mu.RLock(opDescribeTableReplicaAutoScaling) defer table.mu.RUnlock() var write *types.AutoScalingSettingsDescription - var gsiDescriptions []types.ReplicaGlobalSecondaryIndexAutoScalingDescription + gsiWrite := map[string]*types.AutoScalingSettingsDescription{} if table.AutoScaling != nil { write = sdkAutoScalingSettingsDescription(table.AutoScaling.Write) - if len(table.AutoScaling.GlobalSecondaryIndexes) > 0 { - gsiDescriptions = make( - []types.ReplicaGlobalSecondaryIndexAutoScalingDescription, - 0, - len(table.AutoScaling.GlobalSecondaryIndexes), - ) - for name, throughput := range table.AutoScaling.GlobalSecondaryIndexes { - idxName := name - gsiDescriptions = append(gsiDescriptions, types.ReplicaGlobalSecondaryIndexAutoScalingDescription{ - IndexName: &idxName, - IndexStatus: types.IndexStatusActive, - ProvisionedWriteCapacityAutoScalingSettings: sdkAutoScalingSettingsDescription(throughput), - }) - } - sort.Slice(gsiDescriptions, func(i, j int) bool { - return *gsiDescriptions[i].IndexName < *gsiDescriptions[j].IndexName - }) + for name, throughput := range table.AutoScaling.GlobalSecondaryIndexes { + gsiWrite[name] = sdkAutoScalingSettingsDescription(throughput) } } - replicas := make([]types.ReplicaAutoScalingDescription, 0, len(table.Replicas)) - for _, r := range table.Replicas { - region := r.RegionName - status := r.ReplicaStatus + entries := autoScalingReplicaEntries(table) + replicas := make([]types.ReplicaAutoScalingDescription, 0, len(entries)) + + for _, e := range entries { + region := e.region + status := e.status + + var read *types.AutoScalingSettingsDescription + gsiRead := map[string]*types.AutoScalingSettingsDescription{} + if rs := table.ReplicaAutoScaling[region]; rs != nil { + read = sdkAutoScalingSettingsDescription(rs.Read) + for name, throughput := range rs.GlobalSecondaryIndexes { + gsiRead[name] = sdkAutoScalingSettingsDescription(throughput) + } + } replicas = append(replicas, types.ReplicaAutoScalingDescription{ RegionName: ®ion, ReplicaStatus: types.ReplicaStatus(status), ReplicaProvisionedWriteCapacityAutoScalingSettings: write, - GlobalSecondaryIndexes: gsiDescriptions, + ReplicaProvisionedReadCapacityAutoScalingSettings: read, + GlobalSecondaryIndexes: buildReplicaGSIAutoScalingDescriptions( + gsiWrite, + gsiRead, + ), }) } diff --git a/services/dynamodb/autoscaling_replica_updates_test.go b/services/dynamodb/autoscaling_replica_updates_test.go new file mode 100644 index 0000000000..907205643a --- /dev/null +++ b/services/dynamodb/autoscaling_replica_updates_test.go @@ -0,0 +1,276 @@ +package dynamodb_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + sdk "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/dynamodb" +) + +// TestUpdateTableReplicaAutoScaling_ScalingPolicyAndRoleArn_SurvivesWireConversion +// verifies AutoScalingRoleArn and ScalingPolicyUpdate (a real, wire-serialized +// input member per api_op_UpdateTableReplicaAutoScaling.go's +// AutoScalingSettingsUpdate) round-trip through Update and Describe as +// AutoScalingRoleArn/ScalingPolicies on the response -- echoed exactly as the +// caller supplied them, not fabricated (this backend has no IAM/scaling-policy +// engine behind them). +func TestUpdateTableReplicaAutoScaling_ScalingPolicyAndRoleArn_SurvivesWireConversion(t *testing.T) { + t.Parallel() + + client := newTestDynamoDBClient(t, dynamodb.NewHandler(dynamodb.NewInMemoryDB())) + + _, err := client.CreateGlobalTable(t.Context(), &sdk.CreateGlobalTableInput{ + GlobalTableName: aws.String("gt-policy-table"), + ReplicationGroup: []types.Replica{ + {RegionName: aws.String("us-east-1")}, + {RegionName: aws.String("eu-west-1")}, + }, + }) + require.NoError(t, err) + + out, err := client.UpdateTableReplicaAutoScaling(t.Context(), &sdk.UpdateTableReplicaAutoScalingInput{ + TableName: aws.String("gt-policy-table"), + ProvisionedWriteCapacityAutoScalingUpdate: &types.AutoScalingSettingsUpdate{ + MinimumUnits: aws.Int64(5), + MaximumUnits: aws.Int64(500), + AutoScalingRoleArn: aws.String("arn:aws:iam::123456789012:role/DynamoDBAutoscaleRole"), + ScalingPolicyUpdate: &types.AutoScalingPolicyUpdate{ + PolicyName: aws.String("my-write-policy"), + TargetTrackingScalingPolicyConfiguration: &types.AutoScalingTargetTrackingScalingPolicyConfigurationUpdate{ + TargetValue: aws.Float64(70.0), + DisableScaleIn: aws.Bool(true), + ScaleInCooldown: aws.Int32(60), + ScaleOutCooldown: aws.Int32(30), + }, + }, + }, + }) + require.NoError(t, err) + require.Len(t, out.TableAutoScalingDescription.Replicas, 1) + + settings := out.TableAutoScalingDescription.Replicas[0].ReplicaProvisionedWriteCapacityAutoScalingSettings + require.NotNil(t, settings) + assert.Equal(t, "arn:aws:iam::123456789012:role/DynamoDBAutoscaleRole", aws.ToString(settings.AutoScalingRoleArn)) + require.Len(t, settings.ScalingPolicies, 1) + policy := settings.ScalingPolicies[0] + assert.Equal(t, "my-write-policy", aws.ToString(policy.PolicyName)) + require.NotNil(t, policy.TargetTrackingScalingPolicyConfiguration) + assert.InDelta(t, 70.0, aws.ToFloat64(policy.TargetTrackingScalingPolicyConfiguration.TargetValue), 0.001) + assert.True(t, aws.ToBool(policy.TargetTrackingScalingPolicyConfiguration.DisableScaleIn)) + assert.Equal(t, int32(60), aws.ToInt32(policy.TargetTrackingScalingPolicyConfiguration.ScaleInCooldown)) + assert.Equal(t, int32(30), aws.ToInt32(policy.TargetTrackingScalingPolicyConfiguration.ScaleOutCooldown)) + + desc, err := client.DescribeTableReplicaAutoScaling(t.Context(), &sdk.DescribeTableReplicaAutoScalingInput{ + TableName: aws.String("gt-policy-table"), + }) + require.NoError(t, err) + require.Len(t, desc.TableAutoScalingDescription.Replicas, 1) + + descSettings := desc.TableAutoScalingDescription.Replicas[0].ReplicaProvisionedWriteCapacityAutoScalingSettings + require.NotNil(t, descSettings) + assert.Equal( + t, + "arn:aws:iam::123456789012:role/DynamoDBAutoscaleRole", + aws.ToString(descSettings.AutoScalingRoleArn), + "AutoScalingRoleArn must also survive on DescribeTableReplicaAutoScaling", + ) + require.Len(t, descSettings.ScalingPolicies, 1) + assert.Equal(t, "my-write-policy", aws.ToString(descSettings.ScalingPolicies[0].PolicyName)) +} + +// TestUpdateTableReplicaAutoScaling_ReplicaUpdates_ReadCapacity_RoundTrip +// verifies ReplicaUpdates -- previously entirely dropped at the wire layer +// (handler_autoscaling.go's updateTableReplicaAutoScalingInput declared no +// field for it) -- now reaches the backend and is reflected back as +// ReplicaProvisionedReadCapacityAutoScalingSettings on that replica alone. +func TestUpdateTableReplicaAutoScaling_ReplicaUpdates_ReadCapacity_RoundTrip(t *testing.T) { + t.Parallel() + + client := newTestDynamoDBClient(t, dynamodb.NewHandler(dynamodb.NewInMemoryDB())) + + _, err := client.CreateGlobalTable(t.Context(), &sdk.CreateGlobalTableInput{ + GlobalTableName: aws.String("gt-replica-read"), + ReplicationGroup: []types.Replica{ + {RegionName: aws.String("us-east-1")}, + {RegionName: aws.String("eu-west-1")}, + {RegionName: aws.String("ap-southeast-2")}, + }, + }) + require.NoError(t, err) + + out, err := client.UpdateTableReplicaAutoScaling(t.Context(), &sdk.UpdateTableReplicaAutoScalingInput{ + TableName: aws.String("gt-replica-read"), + ReplicaUpdates: []types.ReplicaAutoScalingUpdate{ + { + RegionName: aws.String("eu-west-1"), + ReplicaProvisionedReadCapacityAutoScalingUpdate: &types.AutoScalingSettingsUpdate{ + MinimumUnits: aws.Int64(3), + MaximumUnits: aws.Int64(300), + }, + ReplicaGlobalSecondaryIndexUpdates: []types.ReplicaGlobalSecondaryIndexAutoScalingUpdate{ + { + IndexName: aws.String("gsi-1"), + ProvisionedReadCapacityAutoScalingUpdate: &types.AutoScalingSettingsUpdate{ + MinimumUnits: aws.Int64(1), + MaximumUnits: aws.Int64(100), + }, + }, + }, + }, + }, + }) + require.NoError(t, err) + + byRegion := replicaAutoScalingByRegion(out.TableAutoScalingDescription.Replicas) + + untouched := byRegion["ap-southeast-2"] + require.NotNil(t, untouched) + assert.Nil(t, untouched.ReplicaProvisionedReadCapacityAutoScalingSettings, + "a replica not named in ReplicaUpdates must not get another replica's read settings") + + euWest := byRegion["eu-west-1"] + require.NotNil(t, euWest) + require.NotNil(t, euWest.ReplicaProvisionedReadCapacityAutoScalingSettings) + assert.Equal(t, int64(3), aws.ToInt64(euWest.ReplicaProvisionedReadCapacityAutoScalingSettings.MinimumUnits)) + assert.Equal(t, int64(300), aws.ToInt64(euWest.ReplicaProvisionedReadCapacityAutoScalingSettings.MaximumUnits)) + require.Len(t, euWest.GlobalSecondaryIndexes, 1) + assert.Equal(t, "gsi-1", aws.ToString(euWest.GlobalSecondaryIndexes[0].IndexName)) + require.NotNil(t, euWest.GlobalSecondaryIndexes[0].ProvisionedReadCapacityAutoScalingSettings) + assert.Equal( + t, + int64(1), + aws.ToInt64(euWest.GlobalSecondaryIndexes[0].ProvisionedReadCapacityAutoScalingSettings.MinimumUnits), + ) + + desc, err := client.DescribeTableReplicaAutoScaling(t.Context(), &sdk.DescribeTableReplicaAutoScalingInput{ + TableName: aws.String("gt-replica-read"), + }) + require.NoError(t, err) + + descByRegion := replicaAutoScalingByRegion(desc.TableAutoScalingDescription.Replicas) + descEuWest := descByRegion["eu-west-1"] + require.NotNil(t, descEuWest) + require.NotNil(t, descEuWest.ReplicaProvisionedReadCapacityAutoScalingSettings) + assert.Equal(t, int64(3), aws.ToInt64(descEuWest.ReplicaProvisionedReadCapacityAutoScalingSettings.MinimumUnits)) +} + +func replicaAutoScalingByRegion( + replicas []types.ReplicaAutoScalingDescription, +) map[string]*types.ReplicaAutoScalingDescription { + out := make(map[string]*types.ReplicaAutoScalingDescription, len(replicas)) + for i := range replicas { + out[aws.ToString(replicas[i].RegionName)] = &replicas[i] + } + + return out +} + +// TestUpdateTableReplicaAutoScaling_ReplicaUpdates_UnknownRegion_ResourceNotFound +// verifies a ReplicaUpdates entry naming a region that isn't one of the +// table's replicas is rejected with ResourceNotFoundException -- a real, +// documented error for this op (confirmed against +// awsAwsjson10_deserializeOpErrorUpdateTableReplicaAutoScaling in the pinned SDK). +func TestUpdateTableReplicaAutoScaling_ReplicaUpdates_UnknownRegion_ResourceNotFound(t *testing.T) { + t.Parallel() + + client := newTestDynamoDBClient(t, dynamodb.NewHandler(dynamodb.NewInMemoryDB())) + + _, err := client.CreateGlobalTable(t.Context(), &sdk.CreateGlobalTableInput{ + GlobalTableName: aws.String("gt-no-replica"), + ReplicationGroup: []types.Replica{ + {RegionName: aws.String("us-east-1")}, + }, + }) + require.NoError(t, err) + + _, err = client.UpdateTableReplicaAutoScaling(t.Context(), &sdk.UpdateTableReplicaAutoScalingInput{ + TableName: aws.String("gt-no-replica"), + ReplicaUpdates: []types.ReplicaAutoScalingUpdate{ + { + RegionName: aws.String("ap-southeast-1"), + ReplicaProvisionedReadCapacityAutoScalingUpdate: &types.AutoScalingSettingsUpdate{ + MinimumUnits: aws.Int64(1), + MaximumUnits: aws.Int64(10), + }, + }, + }, + }) + require.Error(t, err) + + var nf *types.ResourceNotFoundException + require.ErrorAs(t, err, &nf) +} + +// TestUpdateTableReplicaAutoScaling_Validation covers the table-driven +// validation error cases: billing mode gate, and MinimumUnits > MaximumUnits. +func TestUpdateTableReplicaAutoScaling_Validation(t *testing.T) { + t.Parallel() + + tests := []struct { + update *types.AutoScalingSettingsUpdate + name string + billingMode types.BillingMode + }{ + { + name: "on_demand_table_rejects_autoscaling_settings", + billingMode: types.BillingModePayPerRequest, + update: &types.AutoScalingSettingsUpdate{ + MinimumUnits: aws.Int64(5), + MaximumUnits: aws.Int64(50), + }, + }, + { + name: "minimum_greater_than_maximum", + billingMode: types.BillingModeProvisioned, + update: &types.AutoScalingSettingsUpdate{ + MinimumUnits: aws.Int64(500), + MaximumUnits: aws.Int64(5), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + client := newTestDynamoDBClient(t, dynamodb.NewHandler(dynamodb.NewInMemoryDB())) + tableName := "as-validation-" + tt.name + + rc, wc := int64(5), int64(5) + createInput := &sdk.CreateTableInput{ + TableName: aws.String(tableName), + KeySchema: []types.KeySchemaElement{ + {AttributeName: aws.String("pk"), KeyType: types.KeyTypeHash}, + }, + AttributeDefinitions: []types.AttributeDefinition{ + {AttributeName: aws.String("pk"), AttributeType: types.ScalarAttributeTypeS}, + }, + BillingMode: tt.billingMode, + } + if tt.billingMode == types.BillingModeProvisioned { + createInput.ProvisionedThroughput = &types.ProvisionedThroughput{ + ReadCapacityUnits: &rc, + WriteCapacityUnits: &wc, + } + } + _, err := client.CreateTable(t.Context(), createInput) + require.NoError(t, err) + + _, err = client.UpdateTableReplicaAutoScaling(t.Context(), &sdk.UpdateTableReplicaAutoScalingInput{ + TableName: aws.String(tableName), + ProvisionedWriteCapacityAutoScalingUpdate: tt.update, + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr, "expected a smithy.APIError") + assert.Equal(t, "ValidationException", apiErr.ErrorCode()) + }) + } +} diff --git a/services/dynamodb/autoscaling_status_agreement_internal_test.go b/services/dynamodb/autoscaling_status_agreement_internal_test.go index d3bb073a6b..d8670dda8c 100644 --- a/services/dynamodb/autoscaling_status_agreement_internal_test.go +++ b/services/dynamodb/autoscaling_status_agreement_internal_test.go @@ -200,10 +200,11 @@ func TestTableAutoScaling_TableStatusSourcedFromSameField(t *testing.T) { mu: lockmetrics.New("test.table"), } - _, updStatus := applyAutoScalingSettingsLocked( + _, updStatus, err := applyAutoScalingSettingsLocked( table, &sdkdynamodb.UpdateTableReplicaAutoScalingInput{TableName: aws.String(table.Name)}, ) + require.NoError(t, err) descStatus, _ := replicaAutoScalingDescriptionsRLocked(table) assert.Equal(t, descStatus, updStatus, "both helpers must agree on TableStatus") diff --git a/services/dynamodb/backup_replica_test.go b/services/dynamodb/backup_replica_test.go index 78ce0320ce..92f6f2b025 100644 --- a/services/dynamodb/backup_replica_test.go +++ b/services/dynamodb/backup_replica_test.go @@ -761,7 +761,9 @@ func TestDescribeTableReplicaAutoScaling(t *testing.T) { name string }{ { - name: "DescribeTableReplicaAutoScaling_NoReplicas", + // A plain (non-global) table still reports its own region as one + // replica, matching real DynamoDB (see autoScalingReplicaEntries). + name: "DescribeTableReplicaAutoScaling_NoExplicitReplicas_ReportsHomeRegion", setup: func(t *testing.T, h *dynamodb.DynamoDBHandler) { t.Helper() createTable(t, h.Backend.(*dynamodb.InMemoryDB), "AutoScaleTable") @@ -779,8 +781,9 @@ func TestDescribeTableReplicaAutoScaling(t *testing.T) { require.Equal(t, http.StatusOK, code) desc := resp["TableAutoScalingDescription"].(map[string]any) assert.Equal(t, "AutoScaleTable", desc["TableName"]) - // No replicas configured - assert.Nil(t, desc["Replicas"]) + replicas := desc["Replicas"].([]any) + require.Len(t, replicas, 1) + assert.Equal(t, "us-east-1", replicas[0].(map[string]any)["RegionName"]) }, }, { diff --git a/services/dynamodb/bench_extract_resource_test.go b/services/dynamodb/bench_extract_resource_test.go new file mode 100644 index 0000000000..4a3a1b649e --- /dev/null +++ b/services/dynamodb/bench_extract_resource_test.go @@ -0,0 +1,46 @@ +package dynamodb_test + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/services/dynamodb" +) + +// BenchmarkExtractResource measures ExtractResource, called on every request. +func BenchmarkExtractResource(b *testing.B) { + db := dynamodb.NewInMemoryDB() + h := dynamodb.NewHandler(db) + e := echo.New() + + item := make(map[string]any, 20) + for i := range 20 { + item[fmt.Sprintf("attr%d", i)] = map[string]any{"S": fmt.Sprintf("value-%d", i)} + } + + body := map[string]any{ + "TableName": "BenchTable", + "Item": item, + } + bodyBytes, err := json.Marshal(body) + if err != nil { + b.Fatalf("marshal request body: %v", err) + } + + b.ReportAllocs() + for b.Loop() { + req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(bodyBytes)) + req.Header.Set("Content-Type", "application/x-amz-json-1.0") + c := e.NewContext(req, httptest.NewRecorder()) + + if got := h.ExtractResource(c); got != "BenchTable" { + b.Fatalf("ExtractResource: got %q", got) + } + } +} diff --git a/services/dynamodb/bench_update_item_test.go b/services/dynamodb/bench_update_item_test.go new file mode 100644 index 0000000000..9991627f51 --- /dev/null +++ b/services/dynamodb/bench_update_item_test.go @@ -0,0 +1,38 @@ +package dynamodb_test + +import "testing" + +// BenchmarkUpdateItem_UpdateExpression runs UpdateItem with SET arithmetic and ADD. +func BenchmarkUpdateItem_UpdateExpression(b *testing.B) { + h := newBenchHandlerWithGSI(b, "BenchUpdateTable") + seedBenchItem(b, h, "BenchUpdateTable", map[string]any{ + "pk": map[string]any{"S": "cust#1"}, + "sk": map[string]any{"S": "order#0001"}, + "tally": map[string]any{"N": "0"}, + "gsipk": map[string]any{"S": "g0"}, + "nested": map[string]any{"M": map[string]any{ + "x": map[string]any{"N": "1"}, + }}, + }) + + req := map[string]any{ + "TableName": "BenchUpdateTable", + "Key": map[string]any{ + "pk": map[string]any{"S": "cust#1"}, + "sk": map[string]any{"S": "order#0001"}, + }, + "UpdateExpression": "SET nested.x = nested.x + :inc, updatedAt = :now ADD tally :inc", + "ExpressionAttributeValues": map[string]any{ + ":inc": map[string]any{"N": "1"}, + ":now": map[string]any{"S": "2026-09-26T00:00:00Z"}, + }, + } + + b.ReportAllocs() + for b.Loop() { + code, resp := invokeOpB(b, h, "UpdateItem", req) + if code != 200 { + b.Fatalf("UpdateItem failed: %v", resp) + } + } +} diff --git a/services/dynamodb/concurrency_test.go b/services/dynamodb/concurrency_test.go index e4a5232195..4eef778332 100644 --- a/services/dynamodb/concurrency_test.go +++ b/services/dynamodb/concurrency_test.go @@ -4,7 +4,6 @@ import ( "fmt" "sync" "testing" - "time" "github.com/blackbirdworks/gopherstack/services/dynamodb" @@ -77,7 +76,6 @@ func TestBatchConcurrency(t *testing.T) { } _, writeErr := db.BatchWriteItem(ctx, input) require.NoError(t, writeErr) - time.Sleep(1 * time.Millisecond) } }) } @@ -100,7 +98,6 @@ func TestBatchConcurrency(t *testing.T) { } _, readErr := db.BatchGetItem(ctx, input) require.NoError(t, readErr) - time.Sleep(1 * time.Millisecond) } }) } diff --git a/services/dynamodb/expr/evaluator.go b/services/dynamodb/expr/evaluator.go index dc328714a5..7373a67f4e 100644 --- a/services/dynamodb/expr/evaluator.go +++ b/services/dynamodb/expr/evaluator.go @@ -776,6 +776,10 @@ func (e *Evaluator) parseNumeric(v any) (float64, bool) { case int64: return float64(val), true case string: + // Skip ParseFloat (allocates on failure) for plainly non-numeric strings. + if !couldBeNumeric(val) { + return 0, false + } if f, parseErr := strconv.ParseFloat(val, 64); parseErr == nil { return f, true } @@ -784,6 +788,24 @@ func (e *Evaluator) parseNumeric(v any) (float64, bool) { return 0, false } +// couldBeNumeric reports whether s has only decimal-float characters. +func couldBeNumeric(s string) bool { + if s == "" { + return false + } + + for i := range len(s) { + switch c := s[i]; { + case c >= '0' && c <= '9': + case c == '-' || c == '+' || c == '.' || c == 'e' || c == 'E': + default: + return false + } + } + + return true +} + // formatDynamoNumber formats a float64 as a plain decimal string without // scientific notation, matching DynamoDB's number representation. func formatDynamoNumber(f float64) string { diff --git a/services/dynamodb/fis_test.go b/services/dynamodb/fis_test.go index 2a754b50ec..27fa9a11a6 100644 --- a/services/dynamodb/fis_test.go +++ b/services/dynamodb/fis_test.go @@ -4,6 +4,7 @@ import ( "context" "net/http" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -74,34 +75,37 @@ func TestDynamoDB_ExecuteFISAction_PauseReplication(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - db := dynamodb.NewInMemoryDB() - h := dynamodb.NewHandler(db) - - err := h.ExecuteFISAction(t.Context(), service.FISActionExecution{ - ActionID: "aws:dynamodb:global-table-pause-replication", - Targets: tt.targets, - Duration: tt.duration, + synctest.Test(t, func(t *testing.T) { + db := dynamodb.NewInMemoryDB() + h := dynamodb.NewHandler(db) + + err := h.ExecuteFISAction(t.Context(), service.FISActionExecution{ + ActionID: "aws:dynamodb:global-table-pause-replication", + Targets: tt.targets, + Duration: tt.duration, + }) + + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + // Verify replication pause state is recorded. + if len(tt.targets) > 0 { + assert.True(t, db.IsReplicationPaused(tt.targets[0]), + "replication should be marked as paused for target %s", tt.targets[0]) + } + + // Verify the pause clears after the duration. + if tt.duration > 0 && len(tt.targets) > 0 { + time.Sleep(tt.duration + 50*time.Millisecond) + synctest.Wait() + + assert.False(t, db.IsReplicationPaused(tt.targets[0]), + "replication pause should have expired after duration") + } }) - - if tt.wantErr { - require.Error(t, err) - } else { - require.NoError(t, err) - } - - // Verify replication pause state is recorded. - if len(tt.targets) > 0 { - assert.True(t, db.IsReplicationPaused(tt.targets[0]), - "replication should be marked as paused for target %s", tt.targets[0]) - } - - // Verify the pause clears after the duration. - if tt.duration > 0 && len(tt.targets) > 0 { - time.Sleep(tt.duration + 50*time.Millisecond) - - assert.False(t, db.IsReplicationPaused(tt.targets[0]), - "replication pause should have expired after duration") - } }) } } @@ -122,29 +126,30 @@ func TestDynamoDB_ExecuteFISAction_Unknown(t *testing.T) { func TestDynamoDB_ExecuteFISAction_PauseReplication_CtxCancel(t *testing.T) { t.Parallel() - db := dynamodb.NewInMemoryDB() - h := dynamodb.NewHandler(db) + synctest.Test(t, func(t *testing.T) { + db := dynamodb.NewInMemoryDB() + h := dynamodb.NewHandler(db) - const tableARN = "arn:aws:dynamodb:us-east-1:000000000000:table/CancelTable" + const tableARN = "arn:aws:dynamodb:us-east-1:000000000000:table/CancelTable" - ctx, cancel := context.WithCancel(t.Context()) + ctx, cancel := context.WithCancel(t.Context()) - // Activate indefinite pause (dur==0). - err := h.ExecuteFISAction(ctx, service.FISActionExecution{ - ActionID: "aws:dynamodb:global-table-pause-replication", - Targets: []string{tableARN}, - Duration: 0, - }) - require.NoError(t, err) + // Activate indefinite pause (dur==0). + err := h.ExecuteFISAction(ctx, service.FISActionExecution{ + ActionID: "aws:dynamodb:global-table-pause-replication", + Targets: []string{tableARN}, + Duration: 0, + }) + require.NoError(t, err) - assert.True(t, db.IsReplicationPaused(tableARN), "pause should be active") + assert.True(t, db.IsReplicationPaused(tableARN), "pause should be active") - // Cancel ctx (simulates StopExperiment). - cancel() + // Cancel ctx (simulates StopExperiment). + cancel() + synctest.Wait() - require.Eventually(t, func() bool { - return !db.IsReplicationPaused(tableARN) - }, 2*time.Second, 20*time.Millisecond, "pause should clear after ctx cancel") + assert.False(t, db.IsReplicationPaused(tableARN), "pause should clear after ctx cancel") + }) } func TestDynamoDB_IsReplicationPaused_LazyEviction(t *testing.T) { diff --git a/services/dynamodb/global_tables.go b/services/dynamodb/global_tables.go index 6b12fc9eba..c35626f6d6 100644 --- a/services/dynamodb/global_tables.go +++ b/services/dynamodb/global_tables.go @@ -147,17 +147,35 @@ func (db *InMemoryDB) ensureReplicaTablesLocked( replica.GlobalTableName = name db.tables.Put(replica) } else { - existing.GlobalTableName = name + setTableGlobalTableNameLocked(existing, name) } } for _, region := range regions { if t, ok := db.tables.Get(tableKey(region, name)); ok { - t.Replicas = buildReplicasExcluding(allReplicas, region) + setTableReplicasLocked(t, buildReplicasExcluding(allReplicas, region)) } } } +// setTableGlobalTableNameLocked sets GlobalTableName under table.mu; readers use +// table.mu, so holding db.mu alone is not enough. +func setTableGlobalTableNameLocked(table *Table, name string) { + table.mu.Lock("GlobalTable.setName") + defer table.mu.Unlock() + + table.GlobalTableName = name +} + +// setTableReplicasLocked sets table.Replicas under a defer-protected +// table.mu.Lock, for the same reason as setTableGlobalTableNameLocked. +func setTableReplicasLocked(table *Table, replicas []models.ReplicaDescription) { + table.mu.Lock("GlobalTable.setReplicas") + defer table.mu.Unlock() + + table.Replicas = replicas +} + // buildReplicaTable creates a new Table for use as a global table replica. // If a source table exists it is cloned; otherwise a placeholder table is created. func (db *InMemoryDB) buildReplicaTable(name, region string, source *Table, now time.Time) *Table { @@ -487,7 +505,7 @@ func (db *InMemoryDB) applyGlobalTableReplicaCreate( replica.GlobalTableName = name db.tables.Put(replica) } else { - existing.GlobalTableName = name + setTableGlobalTableNameLocked(existing, name) } return nil @@ -524,7 +542,7 @@ func (db *InMemoryDB) rebuildGlobalTableReplicasLocked(name string, regions []st allReplicas := buildAllReplicas(regions) for _, region := range regions { if t, tableExists := db.tables.Get(tableKey(region, name)); tableExists { - t.Replicas = buildReplicasExcluding(allReplicas, region) + setTableReplicasLocked(t, buildReplicasExcluding(allReplicas, region)) } } } diff --git a/services/dynamodb/global_tables_replica_metadata_race_test.go b/services/dynamodb/global_tables_replica_metadata_race_test.go new file mode 100644 index 0000000000..d8fe0d1c9e --- /dev/null +++ b/services/dynamodb/global_tables_replica_metadata_race_test.go @@ -0,0 +1,110 @@ +package dynamodb_test + +import ( + "sync" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + sdk "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/dynamodb" +) + +// TestGlobalTableReplicaMetadataRace checks replica metadata writes take table.mu, +// which DescribeTable reads under. +func TestGlobalTableReplicaMetadataRace(t *testing.T) { + t.Parallel() + + tests := []struct { + mutate func(t *testing.T, db *dynamodb.InMemoryDB) + name string + }{ + { + name: "CreateGlobalTable_adopts_existing_replica", + mutate: func(t *testing.T, db *dynamodb.InMemoryDB) { + t.Helper() + + _, err := db.CreateGlobalTable(t.Context(), &sdk.CreateGlobalTableInput{ + GlobalTableName: aws.String("RaceTable"), + ReplicationGroup: []types.Replica{ + {RegionName: aws.String("us-east-1")}, + {RegionName: aws.String("us-west-2")}, + }, + }) + require.NoError(t, err) + }, + }, + { + name: "UpdateGlobalTable_adds_replica_region", + mutate: func(t *testing.T, db *dynamodb.InMemoryDB) { + t.Helper() + + _, err := db.CreateGlobalTable(t.Context(), &sdk.CreateGlobalTableInput{ + GlobalTableName: aws.String("RaceTable"), + ReplicationGroup: []types.Replica{{RegionName: aws.String("us-east-1")}}, + }) + require.NoError(t, err) + + _, err = db.UpdateGlobalTable(t.Context(), &sdk.UpdateGlobalTableInput{ + GlobalTableName: aws.String("RaceTable"), + ReplicaUpdates: []types.ReplicaUpdate{ + {Create: &types.CreateReplicaAction{ + RegionName: aws.String("us-west-2"), + }}, + }, + }) + require.NoError(t, err) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + db := newTestDBWithCleanup(t) + + // Pre-create the table in us-west-2 so the mutate step's + // global-table op adopts an *existing* Table (the in-place + // "existing.GlobalTableName = name" / "t.Replicas = ..." path) + // instead of building a fresh, not-yet-published one. + _, err := db.CreateTableInRegion(t.Context(), &sdk.CreateTableInput{ + TableName: aws.String("RaceTable"), + KeySchema: []types.KeySchemaElement{ + {AttributeName: aws.String("pk"), KeyType: types.KeyTypeHash}, + }, + AttributeDefinitions: []types.AttributeDefinition{ + {AttributeName: aws.String("pk"), AttributeType: types.ScalarAttributeTypeS}, + }, + BillingMode: types.BillingModePayPerRequest, + }, "us-west-2") + require.NoError(t, err) + + readCtx := dynamodb.WithRegion(t.Context(), "us-west-2") + + var wg sync.WaitGroup + + stop := make(chan struct{}) + + wg.Go(func() { + for { + select { + case <-stop: + return + default: + } + + _, _ = db.DescribeTable(readCtx, &sdk.DescribeTableInput{ + TableName: aws.String("RaceTable"), + }) + } + }) + + tt.mutate(t, db) + close(stop) + wg.Wait() + }) + } +} diff --git a/services/dynamodb/handler.go b/services/dynamodb/handler.go index 560baff127..f541e5a55e 100644 --- a/services/dynamodb/handler.go +++ b/services/dynamodb/handler.go @@ -434,22 +434,22 @@ func (h *DynamoDBHandler) ExtractResource(c *echo.Context) string { return "" } - var data map[string]any + // Struct decode, not map[string]any: this runs on every request. + var data struct { + TableName string `json:"TableName"` + BackupArn string `json:"BackupArn"` + } if uerr := json.Unmarshal(body, &data); uerr != nil { return "" } - if tbl, exists := data["TableName"]; exists { - if tblStr, ok := tbl.(string); ok && tblStr != "" { - return tblStr - } + if data.TableName != "" { + return data.TableName } // Backup operations carry BackupArn instead of TableName. - if arnVal, exists := data["BackupArn"]; exists { - if arnStr, ok := arnVal.(string); ok && arnStr != "" { - return extractTableFromBackupARN(arnStr) - } + if data.BackupArn != "" { + return extractTableFromBackupARN(data.BackupArn) } return "" diff --git a/services/dynamodb/handler_autoscaling.go b/services/dynamodb/handler_autoscaling.go index e69418b063..1221bacc14 100644 --- a/services/dynamodb/handler_autoscaling.go +++ b/services/dynamodb/handler_autoscaling.go @@ -17,10 +17,10 @@ import ( // autoScalingSettingsUpdateWire is the wire format for // types.AutoScalingSettingsUpdate (see serializers.go's -// awsAwsjson10_serializeDocumentAutoScalingSettingsUpdate). AutoScalingRoleArn -// is omitted: this emulator does not model IAM roles for scaling policies. +// awsAwsjson10_serializeDocumentAutoScalingSettingsUpdate). type autoScalingSettingsUpdateWire struct { ScalingPolicyUpdate *autoScalingPolicyUpdateWire `json:"ScalingPolicyUpdate,omitempty"` + AutoScalingRoleArn *string `json:"AutoScalingRoleArn,omitempty"` MinimumUnits *int64 `json:"MinimumUnits,omitempty"` MaximumUnits *int64 `json:"MaximumUnits,omitempty"` AutoScalingDisabled *bool `json:"AutoScalingDisabled,omitempty"` @@ -32,8 +32,10 @@ type autoScalingPolicyUpdateWire struct { } type autoScalingTargetTrackingUpdateWire struct { - TargetValue float64 `json:"TargetValue"` - DisableScaleIn bool `json:"DisableScaleIn,omitempty"` + TargetValue float64 `json:"TargetValue"` + DisableScaleIn bool `json:"DisableScaleIn,omitempty"` + ScaleInCooldown int32 `json:"ScaleInCooldown,omitempty"` + ScaleOutCooldown int32 `json:"ScaleOutCooldown,omitempty"` } // gsiAutoScalingUpdateWire is the wire format for @@ -43,15 +45,47 @@ type gsiAutoScalingUpdateWire struct { IndexName string `json:"IndexName,omitempty"` } +// replicaGSIAutoScalingUpdateWire is the wire format for +// types.ReplicaGlobalSecondaryIndexAutoScalingUpdate. +type replicaGSIAutoScalingUpdateWire struct { + ReadCapacityUpdate *autoScalingSettingsUpdateWire `json:"ProvisionedReadCapacityAutoScalingUpdate,omitempty"` + IndexName string `json:"IndexName,omitempty"` +} + +// replicaAutoScalingUpdateWire is the wire format for +// types.ReplicaAutoScalingUpdate. +type replicaAutoScalingUpdateWire struct { + ReadCapacityUpdate *autoScalingSettingsUpdateWire `json:"ReplicaProvisionedReadCapacityAutoScalingUpdate,omitempty"` + RegionName string `json:"RegionName"` + GSIUpdates []replicaGSIAutoScalingUpdateWire `json:"ReplicaGlobalSecondaryIndexUpdates,omitempty"` +} + // updateTableReplicaAutoScalingInput is the wire format for -// UpdateTableReplicaAutoScaling. ReplicaUpdates (per-replica overrides) is -// deliberately not modeled: this emulator's replica lifecycle is owned by -// UpdateGlobalTable/CreateGlobalTable, and per-replica autoscaling overrides -// don't map onto that model without a larger redesign. +// UpdateTableReplicaAutoScaling. type updateTableReplicaAutoScalingInput struct { WriteCapacityUpdate *autoScalingSettingsUpdateWire `json:"ProvisionedWriteCapacityAutoScalingUpdate,omitempty"` TableName string `json:"TableName"` GlobalSecondaryIndexUpdates []gsiAutoScalingUpdateWire `json:"GlobalSecondaryIndexUpdates,omitempty"` + ReplicaUpdates []replicaAutoScalingUpdateWire `json:"ReplicaUpdates,omitempty"` +} + +// int32PtrIfNonZero returns nil for a zero value so an omitted wire field +// (Go zero value) doesn't turn into an explicit SDK zero-cooldown. +func int32PtrIfNonZero(v int32) *int32 { + if v == 0 { + return nil + } + + return &v +} + +// int32Val dereferences an *int32, returning 0 for nil. +func int32Val(v *int32) int32 { + if v == nil { + return 0 + } + + return *v } // toSDKAutoScalingSettingsUpdate converts the wire form to the SDK type. w may @@ -65,6 +99,7 @@ func toSDKAutoScalingSettingsUpdate(w *autoScalingSettingsUpdateWire) *types.Aut MinimumUnits: w.MinimumUnits, MaximumUnits: w.MaximumUnits, AutoScalingDisabled: w.AutoScalingDisabled, + AutoScalingRoleArn: w.AutoScalingRoleArn, } if w.ScalingPolicyUpdate != nil && w.ScalingPolicyUpdate.TargetTracking != nil { @@ -72,8 +107,10 @@ func toSDKAutoScalingSettingsUpdate(w *autoScalingSettingsUpdateWire) *types.Aut out.ScalingPolicyUpdate = &types.AutoScalingPolicyUpdate{ PolicyName: ptrconv.NilIfEmpty(w.ScalingPolicyUpdate.PolicyName), TargetTrackingScalingPolicyConfiguration: &types.AutoScalingTargetTrackingScalingPolicyConfigurationUpdate{ - TargetValue: &tt.TargetValue, - DisableScaleIn: &tt.DisableScaleIn, + TargetValue: &tt.TargetValue, + DisableScaleIn: &tt.DisableScaleIn, + ScaleInCooldown: int32PtrIfNonZero(tt.ScaleInCooldown), + ScaleOutCooldown: int32PtrIfNonZero(tt.ScaleOutCooldown), }, } } @@ -101,20 +138,76 @@ func toSDKGlobalSecondaryIndexAutoScalingUpdates( return out } +// toSDKReplicaAutoScalingUpdates converts the wire ReplicaUpdates slice to SDK form. +func toSDKReplicaAutoScalingUpdates(w []replicaAutoScalingUpdateWire) []types.ReplicaAutoScalingUpdate { + if len(w) == 0 { + return nil + } + + out := make([]types.ReplicaAutoScalingUpdate, len(w)) + for i, r := range w { + regionName := r.RegionName + out[i] = types.ReplicaAutoScalingUpdate{ + RegionName: ®ionName, + ReplicaProvisionedReadCapacityAutoScalingUpdate: toSDKAutoScalingSettingsUpdate(r.ReadCapacityUpdate), + ReplicaGlobalSecondaryIndexUpdates: toSDKReplicaGSIAutoScalingUpdates(r.GSIUpdates), + } + } + + return out +} + +// toSDKReplicaGSIAutoScalingUpdates converts the wire per-replica GSI slice to SDK form. +func toSDKReplicaGSIAutoScalingUpdates( + w []replicaGSIAutoScalingUpdateWire, +) []types.ReplicaGlobalSecondaryIndexAutoScalingUpdate { + if len(w) == 0 { + return nil + } + + out := make([]types.ReplicaGlobalSecondaryIndexAutoScalingUpdate, len(w)) + for i, g := range w { + indexName := g.IndexName + out[i] = types.ReplicaGlobalSecondaryIndexAutoScalingUpdate{ + IndexName: &indexName, + ProvisionedReadCapacityAutoScalingUpdate: toSDKAutoScalingSettingsUpdate(g.ReadCapacityUpdate), + } + } + + return out +} + // autoScalingSettingsDescWire is the wire format for -// types.AutoScalingSettingsDescription, trimmed to the members this emulator -// tracks (min/max/disabled). AutoScalingRoleArn and ScalingPolicies' -// full nested policy list are not modeled. +// types.AutoScalingSettingsDescription. type autoScalingSettingsDescWire struct { - MinimumUnits *int64 `json:"MinimumUnits,omitempty"` - MaximumUnits *int64 `json:"MaximumUnits,omitempty"` - AutoScalingDisabled *bool `json:"AutoScalingDisabled,omitempty"` + MinimumUnits *int64 `json:"MinimumUnits,omitempty"` + MaximumUnits *int64 `json:"MaximumUnits,omitempty"` + AutoScalingRoleArn *string `json:"AutoScalingRoleArn,omitempty"` + AutoScalingDisabled *bool `json:"AutoScalingDisabled,omitempty"` + ScalingPolicies []autoScalingPolicyDescWire `json:"ScalingPolicies,omitempty"` +} + +// autoScalingPolicyDescWire is the wire format for +// types.AutoScalingPolicyDescription. +type autoScalingPolicyDescWire struct { + TargetTracking *autoScalingTargetTrackingDescWire `json:"TargetTrackingScalingPolicyConfiguration,omitempty"` + PolicyName string `json:"PolicyName,omitempty"` +} + +// autoScalingTargetTrackingDescWire is the wire format for +// types.AutoScalingTargetTrackingScalingPolicyConfigurationDescription. +type autoScalingTargetTrackingDescWire struct { + TargetValue float64 `json:"TargetValue"` + DisableScaleIn bool `json:"DisableScaleIn,omitempty"` + ScaleInCooldown int32 `json:"ScaleInCooldown,omitempty"` + ScaleOutCooldown int32 `json:"ScaleOutCooldown,omitempty"` } // replicaGSIAutoScalingDescWire is the wire format for // types.ReplicaGlobalSecondaryIndexAutoScalingDescription. type replicaGSIAutoScalingDescWire struct { WriteCap *autoScalingSettingsDescWire `json:"ProvisionedWriteCapacityAutoScalingSettings,omitempty"` + ReadCap *autoScalingSettingsDescWire `json:"ProvisionedReadCapacityAutoScalingSettings,omitempty"` IndexName string `json:"IndexName,omitempty"` IndexStatus string `json:"IndexStatus,omitempty"` } @@ -123,6 +216,7 @@ type replicaGSIAutoScalingDescWire struct { // types.ReplicaAutoScalingDescription. type replicaAutoScalingDescWire struct { WriteCap *autoScalingSettingsDescWire `json:"ReplicaProvisionedWriteCapacityAutoScalingSettings,omitempty"` + ReadCap *autoScalingSettingsDescWire `json:"ReplicaProvisionedReadCapacityAutoScalingSettings,omitempty"` RegionName string `json:"RegionName,omitempty"` ReplicaStatus string `json:"ReplicaStatus,omitempty"` GSIs []replicaGSIAutoScalingDescWire `json:"GlobalSecondaryIndexes,omitempty"` @@ -155,6 +249,7 @@ func (h *DynamoDBHandler) handleUpdateTableReplicaAutoScaling( GlobalSecondaryIndexUpdates: toSDKGlobalSecondaryIndexAutoScalingUpdates( req.GlobalSecondaryIndexUpdates, ), + ReplicaUpdates: toSDKReplicaAutoScalingUpdates(req.ReplicaUpdates), }, ) if err != nil { @@ -190,6 +285,9 @@ func buildTableAutoScalingDescWire(d *types.TableAutoScalingDescription) tableAu WriteCap: autoScalingSettingsDescWireFromSDK( g.ProvisionedWriteCapacityAutoScalingSettings, ), + ReadCap: autoScalingSettingsDescWireFromSDK( + g.ProvisionedReadCapacityAutoScalingSettings, + ), }) } } @@ -199,6 +297,9 @@ func buildTableAutoScalingDescWire(d *types.TableAutoScalingDescription) tableAu WriteCap: autoScalingSettingsDescWireFromSDK( r.ReplicaProvisionedWriteCapacityAutoScalingSettings, ), + ReadCap: autoScalingSettingsDescWireFromSDK( + r.ReplicaProvisionedReadCapacityAutoScalingSettings, + ), GSIs: gsis, }) } @@ -206,16 +307,34 @@ func buildTableAutoScalingDescWire(d *types.TableAutoScalingDescription) tableAu return desc } -// autoScalingSettingsDescWireFromSDK converts the SDK description to the wire -// shape, trimmed the same way autoScalingSettingsDescWire is. +// autoScalingSettingsDescWireFromSDK converts the SDK description to the wire shape. func autoScalingSettingsDescWireFromSDK(d *types.AutoScalingSettingsDescription) *autoScalingSettingsDescWire { if d == nil { return nil } - return &autoScalingSettingsDescWire{ + out := &autoScalingSettingsDescWire{ MinimumUnits: d.MinimumUnits, MaximumUnits: d.MaximumUnits, AutoScalingDisabled: d.AutoScalingDisabled, + AutoScalingRoleArn: d.AutoScalingRoleArn, } + + if len(d.ScalingPolicies) > 0 { + out.ScalingPolicies = make([]autoScalingPolicyDescWire, 0, len(d.ScalingPolicies)) + for _, p := range d.ScalingPolicies { + pw := autoScalingPolicyDescWire{PolicyName: ptrconv.String(p.PolicyName)} + if tt := p.TargetTrackingScalingPolicyConfiguration; tt != nil { + pw.TargetTracking = &autoScalingTargetTrackingDescWire{ + TargetValue: ptrconv.Float64(tt.TargetValue), + DisableScaleIn: ptrconv.Bool(tt.DisableScaleIn), + ScaleInCooldown: int32Val(tt.ScaleInCooldown), + ScaleOutCooldown: int32Val(tt.ScaleOutCooldown), + } + } + out.ScalingPolicies = append(out.ScalingPolicies, pw) + } + } + + return out } diff --git a/services/dynamodb/import_export_s3_test.go b/services/dynamodb/import_export_s3_test.go index 0af67ae920..f58f10f9e6 100644 --- a/services/dynamodb/import_export_s3_test.go +++ b/services/dynamodb/import_export_s3_test.go @@ -10,6 +10,7 @@ import ( "sort" "strings" "testing" + "testing/synctest" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -135,153 +136,161 @@ func waitForExport(t *testing.T, h *dynamodb.DynamoDBHandler, arn string) { func TestImportTable_FromS3_DynamoDBJSON(t *testing.T) { t.Parallel() - db := dynamodb.NewInMemoryDB() - s3 := newMockS3() - db.SetS3Backend(s3) + synctest.Test(t, func(t *testing.T) { + db := dynamodb.NewInMemoryDB() + s3 := newMockS3() + db.SetS3Backend(s3) - s3.put("src", "data/part-1.json.gz", gzipBytes(t, - `{"Item":{"pk":{"S":"a"},"v":{"N":"1"}}}`+"\n"+ - `{"Item":{"pk":{"S":"b"},"v":{"N":"2"}}}`+"\n")) + s3.put("src", "data/part-1.json.gz", gzipBytes(t, + `{"Item":{"pk":{"S":"a"},"v":{"N":"1"}}}`+"\n"+ + `{"Item":{"pk":{"S":"b"},"v":{"N":"2"}}}`+"\n")) - out, err := db.ImportTable(t.Context(), &sdk.ImportTableInput{ - S3BucketSource: &ddbtypes.S3BucketSource{ - S3Bucket: aws.String("src"), - S3KeyPrefix: aws.String("data/"), - }, - InputFormat: ddbtypes.InputFormatDynamodbJson, - InputCompressionType: ddbtypes.InputCompressionTypeGzip, - TableCreationParameters: importCreationParams("ImportedJSON"), - }) - require.NoError(t, err) + out, err := db.ImportTable(t.Context(), &sdk.ImportTableInput{ + S3BucketSource: &ddbtypes.S3BucketSource{ + S3Bucket: aws.String("src"), + S3KeyPrefix: aws.String("data/"), + }, + InputFormat: ddbtypes.InputFormatDynamodbJson, + InputCompressionType: ddbtypes.InputCompressionTypeGzip, + TableCreationParameters: importCreationParams("ImportedJSON"), + }) + require.NoError(t, err) - importDesc := waitForImport(t, db, aws.ToString(out.ImportTableDescription.ImportArn)) + importDesc := waitForImport(t, db, aws.ToString(out.ImportTableDescription.ImportArn)) - assert.Equal(t, ddbtypes.ImportStatusCompleted, importDesc.ImportTableDescription.ImportStatus) - assert.Equal(t, int64(2), importDesc.ImportTableDescription.ImportedItemCount) - assert.Equal(t, int64(2), importDesc.ImportTableDescription.ProcessedItemCount) + assert.Equal(t, ddbtypes.ImportStatusCompleted, importDesc.ImportTableDescription.ImportStatus) + assert.Equal(t, int64(2), importDesc.ImportTableDescription.ImportedItemCount) + assert.Equal(t, int64(2), importDesc.ImportTableDescription.ProcessedItemCount) - got, err := db.GetItem(t.Context(), &sdk.GetItemInput{ - TableName: aws.String("ImportedJSON"), - Key: map[string]ddbtypes.AttributeValue{ - "pk": &ddbtypes.AttributeValueMemberS{Value: "a"}, - }, + got, err := db.GetItem(t.Context(), &sdk.GetItemInput{ + TableName: aws.String("ImportedJSON"), + Key: map[string]ddbtypes.AttributeValue{ + "pk": &ddbtypes.AttributeValueMemberS{Value: "a"}, + }, + }) + require.NoError(t, err) + require.NotEmpty(t, got.Item) + assert.Equal(t, "1", got.Item["v"].(*ddbtypes.AttributeValueMemberN).Value) }) - require.NoError(t, err) - require.NotEmpty(t, got.Item) - assert.Equal(t, "1", got.Item["v"].(*ddbtypes.AttributeValueMemberN).Value) } // TestImportTable_FromS3_CSV verifies CSV ingestion with a header row. func TestImportTable_FromS3_CSV(t *testing.T) { t.Parallel() - db := dynamodb.NewInMemoryDB() - s3 := newMockS3() - db.SetS3Backend(s3) + synctest.Test(t, func(t *testing.T) { + db := dynamodb.NewInMemoryDB() + s3 := newMockS3() + db.SetS3Backend(s3) - s3.put("src", "csv/rows.csv", []byte("pk,name\na,Alice\nb,Bob\n")) + s3.put("src", "csv/rows.csv", []byte("pk,name\na,Alice\nb,Bob\n")) - out, err := db.ImportTable(t.Context(), &sdk.ImportTableInput{ - S3BucketSource: &ddbtypes.S3BucketSource{ - S3Bucket: aws.String("src"), - S3KeyPrefix: aws.String("csv/"), - }, - InputFormat: ddbtypes.InputFormatCsv, - TableCreationParameters: importCreationParams("ImportedCSV"), - }) - require.NoError(t, err) - importDesc := waitForImport(t, db, aws.ToString(out.ImportTableDescription.ImportArn)) - assert.Equal(t, ddbtypes.ImportStatusCompleted, importDesc.ImportTableDescription.ImportStatus) - assert.Equal(t, int64(2), importDesc.ImportTableDescription.ImportedItemCount) - - got, err := db.GetItem(t.Context(), &sdk.GetItemInput{ - TableName: aws.String("ImportedCSV"), - Key: map[string]ddbtypes.AttributeValue{ - "pk": &ddbtypes.AttributeValueMemberS{Value: "b"}, - }, + out, err := db.ImportTable(t.Context(), &sdk.ImportTableInput{ + S3BucketSource: &ddbtypes.S3BucketSource{ + S3Bucket: aws.String("src"), + S3KeyPrefix: aws.String("csv/"), + }, + InputFormat: ddbtypes.InputFormatCsv, + TableCreationParameters: importCreationParams("ImportedCSV"), + }) + require.NoError(t, err) + importDesc := waitForImport(t, db, aws.ToString(out.ImportTableDescription.ImportArn)) + assert.Equal(t, ddbtypes.ImportStatusCompleted, importDesc.ImportTableDescription.ImportStatus) + assert.Equal(t, int64(2), importDesc.ImportTableDescription.ImportedItemCount) + + got, err := db.GetItem(t.Context(), &sdk.GetItemInput{ + TableName: aws.String("ImportedCSV"), + Key: map[string]ddbtypes.AttributeValue{ + "pk": &ddbtypes.AttributeValueMemberS{Value: "b"}, + }, + }) + require.NoError(t, err) + require.NotEmpty(t, got.Item) + assert.Equal(t, "Bob", got.Item["name"].(*ddbtypes.AttributeValueMemberS).Value) }) - require.NoError(t, err) - require.NotEmpty(t, got.Item) - assert.Equal(t, "Bob", got.Item["name"].(*ddbtypes.AttributeValueMemberS).Value) } // TestImportTable_ION_Unsupported verifies that ION input fails the import cleanly. func TestImportTable_ION_Unsupported(t *testing.T) { t.Parallel() - db := dynamodb.NewInMemoryDB() - s3 := newMockS3() - db.SetS3Backend(s3) - s3.put("src", "ion/data.ion", []byte("{pk: \"a\"}")) + synctest.Test(t, func(t *testing.T) { + db := dynamodb.NewInMemoryDB() + s3 := newMockS3() + db.SetS3Backend(s3) + s3.put("src", "ion/data.ion", []byte("{pk: \"a\"}")) - out, err := db.ImportTable(t.Context(), &sdk.ImportTableInput{ - S3BucketSource: &ddbtypes.S3BucketSource{ - S3Bucket: aws.String("src"), - S3KeyPrefix: aws.String("ion/"), - }, - InputFormat: ddbtypes.InputFormatIon, - TableCreationParameters: importCreationParams("ImportedION"), + out, err := db.ImportTable(t.Context(), &sdk.ImportTableInput{ + S3BucketSource: &ddbtypes.S3BucketSource{ + S3Bucket: aws.String("src"), + S3KeyPrefix: aws.String("ion/"), + }, + InputFormat: ddbtypes.InputFormatIon, + TableCreationParameters: importCreationParams("ImportedION"), + }) + require.NoError(t, err) + importDesc := waitForImport(t, db, aws.ToString(out.ImportTableDescription.ImportArn)) + assert.Equal(t, ddbtypes.ImportStatusFailed, importDesc.ImportTableDescription.ImportStatus) + assert.NotEmpty(t, aws.ToString(importDesc.ImportTableDescription.FailureCode)) }) - require.NoError(t, err) - importDesc := waitForImport(t, db, aws.ToString(out.ImportTableDescription.ImportArn)) - assert.Equal(t, ddbtypes.ImportStatusFailed, importDesc.ImportTableDescription.ImportStatus) - assert.NotEmpty(t, aws.ToString(importDesc.ImportTableDescription.FailureCode)) } // TestExportImport_RoundTrip exports a populated table to S3 and re-imports it. func TestExportImport_RoundTrip(t *testing.T) { t.Parallel() - db := dynamodb.NewInMemoryDB() - s3 := newMockS3() - db.SetS3Backend(s3) - h := dynamodb.NewHandler(db) - - createTableHelper(t, db, "SourceTbl", "pk") - for _, id := range []string{"x", "y", "z"} { - _, err := db.PutItem(t.Context(), &sdk.PutItemInput{ - TableName: aws.String("SourceTbl"), - Item: map[string]ddbtypes.AttributeValue{ - "pk": &ddbtypes.AttributeValueMemberS{Value: id}, - }, - }) - require.NoError(t, err) - } + synctest.Test(t, func(t *testing.T) { + db := dynamodb.NewInMemoryDB() + s3 := newMockS3() + db.SetS3Backend(s3) + h := dynamodb.NewHandler(db) + + createTableHelper(t, db, "SourceTbl", "pk") + for _, id := range []string{"x", "y", "z"} { + _, err := db.PutItem(t.Context(), &sdk.PutItemInput{ + TableName: aws.String("SourceTbl"), + Item: map[string]ddbtypes.AttributeValue{ + "pk": &ddbtypes.AttributeValueMemberS{Value: id}, + }, + }) + require.NoError(t, err) + } - tbl, ok := db.GetTable("SourceTbl") - require.True(t, ok) + tbl, ok := db.GetTable("SourceTbl") + require.True(t, ok) - // Export to S3 via the handler. - code, res := invokeOp(t, h, "ExportTableToPointInTime", map[string]any{ - "TableArn": tbl.TableArn, - "S3Bucket": "exb", - "S3Prefix": "out", - }) - require.Equal(t, 200, code) - waitForExport(t, h, res["ExportDescription"].(map[string]any)["ExportArn"].(string)) - - // Re-import the exported data into a new table from the data/ prefix. - var dataPrefix string - for k := range s3.objects { - if strings.Contains(k, "/data/") { - _, key, _ := strings.Cut(k, "/") - dataPrefix = strings.TrimSuffix(key, "00000.json.gz") + // Export to S3 via the handler. + code, res := invokeOp(t, h, "ExportTableToPointInTime", map[string]any{ + "TableArn": tbl.TableArn, + "S3Bucket": "exb", + "S3Prefix": "out", + }) + require.Equal(t, 200, code) + waitForExport(t, h, res["ExportDescription"].(map[string]any)["ExportArn"].(string)) + + // Re-import the exported data into a new table from the data/ prefix. + var dataPrefix string + for k := range s3.objects { + if strings.Contains(k, "/data/") { + _, key, _ := strings.Cut(k, "/") + dataPrefix = strings.TrimSuffix(key, "00000.json.gz") + } } - } - require.NotEmpty(t, dataPrefix, "export must write a data object") + require.NotEmpty(t, dataPrefix, "export must write a data object") - out, err := db.ImportTable(t.Context(), &sdk.ImportTableInput{ - S3BucketSource: &ddbtypes.S3BucketSource{ - S3Bucket: aws.String("exb"), - S3KeyPrefix: aws.String(dataPrefix), - }, - InputFormat: ddbtypes.InputFormatDynamodbJson, - InputCompressionType: ddbtypes.InputCompressionTypeGzip, - TableCreationParameters: importCreationParams("RoundTripTbl"), + out, err := db.ImportTable(t.Context(), &sdk.ImportTableInput{ + S3BucketSource: &ddbtypes.S3BucketSource{ + S3Bucket: aws.String("exb"), + S3KeyPrefix: aws.String(dataPrefix), + }, + InputFormat: ddbtypes.InputFormatDynamodbJson, + InputCompressionType: ddbtypes.InputCompressionTypeGzip, + TableCreationParameters: importCreationParams("RoundTripTbl"), + }) + require.NoError(t, err) + importDesc := waitForImport(t, db, aws.ToString(out.ImportTableDescription.ImportArn)) + assert.Equal(t, int64(3), importDesc.ImportTableDescription.ImportedItemCount) }) - require.NoError(t, err) - importDesc := waitForImport(t, db, aws.ToString(out.ImportTableDescription.ImportArn)) - assert.Equal(t, int64(3), importDesc.ImportTableDescription.ImportedItemCount) } func TestImportTable_MissingTableCreationParameters(t *testing.T) { diff --git a/services/dynamodb/memory_fixes_test.go b/services/dynamodb/memory_fixes_test.go index 79f8c01fc3..dd8dfa49e2 100644 --- a/services/dynamodb/memory_fixes_test.go +++ b/services/dynamodb/memory_fixes_test.go @@ -12,6 +12,7 @@ import ( "strings" "sync" "testing" + "testing/synctest" "time" "github.com/blackbirdworks/gopherstack/services/dynamodb" @@ -124,15 +125,17 @@ func TestExpressionCacheTTL_LazyEvictionOnGet(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - cache := dynamodb.NewExpressionCacheWithTTL(100, tt.ttl) - cache.Put("my-key", "my-value") + synctest.Test(t, func(t *testing.T) { + cache := dynamodb.NewExpressionCacheWithTTL(100, tt.ttl) + cache.Put("my-key", "my-value") - if tt.sleepFor > 0 { - time.Sleep(tt.sleepFor) - } + if tt.sleepFor > 0 { + time.Sleep(tt.sleepFor) + } - _, found := cache.Get("my-key") - assert.Equal(t, tt.wantFound, found) + _, found := cache.Get("my-key") + assert.Equal(t, tt.wantFound, found) + }) }) } } @@ -161,42 +164,44 @@ func TestExpressionCacheTTL_SweepRemovesExpiredEntries(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - // Use a cache with a very short TTL so entries expire quickly. - cache := dynamodb.NewExpressionCacheWithTTL(200, 1*time.Millisecond) + synctest.Test(t, func(t *testing.T) { + // Use a cache with a very short TTL so entries expire quickly. + cache := dynamodb.NewExpressionCacheWithTTL(200, 1*time.Millisecond) - // Add entries with the short TTL — they will expire. - for i := range tt.nExpired { - cache.Put(fmt.Sprintf("expired-%d", i), i) - } + // Add entries with the short TTL — they will expire. + for i := range tt.nExpired { + cache.Put(fmt.Sprintf("expired-%d", i), i) + } - // Wait for the short-TTL entries to expire. - time.Sleep(5 * time.Millisecond) + // Wait for the short-TTL entries to expire. + time.Sleep(5 * time.Millisecond) - // Add fresh entries into a SEPARATE long-TTL cache. Using a separate - // instance avoids TTL races with the short-TTL cache above and lets us - // assert independently. For mixed-cache behaviour (expired + fresh in the - // same cache instance), see TestExpressionCacheTTL_SweepMixedInSameCache. - freshCache := dynamodb.NewExpressionCacheWithTTL(200, 1*time.Hour) + // Add fresh entries into a SEPARATE long-TTL cache. Using a separate + // instance avoids TTL races with the short-TTL cache above and lets us + // assert independently. For mixed-cache behaviour (expired + fresh in the + // same cache instance), see TestExpressionCacheTTL_SweepMixedInSameCache. + freshCache := dynamodb.NewExpressionCacheWithTTL(200, 1*time.Hour) - for i := range tt.nFresh { - freshCache.Put(fmt.Sprintf("fresh-%d", i), i) - } + for i := range tt.nFresh { + freshCache.Put(fmt.Sprintf("fresh-%d", i), i) + } - // Sweep the short-TTL cache — all expired entries should be removed. - cache.Sweep() + // Sweep the short-TTL cache — all expired entries should be removed. + cache.Sweep() - for i := range tt.nExpired { - _, found := cache.Get(fmt.Sprintf("expired-%d", i)) - assert.False(t, found, "expired entry %d should be gone after Sweep", i) - } + for i := range tt.nExpired { + _, found := cache.Get(fmt.Sprintf("expired-%d", i)) + assert.False(t, found, "expired entry %d should be gone after Sweep", i) + } - // The long-TTL cache entries should survive their own sweep. - freshCache.Sweep() + // The long-TTL cache entries should survive their own sweep. + freshCache.Sweep() - for i := range tt.nFresh { - _, found := freshCache.Get(fmt.Sprintf("fresh-%d", i)) - assert.True(t, found, "fresh entry %d should survive Sweep", i) - } + for i := range tt.nFresh { + _, found := freshCache.Get(fmt.Sprintf("fresh-%d", i)) + assert.True(t, found, "fresh entry %d should survive Sweep", i) + } + }) }) } } diff --git a/services/dynamodb/perf_fixes_test.go b/services/dynamodb/perf_fixes_test.go index 12b6386911..25f03a163c 100644 --- a/services/dynamodb/perf_fixes_test.go +++ b/services/dynamodb/perf_fixes_test.go @@ -679,9 +679,9 @@ func BenchmarkScanWithLimit(b *testing.B) { } } - b.ResetTimer() + b.ReportAllocs() - for range b.N { + for b.Loop() { var out *sdk.ScanOutput out, err = db.Scan(b.Context(), &sdk.ScanInput{ TableName: aws.String("BenchScanTable"), @@ -696,3 +696,111 @@ func BenchmarkScanWithLimit(b *testing.B) { } } } + +// BenchmarkScanWithLimit_ExclusiveStartKey exercises the paginated path +// (ExclusiveStartKey set) on the same table shape as BenchmarkScanWithLimit. +func BenchmarkScanWithLimit_ExclusiveStartKey(b *testing.B) { + const ( + numItems = 5000 + limit = 5 + ) + + db := dynamodb.NewInMemoryDB() + + _, err := db.CreateTable(b.Context(), &sdk.CreateTableInput{ + TableName: aws.String("BenchScanESKTable"), + KeySchema: []types.KeySchemaElement{ + {AttributeName: aws.String("pk"), KeyType: types.KeyTypeHash}, + }, + AttributeDefinitions: []types.AttributeDefinition{ + {AttributeName: aws.String("pk"), AttributeType: types.ScalarAttributeTypeS}, + }, + }) + if err != nil { + b.Fatal(err) + } + + for i := range numItems { + _, err = db.PutItem(b.Context(), &sdk.PutItemInput{ + TableName: aws.String("BenchScanESKTable"), + Item: map[string]types.AttributeValue{ + "pk": &types.AttributeValueMemberS{Value: fmt.Sprintf("item-%05d", i)}, + "data": &types.AttributeValueMemberS{Value: "padding-to-simulate-real-item-size"}, + }, + }) + if err != nil { + b.Fatal(err) + } + } + + startKey := map[string]types.AttributeValue{ + "pk": &types.AttributeValueMemberS{Value: "item-02500"}, + } + + b.ReportAllocs() + + for b.Loop() { + _, err = db.Scan(b.Context(), &sdk.ScanInput{ + TableName: aws.String("BenchScanESKTable"), + Limit: aws.Int32(limit), + ExclusiveStartKey: startKey, + }) + if err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkScanWithLimit_100k is BenchmarkScanWithLimit at a larger table +// size; per-item attribute unwrapping, not sorting, dominates its profile. +func BenchmarkScanWithLimit_100k(b *testing.B) { + const ( + numItems = 100_000 + limit = 10 + ) + + db := dynamodb.NewInMemoryDB() + + _, err := db.CreateTable(b.Context(), &sdk.CreateTableInput{ + TableName: aws.String("BenchScanTable100k"), + KeySchema: []types.KeySchemaElement{ + {AttributeName: aws.String("pk"), KeyType: types.KeyTypeHash}, + }, + AttributeDefinitions: []types.AttributeDefinition{ + {AttributeName: aws.String("pk"), AttributeType: types.ScalarAttributeTypeS}, + }, + }) + if err != nil { + b.Fatal(err) + } + + for i := range numItems { + _, err = db.PutItem(b.Context(), &sdk.PutItemInput{ + TableName: aws.String("BenchScanTable100k"), + Item: map[string]types.AttributeValue{ + "pk": &types.AttributeValueMemberS{Value: fmt.Sprintf("item-%06d", i)}, + "data": &types.AttributeValueMemberS{Value: "padding-to-simulate-real-item-size"}, + }, + }) + if err != nil { + b.Fatal(err) + } + } + + b.ReportAllocs() + + for b.Loop() { + var out *sdk.ScanOutput + out, err = db.Scan(b.Context(), &sdk.ScanInput{ + TableName: aws.String("BenchScanTable100k"), + Limit: aws.Int32(limit), + }) + if err != nil { + b.Fatal(err) + } + + if out.Count != limit { + b.Fatalf("expected %d items, got %d", limit, out.Count) + } + } +} diff --git a/services/dynamodb/persistence.go b/services/dynamodb/persistence.go index 654a650834..6d7f7f1bee 100644 --- a/services/dynamodb/persistence.go +++ b/services/dynamodb/persistence.go @@ -30,6 +30,17 @@ type dbSnapshot struct { Version int `json:"version"` } +// dbSnapshotWire mirrors dbSnapshot but carries Tables as pre-marshaled JSON. +// This lets Snapshot serialise each table under its own table.mu instead of racing PutItem/UpdateTable. +type dbSnapshotWire struct { + DefaultRegion string `json:"defaultRegion"` + AccountID string `json:"accountID"` + Tables json.RawMessage `json:"tables"` + Backups []*Backup `json:"backups,omitempty"` + GlobalTables []*StoredGlobalTable `json:"globalTables,omitempty"` + Version int `json:"version"` +} + // Snapshot serialises the backend state to JSON; implements persistence.Persistable. // streamSeq is unexported and not serialised -- Restore reconstructs it from // the highest SequenceNumber in each table's StreamRecords ring buffer. @@ -37,9 +48,19 @@ func (db *InMemoryDB) Snapshot(ctx context.Context) []byte { db.mu.RLock("Snapshot") defer db.mu.RUnlock() - snap := dbSnapshot{ + tablesJSON, err := marshalTablesRLocked(db.tables.Snapshot()) + if err != nil { + logger.Load(ctx).WarnContext(ctx, + "DynamoDB: failed to serialise snapshot; state will not be persisted", + slog.String("error", err.Error()), + ) + + return nil + } + + snap := dbSnapshotWire{ Version: dynamodbSnapshotVersion, - Tables: db.tables.Snapshot(), + Tables: tablesJSON, Backups: db.backups.Snapshot(), GlobalTables: db.globalTables.Snapshot(), DefaultRegion: db.defaultRegion, @@ -59,6 +80,30 @@ func (db *InMemoryDB) Snapshot(ctx context.Context) []byte { return data } +// marshalTablesRLocked marshals each table under its own table.mu.RLock, then reassembles the JSON array. +// Reassembling per-table bytes, rather than marshaling the slice directly, keeps the wire format byte-identical. +func marshalTablesRLocked(tables []*Table) (json.RawMessage, error) { + if len(tables) == 0 { + return json.RawMessage("null"), nil + } + + parts := make([]json.RawMessage, len(tables)) + + for i, t := range tables { + t.mu.RLock("Snapshot") + data, err := json.Marshal(t) + t.mu.RUnlock() + + if err != nil { + return nil, err + } + + parts[i] = data + } + + return json.Marshal(parts) +} + // Restore loads backend state from a JSON snapshot. // It implements persistence.Persistable. func (db *InMemoryDB) Restore(ctx context.Context, data []byte) error { diff --git a/services/dynamodb/persistence_race_test.go b/services/dynamodb/persistence_race_test.go new file mode 100644 index 0000000000..23cf11e4b0 --- /dev/null +++ b/services/dynamodb/persistence_race_test.go @@ -0,0 +1,103 @@ +package dynamodb_test + +import ( + "strconv" + "sync" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + sdkddb "github.com/aws/aws-sdk-go-v2/service/dynamodb" + sdktypes "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/dynamodb" +) + +// TestSnapshot_RacesWithItemAndTableWrites reproduces gopherstack-fwd0g: Snapshot marshals tables under db.mu.RLock. +// PutItem/UpdateTable mutate table fields under table.mu alone; run with -race. +func TestSnapshot_RacesWithItemAndTableWrites(t *testing.T) { + t.Parallel() + + tests := []struct { + mutate func(t *testing.T, db *dynamodb.InMemoryDB, tableName string) + name string + }{ + { + name: "concurrent_snapshot_and_put_item", + mutate: func(t *testing.T, db *dynamodb.InMemoryDB, tableName string) { + t.Helper() + + for i := range 200 { + _, err := db.PutItem(t.Context(), &sdkddb.PutItemInput{ + TableName: aws.String(tableName), + Item: map[string]sdktypes.AttributeValue{ + "pk": &sdktypes.AttributeValueMemberS{Value: "item-" + strconv.Itoa(i)}, + }, + }) + require.NoError(t, err) + } + }, + }, + { + name: "concurrent_snapshot_and_update_table", + mutate: func(t *testing.T, db *dynamodb.InMemoryDB, tableName string) { + t.Helper() + + for range 200 { + _, err := db.UpdateTable(t.Context(), &sdkddb.UpdateTableInput{ + TableName: aws.String(tableName), + ProvisionedThroughput: &sdktypes.ProvisionedThroughput{ + ReadCapacityUnits: aws.Int64(5), + WriteCapacityUnits: aws.Int64(5), + }, + }) + require.NoError(t, err) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + db := newTestDBWithCleanup(t) + const tableName = "race-snapshot-table" + + _, err := db.CreateTable(t.Context(), &sdkddb.CreateTableInput{ + TableName: aws.String(tableName), + KeySchema: []sdktypes.KeySchemaElement{ + {AttributeName: aws.String("pk"), KeyType: sdktypes.KeyTypeHash}, + }, + AttributeDefinitions: []sdktypes.AttributeDefinition{ + {AttributeName: aws.String("pk"), AttributeType: sdktypes.ScalarAttributeTypeS}, + }, + ProvisionedThroughput: &sdktypes.ProvisionedThroughput{ + ReadCapacityUnits: aws.Int64(5), + WriteCapacityUnits: aws.Int64(5), + }, + }) + require.NoError(t, err) + + var wg sync.WaitGroup + + stop := make(chan struct{}) + + wg.Go(func() { + for { + select { + case <-stop: + return + default: + } + + _ = db.Snapshot(t.Context()) + } + }) + + tt.mutate(t, db, tableName) + close(stop) + wg.Wait() + }) + } +} diff --git a/services/dynamodb/pitr_test.go b/services/dynamodb/pitr_test.go index dd384e2036..3ea50ab9fc 100644 --- a/services/dynamodb/pitr_test.go +++ b/services/dynamodb/pitr_test.go @@ -9,6 +9,7 @@ import ( "context" "net/http" "testing" + "testing/synctest" "time" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" @@ -91,36 +92,39 @@ func TestPITR_SnapshotsSurvivePersistenceRoundTrip(t *testing.T) { // for well over 60 fires and asserting no PITR snapshot was taken. func TestPITR_SnapshotCadenceDecoupledFromMainSweep(t *testing.T) { t.Parallel() - ctx, cancel := context.WithCancel(t.Context()) - db := newInMemoryTestDB(t) - h := dynamodb.NewHandler(db) - createSimpleTestTable(t, db, "PITRCadenceTable") - enablePITR(t, h, "PITRCadenceTable") - - j := dynamodb.NewJanitor(db, dynamodb.Settings{JanitorInterval: 2 * time.Millisecond}) - - done := make(chan struct{}) - go func() { - defer close(done) - j.Run(ctx) - }() - - // 300ms at a 2ms housekeeping interval is >100 fast-ticker fires -- far - // more than the 60-slot ring's capacity -- while the PITR ticker - // (1 minute) cannot have fired even once. - time.Sleep(300 * time.Millisecond) - cancel() - <-done - - tbl, ok := db.GetTableInRegion("PITRCadenceTable", "us-east-1") - require.True(t, ok) - assert.Empty( - t, - tbl.PITRSnapshots, - "PITR snapshot must not be taken by the fast housekeeping ticker; "+ - "it must only fire on its own slower, decoupled ticker", - ) + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + + db := newInMemoryTestDB(t) + h := dynamodb.NewHandler(db) + createSimpleTestTable(t, db, "PITRCadenceTable") + enablePITR(t, h, "PITRCadenceTable") + + j := dynamodb.NewJanitor(db, dynamodb.Settings{JanitorInterval: 2 * time.Millisecond}) + + done := make(chan struct{}) + go func() { + defer close(done) + j.Run(ctx) + }() + + // 300ms at a 2ms housekeeping interval is >100 fast-ticker fires -- far + // more than the 60-slot ring's capacity -- while the PITR ticker + // (1 minute) cannot have fired even once. + time.Sleep(300 * time.Millisecond) + cancel() + <-done + + tbl, ok := db.GetTableInRegion("PITRCadenceTable", "us-east-1") + require.True(t, ok) + assert.Empty( + t, + tbl.PITRSnapshots, + "PITR snapshot must not be taken by the fast housekeeping ticker; "+ + "it must only fire on its own slower, decoupled ticker", + ) + }) } // TestPITR_RestoreOutsideWindow_ReturnsInvalidRestoreTimeException is a diff --git a/services/dynamodb/store.go b/services/dynamodb/store.go index b5305fd974..615e1517f4 100644 --- a/services/dynamodb/store.go +++ b/services/dynamodb/store.go @@ -139,13 +139,35 @@ type autoScalingSettings struct { } // autoScalingThroughput captures the min/max/target settings for one direction -// (read or write). Mirrors types.AutoScalingSettingsUpdate but stripped to the -// fields LocalStack and most callers care about. +// (read or write), plus the scaling-policy fields real DynamoDB carries on +// AutoScalingSettingsDescription/Update (types.go:314/338): AutoScalingRoleArn +// and the single TargetTrackingScalingPolicyConfiguration a v1 update accepts +// (ScalingPolicyUpdate is singular on the update side; DisableScaleIn is captured +// alongside TargetUtilizPct). These are echoed back exactly as the client sent +// them, never fabricated -- this emulator has no real IAM-role or scaling-policy +// engine behind them. type autoScalingThroughput struct { - MinCapacity *int64 `json:"MinCapacity,omitempty"` - MaxCapacity *int64 `json:"MaxCapacity,omitempty"` - TargetUtilizPct *float64 `json:"TargetUtilizationPct,omitempty"` - Disabled bool `json:"AutoScalingDisabled,omitempty"` + MinCapacity *int64 `json:"MinCapacity,omitempty"` + MaxCapacity *int64 `json:"MaxCapacity,omitempty"` + TargetUtilizPct *float64 `json:"TargetUtilizationPct,omitempty"` + DisableScaleIn *bool `json:"DisableScaleIn,omitempty"` + ScaleInCooldown *int32 `json:"ScaleInCooldown,omitempty"` + ScaleOutCooldown *int32 `json:"ScaleOutCooldown,omitempty"` + RoleArn *string `json:"AutoScalingRoleArn,omitempty"` + PolicyName *string `json:"PolicyName,omitempty"` + Disabled bool `json:"AutoScalingDisabled,omitempty"` +} + +// replicaAutoScalingSettings records the per-replica read-capacity autoscaling +// settings from UpdateTableReplicaAutoScaling's ReplicaUpdates +// (types.ReplicaAutoScalingUpdate). Read capacity is per-replica in the v1 +// global tables API, unlike write capacity, which this emulator applies +// table-wide via autoScalingSettings.Write and echoes identically to every +// replica (matches AWS: a v1 global table has one write capacity shared by +// all replicas). +type replicaAutoScalingSettings struct { + Read *autoScalingThroughput `json:"Read,omitempty"` + GlobalSecondaryIndexes map[string]*autoScalingThroughput `json:"GlobalSecondaryIndexes,omitempty"` } // pitrSnapshot captures the items of a PITR-enabled table at a point in time. @@ -288,37 +310,40 @@ type Table struct { // Table built per-Query call (see snapshotTableForQuery); it holds a // deep copy of the one GSI/LSI index the query targets, same role as // itemsByOffset plays for primary-key queries. - activeSecondaryIndex *secondaryIndex - itemsByOffset map[int]map[string]any - mu *lockmetrics.RWMutex - activateTimer *time.Timer - Tags *tags.Tags `json:"Tags,omitempty"` - AutoScaling *autoScalingSettings `json:"AutoScaling,omitempty"` - OnDemandMaxWriteRRU *int64 `json:"OnDemandMaxWriteRRU,omitempty"` - OnDemandMaxReadRRU *int64 `json:"OnDemandMaxReadRRU,omitempty"` - ResourcePolicy string `json:"ResourcePolicy,omitempty"` - ResourcePolicyRevision string `json:"ResourcePolicyRevision,omitempty"` - TTLAttribute string `json:"TTLAttribute,omitempty"` - StreamViewType string `json:"StreamViewType,omitempty"` - StreamARN string `json:"StreamARN,omitempty"` - GlobalTableName string `json:"GlobalTableName,omitempty"` - MultiRegionConsistency string `json:"MultiRegionConsistency,omitempty"` - TableArn string `json:"TableArn"` - Status string `json:"Status"` - TableID string `json:"TableID"` - SSEType string `json:"SSEType,omitempty"` - TableClass string `json:"TableClass,omitempty"` - BillingMode string `json:"BillingMode,omitempty"` - Name string `json:"Name"` - SSEKMSMasterKeyArn string `json:"SSEKMSMasterKeyArn,omitempty"` - ContributorInsightsMode string `json:"ContributorInsightsMode,omitempty"` - AttributeDefinitions []models.AttributeDefinition `json:"AttributeDefinitions"` - GlobalSecondaryIndexes []models.GlobalSecondaryIndex `json:"GlobalSecondaryIndexes,omitempty"` - Replicas []models.ReplicaDescription `json:"Replicas,omitempty"` - LocalSecondaryIndexes []models.LocalSecondaryIndex `json:"LocalSecondaryIndexes,omitempty"` - KeySchema []models.KeySchemaElement `json:"KeySchema"` - KinesisDestinations []KinesisDestinationEntry `json:"KinesisDestinations,omitempty"` - Items []map[string]any `json:"Items"` + activeSecondaryIndex *secondaryIndex + itemsByOffset map[int]map[string]any + mu *lockmetrics.RWMutex + activateTimer *time.Timer + Tags *tags.Tags `json:"Tags,omitempty"` + AutoScaling *autoScalingSettings `json:"AutoScaling,omitempty"` + // ReplicaAutoScaling holds per-replica read-capacity autoscaling settings, + // keyed by RegionName (see replicaAutoScalingSettings doc). + ReplicaAutoScaling map[string]*replicaAutoScalingSettings `json:"ReplicaAutoScaling,omitempty"` + OnDemandMaxWriteRRU *int64 `json:"OnDemandMaxWriteRRU,omitempty"` + OnDemandMaxReadRRU *int64 `json:"OnDemandMaxReadRRU,omitempty"` + ResourcePolicy string `json:"ResourcePolicy,omitempty"` + ResourcePolicyRevision string `json:"ResourcePolicyRevision,omitempty"` + TTLAttribute string `json:"TTLAttribute,omitempty"` + StreamViewType string `json:"StreamViewType,omitempty"` + StreamARN string `json:"StreamARN,omitempty"` + GlobalTableName string `json:"GlobalTableName,omitempty"` + MultiRegionConsistency string `json:"MultiRegionConsistency,omitempty"` + TableArn string `json:"TableArn"` + Status string `json:"Status"` + TableID string `json:"TableID"` + SSEType string `json:"SSEType,omitempty"` + TableClass string `json:"TableClass,omitempty"` + BillingMode string `json:"BillingMode,omitempty"` + Name string `json:"Name"` + SSEKMSMasterKeyArn string `json:"SSEKMSMasterKeyArn,omitempty"` + ContributorInsightsMode string `json:"ContributorInsightsMode,omitempty"` + AttributeDefinitions []models.AttributeDefinition `json:"AttributeDefinitions"` + GlobalSecondaryIndexes []models.GlobalSecondaryIndex `json:"GlobalSecondaryIndexes,omitempty"` + Replicas []models.ReplicaDescription `json:"Replicas,omitempty"` + LocalSecondaryIndexes []models.LocalSecondaryIndex `json:"LocalSecondaryIndexes,omitempty"` + KeySchema []models.KeySchemaElement `json:"KeySchema"` + KinesisDestinations []KinesisDestinationEntry `json:"KinesisDestinations,omitempty"` + Items []map[string]any `json:"Items"` itemSizes []int // PITRSnapshots is the per-table PITR ring buffer (see pitrSnapshot). It must be // exported with a json tag -- encoding/json silently skips unexported fields, so an diff --git a/services/dynamodb/table_ops.go b/services/dynamodb/table_ops.go index 1762baa308..f7d9faa649 100644 --- a/services/dynamodb/table_ops.go +++ b/services/dynamodb/table_ops.go @@ -1159,7 +1159,7 @@ func (db *InMemoryDB) applyOneReplicaTableEntry( db.tables.Put(replica) } else { - existing.GlobalTableName = tableName + setTableGlobalTableNameLocked(existing, tableName) } case update.Delete != nil: diff --git a/services/dynamodb/table_status_test.go b/services/dynamodb/table_status_test.go index 21c974303f..e7114cb552 100644 --- a/services/dynamodb/table_status_test.go +++ b/services/dynamodb/table_status_test.go @@ -2,6 +2,7 @@ package dynamodb_test import ( "testing" + "testing/synctest" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -60,31 +61,34 @@ func TestTableStatus(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - db := ddb.NewInMemoryDB() - if tt.createDelay > 0 { - db.SetCreateDelay(tt.createDelay) - } + synctest.Test(t, func(t *testing.T) { + db := ddb.NewInMemoryDB() + if tt.createDelay > 0 { + db.SetCreateDelay(tt.createDelay) + } - out, err := db.CreateTable(t.Context(), createInput(tt.tableName)) - require.NoError(t, err) - assert.Equal(t, tt.wantInitStatus, out.TableDescription.TableStatus) + out, err := db.CreateTable(t.Context(), createInput(tt.tableName)) + require.NoError(t, err) + assert.Equal(t, tt.wantInitStatus, out.TableDescription.TableStatus) - desc, err := db.DescribeTable(t.Context(), &sdk.DescribeTableInput{ - TableName: aws.String(tt.tableName), - }) - require.NoError(t, err) - assert.Equal(t, tt.wantInitStatus, desc.Table.TableStatus) - - if tt.finalSleep > 0 { - time.Sleep(tt.finalSleep) - - desc2, err2 := db.DescribeTable(t.Context(), &sdk.DescribeTableInput{ + desc, err := db.DescribeTable(t.Context(), &sdk.DescribeTableInput{ TableName: aws.String(tt.tableName), }) - require.NoError(t, err2) - assert.Equal(t, tt.wantFinalStatus, desc2.Table.TableStatus, - "expected ACTIVE after delay elapsed") - } + require.NoError(t, err) + assert.Equal(t, tt.wantInitStatus, desc.Table.TableStatus) + + if tt.finalSleep > 0 { + time.Sleep(tt.finalSleep) + synctest.Wait() + + desc2, err2 := db.DescribeTable(t.Context(), &sdk.DescribeTableInput{ + TableName: aws.String(tt.tableName), + }) + require.NoError(t, err2) + assert.Equal(t, tt.wantFinalStatus, desc2.Table.TableStatus, + "expected ACTIVE after delay elapsed") + } + }) }) } } @@ -96,33 +100,37 @@ func TestTableStatus(t *testing.T) { func TestDeleteWhileCreating(t *testing.T) { t.Parallel() - db := ddb.NewInMemoryDB() - db.SetCreateDelay(150 * time.Millisecond) + synctest.Test(t, func(t *testing.T) { + db := ddb.NewInMemoryDB() + db.SetCreateDelay(150 * time.Millisecond) - out, err := db.CreateTable(t.Context(), createInput("timer-cancel-table")) - require.NoError(t, err) - require.Equal(t, types.TableStatusCreating, out.TableDescription.TableStatus) + out, err := db.CreateTable(t.Context(), createInput("timer-cancel-table")) + require.NoError(t, err) + require.Equal(t, types.TableStatusCreating, out.TableDescription.TableStatus) - // Delete while still CREATING must be rejected. - _, err = db.DeleteTable(t.Context(), &sdk.DeleteTableInput{ - TableName: aws.String("timer-cancel-table"), - }) - require.Error(t, err) - var ddbErr *ddb.Error - require.ErrorAs(t, err, &ddbErr) - assert.Contains(t, ddbErr.Type, "ResourceInUseException") + // Delete while still CREATING must be rejected. + _, err = db.DeleteTable(t.Context(), &sdk.DeleteTableInput{ + TableName: aws.String("timer-cancel-table"), + }) + require.Error(t, err) + var ddbErr *ddb.Error + require.ErrorAs(t, err, &ddbErr) + assert.Contains(t, ddbErr.Type, "ResourceInUseException") + + time.Sleep(200 * time.Millisecond) + synctest.Wait() - require.Eventually(t, func() bool { desc, descErr := db.DescribeTable(t.Context(), &sdk.DescribeTableInput{ TableName: aws.String("timer-cancel-table"), }) + require.NoError(t, descErr) + require.Equal(t, types.TableStatusActive, desc.Table.TableStatus, + "table should become ACTIVE after the create delay elapses") - return descErr == nil && desc.Table.TableStatus == types.TableStatusActive - }, time.Second, 10*time.Millisecond, "table should become ACTIVE after the create delay elapses") - - // Now that the table is ACTIVE, deletion must succeed. - _, err = db.DeleteTable(t.Context(), &sdk.DeleteTableInput{ - TableName: aws.String("timer-cancel-table"), + // Now that the table is ACTIVE, deletion must succeed. + _, err = db.DeleteTable(t.Context(), &sdk.DeleteTableInput{ + TableName: aws.String("timer-cancel-table"), + }) + require.NoError(t, err) }) - require.NoError(t, err) } diff --git a/services/ec2/lifecycle_test.go b/services/ec2/lifecycle_test.go index f65d418385..431143116d 100644 --- a/services/ec2/lifecycle_test.go +++ b/services/ec2/lifecycle_test.go @@ -3,6 +3,7 @@ package ec2_test import ( "context" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -166,29 +167,28 @@ func TestEC2Lifecycle_StopPendingInstance(t *testing.T) { func TestEC2Lifecycle_BackgroundReconciler(t *testing.T) { t.Parallel() - b := ec2.NewInMemoryBackend("000000000000", "us-east-1") - // This test exercises the production background reconciler, so it starts the - // goroutine explicitly and stops it on cleanup. All other tests drive - // lifecycle transitions via TickLifecycleForTest and leave it stopped. - b.StartLifecycleReconciler(context.Background()) - t.Cleanup(b.StopLifecycleReconciler) + synctest.Test(t, func(t *testing.T) { + b := ec2.NewInMemoryBackend("000000000000", "us-east-1") + // This test exercises the production background reconciler, so it + // starts the goroutine explicitly and stops it before the bubble + // exits (StopLifecycleReconciler must run inside the bubble, or the + // still-running ticker goroutine deadlocks the bubble on exit). All + // other tests drive lifecycle transitions via TickLifecycleForTest + // and leave it stopped. + b.StartLifecycleReconciler(context.Background()) - instances, err := b.RunInstances("ami-123", "t2.micro", "", 1) - require.NoError(t, err) + instances, err := b.RunInstances("ami-123", "t2.micro", "", 1) + require.NoError(t, err) - // Wait up to 500ms for the goroutine to advance state. - deadline := time.Now().Add(500 * time.Millisecond) + // lifecycleReconcileInterval is 50ms; cross a few ticks. + time.Sleep(200 * time.Millisecond) + synctest.Wait() - for time.Now().Before(deadline) { all := b.DescribeInstances([]string{instances[0].ID}, "") require.Len(t, all, 1) + assert.Equal(t, "running", all[0].State.Name, "instance did not advance from pending to running") - if all[0].State.Name == "running" { - return - } - - time.Sleep(10 * time.Millisecond) - } - - t.Fatal("instance did not advance from pending to running within 500ms") + b.StopLifecycleReconciler() + synctest.Wait() + }) } diff --git a/services/ecr/replication_test.go b/services/ecr/replication_test.go index c6fa003b1e..11b92e1ae4 100644 --- a/services/ecr/replication_test.go +++ b/services/ecr/replication_test.go @@ -9,6 +9,7 @@ import ( "context" "net/http" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -259,43 +260,45 @@ func TestReplicationConfiguration_Clear(t *testing.T) { func TestDescribeImageReplicationStatus_ReturnsStatus(t *testing.T) { t.Parallel() - h := newAccuracyHandler() - mustCreateRepo(t, h, "replication-repo") - - // A replication status is reported per configured destination; with no - // replication configuration the list is (correctly) empty, so configure one. - repCfg := doAccuracy(t, h, "PutReplicationConfiguration", map[string]any{ - "replicationConfiguration": map[string]any{ - "rules": []any{ - map[string]any{ - "destinations": []any{ - map[string]any{"region": "us-west-2", "registryId": "000000000000"}, + synctest.Test(t, func(t *testing.T) { + h := newAccuracyHandler() + mustCreateRepo(t, h, "replication-repo") + + // A replication status is reported per configured destination; with no + // replication configuration the list is (correctly) empty, so configure one. + repCfg := doAccuracy(t, h, "PutReplicationConfiguration", map[string]any{ + "replicationConfiguration": map[string]any{ + "rules": []any{ + map[string]any{ + "destinations": []any{ + map[string]any{"region": "us-west-2", "registryId": "000000000000"}, + }, }, }, }, - }, - }) - require.Equal(t, http.StatusOK, repCfg.Code) + }) + require.Equal(t, http.StatusOK, repCfg.Code) - digest := mustPutImage(t, h, "replication-repo", "v1.0", `{"schemaVersion":2,"repl":"test"}`) + digest := mustPutImage(t, h, "replication-repo", "v1.0", `{"schemaVersion":2,"repl":"test"}`) - // Wait briefly for async replication to complete - time.Sleep(20 * time.Millisecond) + // Wait briefly for async replication to complete + time.Sleep(20 * time.Millisecond) - rec := doAccuracy(t, h, "DescribeImageReplicationStatus", map[string]any{ - "repositoryName": "replication-repo", - "imageId": map[string]any{ - "imageDigest": digest, - }, + rec := doAccuracy(t, h, "DescribeImageReplicationStatus", map[string]any{ + "repositoryName": "replication-repo", + "imageId": map[string]any{ + "imageDigest": digest, + }, + }) + require.Equal(t, http.StatusOK, rec.Code) + + out := parseAccuracy(t, rec) + assert.Equal(t, "replication-repo", out["repositoryName"]) + statuses, _ := out["replicationStatuses"].([]any) + require.NotEmpty(t, statuses, "replicationStatuses must be present") + status := statuses[0].(map[string]any) + assert.NotEmpty(t, status["status"], "replication status must not be empty") }) - require.Equal(t, http.StatusOK, rec.Code) - - out := parseAccuracy(t, rec) - assert.Equal(t, "replication-repo", out["repositoryName"]) - statuses, _ := out["replicationStatuses"].([]any) - require.NotEmpty(t, statuses, "replicationStatuses must be present") - status := statuses[0].(map[string]any) - assert.NotEmpty(t, status["status"], "replication status must not be empty") } func TestDescribeImageReplicationStatus_ByTag(t *testing.T) { diff --git a/services/ecrpublic/PARITY.md b/services/ecrpublic/PARITY.md new file mode 100644 index 0000000000..ce4e43e0fd --- /dev/null +++ b/services/ecrpublic/PARITY.md @@ -0,0 +1,93 @@ +--- +service: ecrpublic +sdk_module: aws-sdk-go-v2/service/ecrpublic@v1.47.1 +last_audit_commit: 709187947 +last_audit_date: 2026-09-25 +overall: B # new service, control plane + honest layer/image metadata tracking, unit-tested against the real SDK client +ops: + CreateRepository: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeRepositories: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteRepository: {wire: ok, errors: ok, state: ok, persist: ok, note: "force required to delete a non-empty repository"} + GetRepositoryCatalogData: {wire: ok, errors: ok, state: ok, persist: ok} + PutRepositoryCatalogData: {wire: ok, errors: ok, state: ok, persist: ok} + SetRepositoryPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + GetRepositoryPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + DeleteRepositoryPolicy: {wire: ok, errors: ok, state: ok, persist: ok} + TagResource: {wire: ok, errors: ok, state: ok, persist: ok} + UntagResource: {wire: ok, errors: ok, state: ok, persist: ok} + ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeRegistries: {wire: ok, errors: ok, state: ok, persist: n/a, note: "single-tenant emulator: always returns exactly the caller's own registry -- see items_still_open"} + GetRegistryCatalogData: {wire: ok, errors: ok, state: ok, persist: ok} + PutRegistryCatalogData: {wire: ok, errors: ok, state: ok, persist: ok} + GetAuthorizationToken: {wire: ok, errors: ok, state: ok, persist: n/a, note: "stable dummy AWS:password credential, matches services/ecr's convention -- see items_still_open"} + DescribeImages: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeImageTags: {wire: ok, errors: ok, state: ok, persist: ok} + BatchCheckLayerAvailability: {wire: ok, errors: ok, state: ok, persist: ok} + InitiateLayerUpload: {wire: ok, errors: ok, state: ok, persist: n/a, note: "in-flight sessions never persisted, matching AWS; abandoned sessions pruned lazily after layerUploadTTL (24h)"} + UploadLayerPart: {wire: ok, errors: ok, state: ok, persist: n/a} + CompleteLayerUpload: {wire: ok, errors: ok, state: ok, persist: ok, note: "SHA256 computed from accumulated bytes; verified against a caller-supplied full digest"} + PutImage: {wire: ok, errors: ok, state: ok, persist: ok, note: "rejects a manifest referencing layer/config digests never uploaded (LayersNotFoundException)"} + BatchDeleteImage: {wire: ok, errors: ok, state: ok, persist: ok} +families: + Repository: {status: ok, note: "Create/Describe/Delete verified end-to-end against the real aws-sdk-go-v2 client over an httptest server -- ARN (arn:aws:ecr-public:::repository/, no region segment), repositoryUri (public.ecr.aws//), and epoch createdAt all round-trip cleanly."} + CatalogData: {status: ok, note: "Repository- and registry-level catalog data round-trip the full shape (aboutText/description/usageText/architectures/operatingSystems/logoImageBlob). logoUrl is a synthetic placeholder host, not a real asset store."} + RepositoryPolicy: {status: ok, note: "Set/Get/Delete round-trip opaque policy text; RepositoryPolicyNotFoundException on a repository with no policy."} + Tags: {status: ok, note: "TagResource/UntagResource/ListTagsForResource key off the repository ARN, the only taggable resource in this API."} + Registry: {status: ok, note: "DescribeRegistries/Get+PutRegistryCatalogData/GetAuthorizationToken -- see items_still_open for the single-tenant and dummy-credential simplifications."} + ImagesAndLayers: {status: ok, note: "BatchCheckLayerAvailability/InitiateLayerUpload/UploadLayerPart/CompleteLayerUpload/PutImage/BatchDeleteImage/DescribeImages/DescribeImageTags all mutate real in-memory state: layer bytes are buffered and SHA256-verified, PutImage verifies every layer/config digest a manifest references was actually uploaded first, and BatchDeleteImage's by-tag vs by-digest semantics match AWS (by-tag removes only the binding; the image survives untagged)."} +gaps: [] +items_still_open: + - "There is no embedded Docker Registry v2 HTTP API (unlike services/ecr's optional + GOPHERSTACK_ENABLE_LOCAL_REGISTRY local registry): the control-plane layer/image + operations (BatchCheckLayerAvailability, InitiateLayerUpload, UploadLayerPart, + CompleteLayerUpload, PutImage, BatchDeleteImage) track real layer/image metadata, but + nothing serves the resulting blobs over /v2/... for an actual `docker pull` against a + public.ecr.aws-style host. Structural: out of scope for this pass." + - "GetAuthorizationToken returns a stable dummy AWS:password credential and does not + enforce docker-login authentication against it, matching services/ecr's existing + convention for the same operation." + - "DescribeRegistries is single-tenant: it always returns exactly the caller's own + registry, never other accounts' registries. There is no cross-account Amazon ECR + Public Gallery directory modeled (that surface is the public gallery.ecr.aws website, + not this control-plane API, but even the multi-account admin view this operation can + return for a verified account is not modeled)." + - "Registry/repository 'verified' and marketplaceCertified badges are always false -- + the Amazon Web Services Marketplace vendor verification workflow is not modeled." + - "PutImage's manifest-layer verification only understands a plain OCI/Docker image + manifest ({config.digest, layers[].digest}); a manifest list / OCI index (multi-arch) + is not parsed for referenced digests and is pushed without that check. Real docker + clients pushing multi-arch images would not get LayersNotFoundException protection + for the top-level manifest list, only for each per-platform manifest they also push." +--- + +## Notes + +Initial implementation (2026-09-25): JSON-RPC (awsjson1.1, X-Amz-Target prefix +`SpencerFrontendService.`) control-plane API modeled after services/kinesisvideo's +pkgs/store + pkgs/lockmetrics + pkgs/persistence layout, and after services/ecr's +X-Amz-Target dispatch via pkgs/service.HandleTarget/WrapOp for the shared protocol. +Every operation mutates/reads real in-memory state, with JSON snapshot/restore wired +into pkgs/persistence. + +Wire shapes and errors were verified directly against the pinned +aws-sdk-go-v2/service/ecrpublic v1.47.1 request_snapshot/ and response_snapshot/ +fixtures -- that SDK version generates via smithy schemas rather than the older +serializers.go/deserializers.go, so those exact-wire-JSON snapshot fixtures (one +per operation, request and response, plus one per reachable exception) are the +authoritative source, not a serializer function body. Confirmed from AWS docs and +the terraform-provider-aws `aws_ecrpublic_repository` resource: unlike private ECR, +the repository ARN omits the region segment +(`arn:aws:ecr-public:::repository/`), and repositoryUri follows +`public.ecr.aws//` with a registry alias derived +deterministically per account (a real alias is an opaque, AWS-assigned string). + +### 2026-09-26: InitiateLayerUpload session leak fixed + +Unfinished `InitiateLayerUpload` sessions (never reaching +`CompleteLayerUpload`) were retained in `layerUploads` forever, leaking +memory in a long-running server. AWS does not document an explicit expiry +window for unfinished layer uploads, so this follows services/ecr's own +`layerUploadTTL` precedent: sessions older than 24h are now pruned lazily on +the next `InitiateLayerUpload` call (`pruneExpiredLayerUploadsLocked`, in +layers.go). Covered by `layer_upload_ttl_test.go` +(`testing/synctest`-driven: kept within the window, evicted just past it). diff --git a/services/ecrpublic/README.md b/services/ecrpublic/README.md new file mode 100644 index 0000000000..17b066f0b2 --- /dev/null +++ b/services/ecrpublic/README.md @@ -0,0 +1,27 @@ + +# Ecrpublic + +**Parity grade: B** · SDK `aws-sdk-go-v2/service/ecrpublic@v1.47.1` · last audited 2026-09-25 (`709187947`) + +## Coverage + +| Metric | Value | +| --- | --- | +| PARITY entries audited | 23 (23 ok) | +| Feature families | 6 (6 ok) | +| Known gaps | 5 | +| Deferred items | 0 | +| Resource leaks | unknown | + +### Known gaps + +- "There is no embedded Docker Registry v2 HTTP API (unlike services/ecr's optional GOPHERSTACK_ENABLE_LOCAL_REGISTRY local registry): the control-plane layer/image operations (BatchCheckLayerAvailability, InitiateLayerUpload, UploadLayerPart, CompleteLayerUpload, PutImage, BatchDeleteImage) track real layer/image metadata, but nothing serves the resulting blobs over /v2/... for an actual `docker pull` against a public.ecr.aws-style host. Structural: out of scope for this pass." +- "GetAuthorizationToken returns a stable dummy AWS:password credential and does not enforce docker-login authentication against it, matching services/ecr's existing convention for the same operation." +- "DescribeRegistries is single-tenant: it always returns exactly the caller's own registry, never other accounts' registries. There is no cross-account Amazon ECR Public Gallery directory modeled (that surface is the public gallery.ecr.aws website, not this control-plane API, but even the multi-account admin view this operation can return for a verified account is not modeled)." +- "Registry/repository 'verified' and marketplaceCertified badges are always false -- the Amazon Web Services Marketplace vendor verification workflow is not modeled." +- "PutImage's manifest-layer verification only understands a plain OCI/Docker image manifest ({config.digest, layers[].digest}); a manifest list / OCI index (multi-arch) is not parsed for referenced digests and is pushed without that check. Real docker clients pushing multi-arch images would not get LayersNotFoundException protection for the top-level manifest list, only for each per-platform manifest they also push." + +## More + +- [Full parity audit](PARITY.md) +- [All services](../../README.md#services) diff --git a/services/ecrpublic/auth.go b/services/ecrpublic/auth.go new file mode 100644 index 0000000000..53f4fac4e3 --- /dev/null +++ b/services/ecrpublic/auth.go @@ -0,0 +1,28 @@ +package ecrpublic + +import ( + "context" + "encoding/base64" + "time" +) + +const ( + authTokenTTL = 12 * time.Hour + authTokenUser = "AWS" + // authTokenPassword is a stable dummy credential, not a real secret; this + // emulator does not enforce docker-login authentication against it + // (matches services/ecr). + //nolint:gosec // dummy emulator credential, not a real secret + authTokenPassword = "gopherstack-ecr-public-dummy-password" +) + +// GetAuthorizationToken returns a base64(user:password) authorization token +// and its expiry. The emulator does not model per-principal credentials -- +// every caller gets the same stable token, honestly reflecting that this is +// not a real authentication backend (see PARITY.md). +func (b *InMemoryBackend) GetAuthorizationToken(_ context.Context) (string, int64, error) { + token := base64.StdEncoding.EncodeToString([]byte(authTokenUser + ":" + authTokenPassword)) + expiresAt := time.Now().Add(authTokenTTL).Unix() + + return token, expiresAt, nil +} diff --git a/services/ecrpublic/auth_test.go b/services/ecrpublic/auth_test.go new file mode 100644 index 0000000000..5c8ec0ee44 --- /dev/null +++ b/services/ecrpublic/auth_test.go @@ -0,0 +1,33 @@ +package ecrpublic_test + +import ( + "encoding/base64" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + ecrpublicsdk "github.com/aws/aws-sdk-go-v2/service/ecrpublic" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetAuthorizationToken(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + out, err := client.GetAuthorizationToken(t.Context(), &ecrpublicsdk.GetAuthorizationTokenInput{}) + require.NoError(t, err) + require.NotNil(t, out.AuthorizationData) + + token := aws.ToString(out.AuthorizationData.AuthorizationToken) + require.NotEmpty(t, token) + + decoded, err := base64.StdEncoding.DecodeString(token) + require.NoError(t, err) + assert.True(t, strings.HasPrefix(string(decoded), "AWS:")) + + require.NotNil(t, out.AuthorizationData.ExpiresAt) + assert.WithinDuration(t, time.Now().Add(12*time.Hour), *out.AuthorizationData.ExpiresAt, time.Minute) +} diff --git a/services/ecrpublic/catalog.go b/services/ecrpublic/catalog.go new file mode 100644 index 0000000000..7fd8d4ced6 --- /dev/null +++ b/services/ecrpublic/catalog.go @@ -0,0 +1,91 @@ +package ecrpublic + +import "fmt" + +// GetRepositoryCatalogData returns the Gallery-visible catalog metadata for a repository. +func (b *InMemoryBackend) GetRepositoryCatalogData(registryID, name string) (*CatalogData, error) { + b.mu.RLock("GetRepositoryCatalogData") + defer b.mu.RUnlock() + + if err := b.resolveRegistryIDLocked(registryID); err != nil { + return nil, err + } + + repo, ok := b.repos.Get(name) + if !ok { + return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, name) + } + + cd := repo.CatalogData + + return &cd, nil +} + +// PutRepositoryCatalogData replaces the Gallery-visible catalog metadata for a repository. +func (b *InMemoryBackend) PutRepositoryCatalogData( + registryID, name string, catalogData *CatalogData, +) (*CatalogData, error) { + b.mu.Lock("PutRepositoryCatalogData") + defer b.mu.Unlock() + + if err := b.resolveRegistryIDLocked(registryID); err != nil { + return nil, err + } + + repo, ok := b.repos.Get(name) + if !ok { + return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, name) + } + + if catalogData != nil { + repo.CatalogData = *catalogData + } + + cd := repo.CatalogData + + return &cd, nil +} + +// DescribeRegistries returns the public registries visible to the caller. +// This is a single-tenant emulator: it always returns exactly the caller's +// own registry, never other accounts' registries (there is no cross-account +// Gallery directory modeled here -- see PARITY.md). +func (b *InMemoryBackend) DescribeRegistries() ([]RegistryInfo, error) { + b.mu.RLock("DescribeRegistries") + defer b.mu.RUnlock() + + info := RegistryInfo{ + RegistryArn: registryARN(b.region, b.accountID), + RegistryID: b.accountID, + RegistryURI: repositoryURIHost + "/" + b.registryAlias, + Verified: false, + Aliases: []RegistryAliasInfo{ + { + Name: b.registryAlias, + DefaultRegistryAlias: true, + PrimaryRegistryAlias: true, + Status: "ACTIVE", + }, + }, + } + + return []RegistryInfo{info}, nil +} + +// GetRegistryCatalogData returns the account-wide Gallery display metadata. +func (b *InMemoryBackend) GetRegistryCatalogData() (RegistryCatalogData, error) { + b.mu.RLock("GetRegistryCatalogData") + defer b.mu.RUnlock() + + return b.registryCatalogData, nil +} + +// PutRegistryCatalogData replaces the account-wide Gallery display metadata. +func (b *InMemoryBackend) PutRegistryCatalogData(displayName string) (RegistryCatalogData, error) { + b.mu.Lock("PutRegistryCatalogData") + defer b.mu.Unlock() + + b.registryCatalogData = RegistryCatalogData{DisplayName: displayName} + + return b.registryCatalogData, nil +} diff --git a/services/ecrpublic/catalog_test.go b/services/ecrpublic/catalog_test.go new file mode 100644 index 0000000000..4df1293250 --- /dev/null +++ b/services/ecrpublic/catalog_test.go @@ -0,0 +1,102 @@ +package ecrpublic_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + ecrpublicsdk "github.com/aws/aws-sdk-go-v2/service/ecrpublic" + "github.com/aws/aws-sdk-go-v2/service/ecrpublic/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRepositoryCatalogDataLifecycle(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateRepository( + ctx, + &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String("catalog-repo")}, + ) + require.NoError(t, err) + + got, err := client.GetRepositoryCatalogData(ctx, &ecrpublicsdk.GetRepositoryCatalogDataInput{ + RepositoryName: aws.String("catalog-repo"), + }) + require.NoError(t, err) + assert.Empty(t, aws.ToString(got.CatalogData.AboutText)) + + put, err := client.PutRepositoryCatalogData(ctx, &ecrpublicsdk.PutRepositoryCatalogDataInput{ + RepositoryName: aws.String("catalog-repo"), + CatalogData: &types.RepositoryCatalogDataInput{ + AboutText: aws.String("new about"), + Architectures: []string{"x86-64"}, + }, + }) + require.NoError(t, err) + assert.Equal(t, "new about", aws.ToString(put.CatalogData.AboutText)) + + after, err := client.GetRepositoryCatalogData(ctx, &ecrpublicsdk.GetRepositoryCatalogDataInput{ + RepositoryName: aws.String("catalog-repo"), + }) + require.NoError(t, err) + assert.Equal(t, "new about", aws.ToString(after.CatalogData.AboutText)) + assert.Equal(t, []string{"x86-64"}, after.CatalogData.Architectures) +} + +func TestGetRepositoryCatalogData_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.GetRepositoryCatalogData(t.Context(), &ecrpublicsdk.GetRepositoryCatalogDataInput{ + RepositoryName: aws.String("missing"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "RepositoryNotFoundException", apiErr.ErrorCode()) +} + +func TestDescribeRegistries(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + out, err := client.DescribeRegistries(t.Context(), &ecrpublicsdk.DescribeRegistriesInput{}) + require.NoError(t, err) + require.Len(t, out.Registries, 1) + + reg := out.Registries[0] + assert.Equal(t, testAccountID, aws.ToString(reg.RegistryId)) + assert.Equal(t, "arn:aws:ecr-public::123456789012:registry", aws.ToString(reg.RegistryArn)) + require.Len(t, reg.Aliases, 1) + assert.True(t, reg.Aliases[0].DefaultRegistryAlias) + assert.True(t, reg.Aliases[0].PrimaryRegistryAlias) + assert.Equal(t, types.RegistryAliasStatusActive, reg.Aliases[0].Status) +} + +func TestRegistryCatalogDataLifecycle(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + got, err := client.GetRegistryCatalogData(ctx, &ecrpublicsdk.GetRegistryCatalogDataInput{}) + require.NoError(t, err) + assert.Empty(t, aws.ToString(got.RegistryCatalogData.DisplayName)) + + put, err := client.PutRegistryCatalogData(ctx, &ecrpublicsdk.PutRegistryCatalogDataInput{ + DisplayName: aws.String("My Org"), + }) + require.NoError(t, err) + assert.Equal(t, "My Org", aws.ToString(put.RegistryCatalogData.DisplayName)) + + after, err := client.GetRegistryCatalogData(ctx, &ecrpublicsdk.GetRegistryCatalogDataInput{}) + require.NoError(t, err) + assert.Equal(t, "My Org", aws.ToString(after.RegistryCatalogData.DisplayName)) +} diff --git a/services/ecrpublic/digest.go b/services/ecrpublic/digest.go new file mode 100644 index 0000000000..c888625042 --- /dev/null +++ b/services/ecrpublic/digest.go @@ -0,0 +1,70 @@ +package ecrpublic + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" +) + +func sha256Digest(data []byte) string { + sum := sha256.Sum256(data) + + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// isFullSHA256Digest returns true when s is a properly-formed "sha256:<64 hex>" digest. +func isFullSHA256Digest(s string) bool { + const prefix = "sha256:" + if len(s) != len(prefix)+sha256.Size*2 { + return false + } + + if s[:len(prefix)] != prefix { + return false + } + + for _, c := range s[len(prefix):] { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') && (c < 'A' || c > 'F') { + return false + } + } + + return true +} + +// manifestLayerDigests is the minimal subset of an OCI/Docker image manifest +// needed to verify that every layer (and the config blob) a PutImage call +// references was actually uploaded first. A manifest this loose parser +// cannot decode (e.g. a manifest list / OCI index for multi-arch images) is +// treated as having no checkable layer references, since a real manifest +// list references per-platform manifests rather than layer digests directly. +type manifestLayers struct { + Config struct { + Digest string `json:"digest"` + } `json:"config"` + Layers []struct { + Digest string `json:"digest"` + } `json:"layers"` +} + +// referencedDigests returns every layer/config digest manifest references, +// or nil if manifest is not a shape this parser understands. +func referencedDigests(manifest string) []string { + var m manifestLayers + if err := json.Unmarshal([]byte(manifest), &m); err != nil { + return nil + } + + var out []string + if m.Config.Digest != "" { + out = append(out, m.Config.Digest) + } + + for _, l := range m.Layers { + if l.Digest != "" { + out = append(out, l.Digest) + } + } + + return out +} diff --git a/services/ecrpublic/errors.go b/services/ecrpublic/errors.go new file mode 100644 index 0000000000..fa7ce84efd --- /dev/null +++ b/services/ecrpublic/errors.go @@ -0,0 +1,29 @@ +package ecrpublic + +import "github.com/blackbirdworks/gopherstack/pkgs/awserr" + +// Sentinel errors, wrapped so callers can match with [errors.Is] while the +// message carries the real AWS exception name for classifyError. +var ( + ErrRepositoryNotFound = awserr.New("RepositoryNotFoundException", awserr.ErrNotFound) + ErrRepositoryAlreadyExists = awserr.New("RepositoryAlreadyExistsException", awserr.ErrAlreadyExists) + ErrRepositoryNotEmpty = awserr.New("RepositoryNotEmptyException", awserr.ErrConflict) + ErrRepositoryPolicyNotFound = awserr.New("RepositoryPolicyNotFoundException", awserr.ErrNotFound) + ErrRegistryNotFound = awserr.New("RegistryNotFoundException", awserr.ErrNotFound) + ErrInvalidParameter = awserr.New("InvalidParameterException", awserr.ErrInvalidParameter) + ErrTooManyTags = awserr.New("TooManyTagsException", awserr.ErrInvalidParameter) + ErrInvalidTagParameter = awserr.New("InvalidTagParameterException", awserr.ErrInvalidParameter) + + ErrUploadNotFound = awserr.New("UploadNotFoundException", awserr.ErrNotFound) + ErrEmptyUpload = awserr.New("EmptyUploadException", awserr.ErrInvalidParameter) + ErrLayerPartTooSmall = awserr.New("LayerPartTooSmallException", awserr.ErrInvalidParameter) + ErrInvalidLayerPart = awserr.New("InvalidLayerPartException", awserr.ErrInvalidParameter) + ErrLayerAlreadyExists = awserr.New("LayerAlreadyExistsException", awserr.ErrAlreadyExists) + ErrLayersNotFound = awserr.New("LayersNotFoundException", awserr.ErrInvalidParameter) + ErrInvalidLayer = awserr.New("InvalidLayerException", awserr.ErrInvalidParameter) + + ErrImageNotFound = awserr.New("ImageNotFoundException", awserr.ErrNotFound) + ErrImageAlreadyExists = awserr.New("ImageAlreadyExistsException", awserr.ErrAlreadyExists) + ErrImageDigestDoesNotMatch = awserr.New("ImageDigestDoesNotMatchException", awserr.ErrInvalidParameter) + ErrImageTagAlreadyExists = awserr.New("ImageTagAlreadyExistsException", awserr.ErrConflict) +) diff --git a/services/ecrpublic/handler.go b/services/ecrpublic/handler.go new file mode 100644 index 0000000000..cd3c28ae3c --- /dev/null +++ b/services/ecrpublic/handler.go @@ -0,0 +1,210 @@ +package ecrpublic + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "net/http" + "strings" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/pkgs/awserr" + "github.com/blackbirdworks/gopherstack/pkgs/httputils" + "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/service" +) + +// ecrPublicTargetPrefix is the X-Amz-Target service prefix, confirmed against +// aws-sdk-go-v2/service/ecrpublic@v1.47.1 api_client.go: +// options.Protocol = awsjson.New11(schemas.SpencerFrontendService). +const ecrPublicTargetPrefix = "SpencerFrontendService." + +var errUnknownAction = errors.New("UnknownOperationException") + +// Handler is the HTTP handler for the Amazon ECR Public control-plane API. +type Handler struct { + Backend Backend + ops map[string]service.JSONOpFunc + AccountID string + DefaultRegion string +} + +// NewHandler creates a new Amazon ECR Public handler. +func NewHandler(backend Backend) *Handler { + h := &Handler{Backend: backend} + h.ops = h.buildOps() + + return h +} + +// Reset clears backend state. +func (h *Handler) Reset() { h.Backend.Reset() } + +// Name returns the service name. +func (h *Handler) Name() string { return "ECRPublic" } + +// GetSupportedOperations returns the list of supported operations, matching +// aws-sdk-go-v2/service/ecrpublic@v1.47.1's client method set exactly. +func (h *Handler) GetSupportedOperations() []string { + ops := make([]string, 0, len(h.ops)) + for op := range h.ops { + ops = append(ops, op) + } + + return ops +} + +// ChaosServiceName returns the lowercase AWS service name for fault rule matching. +func (h *Handler) ChaosServiceName() string { return "ecrpublic" } + +// ChaosOperations returns all operations that can be fault-injected. +func (h *Handler) ChaosOperations() []string { return h.GetSupportedOperations() } + +// ChaosRegions returns all regions this handler handles. Amazon ECR Public is +// us-east-1-only, matching the real service. +func (h *Handler) ChaosRegions() []string { return []string{"us-east-1"} } + +// RouteMatcher matches requests carrying the SpencerFrontendService X-Amz-Target prefix. +func (h *Handler) RouteMatcher() service.Matcher { + return func(c *echo.Context) bool { + return strings.HasPrefix(c.Request().Header.Get("X-Amz-Target"), ecrPublicTargetPrefix) + } +} + +// MatchPriority returns the routing priority for header-exact matching. +func (h *Handler) MatchPriority() int { return service.PriorityHeaderExact } + +// ExtractOperation extracts the action name from the X-Amz-Target header. +func (h *Handler) ExtractOperation(c *echo.Context) string { + target := c.Request().Header.Get("X-Amz-Target") + + return strings.TrimPrefix(target, ecrPublicTargetPrefix) +} + +// ExtractResource extracts the repository name from the request body, when present. +func (h *Handler) ExtractResource(c *echo.Context) string { + body, err := httputils.ReadBody(c.Request()) + if err != nil { + return "" + } + + var req struct { + RepositoryName string `json:"repositoryName"` + } + + _ = json.Unmarshal(body, &req) + + return req.RepositoryName +} + +// Handler returns the Echo handler function for Amazon ECR Public requests. +func (h *Handler) Handler() echo.HandlerFunc { + return func(c *echo.Context) error { + ctx := c.Request().Context() + + return service.HandleTarget( + c, logger.Load(ctx), + "ECRPublic", "application/x-amz-json-1.1", + h.GetSupportedOperations(), + h.dispatch, + h.handleError, + ) + } +} + +// registryIDOrDefault returns registryID if set, else the backend's own account ID. +func (h *Handler) registryIDOrDefault(registryID string) string { + if registryID != "" { + return registryID + } + + return h.Backend.AccountID() +} + +func (h *Handler) buildOps() map[string]service.JSONOpFunc { + ops := make(map[string]service.JSONOpFunc) + + maps.Copy(ops, h.buildRepositoryOps()) + maps.Copy(ops, h.buildCatalogOps()) + maps.Copy(ops, h.buildPolicyOps()) + maps.Copy(ops, h.buildTagOps()) + maps.Copy(ops, h.buildAuthOps()) + maps.Copy(ops, h.buildImageOps()) + maps.Copy(ops, h.buildLayerOps()) + + return ops +} + +func (h *Handler) dispatch(ctx context.Context, action string, body []byte) ([]byte, error) { + fn, ok := h.ops[action] + if !ok { + return nil, fmt.Errorf("%w: %s", errUnknownAction, action) + } + + result, err := fn(ctx, body) + if err != nil { + return nil, err + } + + return json.Marshal(result) +} + +func (h *Handler) handleError(_ context.Context, c *echo.Context, _ string, err error) error { + status, errType := classifyError(err) + + return c.JSON(status, map[string]string{"__type": errType, "message": err.Error()}) +} + +// classifyError maps a backend/dispatch error to its HTTP status and AWS +// exception name. Every entry's status/name was confirmed against this +// service's response_snapshot/*.error.snap fixtures. +func classifyError(err error) (int, string) { + singleErrStatus := []struct { + err error + errType string + }{ + {ErrRepositoryNotFound, "RepositoryNotFoundException"}, + {ErrRepositoryAlreadyExists, "RepositoryAlreadyExistsException"}, + {ErrRepositoryNotEmpty, "RepositoryNotEmptyException"}, + {ErrRepositoryPolicyNotFound, "RepositoryPolicyNotFoundException"}, + {ErrRegistryNotFound, "RegistryNotFoundException"}, + {ErrTooManyTags, "TooManyTagsException"}, + {ErrInvalidTagParameter, "InvalidTagParameterException"}, + {ErrUploadNotFound, "UploadNotFoundException"}, + {ErrEmptyUpload, "EmptyUploadException"}, + {ErrLayerPartTooSmall, "LayerPartTooSmallException"}, + {ErrInvalidLayerPart, "InvalidLayerPartException"}, + {ErrInvalidLayer, "InvalidLayerException"}, + {ErrLayerAlreadyExists, "LayerAlreadyExistsException"}, + {ErrLayersNotFound, "LayersNotFoundException"}, + {ErrImageNotFound, "ImageNotFoundException"}, + {ErrImageAlreadyExists, "ImageAlreadyExistsException"}, + {ErrImageDigestDoesNotMatch, "ImageDigestDoesNotMatchException"}, + {ErrImageTagAlreadyExists, "ImageTagAlreadyExistsException"}, + {ErrInvalidParameter, "InvalidParameterException"}, + } + + for _, e := range singleErrStatus { + if errors.Is(err, e.err) { + return http.StatusBadRequest, e.errType + } + } + + var syntaxErr *json.SyntaxError + + var typeErr *json.UnmarshalTypeError + + switch { + case errors.Is(err, errUnknownAction): + return http.StatusBadRequest, "UnknownOperationException" + case errors.As(err, &syntaxErr), errors.As(err, &typeErr): + return http.StatusBadRequest, "InvalidParameterException" + case errors.Is(err, awserr.ErrNotFound): + return http.StatusBadRequest, "RepositoryNotFoundException" + default: + return http.StatusInternalServerError, "ServerException" + } +} diff --git a/services/ecrpublic/handler_auth.go b/services/ecrpublic/handler_auth.go new file mode 100644 index 0000000000..57057b738b --- /dev/null +++ b/services/ecrpublic/handler_auth.go @@ -0,0 +1,35 @@ +package ecrpublic + +import ( + "context" + + "github.com/blackbirdworks/gopherstack/pkgs/service" +) + +func (h *Handler) buildAuthOps() map[string]service.JSONOpFunc { + return map[string]service.JSONOpFunc{ + "GetAuthorizationToken": service.WrapOp(h.handleGetAuthorizationToken), + } +} + +type getAuthorizationTokenInput struct{} + +type getAuthorizationTokenOutput struct { + AuthorizationData *AuthorizationDataWire `json:"authorizationData,omitempty"` +} + +func (h *Handler) handleGetAuthorizationToken( + ctx context.Context, _ *getAuthorizationTokenInput, +) (*getAuthorizationTokenOutput, error) { + token, expiresAt, err := h.Backend.GetAuthorizationToken(ctx) + if err != nil { + return nil, err + } + + return &getAuthorizationTokenOutput{ + AuthorizationData: &AuthorizationDataWire{ + AuthorizationToken: token, + ExpiresAt: float64(expiresAt), + }, + }, nil +} diff --git a/services/ecrpublic/handler_catalog.go b/services/ecrpublic/handler_catalog.go new file mode 100644 index 0000000000..42909e5759 --- /dev/null +++ b/services/ecrpublic/handler_catalog.go @@ -0,0 +1,122 @@ +package ecrpublic + +import ( + "context" + + "github.com/blackbirdworks/gopherstack/pkgs/service" +) + +func (h *Handler) buildCatalogOps() map[string]service.JSONOpFunc { + return map[string]service.JSONOpFunc{ + "GetRepositoryCatalogData": service.WrapOp(h.handleGetRepositoryCatalogData), + "PutRepositoryCatalogData": service.WrapOp(h.handlePutRepositoryCatalogData), + "DescribeRegistries": service.WrapOp(h.handleDescribeRegistries), + "GetRegistryCatalogData": service.WrapOp(h.handleGetRegistryCatalogData), + "PutRegistryCatalogData": service.WrapOp(h.handlePutRegistryCatalogData), + } +} + +type getRepositoryCatalogDataInput struct { + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName"` +} + +type getRepositoryCatalogDataOutput struct { + CatalogData CatalogDataWire `json:"catalogData"` +} + +func (h *Handler) handleGetRepositoryCatalogData( + _ context.Context, in *getRepositoryCatalogDataInput, +) (*getRepositoryCatalogDataOutput, error) { + cd, err := h.Backend.GetRepositoryCatalogData(in.RegistryID, in.RepositoryName) + if err != nil { + return nil, err + } + + return &getRepositoryCatalogDataOutput{CatalogData: toCatalogDataWire(cd)}, nil +} + +type putRepositoryCatalogDataInput struct { + CatalogData *CatalogDataInputWire `json:"catalogData,omitempty"` + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName"` +} + +type putRepositoryCatalogDataOutput struct { + CatalogData CatalogDataWire `json:"catalogData"` +} + +func (h *Handler) handlePutRepositoryCatalogData( + _ context.Context, in *putRepositoryCatalogDataInput, +) (*putRepositoryCatalogDataOutput, error) { + cd, err := h.Backend.PutRepositoryCatalogData( + in.RegistryID, in.RepositoryName, toCatalogDataInput(in.CatalogData), + ) + if err != nil { + return nil, err + } + + return &putRepositoryCatalogDataOutput{CatalogData: toCatalogDataWire(cd)}, nil +} + +type describeRegistriesInput struct { + NextToken string `json:"nextToken,omitempty"` + MaxResults int32 `json:"maxResults,omitempty"` +} + +type describeRegistriesOutput struct { + NextToken string `json:"nextToken,omitempty"` + Registries []RegistryWire `json:"registries"` +} + +func (h *Handler) handleDescribeRegistries( + _ context.Context, _ *describeRegistriesInput, +) (*describeRegistriesOutput, error) { + registries, err := h.Backend.DescribeRegistries() + if err != nil { + return nil, err + } + + out := make([]RegistryWire, 0, len(registries)) + for _, r := range registries { + out = append(out, toRegistryWire(r)) + } + + return &describeRegistriesOutput{Registries: out}, nil +} + +type getRegistryCatalogDataInput struct{} + +type getRegistryCatalogDataOutput struct { + RegistryCatalogData RegistryCatalogDataWire `json:"registryCatalogData"` +} + +func (h *Handler) handleGetRegistryCatalogData( + _ context.Context, _ *getRegistryCatalogDataInput, +) (*getRegistryCatalogDataOutput, error) { + cd, err := h.Backend.GetRegistryCatalogData() + if err != nil { + return nil, err + } + + return &getRegistryCatalogDataOutput{RegistryCatalogData: RegistryCatalogDataWire(cd)}, nil +} + +type putRegistryCatalogDataInput struct { + DisplayName string `json:"displayName,omitempty"` +} + +type putRegistryCatalogDataOutput struct { + RegistryCatalogData RegistryCatalogDataWire `json:"registryCatalogData"` +} + +func (h *Handler) handlePutRegistryCatalogData( + _ context.Context, in *putRegistryCatalogDataInput, +) (*putRegistryCatalogDataOutput, error) { + cd, err := h.Backend.PutRegistryCatalogData(in.DisplayName) + if err != nil { + return nil, err + } + + return &putRegistryCatalogDataOutput{RegistryCatalogData: RegistryCatalogDataWire(cd)}, nil +} diff --git a/services/ecrpublic/handler_images.go b/services/ecrpublic/handler_images.go new file mode 100644 index 0000000000..3b6ce83099 --- /dev/null +++ b/services/ecrpublic/handler_images.go @@ -0,0 +1,148 @@ +package ecrpublic + +import ( + "context" + + "github.com/blackbirdworks/gopherstack/pkgs/service" +) + +func (h *Handler) buildImageOps() map[string]service.JSONOpFunc { + return map[string]service.JSONOpFunc{ + "DescribeImages": service.WrapOp(h.handleDescribeImages), + "DescribeImageTags": service.WrapOp(h.handleDescribeImageTags), + "PutImage": service.WrapOp(h.handlePutImage), + "BatchDeleteImage": service.WrapOp(h.handleBatchDeleteImage), + } +} + +type describeImagesInput struct { + NextToken string `json:"nextToken,omitempty"` + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName"` + ImageIDs []ImageIdentifierWire `json:"imageIds,omitempty"` + MaxResults int32 `json:"maxResults,omitempty"` +} + +type describeImagesOutput struct { + NextToken string `json:"nextToken,omitempty"` + ImageDetails []ImageDetailWire `json:"imageDetails"` +} + +func (h *Handler) handleDescribeImages(_ context.Context, in *describeImagesInput) (*describeImagesOutput, error) { + ids := make([]ImageIdentifier, 0, len(in.ImageIDs)) + for _, w := range in.ImageIDs { + ids = append(ids, imageIdentifierFromWire(w)) + } + + details, err := h.Backend.DescribeImages(in.RegistryID, in.RepositoryName, ids) + if err != nil { + return nil, err + } + + out := make([]ImageDetailWire, 0, len(details)) + for _, d := range details { + out = append(out, toImageDetailWire(d)) + } + + return &describeImagesOutput{ImageDetails: out}, nil +} + +type describeImageTagsInput struct { + NextToken string `json:"nextToken,omitempty"` + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName"` + MaxResults int32 `json:"maxResults,omitempty"` +} + +type describeImageTagsOutput struct { + NextToken string `json:"nextToken,omitempty"` + ImageTagDetails []ImageTagDetailWire `json:"imageTagDetails"` +} + +func (h *Handler) handleDescribeImageTags( + _ context.Context, in *describeImageTagsInput, +) (*describeImageTagsOutput, error) { + details, err := h.Backend.DescribeImageTags(in.RegistryID, in.RepositoryName) + if err != nil { + return nil, err + } + + out := make([]ImageTagDetailWire, 0, len(details)) + for _, d := range details { + out = append(out, toImageTagDetailWire(d)) + } + + return &describeImageTagsOutput{ImageTagDetails: out}, nil +} + +type putImageInput struct { + ImageDigest string `json:"imageDigest,omitempty"` + ImageManifest string `json:"imageManifest"` + ImageManifestMediaType string `json:"imageManifestMediaType,omitempty"` + ImageTag string `json:"imageTag,omitempty"` + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName"` +} + +type putImageOutput struct { + Image ImageWire `json:"image"` +} + +func (h *Handler) handlePutImage(_ context.Context, in *putImageInput) (*putImageOutput, error) { + img, err := h.Backend.PutImage(in.RegistryID, in.RepositoryName, PutImageRequest{ + ImageDigest: in.ImageDigest, + ImageManifest: in.ImageManifest, + ImageManifestMediaType: in.ImageManifestMediaType, + ImageTag: in.ImageTag, + }) + if err != nil { + return nil, err + } + + return &putImageOutput{ + Image: ImageWire{ + ImageID: ImageIdentifierWire{ImageDigest: img.ImageDigest, ImageTag: in.ImageTag}, + ImageManifest: img.ImageManifest, + ImageManifestMediaType: img.ImageManifestMediaType, + RegistryID: img.RegistryID, + RepositoryName: img.RepositoryName, + }, + }, nil +} + +type batchDeleteImageInput struct { + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName"` + ImageIDs []ImageIdentifierWire `json:"imageIds"` +} + +type batchDeleteImageOutput struct { + Failures []ImageFailureWire `json:"failures,omitempty"` + ImageIDs []ImageIdentifierWire `json:"imageIds"` +} + +func (h *Handler) handleBatchDeleteImage( + _ context.Context, in *batchDeleteImageInput, +) (*batchDeleteImageOutput, error) { + ids := make([]ImageIdentifier, 0, len(in.ImageIDs)) + for _, w := range in.ImageIDs { + ids = append(ids, imageIdentifierFromWire(w)) + } + + deleted, failures, err := h.Backend.BatchDeleteImage(in.RegistryID, in.RepositoryName, ids) + if err != nil { + return nil, err + } + + outIDs := make([]ImageIdentifierWire, 0, len(deleted)) + for _, id := range deleted { + outIDs = append(outIDs, toImageIdentifierWire(id)) + } + + outFailures := make([]ImageFailureWire, 0, len(failures)) + for _, f := range failures { + outFailures = append(outFailures, toImageFailureWire(f)) + } + + return &batchDeleteImageOutput{Failures: outFailures, ImageIDs: outIDs}, nil +} diff --git a/services/ecrpublic/handler_layers.go b/services/ecrpublic/handler_layers.go new file mode 100644 index 0000000000..747625b50e --- /dev/null +++ b/services/ecrpublic/handler_layers.go @@ -0,0 +1,133 @@ +package ecrpublic + +import ( + "context" + + "github.com/blackbirdworks/gopherstack/pkgs/service" +) + +func (h *Handler) buildLayerOps() map[string]service.JSONOpFunc { + return map[string]service.JSONOpFunc{ + "BatchCheckLayerAvailability": service.WrapOp(h.handleBatchCheckLayerAvailability), + "InitiateLayerUpload": service.WrapOp(h.handleInitiateLayerUpload), + "UploadLayerPart": service.WrapOp(h.handleUploadLayerPart), + "CompleteLayerUpload": service.WrapOp(h.handleCompleteLayerUpload), + } +} + +type batchCheckLayerAvailabilityInput struct { + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName"` + LayerDigests []string `json:"layerDigests"` +} + +type batchCheckLayerAvailabilityOutput struct { + Failures []LayerFailureWire `json:"failures,omitempty"` + Layers []LayerWire `json:"layers"` +} + +func (h *Handler) handleBatchCheckLayerAvailability( + _ context.Context, in *batchCheckLayerAvailabilityInput, +) (*batchCheckLayerAvailabilityOutput, error) { + layers, failures, err := h.Backend.BatchCheckLayerAvailability(in.RegistryID, in.RepositoryName, in.LayerDigests) + if err != nil { + return nil, err + } + + outLayers := make([]LayerWire, 0, len(layers)) + for _, l := range layers { + outLayers = append(outLayers, toLayerWire(l)) + } + + outFailures := make([]LayerFailureWire, 0, len(failures)) + for _, f := range failures { + outFailures = append(outFailures, toLayerFailureWire(f)) + } + + return &batchCheckLayerAvailabilityOutput{Failures: outFailures, Layers: outLayers}, nil +} + +type initiateLayerUploadInput struct { + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName"` +} + +type initiateLayerUploadOutput struct { + UploadID string `json:"uploadId,omitempty"` + PartSize int64 `json:"partSize,omitempty"` +} + +func (h *Handler) handleInitiateLayerUpload( + _ context.Context, in *initiateLayerUploadInput, +) (*initiateLayerUploadOutput, error) { + uploadID, partSize, err := h.Backend.InitiateLayerUpload(in.RegistryID, in.RepositoryName) + if err != nil { + return nil, err + } + + return &initiateLayerUploadOutput{UploadID: uploadID, PartSize: partSize}, nil +} + +type uploadLayerPartInput struct { + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName"` + UploadID string `json:"uploadId"` + LayerPartBlob []byte `json:"layerPartBlob"` + PartFirstByte int64 `json:"partFirstByte"` + PartLastByte int64 `json:"partLastByte"` +} + +type uploadLayerPartOutput struct { + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName,omitempty"` + UploadID string `json:"uploadId,omitempty"` + LastByteReceived int64 `json:"lastByteReceived,omitempty"` +} + +func (h *Handler) handleUploadLayerPart( + _ context.Context, in *uploadLayerPartInput, +) (*uploadLayerPartOutput, error) { + lastByteReceived, err := h.Backend.UploadLayerPart( + in.RegistryID, in.RepositoryName, in.UploadID, in.PartFirstByte, in.PartLastByte, in.LayerPartBlob, + ) + if err != nil { + return nil, err + } + + return &uploadLayerPartOutput{ + LastByteReceived: lastByteReceived, + RegistryID: h.registryIDOrDefault(in.RegistryID), + RepositoryName: in.RepositoryName, + UploadID: in.UploadID, + }, nil +} + +type completeLayerUploadInput struct { + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName"` + UploadID string `json:"uploadId"` + LayerDigests []string `json:"layerDigests"` +} + +type completeLayerUploadOutput struct { + LayerDigest string `json:"layerDigest,omitempty"` + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName,omitempty"` + UploadID string `json:"uploadId,omitempty"` +} + +func (h *Handler) handleCompleteLayerUpload( + _ context.Context, in *completeLayerUploadInput, +) (*completeLayerUploadOutput, error) { + digest, err := h.Backend.CompleteLayerUpload(in.RegistryID, in.RepositoryName, in.UploadID, in.LayerDigests) + if err != nil { + return nil, err + } + + return &completeLayerUploadOutput{ + LayerDigest: digest, + RegistryID: h.registryIDOrDefault(in.RegistryID), + RepositoryName: in.RepositoryName, + UploadID: in.UploadID, + }, nil +} diff --git a/services/ecrpublic/handler_policy.go b/services/ecrpublic/handler_policy.go new file mode 100644 index 0000000000..62194193b8 --- /dev/null +++ b/services/ecrpublic/handler_policy.go @@ -0,0 +1,83 @@ +package ecrpublic + +import ( + "context" + + "github.com/blackbirdworks/gopherstack/pkgs/service" +) + +func (h *Handler) buildPolicyOps() map[string]service.JSONOpFunc { + return map[string]service.JSONOpFunc{ + "GetRepositoryPolicy": service.WrapOp(h.handleGetRepositoryPolicy), + "SetRepositoryPolicy": service.WrapOp(h.handleSetRepositoryPolicy), + "DeleteRepositoryPolicy": service.WrapOp(h.handleDeleteRepositoryPolicy), + } +} + +type getRepositoryPolicyInput struct { + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName"` +} + +type repositoryPolicyOutput struct { + PolicyText string `json:"policyText,omitempty"` + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName,omitempty"` +} + +func (h *Handler) handleGetRepositoryPolicy( + _ context.Context, in *getRepositoryPolicyInput, +) (*repositoryPolicyOutput, error) { + text, err := h.Backend.GetRepositoryPolicy(in.RegistryID, in.RepositoryName) + if err != nil { + return nil, err + } + + return &repositoryPolicyOutput{ + PolicyText: text, + RegistryID: h.registryIDOrDefault(in.RegistryID), + RepositoryName: in.RepositoryName, + }, nil +} + +type setRepositoryPolicyInput struct { + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName"` + PolicyText string `json:"policyText"` + Force bool `json:"force,omitempty"` +} + +func (h *Handler) handleSetRepositoryPolicy( + _ context.Context, in *setRepositoryPolicyInput, +) (*repositoryPolicyOutput, error) { + text, err := h.Backend.SetRepositoryPolicy(in.RegistryID, in.RepositoryName, in.PolicyText) + if err != nil { + return nil, err + } + + return &repositoryPolicyOutput{ + PolicyText: text, + RegistryID: h.registryIDOrDefault(in.RegistryID), + RepositoryName: in.RepositoryName, + }, nil +} + +type deleteRepositoryPolicyInput struct { + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName"` +} + +func (h *Handler) handleDeleteRepositoryPolicy( + _ context.Context, in *deleteRepositoryPolicyInput, +) (*repositoryPolicyOutput, error) { + text, err := h.Backend.DeleteRepositoryPolicy(in.RegistryID, in.RepositoryName) + if err != nil { + return nil, err + } + + return &repositoryPolicyOutput{ + PolicyText: text, + RegistryID: h.registryIDOrDefault(in.RegistryID), + RepositoryName: in.RepositoryName, + }, nil +} diff --git a/services/ecrpublic/handler_repositories.go b/services/ecrpublic/handler_repositories.go new file mode 100644 index 0000000000..a00600517f --- /dev/null +++ b/services/ecrpublic/handler_repositories.go @@ -0,0 +1,93 @@ +package ecrpublic + +import ( + "context" + + "github.com/blackbirdworks/gopherstack/pkgs/service" +) + +func (h *Handler) buildRepositoryOps() map[string]service.JSONOpFunc { + return map[string]service.JSONOpFunc{ + "CreateRepository": service.WrapOp(h.handleCreateRepository), + "DescribeRepositories": service.WrapOp(h.handleDescribeRepositories), + "DeleteRepository": service.WrapOp(h.handleDeleteRepository), + } +} + +type createRepositoryInput struct { + CatalogData *CatalogDataInputWire `json:"catalogData,omitempty"` + RepositoryName string `json:"repositoryName"` + Tags []TagWire `json:"tags,omitempty"` +} + +type createRepositoryOutput struct { + Repository RepositoryWire `json:"repository"` + CatalogData CatalogDataWire `json:"catalogData"` +} + +func (h *Handler) handleCreateRepository( + _ context.Context, in *createRepositoryInput, +) (*createRepositoryOutput, error) { + repo, err := h.Backend.CreateRepository( + in.RepositoryName, + toCatalogDataInput(in.CatalogData), + tagsFromWire(in.Tags), + ) + if err != nil { + return nil, err + } + + return &createRepositoryOutput{ + CatalogData: toCatalogDataWire(&repo.CatalogData), + Repository: toRepositoryWire(repo), + }, nil +} + +type describeRepositoriesInput struct { + NextToken string `json:"nextToken,omitempty"` + RegistryID string `json:"registryId,omitempty"` + RepositoryNames []string `json:"repositoryNames,omitempty"` + MaxResults int32 `json:"maxResults,omitempty"` +} + +type describeRepositoriesOutput struct { + NextToken string `json:"nextToken,omitempty"` + Repositories []RepositoryWire `json:"repositories"` +} + +func (h *Handler) handleDescribeRepositories( + _ context.Context, in *describeRepositoriesInput, +) (*describeRepositoriesOutput, error) { + repos, err := h.Backend.DescribeRepositories(in.RegistryID, in.RepositoryNames) + if err != nil { + return nil, err + } + + out := make([]RepositoryWire, 0, len(repos)) + for _, r := range repos { + out = append(out, toRepositoryWire(r)) + } + + return &describeRepositoriesOutput{Repositories: out}, nil +} + +type deleteRepositoryInput struct { + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName"` + Force bool `json:"force,omitempty"` +} + +type deleteRepositoryOutput struct { + Repository RepositoryWire `json:"repository"` +} + +func (h *Handler) handleDeleteRepository( + _ context.Context, in *deleteRepositoryInput, +) (*deleteRepositoryOutput, error) { + repo, err := h.Backend.DeleteRepository(in.RegistryID, in.RepositoryName, in.Force) + if err != nil { + return nil, err + } + + return &deleteRepositoryOutput{Repository: toRepositoryWire(repo)}, nil +} diff --git a/services/ecrpublic/handler_tags.go b/services/ecrpublic/handler_tags.go new file mode 100644 index 0000000000..a4cfeb058b --- /dev/null +++ b/services/ecrpublic/handler_tags.go @@ -0,0 +1,64 @@ +package ecrpublic + +import ( + "context" + + "github.com/blackbirdworks/gopherstack/pkgs/service" +) + +func (h *Handler) buildTagOps() map[string]service.JSONOpFunc { + return map[string]service.JSONOpFunc{ + "TagResource": service.WrapOp(h.handleTagResource), + "UntagResource": service.WrapOp(h.handleUntagResource), + "ListTagsForResource": service.WrapOp(h.handleListTagsForResource), + } +} + +type tagResourceInput struct { + ResourceArn string `json:"resourceArn"` + Tags []TagWire `json:"tags"` +} + +type tagResourceOutput struct{} + +func (h *Handler) handleTagResource(_ context.Context, in *tagResourceInput) (*tagResourceOutput, error) { + if err := h.Backend.TagResource(in.ResourceArn, tagsFromWire(in.Tags)); err != nil { + return nil, err + } + + return &tagResourceOutput{}, nil +} + +type untagResourceInput struct { + ResourceArn string `json:"resourceArn"` + TagKeys []string `json:"tagKeys"` +} + +type untagResourceOutput struct{} + +func (h *Handler) handleUntagResource(_ context.Context, in *untagResourceInput) (*untagResourceOutput, error) { + if err := h.Backend.UntagResource(in.ResourceArn, in.TagKeys); err != nil { + return nil, err + } + + return &untagResourceOutput{}, nil +} + +type listTagsForResourceInput struct { + ResourceArn string `json:"resourceArn"` +} + +type listTagsForResourceOutput struct { + Tags []TagWire `json:"tags"` +} + +func (h *Handler) handleListTagsForResource( + _ context.Context, in *listTagsForResourceInput, +) (*listTagsForResourceOutput, error) { + tags, err := h.Backend.ListTagsForResource(in.ResourceArn) + if err != nil { + return nil, err + } + + return &listTagsForResourceOutput{Tags: tagsToWire(tags)}, nil +} diff --git a/services/ecrpublic/handler_test.go b/services/ecrpublic/handler_test.go new file mode 100644 index 0000000000..677fbab256 --- /dev/null +++ b/services/ecrpublic/handler_test.go @@ -0,0 +1,58 @@ +package ecrpublic_test + +import ( + "net/http/httptest" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awscfg "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + ecrpublicsdk "github.com/aws/aws-sdk-go-v2/service/ecrpublic" + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/ecrpublic" +) + +const ( + testRegion = "us-east-1" + testAccountID = "123456789012" +) + +// newTestClient stands up the real aws-sdk-go-v2 ecrpublic client against an +// httptest server running this package's Handler, wired through the same +// pkgs/service registry/router used in production. +func newTestClient(t *testing.T, h *ecrpublic.Handler) *ecrpublicsdk.Client { + t.Helper() + + e := echo.New() + registry := service.NewRegistry() + require.NoError(t, registry.Register(h)) + e.Use(service.NewServiceRouter(registry).RouteHandler()) + + srv := httptest.NewServer(e) + t.Cleanup(srv.Close) + + cfg, err := awscfg.LoadDefaultConfig( + t.Context(), + awscfg.WithRegion(testRegion), + awscfg.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider("test", "test", ""), + ), + ) + require.NoError(t, err) + + return ecrpublicsdk.NewFromConfig(cfg, func(o *ecrpublicsdk.Options) { + o.BaseEndpoint = aws.String(srv.URL) + }) +} + +func newTestHandler() *ecrpublic.Handler { + backend := ecrpublic.NewInMemoryBackend(testAccountID, testRegion) + h := ecrpublic.NewHandler(backend) + h.AccountID = testAccountID + h.DefaultRegion = testRegion + + return h +} diff --git a/services/ecrpublic/images.go b/services/ecrpublic/images.go new file mode 100644 index 0000000000..374fc239fb --- /dev/null +++ b/services/ecrpublic/images.go @@ -0,0 +1,292 @@ +package ecrpublic + +import ( + "fmt" + "sort" + "time" +) + +// tagsForDigestLocked returns every tag in repositoryName currently bound to +// digest, sorted for stable output. Caller must hold b.mu. +func (b *InMemoryBackend) tagsForDigestLocked(repositoryName, digest string) []string { + var tags []string + + for tag, binding := range b.tagIndex[repositoryName] { + if binding.Digest == digest { + tags = append(tags, tag) + } + } + + sort.Strings(tags) + + return tags +} + +func toImageDetail(img *Image, tags []string) ImageDetail { + return ImageDetail{ + ArtifactMediaType: img.ArtifactMediaType, + ImageDigest: img.ImageDigest, + ImageManifestMediaType: img.ImageManifestMediaType, + ImagePushedAt: img.ImagePushedAt, + ImageSizeInBytes: img.ImageSizeInBytes, + ImageTags: tags, + RegistryID: img.RegistryID, + RepositoryName: img.RepositoryName, + } +} + +// resolveImageLocked finds an image in repositoryName by digest or tag. +// Caller must hold b.mu. +func (b *InMemoryBackend) resolveImageLocked(repositoryName string, id ImageIdentifier) (*Image, bool) { + if id.ImageDigest != "" { + return b.images.Get(imageTableKey(repositoryName, id.ImageDigest)) + } + + if id.ImageTag != "" { + binding, ok := b.tagIndex[repositoryName][id.ImageTag] + if !ok { + return nil, false + } + + return b.images.Get(imageTableKey(repositoryName, binding.Digest)) + } + + return nil, false +} + +// DescribeImages returns image details for a repository, optionally filtered +// by digest or tag. +func (b *InMemoryBackend) DescribeImages( + registryID, repositoryName string, imageIDs []ImageIdentifier, +) ([]ImageDetail, error) { + b.mu.RLock("DescribeImages") + defer b.mu.RUnlock() + + if err := b.resolveRegistryIDLocked(registryID); err != nil { + return nil, err + } + + if !b.repos.Has(repositoryName) { + return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) + } + + if len(imageIDs) == 0 { + imgs := b.imagesByRepo.Get(repositoryName) + out := make([]ImageDetail, 0, len(imgs)) + + for _, img := range imgs { + out = append(out, toImageDetail(img, b.tagsForDigestLocked(repositoryName, img.ImageDigest))) + } + + sort.Slice(out, func(i, j int) bool { return out[i].ImageDigest < out[j].ImageDigest }) + + return out, nil + } + + out := make([]ImageDetail, 0, len(imageIDs)) + + for _, id := range imageIDs { + img, ok := b.resolveImageLocked(repositoryName, id) + if !ok { + return nil, fmt.Errorf("%w: image not found in %s", ErrImageNotFound, repositoryName) + } + + out = append(out, toImageDetail(img, b.tagsForDigestLocked(repositoryName, img.ImageDigest))) + } + + return out, nil +} + +// DescribeImageTags returns one ImageTagDetail per tag currently bound in the repository. +func (b *InMemoryBackend) DescribeImageTags(registryID, repositoryName string) ([]ImageTagDetail, error) { + b.mu.RLock("DescribeImageTags") + defer b.mu.RUnlock() + + if err := b.resolveRegistryIDLocked(registryID); err != nil { + return nil, err + } + + if !b.repos.Has(repositoryName) { + return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) + } + + tagIdx := b.tagIndex[repositoryName] + out := make([]ImageTagDetail, 0, len(tagIdx)) + + for tag, binding := range tagIdx { + img, ok := b.images.Get(imageTableKey(repositoryName, binding.Digest)) + if !ok { + continue + } + + out = append(out, ImageTagDetail{ + ArtifactMediaType: img.ArtifactMediaType, + CreatedAt: binding.CreatedAt, + ImageDigest: img.ImageDigest, + ImageManifestMediaType: img.ImageManifestMediaType, + ImagePushedAt: img.ImagePushedAt, + ImageSizeInBytes: img.ImageSizeInBytes, + ImageTag: tag, + }) + } + + sort.Slice(out, func(i, j int) bool { return out[i].ImageTag < out[j].ImageTag }) + + return out, nil +} + +// PutImage creates or replaces an image manifest, verifying that every layer +// (and config blob) it references was already uploaded via +// BatchCheckLayerAvailability/CompleteLayerUpload. +func (b *InMemoryBackend) PutImage(registryID, repositoryName string, req PutImageRequest) (*Image, error) { + b.mu.Lock("PutImage") + defer b.mu.Unlock() + + if err := b.resolveRegistryIDLocked(registryID); err != nil { + return nil, err + } + + if !b.repos.Has(repositoryName) { + return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) + } + + digest, err := resolveImageDigest(req) + if err != nil { + return nil, err + } + + if missing := b.missingLayersLocked(repositoryName, req.ImageManifest); len(missing) > 0 { + return nil, fmt.Errorf("%w: %v", ErrLayersNotFound, missing) + } + + if req.ImageTag != "" { + if existing, ok := b.tagIndex[repositoryName][req.ImageTag]; ok { + if existing.Digest == digest { + return nil, fmt.Errorf("%w: tag %s in %s", ErrImageAlreadyExists, req.ImageTag, repositoryName) + } + + return nil, fmt.Errorf("%w: tag %s in %s", ErrImageTagAlreadyExists, req.ImageTag, repositoryName) + } + } + + img := &Image{ + ArtifactMediaType: "", + ImageDigest: digest, + ImageManifest: req.ImageManifest, + ImageManifestMediaType: req.ImageManifestMediaType, + ImagePushedAt: time.Now(), + ImageSizeInBytes: int64(len(req.ImageManifest)), + RegistryID: b.accountID, + RepositoryName: repositoryName, + } + + b.images.Put(img) + + if req.ImageTag != "" { + if b.tagIndex[repositoryName] == nil { + b.tagIndex[repositoryName] = make(map[string]tagBinding) + } + + b.tagIndex[repositoryName][req.ImageTag] = tagBinding{Digest: digest, CreatedAt: time.Now()} + } + + cp := *img + + return &cp, nil +} + +func resolveImageDigest(req PutImageRequest) (string, error) { + computed := sha256Digest([]byte(req.ImageManifest)) + + if req.ImageDigest == "" { + return computed, nil + } + + if isFullSHA256Digest(req.ImageDigest) && req.ImageDigest != computed { + return "", fmt.Errorf("%w: got %s, want %s", ErrImageDigestDoesNotMatch, req.ImageDigest, computed) + } + + return computed, nil +} + +// missingLayersLocked returns every layer/config digest manifest references +// that was never uploaded to repositoryName. Caller must hold b.mu. +func (b *InMemoryBackend) missingLayersLocked(repositoryName, manifest string) []string { + uploaded := b.uploadedLayers[repositoryName] + + var missing []string + + for _, digest := range referencedDigests(manifest) { + if _, ok := uploaded[digest]; !ok { + missing = append(missing, digest) + } + } + + return missing +} + +// BatchDeleteImage deletes images by digest (removing every tag bound to it) +// or by tag (removing only that binding; the image survives if another tag +// still references it). +func (b *InMemoryBackend) BatchDeleteImage( + registryID, repositoryName string, imageIDs []ImageIdentifier, +) ([]ImageIdentifier, []ImageFailure, error) { + b.mu.Lock("BatchDeleteImage") + defer b.mu.Unlock() + + if err := b.resolveRegistryIDLocked(registryID); err != nil { + return nil, nil, err + } + + if !b.repos.Has(repositoryName) { + return nil, nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) + } + + deleted := make([]ImageIdentifier, 0, len(imageIDs)) + failures := make([]ImageFailure, 0, len(imageIDs)) + + for _, id := range imageIDs { + if ok := b.deleteImageIdentifierLocked(repositoryName, id); ok { + deleted = append(deleted, id) + } else { + failures = append(failures, ImageFailure{ + ImageID: id, + FailureCode: "ImageNotFound", + FailureReason: "requested image not found", + }) + } + } + + return deleted, failures, nil +} + +func (b *InMemoryBackend) deleteImageIdentifierLocked(repositoryName string, id ImageIdentifier) bool { + if id.ImageDigest != "" { + key := imageTableKey(repositoryName, id.ImageDigest) + if !b.images.Has(key) { + return false + } + + for tag, binding := range b.tagIndex[repositoryName] { + if binding.Digest == id.ImageDigest { + delete(b.tagIndex[repositoryName], tag) + } + } + + b.images.Delete(key) + + return true + } + + if id.ImageTag != "" { + if _, ok := b.tagIndex[repositoryName][id.ImageTag]; !ok { + return false + } + + delete(b.tagIndex[repositoryName], id.ImageTag) + + return true + } + + return false +} diff --git a/services/ecrpublic/images_and_layers_test.go b/services/ecrpublic/images_and_layers_test.go new file mode 100644 index 0000000000..dbeae5fd91 --- /dev/null +++ b/services/ecrpublic/images_and_layers_test.go @@ -0,0 +1,373 @@ +package ecrpublic_test + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + ecrpublicsdk "github.com/aws/aws-sdk-go-v2/service/ecrpublic" + "github.com/aws/aws-sdk-go-v2/service/ecrpublic/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func layerDigest(data []byte) string { + sum := sha256.Sum256(data) + + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// pushLayer runs the full Initiate/Upload/Complete flow for a single-part +// layer and returns the digest CompleteLayerUpload recorded. +func pushLayer( + ctx context.Context, t *testing.T, client *ecrpublicsdk.Client, repo string, data []byte, +) string { + t.Helper() + + initiated, err := client.InitiateLayerUpload(ctx, &ecrpublicsdk.InitiateLayerUploadInput{ + RepositoryName: aws.String(repo), + }) + require.NoError(t, err) + + _, err = client.UploadLayerPart(ctx, &ecrpublicsdk.UploadLayerPartInput{ + RepositoryName: aws.String(repo), + UploadId: initiated.UploadId, + PartFirstByte: aws.Int64(0), + PartLastByte: aws.Int64(int64(len(data) - 1)), + LayerPartBlob: data, + }) + require.NoError(t, err) + + completed, err := client.CompleteLayerUpload(ctx, &ecrpublicsdk.CompleteLayerUploadInput{ + RepositoryName: aws.String(repo), + UploadId: initiated.UploadId, + LayerDigests: []string{layerDigest(data)}, + }) + require.NoError(t, err) + + return aws.ToString(completed.LayerDigest) +} + +func buildManifest(configDigest string, layerDigests ...string) string { + type layer struct { + Digest string `json:"digest"` + } + + m := struct { + Config struct { + Digest string `json:"digest"` + } `json:"config"` + Layers []layer `json:"layers"` + }{} + m.Config.Digest = configDigest + + for _, d := range layerDigests { + m.Layers = append(m.Layers, layer{Digest: d}) + } + + b, _ := json.Marshal(m) + + return string(b) +} + +func TestImagePushLifecycle(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateRepository(ctx, &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String("push-repo")}) + require.NoError(t, err) + + configDigest := pushLayer(ctx, t, client, "push-repo", []byte("config-blob")) + layerADigest := pushLayer(ctx, t, client, "push-repo", []byte("layer-a-blob")) + + avail, err := client.BatchCheckLayerAvailability(ctx, &ecrpublicsdk.BatchCheckLayerAvailabilityInput{ + RepositoryName: aws.String("push-repo"), + LayerDigests: []string{configDigest, layerADigest, "sha256:" + hex.EncodeToString(make([]byte, 32))}, + }) + require.NoError(t, err) + assert.Len(t, avail.Layers, 2) + require.Len(t, avail.Failures, 1) + assert.Equal(t, types.LayerFailureCodeMissingLayerDigest, avail.Failures[0].FailureCode) + + manifest := buildManifest(configDigest, layerADigest) + + putOut, err := client.PutImage(ctx, &ecrpublicsdk.PutImageInput{ + RepositoryName: aws.String("push-repo"), + ImageManifest: aws.String(manifest), + ImageTag: aws.String("v1"), + }) + require.NoError(t, err) + digest := aws.ToString(putOut.Image.ImageId.ImageDigest) + assert.Equal(t, layerDigest([]byte(manifest)), digest) + + described, err := client.DescribeImages(ctx, &ecrpublicsdk.DescribeImagesInput{ + RepositoryName: aws.String("push-repo"), + }) + require.NoError(t, err) + require.Len(t, described.ImageDetails, 1) + assert.Equal(t, []string{"v1"}, described.ImageDetails[0].ImageTags) + assert.Equal(t, digest, aws.ToString(described.ImageDetails[0].ImageDigest)) + + tags, err := client.DescribeImageTags(ctx, &ecrpublicsdk.DescribeImageTagsInput{ + RepositoryName: aws.String("push-repo"), + }) + require.NoError(t, err) + require.Len(t, tags.ImageTagDetails, 1) + assert.Equal(t, "v1", aws.ToString(tags.ImageTagDetails[0].ImageTag)) + assert.Equal(t, digest, aws.ToString(tags.ImageTagDetails[0].ImageDetail.ImageDigest)) + + delByTag, err := client.BatchDeleteImage(ctx, &ecrpublicsdk.BatchDeleteImageInput{ + RepositoryName: aws.String("push-repo"), + ImageIds: []types.ImageIdentifier{{ImageTag: aws.String("v1")}}, + }) + require.NoError(t, err) + assert.Len(t, delByTag.ImageIds, 1) + assert.Empty(t, delByTag.Failures) + + afterUntag, err := client.DescribeImages(ctx, &ecrpublicsdk.DescribeImagesInput{ + RepositoryName: aws.String("push-repo"), + }) + require.NoError(t, err) + require.Len(t, afterUntag.ImageDetails, 1) + assert.Empty(t, afterUntag.ImageDetails[0].ImageTags) + + delByDigest, err := client.BatchDeleteImage(ctx, &ecrpublicsdk.BatchDeleteImageInput{ + RepositoryName: aws.String("push-repo"), + ImageIds: []types.ImageIdentifier{{ImageDigest: aws.String(digest)}}, + }) + require.NoError(t, err) + assert.Len(t, delByDigest.ImageIds, 1) + + empty, err := client.DescribeImages(ctx, &ecrpublicsdk.DescribeImagesInput{ + RepositoryName: aws.String("push-repo"), + }) + require.NoError(t, err) + assert.Empty(t, empty.ImageDetails) +} + +func TestBatchDeleteImage_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateRepository(ctx, &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String("bdi-repo")}) + require.NoError(t, err) + + out, err := client.BatchDeleteImage(ctx, &ecrpublicsdk.BatchDeleteImageInput{ + RepositoryName: aws.String("bdi-repo"), + ImageIds: []types.ImageIdentifier{{ImageTag: aws.String("missing")}}, + }) + require.NoError(t, err) + assert.Empty(t, out.ImageIds) + require.Len(t, out.Failures, 1) + assert.Equal(t, types.ImageFailureCodeImageNotFound, out.Failures[0].FailureCode) +} + +func TestPutImage_LayersNotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateRepository( + ctx, + &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String("missing-layers")}, + ) + require.NoError(t, err) + + manifest := buildManifest( + "sha256:"+hex.EncodeToString(make([]byte, 32)), + "sha256:"+hex.EncodeToString(make([]byte, 32)), + ) + + _, err = client.PutImage(ctx, &ecrpublicsdk.PutImageInput{ + RepositoryName: aws.String("missing-layers"), + ImageManifest: aws.String(manifest), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "LayersNotFoundException", apiErr.ErrorCode()) +} + +func TestPutImage_AlreadyExistsAndTagConflict(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, createErr := client.CreateRepository( + ctx, + &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String("conflict-repo")}, + ) + require.NoError(t, createErr) + + manifestA := `{"schemaVersion":2,"unique":"a"}` + manifestB := `{"schemaVersion":2,"unique":"b"}` + + _, pushErr := client.PutImage(ctx, &ecrpublicsdk.PutImageInput{ + RepositoryName: aws.String("conflict-repo"), + ImageManifest: aws.String(manifestA), + ImageTag: aws.String("latest"), + }) + require.NoError(t, pushErr) + + t.Run("same tag and digest", func(t *testing.T) { + t.Parallel() + + _, err := client.PutImage(ctx, &ecrpublicsdk.PutImageInput{ + RepositoryName: aws.String("conflict-repo"), + ImageManifest: aws.String(manifestA), + ImageTag: aws.String("latest"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ImageAlreadyExistsException", apiErr.ErrorCode()) + }) + + t.Run("same tag different digest", func(t *testing.T) { + t.Parallel() + + _, err := client.PutImage(ctx, &ecrpublicsdk.PutImageInput{ + RepositoryName: aws.String("conflict-repo"), + ImageManifest: aws.String(manifestB), + ImageTag: aws.String("latest"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ImageTagAlreadyExistsException", apiErr.ErrorCode()) + }) +} + +func TestUploadLayerPart_InvalidLayerPart(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateRepository(ctx, &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String("gap-repo")}) + require.NoError(t, err) + + initiated, err := client.InitiateLayerUpload(ctx, &ecrpublicsdk.InitiateLayerUploadInput{ + RepositoryName: aws.String("gap-repo"), + }) + require.NoError(t, err) + + _, err = client.UploadLayerPart(ctx, &ecrpublicsdk.UploadLayerPartInput{ + RepositoryName: aws.String("gap-repo"), + UploadId: initiated.UploadId, + PartFirstByte: aws.Int64(5), + PartLastByte: aws.Int64(10), + LayerPartBlob: []byte("123456"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "InvalidLayerPartException", apiErr.ErrorCode()) +} + +func TestCompleteLayerUpload_UploadNotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateRepository( + ctx, + &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String("no-upload-repo")}, + ) + require.NoError(t, err) + + _, err = client.CompleteLayerUpload(ctx, &ecrpublicsdk.CompleteLayerUploadInput{ + RepositoryName: aws.String("no-upload-repo"), + UploadId: aws.String("bogus-upload-id"), + LayerDigests: []string{"sha256:" + hex.EncodeToString(make([]byte, 32))}, + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "UploadNotFoundException", apiErr.ErrorCode()) +} + +func TestCompleteLayerUpload_EmptyUpload(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateRepository( + ctx, + &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String("empty-upload-repo")}, + ) + require.NoError(t, err) + + initiated, err := client.InitiateLayerUpload(ctx, &ecrpublicsdk.InitiateLayerUploadInput{ + RepositoryName: aws.String("empty-upload-repo"), + }) + require.NoError(t, err) + + _, err = client.CompleteLayerUpload(ctx, &ecrpublicsdk.CompleteLayerUploadInput{ + RepositoryName: aws.String("empty-upload-repo"), + UploadId: initiated.UploadId, + LayerDigests: []string{"sha256:" + hex.EncodeToString(make([]byte, 32))}, + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "EmptyUploadException", apiErr.ErrorCode()) +} + +func TestCompleteLayerUpload_LayerAlreadyExists(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateRepository( + ctx, + &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String("dup-layer-repo")}, + ) + require.NoError(t, err) + + pushLayer(ctx, t, client, "dup-layer-repo", []byte("same-content")) + + initiated, err := client.InitiateLayerUpload(ctx, &ecrpublicsdk.InitiateLayerUploadInput{ + RepositoryName: aws.String("dup-layer-repo"), + }) + require.NoError(t, err) + + _, err = client.UploadLayerPart(ctx, &ecrpublicsdk.UploadLayerPartInput{ + RepositoryName: aws.String("dup-layer-repo"), + UploadId: initiated.UploadId, + PartFirstByte: aws.Int64(0), + PartLastByte: aws.Int64(11), + LayerPartBlob: []byte("same-content"), + }) + require.NoError(t, err) + + _, err = client.CompleteLayerUpload(ctx, &ecrpublicsdk.CompleteLayerUploadInput{ + RepositoryName: aws.String("dup-layer-repo"), + UploadId: initiated.UploadId, + LayerDigests: []string{layerDigest([]byte("same-content"))}, + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "LayerAlreadyExistsException", apiErr.ErrorCode()) +} diff --git a/services/ecrpublic/interfaces.go b/services/ecrpublic/interfaces.go new file mode 100644 index 0000000000..9f0bce8b70 --- /dev/null +++ b/services/ecrpublic/interfaces.go @@ -0,0 +1,51 @@ +package ecrpublic + +import "context" + +// Backend is the interface for the Amazon ECR Public backend. +type Backend interface { + AccountID() string + + CreateRepository(name string, catalogData *CatalogData, tags map[string]string) (*Repository, error) + DescribeRepositories(registryID string, names []string) ([]*Repository, error) + DeleteRepository(registryID, name string, force bool) (*Repository, error) + + GetRepositoryCatalogData(registryID, name string) (*CatalogData, error) + PutRepositoryCatalogData(registryID, name string, catalogData *CatalogData) (*CatalogData, error) + + GetRepositoryPolicy(registryID, name string) (string, error) + SetRepositoryPolicy(registryID, name, policyText string) (string, error) + DeleteRepositoryPolicy(registryID, name string) (string, error) + + TagResource(resourceARN string, tags map[string]string) error + UntagResource(resourceARN string, tagKeys []string) error + ListTagsForResource(resourceARN string) (map[string]string, error) + + DescribeRegistries() ([]RegistryInfo, error) + GetRegistryCatalogData() (RegistryCatalogData, error) + PutRegistryCatalogData(displayName string) (RegistryCatalogData, error) + GetAuthorizationToken(ctx context.Context) (string, int64, error) + + DescribeImages(registryID, repositoryName string, imageIDs []ImageIdentifier) ([]ImageDetail, error) + DescribeImageTags(registryID, repositoryName string) ([]ImageTagDetail, error) + PutImage(registryID, repositoryName string, req PutImageRequest) (*Image, error) + BatchDeleteImage( + registryID, repositoryName string, imageIDs []ImageIdentifier, + ) ([]ImageIdentifier, []ImageFailure, error) + + BatchCheckLayerAvailability( + registryID, repositoryName string, layerDigests []string, + ) ([]LayerInfo, []LayerFailure, error) + InitiateLayerUpload(registryID, repositoryName string) (uploadID string, partSize int64, err error) + UploadLayerPart( + registryID, repositoryName, uploadID string, firstByte, lastByte int64, blob []byte, + ) (lastByteReceived int64, err error) + CompleteLayerUpload( + registryID, repositoryName, uploadID string, layerDigests []string, + ) (digest string, err error) + + Reset() +} + +// Compile-time assertion that InMemoryBackend implements Backend. +var _ Backend = (*InMemoryBackend)(nil) diff --git a/services/ecrpublic/layer_upload_ttl_test.go b/services/ecrpublic/layer_upload_ttl_test.go new file mode 100644 index 0000000000..9f58db9105 --- /dev/null +++ b/services/ecrpublic/layer_upload_ttl_test.go @@ -0,0 +1,66 @@ +package ecrpublic_test + +import ( + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/ecrpublic" +) + +// pastLayerUploadTTL is strictly greater than the modeled 24h +// layerUploadTTL, so the lazy prune always evicts a stale session by the +// time it fires. +const pastLayerUploadTTL = 24*time.Hour + time.Second + +// TestInitiateLayerUpload_KeptWithinTTL proves an abandoned upload session +// survives up to layerUploadTTL, matching a real (if slow) docker push. +func TestInitiateLayerUpload_KeptWithinTTL(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + backend := ecrpublic.NewInMemoryBackend(testAccountID, testRegion) + + _, err := backend.CreateRepository("kept-repo", nil, nil) + require.NoError(t, err) + + uploadID, _, err := backend.InitiateLayerUpload("", "kept-repo") + require.NoError(t, err) + + time.Sleep(24*time.Hour - time.Second) + + _, err = backend.UploadLayerPart("", "kept-repo", uploadID, 0, 3, []byte("data")) + require.NoError(t, err) + }) +} + +// TestInitiateLayerUpload_EvictedAfterTTL locks in the fix for the +// InitiateLayerUpload session leak: sessions that never reach +// CompleteLayerUpload used to be retained forever, growing the backend's +// memory unbounded in a long-running emulator. They are now pruned +// lazily, on the next InitiateLayerUpload call, once layerUploadTTL has +// elapsed. +func TestInitiateLayerUpload_EvictedAfterTTL(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + backend := ecrpublic.NewInMemoryBackend(testAccountID, testRegion) + + _, err := backend.CreateRepository("evict-repo", nil, nil) + require.NoError(t, err) + + uploadID, _, err := backend.InitiateLayerUpload("", "evict-repo") + require.NoError(t, err) + + time.Sleep(pastLayerUploadTTL) + + // The prune runs lazily on the next InitiateLayerUpload call. + _, _, err = backend.InitiateLayerUpload("", "evict-repo") + require.NoError(t, err) + + _, err = backend.UploadLayerPart("", "evict-repo", uploadID, 0, 3, []byte("data")) + require.ErrorIs(t, err, ecrpublic.ErrUploadNotFound) + }) +} diff --git a/services/ecrpublic/layers.go b/services/ecrpublic/layers.go new file mode 100644 index 0000000000..a3e2fdb8cb --- /dev/null +++ b/services/ecrpublic/layers.go @@ -0,0 +1,212 @@ +package ecrpublic + +import ( + "fmt" + "time" +) + +const ( + layerUploadPartSize = 10 * 1024 * 1024 + minLayerPartSize = 5 * 1024 * 1024 + // layerUploadTTL bounds how long an unfinished InitiateLayerUpload + // session is retained before being pruned as abandoned. AWS does not + // document an explicit expiry window for unfinished layer uploads; this + // matches services/ecr's own layerUploadTTL default of 24h. + layerUploadTTL = 24 * time.Hour +) + +// BatchCheckLayerAvailability reports which of the given layer digests have +// already been uploaded (via a completed InitiateLayerUpload session) to repositoryName. +func (b *InMemoryBackend) BatchCheckLayerAvailability( + registryID, repositoryName string, layerDigests []string, +) ([]LayerInfo, []LayerFailure, error) { + b.mu.RLock("BatchCheckLayerAvailability") + defer b.mu.RUnlock() + + if err := b.resolveRegistryIDLocked(registryID); err != nil { + return nil, nil, err + } + + if !b.repos.Has(repositoryName) { + return nil, nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) + } + + uploaded := b.uploadedLayers[repositoryName] + + layers := make([]LayerInfo, 0, len(layerDigests)) + failures := make([]LayerFailure, 0, len(layerDigests)) + + for _, digest := range layerDigests { + if size, ok := uploaded[digest]; ok { + layers = append(layers, LayerInfo{ + LayerDigest: digest, + LayerAvailability: "AVAILABLE", + LayerSize: size, + }) + + continue + } + + failures = append(failures, LayerFailure{ + LayerDigest: digest, + FailureCode: "MissingLayerDigest", + FailureReason: "the layer digest does not exist in the repository", + }) + } + + return layers, failures, nil +} + +// InitiateLayerUpload starts a new layer upload session for repositoryName. +func (b *InMemoryBackend) InitiateLayerUpload(registryID, repositoryName string) (string, int64, error) { + b.mu.Lock("InitiateLayerUpload") + defer b.mu.Unlock() + + if err := b.resolveRegistryIDLocked(registryID); err != nil { + return "", 0, err + } + + if !b.repos.Has(repositoryName) { + return "", 0, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) + } + + b.pruneExpiredLayerUploadsLocked(time.Now()) + + b.layerUploadSeq++ + uploadID := fmt.Sprintf("upload-%d-%d", time.Now().UnixNano(), b.layerUploadSeq) + b.layerUploads[uploadID] = &layerUploadState{RepositoryName: repositoryName, CreatedAt: time.Now()} + + return uploadID, layerUploadPartSize, nil +} + +// UploadLayerPart appends a chunk of layer bytes to a live upload session. +// AWS requires each part's first byte to be consecutive to the bytes already +// received; a gap or overlap is rejected with InvalidLayerPartException. +func (b *InMemoryBackend) UploadLayerPart( + registryID, repositoryName, uploadID string, firstByte, lastByte int64, blob []byte, +) (int64, error) { + b.mu.Lock("UploadLayerPart") + defer b.mu.Unlock() + + if err := b.resolveRegistryIDLocked(registryID); err != nil { + return 0, err + } + + if !b.repos.Has(repositoryName) { + return 0, fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) + } + + upload, ok := b.layerUploads[uploadID] + if !ok || upload.RepositoryName != repositoryName { + return 0, fmt.Errorf("%w: upload %s not found for %s", ErrUploadNotFound, uploadID, repositoryName) + } + + if firstByte != upload.Size { + return 0, fmt.Errorf( + "%w: partFirstByte %d is not consecutive to the %d bytes already received", + ErrInvalidLayerPart, firstByte, upload.Size, + ) + } + + upload.Data = append(upload.Data, blob...) + upload.Size = int64(len(upload.Data)) + upload.PartSizes = append(upload.PartSizes, int64(len(blob))) + + received := lastByte + if received < 0 && len(blob) > 0 { + received = upload.Size - 1 + } + + return received, nil +} + +// CompleteLayerUpload finalizes an upload session, computing (and, if the +// caller supplied one, verifying) the layer's SHA256 digest. +func (b *InMemoryBackend) CompleteLayerUpload( + registryID, repositoryName, uploadID string, layerDigests []string, +) (string, error) { + b.mu.Lock("CompleteLayerUpload") + defer b.mu.Unlock() + + if err := b.resolveRegistryIDLocked(registryID); err != nil { + return "", err + } + + if !b.repos.Has(repositoryName) { + return "", fmt.Errorf("%w: %s", ErrRepositoryNotFound, repositoryName) + } + + upload, ok := b.layerUploads[uploadID] + if !ok || upload.RepositoryName != repositoryName { + return "", fmt.Errorf("%w: upload %s not found for %s", ErrUploadNotFound, uploadID, repositoryName) + } + + if len(upload.Data) == 0 { + return "", fmt.Errorf("%w: upload %s received no layer parts", ErrEmptyUpload, uploadID) + } + + if err := validatePartSizes(upload.PartSizes); err != nil { + return "", err + } + + digest, err := verifiedUploadDigest(upload.Data, layerDigests) + if err != nil { + return "", err + } + + if _, exists := b.uploadedLayers[repositoryName][digest]; exists { + delete(b.layerUploads, uploadID) + + return "", fmt.Errorf("%w: %s", ErrLayerAlreadyExists, digest) + } + + if b.uploadedLayers[repositoryName] == nil { + b.uploadedLayers[repositoryName] = make(map[string]int64) + } + + b.uploadedLayers[repositoryName][digest] = upload.Size + delete(b.layerUploads, uploadID) + + return digest, nil +} + +// pruneExpiredLayerUploadsLocked removes InitiateLayerUpload sessions older +// than layerUploadTTL, preventing an unbounded leak from pushes that are +// initiated but never completed. Caller must hold b.mu. +func (b *InMemoryBackend) pruneExpiredLayerUploadsLocked(now time.Time) { + for id, upload := range b.layerUploads { + if now.Sub(upload.CreatedAt) > layerUploadTTL { + delete(b.layerUploads, id) + } + } +} + +// validatePartSizes enforces the 5MiB minimum-part-size rule against every +// part but the last (which cannot be known until CompleteLayerUpload). +func validatePartSizes(sizes []int64) error { + for _, size := range sizes[:max(0, len(sizes)-1)] { + if size < minLayerPartSize { + return fmt.Errorf( + "%w: layer parts must be at least %d bytes, except for the last part", + ErrLayerPartTooSmall, minLayerPartSize, + ) + } + } + + return nil +} + +func verifiedUploadDigest(data []byte, layerDigests []string) (string, error) { + computed := sha256Digest(data) + + if len(layerDigests) == 0 || layerDigests[0] == "" { + return computed, nil + } + + provided := layerDigests[0] + if isFullSHA256Digest(provided) && provided != computed { + return "", fmt.Errorf("%w: digest mismatch: got %s, want %s", ErrInvalidLayer, provided, computed) + } + + return provided, nil +} diff --git a/services/ecrpublic/models.go b/services/ecrpublic/models.go new file mode 100644 index 0000000000..387254bd22 --- /dev/null +++ b/services/ecrpublic/models.go @@ -0,0 +1,139 @@ +package ecrpublic + +import "time" + +// Repository is the domain model for a public ECR repository. +type Repository struct { + CreatedAt time.Time + Tags map[string]string + RepositoryName string + RepositoryArn string + RegistryID string + RepositoryURI string + PolicyText string + CatalogData CatalogData + HasPolicy bool +} + +// CatalogData is the publicly-visible Gallery metadata for a repository. +type CatalogData struct { + AboutText string + Description string + UsageText string + LogoImageBlob []byte + Architectures []string + OperatingSystems []string + MarketplaceCertified bool +} + +// RegistryCatalogData is the account-wide Gallery display metadata. +type RegistryCatalogData struct { + DisplayName string +} + +// Image is the domain model for a pushed image manifest, keyed by +// (repository, digest) via imageTableKey. +type Image struct { + ImagePushedAt time.Time + RepositoryName string + ImageDigest string + ImageManifest string + ImageManifestMediaType string + ArtifactMediaType string + RegistryID string + ImageSizeInBytes int64 +} + +// tagBinding records which digest a tag currently points to, and when the +// binding was created (surfaced by DescribeImageTags). +type tagBinding struct { + CreatedAt time.Time + Digest string +} + +// layerUploadState tracks an in-progress InitiateLayerUpload session. Never +// persisted: AWS does not guarantee in-flight uploads survive a restart. +type layerUploadState struct { + CreatedAt time.Time + RepositoryName string + Data []byte + PartSizes []int64 + Size int64 +} + +// ImageIdentifier identifies an image by digest, tag, or both. +type ImageIdentifier struct { + ImageDigest string + ImageTag string +} + +// ImageDetail is a computed, request-time view of an Image annotated with its +// current tags; it is not itself persisted state. +type ImageDetail struct { + ImagePushedAt time.Time + ArtifactMediaType string + ImageDigest string + ImageManifestMediaType string + RegistryID string + RepositoryName string + ImageTags []string + ImageSizeInBytes int64 +} + +// ImageTagDetail is a computed, request-time view of a single tag binding. +type ImageTagDetail struct { + CreatedAt time.Time + ImagePushedAt time.Time + ImageTag string + ArtifactMediaType string + ImageDigest string + ImageManifestMediaType string + ImageSizeInBytes int64 +} + +// ImageFailure describes an image that BatchDeleteImage could not process. +type ImageFailure struct { + ImageID ImageIdentifier + FailureCode string + FailureReason string +} + +// LayerInfo describes an available image layer. +type LayerInfo struct { + LayerDigest string + LayerAvailability string + MediaType string + LayerSize int64 +} + +// LayerFailure describes a layer digest BatchCheckLayerAvailability could not find. +type LayerFailure struct { + LayerDigest string + FailureCode string + FailureReason string +} + +// RegistryAliasInfo describes one alias of a public registry. +type RegistryAliasInfo struct { + Name string + Status string + DefaultRegistryAlias bool + PrimaryRegistryAlias bool +} + +// RegistryInfo describes a public registry, computed from backend state. +type RegistryInfo struct { + RegistryArn string + RegistryID string + RegistryURI string + Aliases []RegistryAliasInfo + Verified bool +} + +// PutImageRequest carries the fields PutImage needs from the wire request. +type PutImageRequest struct { + ImageDigest string + ImageManifest string + ImageManifestMediaType string + ImageTag string +} diff --git a/services/ecrpublic/persistence.go b/services/ecrpublic/persistence.go new file mode 100644 index 0000000000..2d5b142862 --- /dev/null +++ b/services/ecrpublic/persistence.go @@ -0,0 +1,145 @@ +package ecrpublic + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + + "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/persistence" +) + +// ErrNoSnapshot is returned when a backend does not support snapshot/restore. +var ErrNoSnapshot = errors.New("backend does not support restore") + +// Snapshottable is an optional interface a Backend may implement to support +// snapshot/restore for persistence or test isolation. +type Snapshottable interface { + Snapshot(ctx context.Context) []byte + Restore(context.Context, []byte) error +} + +// ecrPublicSnapshotVersion identifies the shape of [backendSnapshot]. Bump it +// whenever a change would make an older snapshot unsafe to decode as the +// current shape; Restore discards (rather than partially decodes) any mismatch. +const ecrPublicSnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the backend. Tables +// holds one JSON-encoded array per registered table name ("repos", "images" +// -- see store_setup.go), produced by b.registry.SnapshotAll(). TagIndex and +// UploadedLayers carry no identity field of their own (see store.go's doc) +// and so are persisted directly here instead of as registered tables. +// LayerUploads (in-flight sessions) is deliberately NOT persisted, matching +// AWS: an in-progress upload does not survive a restart. +type backendSnapshot struct { + Tables map[string]json.RawMessage `json:"tables"` + TagIndex map[string]map[string]tagBinding `json:"tagIndex,omitempty"` + UploadedLayers map[string]map[string]int64 `json:"uploadedLayers,omitempty"` + RegistryCatalogData RegistryCatalogData `json:"registryCatalogData"` + Version int `json:"version"` +} + +// Snapshot serializes backend state to JSON. +func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { + b.mu.RLock("Snapshot") + defer b.mu.RUnlock() + + tables, err := b.registry.SnapshotAll() + if err != nil { + logger.Load(ctx).WarnContext(ctx, "ecrpublic: snapshot table marshal failed", "error", err) + + return nil + } + + snap := backendSnapshot{ + Version: ecrPublicSnapshotVersion, + Tables: tables, + TagIndex: copyTagIndex(b.tagIndex), + UploadedLayers: copyLayerSizes(b.uploadedLayers), + RegistryCatalogData: b.registryCatalogData, + } + + return persistence.MarshalSnapshot(ctx, "ecrpublic", &snap) +} + +// Restore deserializes backend state from a JSON snapshot. +func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { + var snap backendSnapshot + + if err := persistence.UnmarshalSnapshot(ctx, "ecrpublic", data, &snap); err != nil { + return err + } + + b.mu.Lock("Restore") + defer b.mu.Unlock() + + if snap.Version != ecrPublicSnapshotVersion { + logger.Load(ctx).WarnContext(ctx, + "ecrpublic: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", ecrPublicSnapshotVersion) + + b.registry.ResetAll() + b.tagIndex = make(map[string]map[string]tagBinding) + b.uploadedLayers = make(map[string]map[string]int64) + b.registryCatalogData = RegistryCatalogData{} + + return nil + } + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("ecrpublic: restore snapshot tables: %w", err) + } + + b.tagIndex = copyTagIndex(snap.TagIndex) + b.uploadedLayers = copyLayerSizes(snap.UploadedLayers) + b.registryCatalogData = snap.RegistryCatalogData + b.layerUploads = make(map[string]*layerUploadState) + + return nil +} + +func copyTagIndex(in map[string]map[string]tagBinding) map[string]map[string]tagBinding { + out := make(map[string]map[string]tagBinding, len(in)) + for repo, tags := range in { + inner := make(map[string]tagBinding, len(tags)) + maps.Copy(inner, tags) + + out[repo] = inner + } + + return out +} + +func copyLayerSizes(in map[string]map[string]int64) map[string]map[string]int64 { + out := make(map[string]map[string]int64, len(in)) + for repo, layers := range in { + inner := make(map[string]int64, len(layers)) + maps.Copy(inner, layers) + + out[repo] = inner + } + + return out +} + +// Snapshot implements persistence by delegating to the backend if it supports it. +func (h *Handler) Snapshot(ctx context.Context) []byte { + s, ok := h.Backend.(Snapshottable) + if !ok { + return nil + } + + return s.Snapshot(ctx) +} + +// Restore implements persistence by delegating to the backend if it supports it. +func (h *Handler) Restore(ctx context.Context, data []byte) error { + s, ok := h.Backend.(Snapshottable) + if !ok { + return ErrNoSnapshot + } + + return s.Restore(ctx, data) +} diff --git a/services/ecrpublic/policy.go b/services/ecrpublic/policy.go new file mode 100644 index 0000000000..20a965acf2 --- /dev/null +++ b/services/ecrpublic/policy.go @@ -0,0 +1,70 @@ +package ecrpublic + +import "fmt" + +// GetRepositoryPolicy returns the repository's resource policy text. +func (b *InMemoryBackend) GetRepositoryPolicy(registryID, name string) (string, error) { + b.mu.RLock("GetRepositoryPolicy") + defer b.mu.RUnlock() + + if err := b.resolveRegistryIDLocked(registryID); err != nil { + return "", err + } + + repo, ok := b.repos.Get(name) + if !ok { + return "", fmt.Errorf("%w: %s", ErrRepositoryNotFound, name) + } + + if !repo.HasPolicy { + return "", fmt.Errorf("%w: %s", ErrRepositoryPolicyNotFound, name) + } + + return repo.PolicyText, nil +} + +// SetRepositoryPolicy sets or replaces the repository's resource policy text. +func (b *InMemoryBackend) SetRepositoryPolicy(registryID, name, policyText string) (string, error) { + b.mu.Lock("SetRepositoryPolicy") + defer b.mu.Unlock() + + if err := b.resolveRegistryIDLocked(registryID); err != nil { + return "", err + } + + repo, ok := b.repos.Get(name) + if !ok { + return "", fmt.Errorf("%w: %s", ErrRepositoryNotFound, name) + } + + repo.PolicyText = policyText + repo.HasPolicy = true + + return repo.PolicyText, nil +} + +// DeleteRepositoryPolicy deletes the repository's resource policy, returning +// the deleted policy text. +func (b *InMemoryBackend) DeleteRepositoryPolicy(registryID, name string) (string, error) { + b.mu.Lock("DeleteRepositoryPolicy") + defer b.mu.Unlock() + + if err := b.resolveRegistryIDLocked(registryID); err != nil { + return "", err + } + + repo, ok := b.repos.Get(name) + if !ok { + return "", fmt.Errorf("%w: %s", ErrRepositoryNotFound, name) + } + + if !repo.HasPolicy { + return "", fmt.Errorf("%w: %s", ErrRepositoryPolicyNotFound, name) + } + + deleted := repo.PolicyText + repo.PolicyText = "" + repo.HasPolicy = false + + return deleted, nil +} diff --git a/services/ecrpublic/policy_test.go b/services/ecrpublic/policy_test.go new file mode 100644 index 0000000000..02d520e845 --- /dev/null +++ b/services/ecrpublic/policy_test.go @@ -0,0 +1,70 @@ +package ecrpublic_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + ecrpublicsdk "github.com/aws/aws-sdk-go-v2/service/ecrpublic" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testPolicyText = `{"Version":"2012-10-17","Statement":[` + + `{"Sid":"AllowPull","Effect":"Allow","Principal":"*","Action":["ecr:BatchGetImage"]}]}` + +func TestRepositoryPolicyLifecycle(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateRepository( + ctx, + &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String("policy-repo")}, + ) + require.NoError(t, err) + + set, err := client.SetRepositoryPolicy(ctx, &ecrpublicsdk.SetRepositoryPolicyInput{ + RepositoryName: aws.String("policy-repo"), + PolicyText: aws.String(testPolicyText), + }) + require.NoError(t, err) + assert.JSONEq(t, testPolicyText, aws.ToString(set.PolicyText)) + + got, err := client.GetRepositoryPolicy(ctx, &ecrpublicsdk.GetRepositoryPolicyInput{ + RepositoryName: aws.String("policy-repo"), + }) + require.NoError(t, err) + assert.JSONEq(t, testPolicyText, aws.ToString(got.PolicyText)) + + deleted, err := client.DeleteRepositoryPolicy(ctx, &ecrpublicsdk.DeleteRepositoryPolicyInput{ + RepositoryName: aws.String("policy-repo"), + }) + require.NoError(t, err) + assert.JSONEq(t, testPolicyText, aws.ToString(deleted.PolicyText)) + + _, err = client.GetRepositoryPolicy(ctx, &ecrpublicsdk.GetRepositoryPolicyInput{ + RepositoryName: aws.String("policy-repo"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "RepositoryPolicyNotFoundException", apiErr.ErrorCode()) +} + +func TestGetRepositoryPolicy_RepositoryNotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.GetRepositoryPolicy(t.Context(), &ecrpublicsdk.GetRepositoryPolicyInput{ + RepositoryName: aws.String("missing"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "RepositoryNotFoundException", apiErr.ErrorCode()) +} diff --git a/services/ecrpublic/provider.go b/services/ecrpublic/provider.go new file mode 100644 index 0000000000..1c35f9973f --- /dev/null +++ b/services/ecrpublic/provider.go @@ -0,0 +1,25 @@ +package ecrpublic + +import "github.com/blackbirdworks/gopherstack/pkgs/service" + +// Provider implements service.Provider for Amazon ECR Public. +type Provider struct{} + +// Name returns the provider name. +func (p *Provider) Name() string { return "ECRPublic" } + +// Init initializes the Amazon ECR Public service backend and handler. Public +// repositories are a us-east-1-only service in real AWS, so the backend is +// always constructed for that region regardless of the configured default. +// +//nolint:ireturn,nolintlint // architecturally required to return interface +func (p *Provider) Init(ctx *service.AppContext) (service.Registerable, error) { + accountID, _ := service.AccountRegionOrDefault(ctx) + + backend := NewInMemoryBackend(accountID, "us-east-1") + handler := NewHandler(backend) + handler.AccountID = accountID + handler.DefaultRegion = "us-east-1" + + return handler, nil +} diff --git a/services/ecrpublic/repositories.go b/services/ecrpublic/repositories.go new file mode 100644 index 0000000000..cf83f35fac --- /dev/null +++ b/services/ecrpublic/repositories.go @@ -0,0 +1,167 @@ +package ecrpublic + +import ( + "fmt" + "maps" + "regexp" + "sort" + "time" +) + +const ( + maxRepositoryNameLen = 205 + minRepositoryNameLen = 2 + maxTagsPerResource = 50 +) + +// repositoryNameRE matches AWS's public repository naming rule: lowercase +// letters, numbers, hyphens, underscores, periods, and forward slashes for +// namespacing (e.g. "project-a/nginx-web-app"), confirmed against +// CreateRepositoryInput's docs in aws-sdk-go-v2/service/ecrpublic@v1.47.1. +var repositoryNameRE = regexp.MustCompile(`^[a-z0-9]+(?:[._-][a-z0-9]+)*(?:/[a-z0-9]+(?:[._-][a-z0-9]+)*)*$`) + +func validateRepositoryName(name string) error { + if len(name) < minRepositoryNameLen || len(name) > maxRepositoryNameLen || !repositoryNameRE.MatchString(name) { + return fmt.Errorf("%w: invalid repository name: %s", ErrInvalidParameter, name) + } + + return nil +} + +func validateTags(tags map[string]string) error { + if len(tags) > maxTagsPerResource { + return fmt.Errorf("%w: a resource can have a maximum of %d tags", ErrTooManyTags, maxTagsPerResource) + } + + for k := range tags { + if k == "" { + return fmt.Errorf("%w: tag key must not be empty", ErrInvalidTagParameter) + } + } + + return nil +} + +// CreateRepository creates a new public repository under the caller's account. +func (b *InMemoryBackend) CreateRepository( + name string, catalogData *CatalogData, tags map[string]string, +) (*Repository, error) { + if err := validateRepositoryName(name); err != nil { + return nil, err + } + + if err := validateTags(tags); err != nil { + return nil, err + } + + b.mu.Lock("CreateRepository") + defer b.mu.Unlock() + + if b.repos.Has(name) { + return nil, fmt.Errorf("%w: %s", ErrRepositoryAlreadyExists, name) + } + + cd := CatalogData{} + if catalogData != nil { + cd = *catalogData + } + + repo := &Repository{ + RepositoryName: name, + RepositoryArn: repositoryARN(b.region, b.accountID, name), + RegistryID: b.accountID, + RepositoryURI: repositoryURI(b.registryAlias, name), + CreatedAt: time.Now(), + CatalogData: cd, + Tags: cloneTagMap(tags), + } + + b.repos.Put(repo) + + cp := *repo + cp.Tags = cloneTagMap(repo.Tags) + + return &cp, nil +} + +// DescribeRepositories returns repositories in the given registry, optionally +// filtered by name. An unknown name in a non-empty filter is a hard error, +// matching AWS. +func (b *InMemoryBackend) DescribeRepositories(registryID string, names []string) ([]*Repository, error) { + b.mu.RLock("DescribeRepositories") + defer b.mu.RUnlock() + + if err := b.resolveRegistryIDLocked(registryID); err != nil { + return nil, err + } + + if len(names) == 0 { + all := b.repos.All() + out := make([]*Repository, 0, len(all)) + + for _, r := range all { + cp := *r + cp.Tags = cloneTagMap(r.Tags) + out = append(out, &cp) + } + + sort.Slice(out, func(i, j int) bool { return out[i].RepositoryName < out[j].RepositoryName }) + + return out, nil + } + + out := make([]*Repository, 0, len(names)) + + for _, name := range names { + r, ok := b.repos.Get(name) + if !ok { + return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, name) + } + + cp := *r + cp.Tags = cloneTagMap(r.Tags) + out = append(out, &cp) + } + + return out, nil +} + +// DeleteRepository deletes a repository. Non-empty repositories are rejected +// unless force is set, matching AWS. +func (b *InMemoryBackend) DeleteRepository(registryID, name string, force bool) (*Repository, error) { + b.mu.Lock("DeleteRepository") + defer b.mu.Unlock() + + if err := b.resolveRegistryIDLocked(registryID); err != nil { + return nil, err + } + + repo, ok := b.repos.Get(name) + if !ok { + return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, name) + } + + if !force && len(b.imagesByRepo.Get(name)) > 0 { + return nil, fmt.Errorf("%w: %s", ErrRepositoryNotEmpty, name) + } + + for _, img := range b.imagesByRepo.Get(name) { + b.images.Delete(imageTableKey(name, img.ImageDigest)) + } + + b.repos.Delete(name) + delete(b.tagIndex, name) + delete(b.uploadedLayers, name) + + cp := *repo + cp.Tags = cloneTagMap(repo.Tags) + + return &cp, nil +} + +func cloneTagMap(tags map[string]string) map[string]string { + out := make(map[string]string, len(tags)) + maps.Copy(out, tags) + + return out +} diff --git a/services/ecrpublic/repositories_test.go b/services/ecrpublic/repositories_test.go new file mode 100644 index 0000000000..d1ff63a2f0 --- /dev/null +++ b/services/ecrpublic/repositories_test.go @@ -0,0 +1,166 @@ +package ecrpublic_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + ecrpublicsdk "github.com/aws/aws-sdk-go-v2/service/ecrpublic" + "github.com/aws/aws-sdk-go-v2/service/ecrpublic/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateRepository(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + out, err := client.CreateRepository(ctx, &ecrpublicsdk.CreateRepositoryInput{ + RepositoryName: aws.String("my-app"), + CatalogData: &types.RepositoryCatalogDataInput{ + AboutText: aws.String("about"), + Description: aws.String("desc"), + UsageText: aws.String("usage"), + Architectures: []string{"ARM"}, + OperatingSystems: []string{"Linux"}, + LogoImageBlob: []byte("logo-bytes"), + }, + Tags: []types.Tag{{Key: aws.String("team"), Value: aws.String("video")}}, + }) + require.NoError(t, err) + require.NotNil(t, out.Repository) + assert.Equal(t, "my-app", aws.ToString(out.Repository.RepositoryName)) + assert.Equal(t, "123456789012", aws.ToString(out.Repository.RegistryId)) + assert.Equal(t, "arn:aws:ecr-public::123456789012:repository/my-app", aws.ToString(out.Repository.RepositoryArn)) + assert.Regexp(t, `^public\.ecr\.aws/[0-9a-f]+/my-app$`, aws.ToString(out.Repository.RepositoryUri)) + require.NotNil(t, out.CatalogData) + assert.Equal(t, "about", aws.ToString(out.CatalogData.AboutText)) + assert.NotEmpty(t, aws.ToString(out.CatalogData.LogoUrl)) +} + +func TestCreateRepository_AlreadyExists(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateRepository(ctx, &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String("dup")}) + require.NoError(t, err) + + _, err = client.CreateRepository(ctx, &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String("dup")}) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "RepositoryAlreadyExistsException", apiErr.ErrorCode()) +} + +func TestDescribeRepositories(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + for _, name := range []string{"repo-a", "repo-b"} { + _, err := client.CreateRepository(ctx, &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String(name)}) + require.NoError(t, err) + } + + t.Run("list all", func(t *testing.T) { + t.Parallel() + + out, err := client.DescribeRepositories(ctx, &ecrpublicsdk.DescribeRepositoriesInput{}) + require.NoError(t, err) + assert.Len(t, out.Repositories, 2) + }) + + t.Run("filtered", func(t *testing.T) { + t.Parallel() + + out, err := client.DescribeRepositories(ctx, &ecrpublicsdk.DescribeRepositoriesInput{ + RepositoryNames: []string{"repo-a"}, + }) + require.NoError(t, err) + require.Len(t, out.Repositories, 1) + assert.Equal(t, "repo-a", aws.ToString(out.Repositories[0].RepositoryName)) + }) + + t.Run("not found", func(t *testing.T) { + t.Parallel() + + _, err := client.DescribeRepositories(ctx, &ecrpublicsdk.DescribeRepositoriesInput{ + RepositoryNames: []string{"missing"}, + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "RepositoryNotFoundException", apiErr.ErrorCode()) + }) +} + +func TestDeleteRepository(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateRepository(ctx, &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String("gone")}) + require.NoError(t, err) + + out, err := client.DeleteRepository(ctx, &ecrpublicsdk.DeleteRepositoryInput{RepositoryName: aws.String("gone")}) + require.NoError(t, err) + assert.Equal(t, "gone", aws.ToString(out.Repository.RepositoryName)) + + _, err = client.DescribeRepositories(ctx, &ecrpublicsdk.DescribeRepositoriesInput{ + RepositoryNames: []string{"gone"}, + }) + require.Error(t, err) +} + +func TestDeleteRepository_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.DeleteRepository(t.Context(), &ecrpublicsdk.DeleteRepositoryInput{ + RepositoryName: aws.String("missing"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "RepositoryNotFoundException", apiErr.ErrorCode()) +} + +func TestDeleteRepository_NotEmptyWithoutForce(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateRepository(ctx, &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String("has-image")}) + require.NoError(t, err) + + _, err = client.PutImage(ctx, &ecrpublicsdk.PutImageInput{ + RepositoryName: aws.String("has-image"), + ImageManifest: aws.String(`{"schemaVersion":2}`), + }) + require.NoError(t, err) + + _, err = client.DeleteRepository(ctx, &ecrpublicsdk.DeleteRepositoryInput{RepositoryName: aws.String("has-image")}) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "RepositoryNotEmptyException", apiErr.ErrorCode()) + + out, err := client.DeleteRepository(ctx, &ecrpublicsdk.DeleteRepositoryInput{ + RepositoryName: aws.String("has-image"), + Force: true, + }) + require.NoError(t, err) + assert.Equal(t, "has-image", aws.ToString(out.Repository.RepositoryName)) +} diff --git a/services/ecrpublic/store.go b/services/ecrpublic/store.go new file mode 100644 index 0000000000..51fa0adb29 --- /dev/null +++ b/services/ecrpublic/store.go @@ -0,0 +1,132 @@ +package ecrpublic + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + + "github.com/blackbirdworks/gopherstack/pkgs/arn" + "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +const ( + ecrPublicService = "ecr-public" + + // repositoryURIHost is the fixed public pull host; see + // https://docs.aws.amazon.com/AmazonECR/latest/public/public-getting-started.html -- + // "public.ecr.aws/registry_alias/repository_name". + repositoryURIHost = "public.ecr.aws" +) + +// InMemoryBackend is the in-memory implementation of Backend. Amazon ECR +// Public is a single-region (us-east-1), single-registry-per-account service: +// there is no per-region partitioning of repositories, matching the real API. +type InMemoryBackend struct { + registry *store.Registry + repos *store.Table[Repository] + images *store.Table[Image] + imagesByRepo *store.Index[Image] + + // tagIndex, uploadedLayers, and layerUploads carry no identity field of + // their own (see services/ecr/store_setup.go's registerAllTables doc for + // the same exemption pattern) and are left as plain maps rather than + // store.Table entries. + tagIndex map[string]map[string]tagBinding + uploadedLayers map[string]map[string]int64 + layerUploads map[string]*layerUploadState + + registryCatalogData RegistryCatalogData + + mu *lockmetrics.RWMutex + accountID string + region string + registryAlias string + layerUploadSeq uint64 +} + +// NewInMemoryBackend creates a new Amazon ECR Public backend for accountID. +func NewInMemoryBackend(accountID, region string) *InMemoryBackend { + b := &InMemoryBackend{ + registry: store.NewRegistry(), + tagIndex: make(map[string]map[string]tagBinding), + uploadedLayers: make(map[string]map[string]int64), + layerUploads: make(map[string]*layerUploadState), + mu: lockmetrics.New("ecrpublic"), + accountID: accountID, + region: region, + registryAlias: registryAliasForAccount(accountID), + } + + registerAllTables(b) + + return b +} + +// Reset clears all backend state. +func (b *InMemoryBackend) Reset() { + b.mu.Lock("Reset") + defer b.mu.Unlock() + + b.registry.ResetAll() + b.tagIndex = make(map[string]map[string]tagBinding) + b.uploadedLayers = make(map[string]map[string]int64) + b.layerUploads = make(map[string]*layerUploadState) + b.layerUploadSeq = 0 + b.registryCatalogData = RegistryCatalogData{} +} + +// AccountID returns the AWS account ID this backend is configured for. +func (b *InMemoryBackend) AccountID() string { + b.mu.RLock("AccountID") + defer b.mu.RUnlock() + + return b.accountID +} + +// shortHash returns a short, deterministic hex digest of s, used to derive a +// stable-looking pseudo-random registry alias from an account ID. +func shortHash(s string) string { + sum := sha256.Sum256([]byte(s)) + + return hex.EncodeToString(sum[:])[:10] +} + +// registryAliasForAccount derives a stable default registry alias for an +// account. Real aliases are opaque, account-scoped strings assigned by AWS +// (e.g. "a1b2c3d4e5"); this emulator derives one deterministically so it is +// stable across restarts (and across Restore) without persisting it. +func registryAliasForAccount(accountID string) string { + return shortHash(accountID) +} + +// repositoryARN builds the ARN for a public repository. Confirmed against AWS +// docs and the terraform-provider-aws ecrpublic_repository resource: unlike +// private ECR, the region segment is empty -- +// arn:aws:ecr-public:::repository/. +func repositoryARN(region, accountID, name string) string { + return arn.BuildGlobal(ecrPublicService, region, accountID, "repository/"+name) +} + +// registryARN builds the ARN for the caller's public registry itself +// (as opposed to a repository within it). +func registryARN(region, accountID string) string { + return arn.BuildGlobal(ecrPublicService, region, accountID, "registry") +} + +// repositoryURI builds the docker pull URI for a public repository: +// public.ecr.aws//. +func repositoryURI(alias, name string) string { + return fmt.Sprintf("%s/%s/%s", repositoryURIHost, alias, name) +} + +// resolveRegistryIDLocked validates a caller-supplied registryId against this +// single-tenant emulator's own account. An empty registryId defaults to the +// caller's own account, matching AWS. Caller must hold b.mu. +func (b *InMemoryBackend) resolveRegistryIDLocked(registryID string) error { + if registryID == "" || registryID == b.accountID { + return nil + } + + return fmt.Errorf("%w: %s", ErrRegistryNotFound, registryID) +} diff --git a/services/ecrpublic/store_setup.go b/services/ecrpublic/store_setup.go new file mode 100644 index 0000000000..a5ec4f5450 --- /dev/null +++ b/services/ecrpublic/store_setup.go @@ -0,0 +1,25 @@ +package ecrpublic + +import "github.com/blackbirdworks/gopherstack/pkgs/store" + +func repoKeyFn(r *Repository) string { return r.RepositoryName } + +// imageTableKey returns the store.Table primary key for an image: repository +// name and digest joined by "@", matching the OCI image-reference convention. +// "@" never appears in a repository name or a "sha256:..." digest. +func imageTableKey(repositoryName, digest string) string { return repositoryName + "@" + digest } + +func imageKeyFn(img *Image) string { return imageTableKey(img.RepositoryName, img.ImageDigest) } + +func imageRepoIndexKeyFn(img *Image) string { return img.RepositoryName } + +// registerAllTables registers every backend resource table exactly once. +// Must be called during construction only -- store.Register panics on a +// duplicate name. +func registerAllTables(b *InMemoryBackend) { + b.repos = store.Register(b.registry, "repos", store.New(repoKeyFn)) + + imagesT := store.Register(b.registry, "images", store.New(imageKeyFn)) + b.images = imagesT + b.imagesByRepo = imagesT.AddIndex("repo", imageRepoIndexKeyFn) +} diff --git a/services/ecrpublic/tags.go b/services/ecrpublic/tags.go new file mode 100644 index 0000000000..25b513391b --- /dev/null +++ b/services/ecrpublic/tags.go @@ -0,0 +1,89 @@ +package ecrpublic + +import ( + "fmt" + "maps" + "strings" +) + +// repositoryNameFromARN extracts the repository name from a well-formed +// public-repository ARN (arn:{partition}:ecr-public::{account}:repository/{name}). +func repositoryNameFromARN(resourceARN string) (string, bool) { + parts := strings.SplitN(resourceARN, ":", 6) //nolint:mnd // arn:partition:service::account:resource + if len(parts) != 6 || !strings.HasPrefix(parts[5], "repository/") { + return "", false + } + + name := strings.TrimPrefix(parts[5], "repository/") + if name == "" { + return "", false + } + + return name, true +} + +func (b *InMemoryBackend) resolveByARNLocked(resourceARN string) (*Repository, error) { + name, ok := repositoryNameFromARN(resourceARN) + if !ok { + return nil, fmt.Errorf("%w: malformed resource ARN: %s", ErrInvalidParameter, resourceARN) + } + + repo, ok := b.repos.Get(name) + if !ok { + return nil, fmt.Errorf("%w: %s", ErrRepositoryNotFound, resourceARN) + } + + return repo, nil +} + +// TagResource adds or replaces tags on a repository. +func (b *InMemoryBackend) TagResource(resourceARN string, tags map[string]string) error { + b.mu.Lock("TagResource") + defer b.mu.Unlock() + + repo, err := b.resolveByARNLocked(resourceARN) + if err != nil { + return err + } + + merged := cloneTagMap(repo.Tags) + maps.Copy(merged, tags) + + if validateErr := validateTags(merged); validateErr != nil { + return validateErr + } + + repo.Tags = merged + + return nil +} + +// UntagResource removes tags from a repository by key. +func (b *InMemoryBackend) UntagResource(resourceARN string, tagKeys []string) error { + b.mu.Lock("UntagResource") + defer b.mu.Unlock() + + repo, err := b.resolveByARNLocked(resourceARN) + if err != nil { + return err + } + + for _, k := range tagKeys { + delete(repo.Tags, k) + } + + return nil +} + +// ListTagsForResource returns all tags on a repository. +func (b *InMemoryBackend) ListTagsForResource(resourceARN string) (map[string]string, error) { + b.mu.RLock("ListTagsForResource") + defer b.mu.RUnlock() + + repo, err := b.resolveByARNLocked(resourceARN) + if err != nil { + return nil, err + } + + return cloneTagMap(repo.Tags), nil +} diff --git a/services/ecrpublic/tags_race_test.go b/services/ecrpublic/tags_race_test.go new file mode 100644 index 0000000000..2ab5fe128e --- /dev/null +++ b/services/ecrpublic/tags_race_test.go @@ -0,0 +1,89 @@ +package ecrpublic_test + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/ecrpublic" +) + +// TestRepositoryTagsConcurrentWithUntagResource proves Describe/Create/Delete +// must not hand back a Repository whose Tags map UntagResource mutates in place. +func TestRepositoryTagsConcurrentWithUntagResource(t *testing.T) { + t.Parallel() + + tests := []struct { + reader func(b *ecrpublic.InMemoryBackend, name, arn string) + name string + }{ + { + name: "DescribeRepositories all", + reader: func(b *ecrpublic.InMemoryBackend, _, _ string) { + repos, err := b.DescribeRepositories("", nil) + if err != nil { + return + } + + for _, r := range repos { + for k := range r.Tags { + _ = k + } + } + }, + }, + { + name: "DescribeRepositories by name", + reader: func(b *ecrpublic.InMemoryBackend, name, _ string) { + repos, err := b.DescribeRepositories("", []string{name}) + if err != nil { + return + } + + for _, r := range repos { + for k := range r.Tags { + _ = k + } + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := ecrpublic.NewInMemoryBackend(testAccountID, testRegion) + + tags := map[string]string{"team": "video", "env": "prod"} + + repo, err := b.CreateRepository("race-repo", nil, tags) + require.NoError(t, err) + + const iterations = 500 + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + for range iterations { + tt.reader(b, repo.RepositoryName, repo.RepositoryArn) + } + }() + + go func() { + defer wg.Done() + + for range iterations { + _ = b.TagResource(repo.RepositoryArn, map[string]string{"env": "prod"}) + _ = b.UntagResource(repo.RepositoryArn, []string{"env"}) + } + }() + + wg.Wait() + }) + } +} diff --git a/services/ecrpublic/tags_test.go b/services/ecrpublic/tags_test.go new file mode 100644 index 0000000000..ab20ac70f7 --- /dev/null +++ b/services/ecrpublic/tags_test.go @@ -0,0 +1,112 @@ +package ecrpublic_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + ecrpublicsdk "github.com/aws/aws-sdk-go-v2/service/ecrpublic" + "github.com/aws/aws-sdk-go-v2/service/ecrpublic/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func tagMap(tags []types.Tag) map[string]string { + out := make(map[string]string, len(tags)) + for _, t := range tags { + out[aws.ToString(t.Key)] = aws.ToString(t.Value) + } + + return out +} + +func TestResourceTagLifecycle(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateRepository(ctx, &ecrpublicsdk.CreateRepositoryInput{ + RepositoryName: aws.String("tagged-repo"), + Tags: []types.Tag{{Key: aws.String("team"), Value: aws.String("video")}}, + }) + require.NoError(t, err) + + repoARN := aws.ToString(created.Repository.RepositoryArn) + + listed, err := client.ListTagsForResource( + ctx, + &ecrpublicsdk.ListTagsForResourceInput{ResourceArn: aws.String(repoARN)}, + ) + require.NoError(t, err) + assert.Equal(t, map[string]string{"team": "video"}, tagMap(listed.Tags)) + + _, err = client.TagResource(ctx, &ecrpublicsdk.TagResourceInput{ + ResourceArn: aws.String(repoARN), + Tags: []types.Tag{{Key: aws.String("env"), Value: aws.String("prod")}}, + }) + require.NoError(t, err) + + afterTag, err := client.ListTagsForResource( + ctx, + &ecrpublicsdk.ListTagsForResourceInput{ResourceArn: aws.String(repoARN)}, + ) + require.NoError(t, err) + assert.Equal(t, map[string]string{"team": "video", "env": "prod"}, tagMap(afterTag.Tags)) + + _, err = client.UntagResource(ctx, &ecrpublicsdk.UntagResourceInput{ + ResourceArn: aws.String(repoARN), + TagKeys: []string{"team"}, + }) + require.NoError(t, err) + + afterUntag, err := client.ListTagsForResource( + ctx, + &ecrpublicsdk.ListTagsForResourceInput{ResourceArn: aws.String(repoARN)}, + ) + require.NoError(t, err) + assert.Equal(t, map[string]string{"env": "prod"}, tagMap(afterUntag.Tags)) +} + +func TestListTagsForResource_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.ListTagsForResource(t.Context(), &ecrpublicsdk.ListTagsForResourceInput{ + ResourceArn: aws.String("arn:aws:ecr-public::123456789012:repository/missing"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "RepositoryNotFoundException", apiErr.ErrorCode()) +} + +func TestTagResource_TooManyTags(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateRepository( + ctx, + &ecrpublicsdk.CreateRepositoryInput{RepositoryName: aws.String("many-tags")}, + ) + require.NoError(t, err) + + tags := make([]types.Tag, 0, 51) + for i := range 51 { + tags = append(tags, types.Tag{Key: aws.String(string(rune('a' + i))), Value: aws.String("v")}) + } + + _, err = client.TagResource(ctx, &ecrpublicsdk.TagResourceInput{ + ResourceArn: created.Repository.RepositoryArn, + Tags: tags, + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "TooManyTagsException", apiErr.ErrorCode()) +} diff --git a/services/ecrpublic/wire.go b/services/ecrpublic/wire.go new file mode 100644 index 0000000000..cba657bf5c --- /dev/null +++ b/services/ecrpublic/wire.go @@ -0,0 +1,288 @@ +package ecrpublic + +// Wire DTOs for the Amazon ECR Public JSON-RPC (awsjson1.1) API. Field names +// and shapes are verified against the pinned aws-sdk-go-v2/service/ecrpublic +// v1.47.1 request_snapshot/ and response_snapshot/ fixtures (that SDK version +// generates via smithy schemas rather than serializers.go/deserializers.go, +// so the snapshot fixtures -- exact captured wire JSON per operation -- are +// the authoritative source here instead of a serializer function body). +// +// Tag.Key/Tag.Value are capitalized on the wire (unlike every other field in +// this API, which is lowerCamelCase) -- confirmed by +// request_snapshot/CreateRepository.request.snap: `{"Key":"...","Value":"..."}`. +import "github.com/blackbirdworks/gopherstack/pkgs/awstime" + +// TagWire mirrors types.Tag. +type TagWire struct { + Key string `json:"Key"` + Value string `json:"Value"` +} + +// RepositoryWire mirrors types.Repository. +type RepositoryWire struct { + RegistryID string `json:"registryId,omitempty"` + RepositoryArn string `json:"repositoryArn,omitempty"` + RepositoryName string `json:"repositoryName,omitempty"` + RepositoryURI string `json:"repositoryUri,omitempty"` + CreatedAt float64 `json:"createdAt,omitempty"` +} + +// CatalogDataInputWire mirrors types.RepositoryCatalogDataInput. +type CatalogDataInputWire struct { + AboutText *string `json:"aboutText,omitempty"` + Description *string `json:"description,omitempty"` + UsageText *string `json:"usageText,omitempty"` + LogoImageBlob []byte `json:"logoImageBlob,omitempty"` + Architectures []string `json:"architectures,omitempty"` + OperatingSystems []string `json:"operatingSystems,omitempty"` +} + +// CatalogDataWire mirrors types.RepositoryCatalogData. +type CatalogDataWire struct { + AboutText string `json:"aboutText,omitempty"` + Description string `json:"description,omitempty"` + LogoURL string `json:"logoUrl,omitempty"` + UsageText string `json:"usageText,omitempty"` + Architectures []string `json:"architectures,omitempty"` + OperatingSystems []string `json:"operatingSystems,omitempty"` + MarketplaceCertified bool `json:"marketplaceCertified,omitempty"` +} + +// RegistryAliasWire mirrors types.RegistryAlias. +type RegistryAliasWire struct { + Name string `json:"name"` + Status string `json:"status"` + DefaultRegistryAlias bool `json:"defaultRegistryAlias"` + PrimaryRegistryAlias bool `json:"primaryRegistryAlias"` +} + +// RegistryWire mirrors types.Registry. +type RegistryWire struct { + RegistryArn string `json:"registryArn"` + RegistryID string `json:"registryId"` + RegistryURI string `json:"registryUri"` + Aliases []RegistryAliasWire `json:"aliases"` + Verified bool `json:"verified"` +} + +// RegistryCatalogDataWire mirrors types.RegistryCatalogData. +type RegistryCatalogDataWire struct { + DisplayName string `json:"displayName,omitempty"` +} + +// AuthorizationDataWire mirrors types.AuthorizationData. +type AuthorizationDataWire struct { + AuthorizationToken string `json:"authorizationToken,omitempty"` + ExpiresAt float64 `json:"expiresAt,omitempty"` +} + +// ImageIdentifierWire mirrors types.ImageIdentifier. +type ImageIdentifierWire struct { + ImageDigest string `json:"imageDigest,omitempty"` + ImageTag string `json:"imageTag,omitempty"` +} + +// ImageDetailWire mirrors types.ImageDetail. +type ImageDetailWire struct { + ArtifactMediaType string `json:"artifactMediaType,omitempty"` + ImageDigest string `json:"imageDigest,omitempty"` + ImageManifestMediaType string `json:"imageManifestMediaType,omitempty"` + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName,omitempty"` + ImageTags []string `json:"imageTags,omitempty"` + ImagePushedAt float64 `json:"imagePushedAt,omitempty"` + ImageSizeInBytes int64 `json:"imageSizeInBytes,omitempty"` +} + +// ReferencedImageDetailWire mirrors types.ReferencedImageDetail. +type ReferencedImageDetailWire struct { + ArtifactMediaType string `json:"artifactMediaType,omitempty"` + ImageDigest string `json:"imageDigest,omitempty"` + ImageManifestMediaType string `json:"imageManifestMediaType,omitempty"` + ImagePushedAt float64 `json:"imagePushedAt,omitempty"` + ImageSizeInBytes int64 `json:"imageSizeInBytes,omitempty"` +} + +// ImageTagDetailWire mirrors types.ImageTagDetail. +type ImageTagDetailWire struct { + ImageDetail *ReferencedImageDetailWire `json:"imageDetail,omitempty"` + ImageTag string `json:"imageTag,omitempty"` + CreatedAt float64 `json:"createdAt,omitempty"` +} + +// ImageWire mirrors types.Image. +type ImageWire struct { + ImageID ImageIdentifierWire `json:"imageId"` + ImageManifest string `json:"imageManifest,omitempty"` + ImageManifestMediaType string `json:"imageManifestMediaType,omitempty"` + RegistryID string `json:"registryId,omitempty"` + RepositoryName string `json:"repositoryName,omitempty"` +} + +// LayerWire mirrors types.Layer. +type LayerWire struct { + LayerAvailability string `json:"layerAvailability,omitempty"` + LayerDigest string `json:"layerDigest,omitempty"` + MediaType string `json:"mediaType,omitempty"` + LayerSize int64 `json:"layerSize,omitempty"` +} + +// LayerFailureWire mirrors types.LayerFailure. +type LayerFailureWire struct { + FailureCode string `json:"failureCode,omitempty"` + FailureReason string `json:"failureReason,omitempty"` + LayerDigest string `json:"layerDigest,omitempty"` +} + +// ImageFailureWire mirrors types.ImageFailure. +type ImageFailureWire struct { + FailureCode string `json:"failureCode,omitempty"` + FailureReason string `json:"failureReason,omitempty"` + ImageID ImageIdentifierWire `json:"imageId"` +} + +func tagsToWire(tags map[string]string) []TagWire { + out := make([]TagWire, 0, len(tags)) + for k, v := range tags { + out = append(out, TagWire{Key: k, Value: v}) + } + + return out +} + +func tagsFromWire(tags []TagWire) map[string]string { + out := make(map[string]string, len(tags)) + for _, t := range tags { + out[t.Key] = t.Value + } + + return out +} + +func toRepositoryWire(r *Repository) RepositoryWire { + return RepositoryWire{ + CreatedAt: awstime.Epoch(r.CreatedAt), + RegistryID: r.RegistryID, + RepositoryArn: r.RepositoryArn, + RepositoryName: r.RepositoryName, + RepositoryURI: r.RepositoryURI, + } +} + +func toCatalogDataInput(w *CatalogDataInputWire) *CatalogData { + if w == nil { + return &CatalogData{} + } + + cd := &CatalogData{ + Architectures: w.Architectures, + OperatingSystems: w.OperatingSystems, + LogoImageBlob: w.LogoImageBlob, + } + + if w.AboutText != nil { + cd.AboutText = *w.AboutText + } + + if w.Description != nil { + cd.Description = *w.Description + } + + if w.UsageText != nil { + cd.UsageText = *w.UsageText + } + + return cd +} + +func toCatalogDataWire(cd *CatalogData) CatalogDataWire { + w := CatalogDataWire{ + AboutText: cd.AboutText, + Architectures: cd.Architectures, + Description: cd.Description, + OperatingSystems: cd.OperatingSystems, + UsageText: cd.UsageText, + MarketplaceCertified: cd.MarketplaceCertified, + } + + if len(cd.LogoImageBlob) > 0 { + w.LogoURL = "https://" + repositoryURIHost + "/logos/" + shortHash(string(cd.LogoImageBlob)) + ".png" + } + + return w +} + +func toImageIdentifierWire(id ImageIdentifier) ImageIdentifierWire { + return ImageIdentifierWire(id) +} + +func imageIdentifierFromWire(w ImageIdentifierWire) ImageIdentifier { + return ImageIdentifier(w) +} + +func toImageDetailWire(d ImageDetail) ImageDetailWire { + return ImageDetailWire{ + ArtifactMediaType: d.ArtifactMediaType, + ImageDigest: d.ImageDigest, + ImageManifestMediaType: d.ImageManifestMediaType, + ImagePushedAt: awstime.Epoch(d.ImagePushedAt), + ImageSizeInBytes: d.ImageSizeInBytes, + ImageTags: d.ImageTags, + RegistryID: d.RegistryID, + RepositoryName: d.RepositoryName, + } +} + +func toImageTagDetailWire(d ImageTagDetail) ImageTagDetailWire { + return ImageTagDetailWire{ + CreatedAt: awstime.Epoch(d.CreatedAt), + ImageDetail: &ReferencedImageDetailWire{ + ArtifactMediaType: d.ArtifactMediaType, + ImageDigest: d.ImageDigest, + ImageManifestMediaType: d.ImageManifestMediaType, + ImagePushedAt: awstime.Epoch(d.ImagePushedAt), + ImageSizeInBytes: d.ImageSizeInBytes, + }, + ImageTag: d.ImageTag, + } +} + +func toImageFailureWire(f ImageFailure) ImageFailureWire { + return ImageFailureWire{ + FailureCode: f.FailureCode, + FailureReason: f.FailureReason, + ImageID: toImageIdentifierWire(f.ImageID), + } +} + +func toLayerWire(l LayerInfo) LayerWire { + return LayerWire{ + LayerAvailability: l.LayerAvailability, + LayerDigest: l.LayerDigest, + LayerSize: l.LayerSize, + MediaType: l.MediaType, + } +} + +func toLayerFailureWire(f LayerFailure) LayerFailureWire { + return LayerFailureWire{ + FailureCode: f.FailureCode, + FailureReason: f.FailureReason, + LayerDigest: f.LayerDigest, + } +} + +func toRegistryWire(info RegistryInfo) RegistryWire { + aliases := make([]RegistryAliasWire, 0, len(info.Aliases)) + for _, a := range info.Aliases { + aliases = append(aliases, RegistryAliasWire(a)) + } + + return RegistryWire{ + Aliases: aliases, + RegistryArn: info.RegistryArn, + RegistryID: info.RegistryID, + RegistryURI: info.RegistryURI, + Verified: info.Verified, + } +} diff --git a/services/ecs/express_gateway.go b/services/ecs/express_gateway.go index f4c30df7df..0499394495 100644 --- a/services/ecs/express_gateway.go +++ b/services/ecs/express_gateway.go @@ -2,7 +2,6 @@ package ecs import ( "fmt" - "strconv" "strings" "time" @@ -205,7 +204,7 @@ func (b *InMemoryBackend) CreateExpressGatewayService( serviceName := input.ServiceName if serviceName == "" { - serviceName = "express-" + strconv.FormatInt(time.Now().UnixNano(), 10) + serviceName = "express-" + uuid.NewString() } serviceArn := fmt.Sprintf( diff --git a/services/ecs/id_generation_test.go b/services/ecs/id_generation_test.go new file mode 100644 index 0000000000..b53456818e --- /dev/null +++ b/services/ecs/id_generation_test.go @@ -0,0 +1,42 @@ +package ecs_test + +import ( + "regexp" + "testing" + "testing/synctest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/ecs" +) + +// expressServiceNamePattern locks in the fix for CreateExpressGatewayService's +// auto-generated name, which used to collide under synctest. +var expressServiceNamePattern = regexp.MustCompile( + `^express-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`, +) + +func TestECSBackend_ExpressGatewayServiceName_Unique(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + b := ecs.NewInMemoryBackend("000000000000", "us-east-1", ecs.NewNoopRunner()) + + input := ecs.CreateExpressGatewayServiceInput{ + InfrastructureRoleArn: "arn:aws:iam::000000000000:role/infra-role", + ExecutionRoleArn: "arn:aws:iam::000000000000:role/exec-role", + } + + svc1, err := b.CreateExpressGatewayService(input) + require.NoError(t, err) + + svc2, err := b.CreateExpressGatewayService(input) + require.NoError(t, err) + + assert.NotEqual(t, svc1.ServiceName, svc2.ServiceName, + "two express gateway services created back-to-back must get distinct names") + assert.Regexp(t, expressServiceNamePattern, svc1.ServiceName) + assert.Regexp(t, expressServiceNamePattern, svc2.ServiceName) + }) +} diff --git a/services/ecs/janitor_test.go b/services/ecs/janitor_test.go index 9c291a56d0..2553e8a43b 100644 --- a/services/ecs/janitor_test.go +++ b/services/ecs/janitor_test.go @@ -3,6 +3,7 @@ package ecs_test import ( "context" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -31,94 +32,100 @@ func setupJanitorBackend(t *testing.T) *ecs.InMemoryBackend { func TestJanitor_SweepsStoppedTasksOlderThanTTL(t *testing.T) { t.Parallel() - backend := setupJanitorBackend(t) + synctest.Test(t, func(t *testing.T) { + backend := setupJanitorBackend(t) - // Run a task then stop it. - tasks, _, err := backend.RunTask(ecs.RunTaskInput{ - Cluster: "test-cluster", - TaskDefinition: "test-family", - Count: 1, - }) - require.NoError(t, err) - require.Len(t, tasks, 1) + // Run a task then stop it. + tasks, _, err := backend.RunTask(ecs.RunTaskInput{ + Cluster: "test-cluster", + TaskDefinition: "test-family", + Count: 1, + }) + require.NoError(t, err) + require.Len(t, tasks, 1) - taskArn := tasks[0].TaskArn - _, err = backend.StopTask("test-cluster", taskArn, "testing") - require.NoError(t, err) + taskArn := tasks[0].TaskArn + _, err = backend.StopTask("test-cluster", taskArn, "testing") + require.NoError(t, err) - // Create a janitor with a very short TTL so the stopped task is immediately stale. - janitor := ecs.NewJanitor(backend, time.Second) - janitor.SetTaskTTL(1 * time.Millisecond) + // Create a janitor with a very short TTL so the stopped task is immediately stale. + janitor := ecs.NewJanitor(backend, time.Second) + janitor.SetTaskTTL(1 * time.Millisecond) - // Allow time for the task to expire. - time.Sleep(5 * time.Millisecond) + // Allow time for the task to expire. + time.Sleep(5 * time.Millisecond) - janitor.SweepOnce(context.Background()) + janitor.SweepOnce(context.Background()) - // The stopped task should have been evicted. - listed, err := backend.ListTasks("test-cluster") - require.NoError(t, err) - assert.Empty(t, listed) + // The stopped task should have been evicted. + listed, err := backend.ListTasks("test-cluster") + require.NoError(t, err) + assert.Empty(t, listed) + }) } func TestJanitor_SweptTaskLosesResourceTags(t *testing.T) { t.Parallel() - backend := setupJanitorBackend(t) + synctest.Test(t, func(t *testing.T) { + backend := setupJanitorBackend(t) - tasks, _, err := backend.RunTask(ecs.RunTaskInput{ - Cluster: "test-cluster", - TaskDefinition: "test-family", - Count: 1, - Tags: []ecs.Tag{{Key: "env", Value: "test"}}, - }) - require.NoError(t, err) - require.Len(t, tasks, 1) + tasks, _, err := backend.RunTask(ecs.RunTaskInput{ + Cluster: "test-cluster", + TaskDefinition: "test-family", + Count: 1, + Tags: []ecs.Tag{{Key: "env", Value: "test"}}, + }) + require.NoError(t, err) + require.Len(t, tasks, 1) - taskArn := tasks[0].TaskArn + taskArn := tasks[0].TaskArn - tags, err := backend.ListTagsForResource(taskArn) - require.NoError(t, err) - require.NotEmpty(t, tags, "tags should be recorded immediately after RunTask") + tags, err := backend.ListTagsForResource(taskArn) + require.NoError(t, err) + require.NotEmpty(t, tags, "tags should be recorded immediately after RunTask") - _, err = backend.StopTask("test-cluster", taskArn, "testing") - require.NoError(t, err) + _, err = backend.StopTask("test-cluster", taskArn, "testing") + require.NoError(t, err) - janitor := ecs.NewJanitor(backend, time.Second) - janitor.SetTaskTTL(1 * time.Millisecond) - time.Sleep(5 * time.Millisecond) + janitor := ecs.NewJanitor(backend, time.Second) + janitor.SetTaskTTL(1 * time.Millisecond) + time.Sleep(5 * time.Millisecond) - janitor.SweepOnce(context.Background()) + janitor.SweepOnce(context.Background()) - // The task is gone; its resourceTags side-map entry must go with it, or a - // stale ARN keeps answering ListTagsForResource forever. - tags, err = backend.ListTagsForResource(taskArn) - require.NoError(t, err) - assert.Empty(t, tags, "resourceTags leaked a ghost row for a swept task") + // The task is gone; its resourceTags side-map entry must go with it, or a + // stale ARN keeps answering ListTagsForResource forever. + tags, err = backend.ListTagsForResource(taskArn) + require.NoError(t, err) + assert.Empty(t, tags, "resourceTags leaked a ghost row for a swept task") + }) } func TestJanitor_DoesNotSweepRunningTasks(t *testing.T) { t.Parallel() - backend := setupJanitorBackend(t) + synctest.Test(t, func(t *testing.T) { + backend := setupJanitorBackend(t) - tasks, _, err := backend.RunTask(ecs.RunTaskInput{ - Cluster: "test-cluster", - TaskDefinition: "test-family", - Count: 1, - }) - require.NoError(t, err) - require.Len(t, tasks, 1) + tasks, _, err := backend.RunTask(ecs.RunTaskInput{ + Cluster: "test-cluster", + TaskDefinition: "test-family", + Count: 1, + }) + require.NoError(t, err) + require.Len(t, tasks, 1) - janitor := ecs.NewJanitor(backend, time.Second) - janitor.SetTaskTTL(1 * time.Millisecond) - time.Sleep(5 * time.Millisecond) + janitor := ecs.NewJanitor(backend, time.Second) + janitor.SetTaskTTL(1 * time.Millisecond) + time.Sleep(5 * time.Millisecond) - janitor.SweepOnce(context.Background()) + janitor.SweepOnce(context.Background()) - listed, err := backend.ListTasks("test-cluster") - require.NoError(t, err) - assert.Len(t, listed, 1) + listed, err := backend.ListTasks("test-cluster") + require.NoError(t, err) + assert.Len(t, listed, 1) + }) } func TestJanitor_DoesNotSweepRecentlyStoppedTasks(t *testing.T) { @@ -154,27 +161,30 @@ func TestJanitor_DoesNotSweepRecentlyStoppedTasks(t *testing.T) { func TestJanitor_RespectsContextCancellation(t *testing.T) { t.Parallel() - backend := setupJanitorBackend(t) + synctest.Test(t, func(t *testing.T) { + backend := setupJanitorBackend(t) - janitor := ecs.NewJanitor(backend, 50*time.Millisecond) + janitor := ecs.NewJanitor(backend, 50*time.Millisecond) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) + done := make(chan struct{}) - go func() { - janitor.Run(ctx) - close(done) - }() + go func() { + janitor.Run(ctx) + close(done) + }() - // Let it tick once. - time.Sleep(100 * time.Millisecond) - cancel() + // Let it tick once. + time.Sleep(100 * time.Millisecond) + synctest.Wait() + cancel() - select { - case <-done: - // Janitor exited as expected. - case <-time.After(2 * time.Second): - t.Fatal("janitor did not exit after context cancellation") - } + select { + case <-done: + // Janitor exited as expected. + case <-time.After(2 * time.Second): + t.Fatal("janitor did not exit after context cancellation") + } + }) } diff --git a/services/ecs/tasks.go b/services/ecs/tasks.go index d83d601605..ebad8dc0b1 100644 --- a/services/ecs/tasks.go +++ b/services/ecs/tasks.go @@ -154,8 +154,7 @@ func (b *InMemoryBackend) RunTask(input RunTaskInput) ([]Task, []Failure, error) tasks := make([]Task, 0, len(work)) for _, w := range work { - cp := *w.task - tasks = append(tasks, cp) + tasks = append(tasks, b.taskWithLiveTagsLocked(w.task)) } return tasks, failures, nil @@ -494,13 +493,13 @@ func (b *InMemoryBackend) DescribeTasks( return out, failures, nil } -// taskWithLiveTagsLocked returns a copy of t with Tags sourced from the -// resourceTags side map instead of t's own creation-time snapshot, so tags -// applied via TagResource/UntagResource after the task was started are -// reflected. Must be called with at least a read lock held. +// taskWithLiveTagsLocked copies t with live tags and deep-copied +// Containers/Attachments (mutated in place elsewhere). Needs at least RLock. func (b *InMemoryBackend) taskWithLiveTagsLocked(t *Task) Task { cp := *t cp.Tags = copyTags(b.resourceTags[resourceTagKey(t.TaskArn)]) + cp.Containers = append([]Container(nil), t.Containers...) + cp.Attachments = append([]TaskAttachment(nil), t.Attachments...) return cp } diff --git a/services/ecs/tasks_race_internal_test.go b/services/ecs/tasks_race_internal_test.go new file mode 100644 index 0000000000..39e9543efa --- /dev/null +++ b/services/ecs/tasks_race_internal_test.go @@ -0,0 +1,103 @@ +package ecs + +import ( + "sync" + "testing" +) + +// TestTaskConcurrentWithStopTask proves DescribeTasks/RunTask must not hand +// back a Task whose Containers slice StopTask mutates in place. +func TestTaskConcurrentWithStopTask(t *testing.T) { + t.Parallel() + + tests := []struct { + reader func(b *InMemoryBackend, taskArn string, runTaskSnapshot Task) func() + name string + }{ + { + name: "DescribeTasks races StopTask", + reader: func(b *InMemoryBackend, taskArn string, _ Task) func() { + return func() { + out, _, err := b.DescribeTasks("race-cluster", []string{taskArn}) + if err != nil || len(out) == 0 { + return + } + + for _, c := range out[0].Containers { + _ = c.LastStatus + _ = c.ExitCode + } + } + }, + }, + { + // Reads the single snapshot RunTask handed back once, mirroring a + // caller that keeps its own RunTask response around. + name: "RunTask snapshot races StopTask", + reader: func(_ *InMemoryBackend, _ string, runTaskSnapshot Task) func() { + return func() { + for _, c := range runTaskSnapshot.Containers { + _ = c.LastStatus + _ = c.ExitCode + } + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b, taskArn, snapshot := newRaceTestTask(t) + reader := tt.reader(b, taskArn, snapshot) + + const iterations = 500 + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + for range iterations { + reader() + } + }() + + go func() { + defer wg.Done() + + for range iterations { + _, _ = b.StopTask("race-cluster", taskArn, "race") + } + }() + + wg.Wait() + }) + } +} + +// newRaceTestTask creates a cluster with one running task and returns the +// backend, the task's ARN, and RunTask's own returned snapshot of it. +func newRaceTestTask(t *testing.T) (*InMemoryBackend, string, Task) { + t.Helper() + + b := newTestBackend() + + tdArn := registerSimpleTaskDef(t, b, "race-app", "nginx") + if _, err := b.CreateCluster(CreateClusterInput{ClusterName: "race-cluster"}); err != nil { + t.Fatalf("CreateCluster: %v", err) + } + + tasks, _, err := b.RunTask(RunTaskInput{ + Cluster: "race-cluster", + TaskDefinition: tdArn, + Count: 1, + }) + if err != nil { + t.Fatalf("RunTask: %v", err) + } + + return b, tasks[0].TaskArn, tasks[0] +} diff --git a/services/eks/handler.go b/services/eks/handler.go index 4f4cc60f9d..8f4019731b 100644 --- a/services/eks/handler.go +++ b/services/eks/handler.go @@ -268,7 +268,7 @@ func (h *Handler) RouteMatcher() service.Matcher { path := c.Request().URL.Path return path == pathClusters || - strings.HasPrefix(path, pathClusters+"/") || + (strings.HasPrefix(path, pathClusters+"/") && !isDSQLVpcEndpointServiceNamePath(path)) || strings.HasPrefix(path, pathEKSTags+"arn:aws:eks:") || path == pathSubscriptions || strings.HasPrefix(path, pathSubscriptions+"/") || @@ -281,6 +281,15 @@ func (h *Handler) RouteMatcher() service.Matcher { } } +// isDSQLVpcEndpointServiceNamePath reports whether path is DSQL's +// GetVpcEndpointServiceName route (/clusters/{id}/vpc-endpoint-service-name), +// which happens to share EKS's "/clusters/" prefix. EKS has no such +// operation, so this exact suffix can safely be excluded from EKS's claim +// (gopherstack-7r6bz). +func isDSQLVpcEndpointServiceNamePath(path string) bool { + return strings.HasSuffix(path, "/vpc-endpoint-service-name") +} + // MatchPriority returns the routing priority. func (h *Handler) MatchPriority() int { return eksMatchPriority } diff --git a/services/eks/id_generation_test.go b/services/eks/id_generation_test.go new file mode 100644 index 0000000000..3623349e8e --- /dev/null +++ b/services/eks/id_generation_test.go @@ -0,0 +1,41 @@ +package eks_test + +import ( + "regexp" + "testing" + "testing/synctest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/config" + "github.com/blackbirdworks/gopherstack/services/eks" +) + +// anywhereSubscriptionIDPattern locks in the fix for CreateEksAnywhereSubscription's ID, +// previously a name+time.Now().UnixNano() hash that collided under synctest. +var anywhereSubscriptionIDPattern = regexp.MustCompile( + `^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`, +) + +func TestEKSBackend_AnywhereSubscriptionID_Unique(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + b := eks.NewInMemoryBackend(t.Context(), "123456789012", config.DefaultRegion) + defer b.Close() + + term := eks.SubscriptionTerm{Duration: 1, Unit: "MONTHS"} + + sub1, err := b.CreateEksAnywhereSubscription("same-name", term, false, 1, "CLUSTER", nil) + require.NoError(t, err) + + sub2, err := b.CreateEksAnywhereSubscription("same-name", term, false, 1, "CLUSTER", nil) + require.NoError(t, err) + + assert.NotEqual(t, sub1.ID, sub2.ID, + "two subscriptions with the same name created back-to-back must get distinct IDs") + assert.Regexp(t, anywhereSubscriptionIDPattern, sub1.ID) + assert.Regexp(t, anywhereSubscriptionIDPattern, sub2.ID) + }) +} diff --git a/services/eks/subscriptions.go b/services/eks/subscriptions.go index c2fec09eed..8d5178d04f 100644 --- a/services/eks/subscriptions.go +++ b/services/eks/subscriptions.go @@ -3,9 +3,10 @@ package eks import ( "fmt" "sort" - "strconv" "time" + "github.com/google/uuid" + "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/tags" ) @@ -31,7 +32,9 @@ func (b *InMemoryBackend) CreateEksAnywhereSubscription( ) } - id := stableID(name + strconv.FormatInt(time.Now().UnixNano(), 10)) + // Id is documented as "UUID identifying a subscription" (types.EksAnywhereSubscription, + // aws-sdk-go-v2/service/eks), not a name-derived hash. + id := uuid.NewString() subARN := arn.Build("eks", b.region, b.accountID, "eks-anywhere-subscription/"+id) t := tags.New("eks.subscription." + id + ".tags") diff --git a/services/elasticbeanstalk/application_versions_test.go b/services/elasticbeanstalk/application_versions_test.go index 4ae14b27f8..35e5b643ca 100644 --- a/services/elasticbeanstalk/application_versions_test.go +++ b/services/elasticbeanstalk/application_versions_test.go @@ -3,6 +3,7 @@ package elasticbeanstalk_test import ( "context" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -143,16 +144,18 @@ func TestInMemoryBackend_DeleteApplicationVersion_RefusesRunningEnvironment(t *t func TestInMemoryBackend_UpdateApplicationVersion_BumpsDateUpdated(t *testing.T) { t.Parallel() - b := newTestBackend() - _, err := b.CreateApplication(context.Background(), "app2", "", nil) - require.NoError(t, err) - ver, err := b.CreateApplicationVersion(context.Background(), "app2", "v1", "orig", "", "", nil) - require.NoError(t, err) - created := ver.DateUpdated + synctest.Test(t, func(t *testing.T) { + b := newTestBackend() + _, err := b.CreateApplication(context.Background(), "app2", "", nil) + require.NoError(t, err) + ver, err := b.CreateApplicationVersion(context.Background(), "app2", "v1", "orig", "", "", nil) + require.NoError(t, err) + created := ver.DateUpdated - time.Sleep(time.Second) + time.Sleep(time.Second) - updated, err := b.UpdateApplicationVersion(context.Background(), "app2", "v1", "new desc") - require.NoError(t, err) - assert.NotEqual(t, created, updated.DateUpdated) + updated, err := b.UpdateApplicationVersion(context.Background(), "app2", "v1", "new desc") + require.NoError(t, err) + assert.NotEqual(t, created, updated.DateUpdated) + }) } diff --git a/services/elasticbeanstalk/applications_test.go b/services/elasticbeanstalk/applications_test.go index 568f38196b..7a9d413899 100644 --- a/services/elasticbeanstalk/applications_test.go +++ b/services/elasticbeanstalk/applications_test.go @@ -3,6 +3,7 @@ package elasticbeanstalk_test import ( "context" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -204,14 +205,16 @@ func TestInMemoryBackend_DeleteApplication_ClearsManagedActionHistory(t *testing func TestInMemoryBackend_UpdateApplication_BumpsDateUpdated(t *testing.T) { t.Parallel() - b := newTestBackend() - app, err := b.CreateApplication(context.Background(), "app1", "orig", nil) - require.NoError(t, err) - created := app.DateUpdated + synctest.Test(t, func(t *testing.T) { + b := newTestBackend() + app, err := b.CreateApplication(context.Background(), "app1", "orig", nil) + require.NoError(t, err) + created := app.DateUpdated - time.Sleep(time.Second) + time.Sleep(time.Second) - updated, err := b.UpdateApplication(context.Background(), "app1", "new desc") - require.NoError(t, err) - assert.NotEqual(t, created, updated.DateUpdated) + updated, err := b.UpdateApplication(context.Background(), "app1", "new desc") + require.NoError(t, err) + assert.NotEqual(t, created, updated.DateUpdated) + }) } diff --git a/services/elasticbeanstalk/configuration_templates_test.go b/services/elasticbeanstalk/configuration_templates_test.go index 38205ed3f4..7e399dccb4 100644 --- a/services/elasticbeanstalk/configuration_templates_test.go +++ b/services/elasticbeanstalk/configuration_templates_test.go @@ -3,6 +3,7 @@ package elasticbeanstalk_test import ( "context" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -17,18 +18,20 @@ import ( func TestInMemoryBackend_UpdateConfigurationTemplate_BumpsDateUpdated(t *testing.T) { t.Parallel() - b := newTestBackend() - _, err := b.CreateApplication(context.Background(), "app3", "", nil) - require.NoError(t, err) - tmpl, err := b.CreateConfigurationTemplate(context.Background(), "app3", "tmpl1", "orig", "", nil) - require.NoError(t, err) - created := tmpl.DateUpdated + synctest.Test(t, func(t *testing.T) { + b := newTestBackend() + _, err := b.CreateApplication(context.Background(), "app3", "", nil) + require.NoError(t, err) + tmpl, err := b.CreateConfigurationTemplate(context.Background(), "app3", "tmpl1", "orig", "", nil) + require.NoError(t, err) + created := tmpl.DateUpdated - time.Sleep(time.Second) + time.Sleep(time.Second) - updated, err := b.UpdateConfigurationTemplate(context.Background(), "app3", "tmpl1", "new desc") - require.NoError(t, err) - assert.NotEqual(t, created, updated.DateUpdated) + updated, err := b.UpdateConfigurationTemplate(context.Background(), "app3", "tmpl1", "new desc") + require.NoError(t, err) + assert.NotEqual(t, created, updated.DateUpdated) + }) } // TestInMemoryBackend_CreateConfigurationTemplate_SeedsFromEnvironment verifies that diff --git a/services/emr/janitor_test.go b/services/emr/janitor_test.go index f756d7e3c7..d82b23e24b 100644 --- a/services/emr/janitor_test.go +++ b/services/emr/janitor_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -15,27 +16,29 @@ import ( func TestEMR_Janitor_SweepsTerminatedClusters(t *testing.T) { t.Parallel() - b := emr.NewInMemoryBackend(testAccountID, testRegion) - cluster, err := b.RunJobFlow( - context.Background(), - emr.RunJobFlowParams{Name: "sweep-test", ReleaseLabel: "emr-6.0.0"}, - ) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := emr.NewInMemoryBackend(testAccountID, testRegion) + cluster, err := b.RunJobFlow( + context.Background(), + emr.RunJobFlowParams{Name: "sweep-test", ReleaseLabel: "emr-6.0.0"}, + ) + require.NoError(t, err) - require.NoError(t, b.TerminateJobFlows(context.Background(), []string{cluster.ID})) + require.NoError(t, b.TerminateJobFlows(context.Background(), []string{cluster.ID})) - janitor := emr.NewJanitor(b, 10*time.Millisecond, 50*time.Millisecond) - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() + janitor := emr.NewJanitor(b, 10*time.Millisecond, 50*time.Millisecond) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() - go janitor.Run(ctx) + go janitor.Run(ctx) - // Wait until the cluster is swept from the backend. - require.Eventually(t, func() bool { - _, descErr := b.DescribeCluster(context.Background(), cluster.ID) + // Cross a ticker interval past the TTL so the sweep has run. + time.Sleep(70 * time.Millisecond) + synctest.Wait() - return descErr != nil - }, 2*time.Second, 20*time.Millisecond, "terminated cluster should be swept") + _, err = b.DescribeCluster(context.Background(), cluster.ID) + require.Error(t, err, "terminated cluster should be swept") + }) } func TestEMR_Janitor_ActiveClusterNotSwept(t *testing.T) { @@ -144,36 +147,38 @@ func TestEMR_Janitor_SweepOnce(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := emr.NewInMemoryBackend(testAccountID, testRegion) - cluster, err := b.RunJobFlow( - context.Background(), - emr.RunJobFlowParams{Name: "sweep-once-test", ReleaseLabel: "emr-6.0.0"}, - ) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := emr.NewInMemoryBackend(testAccountID, testRegion) + cluster, err := b.RunJobFlow( + context.Background(), + emr.RunJobFlowParams{Name: "sweep-once-test", ReleaseLabel: "emr-6.0.0"}, + ) + require.NoError(t, err) - require.NoError(t, b.TerminateJobFlows(context.Background(), []string{cluster.ID})) + require.NoError(t, b.TerminateJobFlows(context.Background(), []string{cluster.ID})) - ttl := 24 * time.Hour - if tt.clusterOld { - ttl = time.Millisecond // effectively expired immediately - } + ttl := 24 * time.Hour + if tt.clusterOld { + ttl = time.Millisecond // effectively expired immediately + } - j := emr.NewJanitor(b, time.Minute, ttl) + j := emr.NewJanitor(b, time.Minute, ttl) - if tt.clusterOld { - // Give the TTL time to expire. - time.Sleep(5 * time.Millisecond) - } + if tt.clusterOld { + // Give the TTL time to expire. + time.Sleep(5 * time.Millisecond) + } - j.SweepOnce(t.Context()) + j.SweepOnce(t.Context()) - _, err = b.DescribeCluster(context.Background(), cluster.ID) + _, err = b.DescribeCluster(context.Background(), cluster.ID) - if tt.wantSwept { - require.Error(t, err, "cluster should have been swept") - } else { - require.NoError(t, err, "cluster should still exist") - } + if tt.wantSwept { + require.Error(t, err, "cluster should have been swept") + } else { + require.NoError(t, err, "cluster should still exist") + } + }) }) } } diff --git a/services/eventbridge/delivery_retry_test.go b/services/eventbridge/delivery_retry_test.go index 6b84fe7bd4..8ad8d4bd88 100644 --- a/services/eventbridge/delivery_retry_test.go +++ b/services/eventbridge/delivery_retry_test.go @@ -3,10 +3,9 @@ package eventbridge_test import ( "context" "errors" - "strings" "sync" "testing" - "time" + "testing/synctest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -35,40 +34,42 @@ func (f *auditFailingSQSSender) SendMessageToQueue(ctx context.Context, queueARN func TestDelivery_DLQCalledOnFailure(t *testing.T) { t.Parallel() - b := newBackend() - dlqSink := newMockSQSSender() - dlqARN := "arn:aws:sqs:us-east-1:123456789012:my-dlq" - targetARN := "arn:aws:sqs:us-east-1:123456789012:my-queue" + synctest.Test(t, func(t *testing.T) { + b := newBackend() - sender := &auditFailingSQSSender{delegate: dlqSink, failARN: targetARN} - b.SetDeliveryTargets(&eventbridge.DeliveryTargets{SQS: sender}) + dlqSink := newMockSQSSender() + dlqARN := "arn:aws:sqs:us-east-1:123456789012:my-dlq" + targetARN := "arn:aws:sqs:us-east-1:123456789012:my-queue" - _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: "rule", - EventPattern: `{"source":["dlq-test"]}`, - }) - require.NoError(t, err) + sender := &auditFailingSQSSender{delegate: dlqSink, failARN: targetARN} + b.SetDeliveryTargets(&eventbridge.DeliveryTargets{SQS: sender}) - _, err = b.PutTargets(context.Background(), "rule", "", []eventbridge.Target{ - { - ID: "t1", - Arn: targetARN, - DeadLetterConfig: &eventbridge.DeadLetterConfig{Arn: dlqARN}, - RetryPolicy: &eventbridge.RetryPolicy{ - MaximumRetryAttempts: 0, + _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: "rule", + EventPattern: `{"source":["dlq-test"]}`, + }) + require.NoError(t, err) + + _, err = b.PutTargets(context.Background(), "rule", "", []eventbridge.Target{ + { + ID: "t1", + Arn: targetARN, + DeadLetterConfig: &eventbridge.DeadLetterConfig{Arn: dlqARN}, + RetryPolicy: &eventbridge.RetryPolicy{ + MaximumRetryAttempts: 0, + }, }, - }, - }) - require.NoError(t, err) + }) + require.NoError(t, err) - b.PutEvents(context.Background(), []eventbridge.EventEntry{ - {Source: "dlq-test", DetailType: "T", Detail: `{}`}, - }) + b.PutEvents(context.Background(), []eventbridge.EventEntry{ + {Source: "dlq-test", DetailType: "T", Detail: `{}`}, + }) + synctest.Wait() - require.Eventually(t, func() bool { - return len(dlqSink.MessagesFor(dlqARN)) > 0 - }, 2*time.Second, 10*time.Millisecond, "DLQ should have received the failed event") + require.NotEmpty(t, dlqSink.MessagesFor(dlqARN), "DLQ should have received the failed event") + }) } // auditCountingSQSSender counts calls per queue and always fails delivery. @@ -94,153 +95,152 @@ func (c *auditCountingSQSSender) CountFor(queueARN string) int { func TestDelivery_RetryPolicyZeroAttemptsNeverRetries(t *testing.T) { t.Parallel() - b := newBackend() - counter := &auditCountingSQSSender{count: make(map[string]int)} - b.SetDeliveryTargets(&eventbridge.DeliveryTargets{SQS: counter}) + synctest.Test(t, func(t *testing.T) { + b := newBackend() - _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: "retry-rule", - EventPattern: `{"source":["retry-test"]}`, - }) - require.NoError(t, err) + counter := &auditCountingSQSSender{count: make(map[string]int)} + b.SetDeliveryTargets(&eventbridge.DeliveryTargets{SQS: counter}) - targetARN := "arn:aws:sqs:us-east-1:123456789012:target-q" - _, err = b.PutTargets(context.Background(), "retry-rule", "", []eventbridge.Target{ - { - ID: "t1", - Arn: targetARN, - RetryPolicy: &eventbridge.RetryPolicy{ - MaximumRetryAttempts: 0, - }, - }, - }) - require.NoError(t, err) + _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: "retry-rule", + EventPattern: `{"source":["retry-test"]}`, + }) + require.NoError(t, err) - b.PutEvents(context.Background(), []eventbridge.EventEntry{ - {Source: "retry-test", DetailType: "T", Detail: `{}`}, - }) + targetARN := "arn:aws:sqs:us-east-1:123456789012:target-q" + _, err = b.PutTargets(context.Background(), "retry-rule", "", []eventbridge.Target{ + { + ID: "t1", + Arn: targetARN, + RetryPolicy: &eventbridge.RetryPolicy{ + MaximumRetryAttempts: 0, + }, + }, + }) + require.NoError(t, err) - // Wait for delivery to complete (1 attempt only). - require.Eventually(t, func() bool { - return counter.CountFor(targetARN) >= 1 - }, 2*time.Second, 10*time.Millisecond) + b.PutEvents(context.Background(), []eventbridge.EventEntry{ + {Source: "retry-test", DetailType: "T", Detail: `{}`}, + }) + synctest.Wait() - time.Sleep(50 * time.Millisecond) - // With 0 retry attempts, should call exactly once. - assert.Equal(t, 1, counter.CountFor(targetARN)) + // With 0 retry attempts, should call exactly once. + assert.Equal(t, 1, counter.CountFor(targetARN)) + }) } func TestDelivery_DefaultRetryAttempts(t *testing.T) { t.Parallel() - b := newBackend() - counter := &auditCountingSQSSender{count: make(map[string]int)} - b.SetDeliveryTargets(&eventbridge.DeliveryTargets{SQS: counter}) + synctest.Test(t, func(t *testing.T) { + b := newBackend() - _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: "default-retry-rule", - EventPattern: `{"source":["default-retry"]}`, - }) - require.NoError(t, err) + counter := &auditCountingSQSSender{count: make(map[string]int)} + b.SetDeliveryTargets(&eventbridge.DeliveryTargets{SQS: counter}) - targetARN := "arn:aws:sqs:us-east-1:123456789012:target-q2" - _, err = b.PutTargets(context.Background(), "default-retry-rule", "", []eventbridge.Target{ - { - ID: "t1", - Arn: targetARN, - // No RetryPolicy set → use defaults (2 retries = 3 total attempts). - }, - }) - require.NoError(t, err) + _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: "default-retry-rule", + EventPattern: `{"source":["default-retry"]}`, + }) + require.NoError(t, err) - b.PutEvents(context.Background(), []eventbridge.EventEntry{ - {Source: "default-retry", DetailType: "T", Detail: `{}`}, - }) + targetARN := "arn:aws:sqs:us-east-1:123456789012:target-q2" + _, err = b.PutTargets(context.Background(), "default-retry-rule", "", []eventbridge.Target{ + { + ID: "t1", + Arn: targetARN, + // No RetryPolicy set → use defaults (2 retries = 3 total attempts). + }, + }) + require.NoError(t, err) - // Default 2 retries = 1 initial + 2 retries = 3 total attempts. - require.Eventually(t, func() bool { - return counter.CountFor(targetARN) >= 3 - }, 2*time.Second, 10*time.Millisecond) + b.PutEvents(context.Background(), []eventbridge.EventEntry{ + {Source: "default-retry", DetailType: "T", Detail: `{}`}, + }) + synctest.Wait() - time.Sleep(50 * time.Millisecond) - assert.Equal(t, 3, counter.CountFor(targetARN)) + // Default 2 retries = 1 initial + 2 retries = 3 total attempts. + assert.Equal(t, 3, counter.CountFor(targetARN)) + }) } func TestCustomBus_DeliverToSQS(t *testing.T) { t.Parallel() - sqsMock := newMockSQSSender() - b := setupDeliveryBackend(t, sqsMock, newMockLambdaInvoker()) - const ( - busName = "my-custom-bus" - queueARN = "arn:aws:sqs:us-east-1:123456789012:custom-bus-queue" - ruleName = "custom-rule" - ) - - _, err := b.CreateEventBus(context.Background(), eventbridge.CreateEventBusParams{Name: busName}) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + sqsMock := newMockSQSSender() + b := setupDeliveryBackend(t, sqsMock, newMockLambdaInvoker()) + const ( + busName = "my-custom-bus" + queueARN = "arn:aws:sqs:us-east-1:123456789012:custom-bus-queue" + ruleName = "custom-rule" + ) + + _, err := b.CreateEventBus(context.Background(), eventbridge.CreateEventBusParams{Name: busName}) + require.NoError(t, err) + + _, err = b.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: ruleName, + EventBusName: busName, + EventPattern: `{"source": ["custom.src"]}`, + State: "ENABLED", + }) + require.NoError(t, err) - _, err = b.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: ruleName, - EventBusName: busName, - EventPattern: `{"source": ["custom.src"]}`, - State: "ENABLED", - }) - require.NoError(t, err) + _, err = b.PutTargets(context.Background(), ruleName, busName, []eventbridge.Target{ + {ID: "t1", Arn: queueARN}, + }) + require.NoError(t, err) - _, err = b.PutTargets(context.Background(), ruleName, busName, []eventbridge.Target{ - {ID: "t1", Arn: queueARN}, - }) - require.NoError(t, err) + b.PutEvents(context.Background(), []eventbridge.EventEntry{ + {Source: "custom.src", DetailType: "Evt", Detail: `{"x":1}`, EventBusName: busName}, + }) + synctest.Wait() - b.PutEvents(context.Background(), []eventbridge.EventEntry{ - {Source: "custom.src", DetailType: "Evt", Detail: `{"x":1}`, EventBusName: busName}, + require.NotEmpty(t, sqsMock.MessagesFor(queueARN), "expected delivery to custom bus SQS target") }) - - require.Eventually(t, func() bool { - return len(sqsMock.MessagesFor(queueARN)) > 0 - }, 2*time.Second, 20*time.Millisecond, "expected delivery to custom bus SQS target") } func TestInputTransformer_TemplateApplied(t *testing.T) { t.Parallel() - sqsMock := newMockSQSSender() - b := setupDeliveryBackend(t, sqsMock, newMockLambdaInvoker()) - const ( - queueARN = "arn:aws:sqs:us-east-1:123456789012:transformer-queue" - ruleName = "transformer-rule" - ) - - _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: ruleName, - EventPattern: `{"source": ["svc"]}`, - State: "ENABLED", - }) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + sqsMock := newMockSQSSender() + b := setupDeliveryBackend(t, sqsMock, newMockLambdaInvoker()) + const ( + queueARN = "arn:aws:sqs:us-east-1:123456789012:transformer-queue" + ruleName = "transformer-rule" + ) + + _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: ruleName, + EventPattern: `{"source": ["svc"]}`, + State: "ENABLED", + }) + require.NoError(t, err) - _, err = b.PutTargets(context.Background(), ruleName, "default", []eventbridge.Target{ - { - ID: "t1", - Arn: queueARN, - InputTransformer: &eventbridge.InputTransformer{ - InputPathsMap: map[string]string{"env": "$.detail.env"}, - InputTemplate: `{"environment": ""}`, + _, err = b.PutTargets(context.Background(), ruleName, "default", []eventbridge.Target{ + { + ID: "t1", + Arn: queueARN, + InputTransformer: &eventbridge.InputTransformer{ + InputPathsMap: map[string]string{"env": "$.detail.env"}, + InputTemplate: `{"environment": ""}`, + }, }, - }, - }) - require.NoError(t, err) + }) + require.NoError(t, err) - b.PutEvents(context.Background(), []eventbridge.EventEntry{ - {Source: "svc", DetailType: "Evt", Detail: `{"env": "production"}`}, - }) + b.PutEvents(context.Background(), []eventbridge.EventEntry{ + {Source: "svc", DetailType: "Evt", Detail: `{"env": "production"}`}, + }) + synctest.Wait() - require.Eventually(t, func() bool { msgs := sqsMock.MessagesFor(queueARN) - - return len(msgs) > 0 && strings.Contains(msgs[0], "production") - }, 2*time.Second, 20*time.Millisecond) + require.NotEmpty(t, msgs) + assert.Contains(t, msgs[0], "production") + }) } var errSimulatedLambdaFailure = errors.New("simulated lambda invocation failure") @@ -285,55 +285,53 @@ func TestDLQ_RoutedOnDeliveryFailure(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - sqsMock := newMockSQSSender() - backend := eventbridge.NewInMemoryBackend() - backend.SetDeliveryTargets(&eventbridge.DeliveryTargets{ - SQS: sqsMock, - Lambda: &failingLambdaInvoker{}, + synctest.Test(t, func(t *testing.T) { + sqsMock := newMockSQSSender() + backend := eventbridge.NewInMemoryBackend() + backend.SetDeliveryTargets(&eventbridge.DeliveryTargets{ + SQS: sqsMock, + Lambda: &failingLambdaInvoker{}, + }) + + _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: "dlq-rule-" + tt.name, + EventPattern: `{"source": ["parity.test"]}`, + State: "ENABLED", + }) + require.NoError(t, err) + + target := eventbridge.Target{ + ID: "t1", + Arn: lambdaARN, + RetryPolicy: &eventbridge.RetryPolicy{ + MaximumRetryAttempts: 0, + }, + } + if tt.dlqARN != "" { + target.DeadLetterConfig = &eventbridge.DeadLetterConfig{Arn: tt.dlqARN} + } + + _, err = backend.PutTargets( + context.Background(), + "dlq-rule-"+tt.name, + "default", + []eventbridge.Target{target}, + ) + require.NoError(t, err) + + backend.PutEvents(context.Background(), []eventbridge.EventEntry{ + {Source: "parity.test", DetailType: "TestEvent", Detail: `{"key": "val"}`}, + }) + synctest.Wait() + + if tt.wantInDLQ { + msgs := sqsMock.MessagesFor(tt.dlqARN) + assert.NotEmpty(t, msgs, "DLQ should receive the failed event") + } else { + // No DLQ configured — nothing should be sent anywhere. + assert.Empty(t, sqsMock.MessagesFor(dlqARN)) + } }) - - _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: "dlq-rule-" + tt.name, - EventPattern: `{"source": ["parity.test"]}`, - State: "ENABLED", - }) - require.NoError(t, err) - - target := eventbridge.Target{ - ID: "t1", - Arn: lambdaARN, - RetryPolicy: &eventbridge.RetryPolicy{ - MaximumRetryAttempts: 0, - }, - } - if tt.dlqARN != "" { - target.DeadLetterConfig = &eventbridge.DeadLetterConfig{Arn: tt.dlqARN} - } - - _, err = backend.PutTargets( - context.Background(), - "dlq-rule-"+tt.name, - "default", - []eventbridge.Target{target}, - ) - require.NoError(t, err) - - backend.PutEvents(context.Background(), []eventbridge.EventEntry{ - {Source: "parity.test", DetailType: "TestEvent", Detail: `{"key": "val"}`}, - }) - - if tt.wantInDLQ { - require.Eventually(t, func() bool { - return len(sqsMock.MessagesFor(tt.dlqARN)) > 0 - }, 2*time.Second, 10*time.Millisecond, "DLQ should receive the failed event") - - msgs := sqsMock.MessagesFor(tt.dlqARN) - assert.NotEmpty(t, msgs) - } else { - time.Sleep(150 * time.Millisecond) - // No DLQ configured — nothing should be sent anywhere. - assert.Empty(t, sqsMock.MessagesFor(dlqARN)) - } }) } } diff --git a/services/eventbridge/delivery_test.go b/services/eventbridge/delivery_test.go index 299da6022b..88a12d4266 100644 --- a/services/eventbridge/delivery_test.go +++ b/services/eventbridge/delivery_test.go @@ -5,7 +5,7 @@ import ( "encoding/json" "sync" "testing" - "time" + "testing/synctest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -228,32 +228,35 @@ func TestDelivery_SQS(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - sqsMock := newMockSQSSender() - backend := setupDeliveryBackend(t, sqsMock, nil) + synctest.Test(t, func(t *testing.T) { + sqsMock := newMockSQSSender() + backend := setupDeliveryBackend(t, sqsMock, nil) - _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: tt.ruleName, - EventPattern: tt.eventPattern, - State: tt.ruleState, - }) - require.NoError(t, err) + _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: tt.ruleName, + EventPattern: tt.eventPattern, + State: tt.ruleState, + }) + require.NoError(t, err) - target := eventbridge.Target{ID: "t1", Arn: tt.queueARN} - if tt.targetInput != "" { - target.Input = tt.targetInput - } - - _, err = backend.PutTargets(context.Background(), tt.ruleName, "default", []eventbridge.Target{target}) - require.NoError(t, err) + target := eventbridge.Target{ID: "t1", Arn: tt.queueARN} + if tt.targetInput != "" { + target.Input = tt.targetInput + } - backend.PutEvents(context.Background(), tt.events) + _, err = backend.PutTargets(context.Background(), tt.ruleName, "default", []eventbridge.Target{target}) + require.NoError(t, err) - if tt.wantDelivered { - require.Eventually(t, func() bool { - return len(sqsMock.MessagesFor(tt.queueARN)) > 0 - }, 2*time.Second, 10*time.Millisecond) + backend.PutEvents(context.Background(), tt.events) + synctest.Wait() msgs := sqsMock.MessagesFor(tt.queueARN) + if !tt.wantDelivered { + assert.Empty(t, msgs) + + return + } + assert.Len(t, msgs, tt.wantLen) if tt.wantContains != "" { @@ -263,11 +266,7 @@ func TestDelivery_SQS(t *testing.T) { if tt.wantJSONEq != "" { assert.JSONEq(t, tt.wantJSONEq, msgs[0]) } - } else { - time.Sleep(100 * time.Millisecond) - msgs := sqsMock.MessagesFor(tt.queueARN) - assert.Empty(t, msgs) - } + }) }) } } @@ -299,30 +298,29 @@ func TestDelivery_Lambda(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - lamMock := newMockLambdaInvoker() - backend := setupDeliveryBackend(t, nil, lamMock) + synctest.Test(t, func(t *testing.T) { + lamMock := newMockLambdaInvoker() + backend := setupDeliveryBackend(t, nil, lamMock) - _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: tt.ruleName, - EventPattern: tt.eventPattern, - State: "ENABLED", - }) - require.NoError(t, err) + _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: tt.ruleName, + EventPattern: tt.eventPattern, + State: "ENABLED", + }) + require.NoError(t, err) - _, err = backend.PutTargets(context.Background(), tt.ruleName, "default", []eventbridge.Target{ - {ID: "t1", Arn: tt.lambdaARN}, - }) - require.NoError(t, err) - - backend.PutEvents(context.Background(), tt.events) + _, err = backend.PutTargets(context.Background(), tt.ruleName, "default", []eventbridge.Target{ + {ID: "t1", Arn: tt.lambdaARN}, + }) + require.NoError(t, err) - require.Eventually(t, func() bool { - return len(lamMock.Invocations()) >= tt.wantInvocations - }, 2*time.Second, 10*time.Millisecond) + backend.PutEvents(context.Background(), tt.events) + synctest.Wait() - invocations := lamMock.Invocations() - assert.Len(t, invocations, tt.wantInvocations) - assert.Equal(t, tt.lambdaARN, invocations[0].name) + invocations := lamMock.Invocations() + assert.Len(t, invocations, tt.wantInvocations) + assert.Equal(t, tt.lambdaARN, invocations[0].name) + }) }) } } @@ -372,34 +370,34 @@ func TestDelivery_FullEnvelope(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - sqsMock := newMockSQSSender() - backend := setupDeliveryBackend(t, sqsMock, nil) - queueARN := "arn:aws:sqs:us-east-1:000000000000:envelope-queue-" + tt.name + synctest.Test(t, func(t *testing.T) { + sqsMock := newMockSQSSender() + backend := setupDeliveryBackend(t, sqsMock, nil) + queueARN := "arn:aws:sqs:us-east-1:000000000000:envelope-queue-" + tt.name - _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: "envelope-rule-" + tt.name, - EventPattern: `{"source": ["test.service"]}`, - State: "ENABLED", - }) - require.NoError(t, err) - - _, err = backend.PutTargets(context.Background(), "envelope-rule-"+tt.name, "default", []eventbridge.Target{ - {ID: "t1", Arn: queueARN}, - }) - require.NoError(t, err) + _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: "envelope-rule-" + tt.name, + EventPattern: `{"source": ["test.service"]}`, + State: "ENABLED", + }) + require.NoError(t, err) - backend.PutEvents(context.Background(), tt.events) + _, err = backend.PutTargets( + context.Background(), "envelope-rule-"+tt.name, "default", + []eventbridge.Target{{ID: "t1", Arn: queueARN}}, + ) + require.NoError(t, err) - require.Eventually(t, func() bool { - return len(sqsMock.MessagesFor(queueARN)) > 0 - }, 2*time.Second, 10*time.Millisecond) + backend.PutEvents(context.Background(), tt.events) + synctest.Wait() - msgs := sqsMock.MessagesFor(queueARN) - require.Len(t, msgs, 1) + msgs := sqsMock.MessagesFor(queueARN) + require.Len(t, msgs, 1) - for _, field := range tt.wantFields { - assert.Contains(t, msgs[0], field, "expected field %q in payload", field) - } + for _, field := range tt.wantFields { + assert.Contains(t, msgs[0], field, "expected field %q in payload", field) + } + }) }) } } @@ -407,50 +405,48 @@ func TestDelivery_FullEnvelope(t *testing.T) { func TestDelivery_SharedEventIDAcrossTargets(t *testing.T) { t.Parallel() - sqsMock1 := newMockSQSSender() - sqsMock2 := newMockSQSSender() + synctest.Test(t, func(t *testing.T) { + sqsMock1 := newMockSQSSender() + sqsMock2 := newMockSQSSender() - backend := eventbridge.NewInMemoryBackend() - backend.SetDeliveryTargets(&eventbridge.DeliveryTargets{ - SQS: &multiQueueSender{senders: map[string]*mockSQSSender{ - "arn:aws:sqs:us-east-1:000000000000:queue-a": sqsMock1, - "arn:aws:sqs:us-east-1:000000000000:queue-b": sqsMock2, - }}, - }) - - _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: "shared-id-rule", - EventPattern: `{"source": ["shared.id.service"]}`, - State: "ENABLED", - }) - require.NoError(t, err) + backend := eventbridge.NewInMemoryBackend() + backend.SetDeliveryTargets(&eventbridge.DeliveryTargets{ + SQS: &multiQueueSender{senders: map[string]*mockSQSSender{ + "arn:aws:sqs:us-east-1:000000000000:queue-a": sqsMock1, + "arn:aws:sqs:us-east-1:000000000000:queue-b": sqsMock2, + }}, + }) - _, err = backend.PutTargets(context.Background(), "shared-id-rule", "default", []eventbridge.Target{ - {ID: "t1", Arn: "arn:aws:sqs:us-east-1:000000000000:queue-a"}, - {ID: "t2", Arn: "arn:aws:sqs:us-east-1:000000000000:queue-b"}, - }) - require.NoError(t, err) + _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: "shared-id-rule", + EventPattern: `{"source": ["shared.id.service"]}`, + State: "ENABLED", + }) + require.NoError(t, err) - backend.PutEvents(context.Background(), []eventbridge.EventEntry{ - {Source: "shared.id.service", DetailType: "Evt", Detail: `{}`}, - }) + _, err = backend.PutTargets(context.Background(), "shared-id-rule", "default", []eventbridge.Target{ + {ID: "t1", Arn: "arn:aws:sqs:us-east-1:000000000000:queue-a"}, + {ID: "t2", Arn: "arn:aws:sqs:us-east-1:000000000000:queue-b"}, + }) + require.NoError(t, err) - require.Eventually(t, func() bool { - return len(sqsMock1.MessagesFor("arn:aws:sqs:us-east-1:000000000000:queue-a")) > 0 && - len(sqsMock2.MessagesFor("arn:aws:sqs:us-east-1:000000000000:queue-b")) > 0 - }, 2*time.Second, 10*time.Millisecond) + backend.PutEvents(context.Background(), []eventbridge.EventEntry{ + {Source: "shared.id.service", DetailType: "Evt", Detail: `{}`}, + }) + synctest.Wait() - var id1, id2 struct { - ID string `json:"id"` - } + var id1, id2 struct { + ID string `json:"id"` + } - msg1 := sqsMock1.MessagesFor("arn:aws:sqs:us-east-1:000000000000:queue-a")[0] - msg2 := sqsMock2.MessagesFor("arn:aws:sqs:us-east-1:000000000000:queue-b")[0] + msg1 := sqsMock1.MessagesFor("arn:aws:sqs:us-east-1:000000000000:queue-a")[0] + msg2 := sqsMock2.MessagesFor("arn:aws:sqs:us-east-1:000000000000:queue-b")[0] - require.NoError(t, json.Unmarshal([]byte(msg1), &id1)) - require.NoError(t, json.Unmarshal([]byte(msg2), &id2)) - assert.NotEmpty(t, id1.ID) - assert.Equal(t, id1.ID, id2.ID, "all targets for the same rule+event must share the same event id") + require.NoError(t, json.Unmarshal([]byte(msg1), &id1)) + require.NoError(t, json.Unmarshal([]byte(msg2), &id2)) + assert.NotEmpty(t, id1.ID) + assert.Equal(t, id1.ID, id2.ID, "all targets for the same rule+event must share the same event id") + }) } // multiQueueSender routes SendMessageToQueue calls to the matching mockSQSSender by ARN. @@ -514,39 +510,38 @@ func TestDelivery_InputPath(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - sqsMock := newMockSQSSender() - backend := setupDeliveryBackend(t, sqsMock, nil) - queueARN := "arn:aws:sqs:us-east-1:000000000000:path-queue-" + tt.name - ruleName := "path-rule-" + tt.name + synctest.Test(t, func(t *testing.T) { + sqsMock := newMockSQSSender() + backend := setupDeliveryBackend(t, sqsMock, nil) + queueARN := "arn:aws:sqs:us-east-1:000000000000:path-queue-" + tt.name + ruleName := "path-rule-" + tt.name - _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: ruleName, - EventPattern: `{"source": ["path.service"]}`, - State: "ENABLED", - }) - require.NoError(t, err) + _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: ruleName, + EventPattern: `{"source": ["path.service"]}`, + State: "ENABLED", + }) + require.NoError(t, err) - _, err = backend.PutTargets(context.Background(), ruleName, "default", []eventbridge.Target{ - {ID: "t1", Arn: queueARN, InputPath: tt.inputPath}, - }) - require.NoError(t, err) - - backend.PutEvents(context.Background(), tt.events) + _, err = backend.PutTargets(context.Background(), ruleName, "default", []eventbridge.Target{ + {ID: "t1", Arn: queueARN, InputPath: tt.inputPath}, + }) + require.NoError(t, err) - require.Eventually(t, func() bool { - return len(sqsMock.MessagesFor(queueARN)) > 0 - }, 2*time.Second, 10*time.Millisecond) + backend.PutEvents(context.Background(), tt.events) + synctest.Wait() - msgs := sqsMock.MessagesFor(queueARN) - require.Len(t, msgs, 1) + msgs := sqsMock.MessagesFor(queueARN) + require.Len(t, msgs, 1) - if tt.wantJSONEq != "" { - assert.JSONEq(t, tt.wantJSONEq, msgs[0]) - } + if tt.wantJSONEq != "" { + assert.JSONEq(t, tt.wantJSONEq, msgs[0]) + } - if tt.wantContains != "" { - assert.Contains(t, msgs[0], tt.wantContains) - } + if tt.wantContains != "" { + assert.Contains(t, msgs[0], tt.wantContains) + } + }) }) } } @@ -620,39 +615,38 @@ func TestDelivery_InputTransformer(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - sqsMock := newMockSQSSender() - backend := setupDeliveryBackend(t, sqsMock, nil) - queueARN := "arn:aws:sqs:us-east-1:000000000000:transform-queue-" + tt.name - ruleName := "transform-rule-" + tt.name + synctest.Test(t, func(t *testing.T) { + sqsMock := newMockSQSSender() + backend := setupDeliveryBackend(t, sqsMock, nil) + queueARN := "arn:aws:sqs:us-east-1:000000000000:transform-queue-" + tt.name + ruleName := "transform-rule-" + tt.name - _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: ruleName, - EventPattern: `{"source": ["transform.service", "order.service", "text.service"]}`, - State: "ENABLED", - }) - require.NoError(t, err) + _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: ruleName, + EventPattern: `{"source": ["transform.service", "order.service", "text.service"]}`, + State: "ENABLED", + }) + require.NoError(t, err) - _, err = backend.PutTargets(context.Background(), ruleName, "default", []eventbridge.Target{ - {ID: "t1", Arn: queueARN, InputTransformer: tt.inputTransformer}, - }) - require.NoError(t, err) + _, err = backend.PutTargets(context.Background(), ruleName, "default", []eventbridge.Target{ + {ID: "t1", Arn: queueARN, InputTransformer: tt.inputTransformer}, + }) + require.NoError(t, err) - backend.PutEvents(context.Background(), tt.events) - - require.Eventually(t, func() bool { - return len(sqsMock.MessagesFor(queueARN)) > 0 - }, 2*time.Second, 10*time.Millisecond) + backend.PutEvents(context.Background(), tt.events) + synctest.Wait() - msgs := sqsMock.MessagesFor(queueARN) - require.Len(t, msgs, 1) + msgs := sqsMock.MessagesFor(queueARN) + require.Len(t, msgs, 1) - if tt.wantJSONEq != "" { - assert.JSONEq(t, tt.wantJSONEq, msgs[0]) - } + if tt.wantJSONEq != "" { + assert.JSONEq(t, tt.wantJSONEq, msgs[0]) + } - if tt.wantContains != "" { - assert.Contains(t, msgs[0], tt.wantContains) - } + if tt.wantContains != "" { + assert.Contains(t, msgs[0], tt.wantContains) + } + }) }) } } @@ -698,44 +692,43 @@ func TestDelivery_SNS(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - snsMock := newMockSNSPublisher() - backend := setupDeliveryBackendFull(t, nil, nil, snsMock) + synctest.Test(t, func(t *testing.T) { + snsMock := newMockSNSPublisher() + backend := setupDeliveryBackendFull(t, nil, nil, snsMock) - state := "ENABLED" - if !tt.wantDelivered { - state = "DISABLED" - } - - _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: tt.ruleName, - EventPattern: tt.eventPattern, - State: state, - }) - require.NoError(t, err) + state := "ENABLED" + if !tt.wantDelivered { + state = "DISABLED" + } - _, err = backend.PutTargets(context.Background(), tt.ruleName, "default", []eventbridge.Target{ - {ID: "t1", Arn: tt.topicARN}, - }) - require.NoError(t, err) + _, err := backend.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: tt.ruleName, + EventPattern: tt.eventPattern, + State: state, + }) + require.NoError(t, err) - backend.PutEvents(context.Background(), tt.events) + _, err = backend.PutTargets(context.Background(), tt.ruleName, "default", []eventbridge.Target{ + {ID: "t1", Arn: tt.topicARN}, + }) + require.NoError(t, err) - if tt.wantDelivered { - require.Eventually(t, func() bool { - return len(snsMock.MessagesFor(tt.topicARN)) > 0 - }, 2*time.Second, 10*time.Millisecond) + backend.PutEvents(context.Background(), tt.events) + synctest.Wait() msgs := snsMock.MessagesFor(tt.topicARN) + if !tt.wantDelivered { + assert.Empty(t, msgs, "expected no messages for disabled rule") + + return + } + assert.Len(t, msgs, tt.wantLen) if tt.wantContains != "" { assert.Contains(t, msgs[0], tt.wantContains) } - } else { - require.Never(t, func() bool { - return len(snsMock.MessagesFor(tt.topicARN)) > 0 - }, 100*time.Millisecond, 10*time.Millisecond, "expected no messages for disabled rule") - } + }) }) } } diff --git a/services/eventbridge/scheduler_test.go b/services/eventbridge/scheduler_test.go index 12da72e41b..597ba85f61 100644 --- a/services/eventbridge/scheduler_test.go +++ b/services/eventbridge/scheduler_test.go @@ -3,6 +3,7 @@ package eventbridge_test import ( "context" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -32,15 +33,20 @@ func TestScheduler(t *testing.T) { runAsync: true, check: func(t *testing.T, backend *eventbridge.InMemoryBackend) { t.Helper() - require.Eventually(t, func() bool { - for _, entry := range backend.GetEventLog(context.Background()) { - if entry.Source == "aws.events" { - return true - } - } + // Advance past the 1-second rate period so the scheduler's + // 50ms ticker fires it at least once, then let delivery settle. + time.Sleep(1100 * time.Millisecond) + synctest.Wait() + + var fired bool + for _, entry := range backend.GetEventLog(context.Background()) { + if entry.Source == "aws.events" { + fired = true - return false - }, 5*time.Second, 100*time.Millisecond, "expected at least one scheduled event to be fired") + break + } + } + assert.True(t, fired, "expected at least one scheduled event to be fired") }, }, { @@ -56,6 +62,7 @@ func TestScheduler(t *testing.T) { t.Helper() // Wait for the context to expire and then a little more to confirm no events fired. time.Sleep(300 * time.Millisecond) + synctest.Wait() for _, e := range backend.GetEventLog(context.Background()) { assert.NotEqual(t, "aws.events", e.Source, "disabled rule should not fire events") } @@ -87,24 +94,26 @@ func TestScheduler(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - backend := eventbridge.NewInMemoryBackend() + synctest.Test(t, func(t *testing.T) { + backend := eventbridge.NewInMemoryBackend() - _, err := backend.PutRule(context.Background(), tt.rule) - require.NoError(t, err) + _, err := backend.PutRule(context.Background(), tt.rule) + require.NoError(t, err) - scheduler := eventbridge.NewScheduler(backend, 50*time.Millisecond) - ctx, cancel := context.WithTimeout(t.Context(), tt.ctxTimeout) - defer cancel() + scheduler := eventbridge.NewScheduler(backend, 50*time.Millisecond) + ctx, cancel := context.WithTimeout(t.Context(), tt.ctxTimeout) + defer cancel() - if tt.runAsync { - go scheduler.Run(ctx) - } else { - scheduler.Run(ctx) - } + if tt.runAsync { + go scheduler.Run(ctx) + } else { + scheduler.Run(ctx) + } - if tt.check != nil { - tt.check(t, backend) - } + if tt.check != nil { + tt.check(t, backend) + } + }) }) } } diff --git a/services/eventbridge/stepfunctions_target_test.go b/services/eventbridge/stepfunctions_target_test.go index a50f894c8c..78b41851d6 100644 --- a/services/eventbridge/stepfunctions_target_test.go +++ b/services/eventbridge/stepfunctions_target_test.go @@ -5,7 +5,7 @@ import ( "errors" "sync" "testing" - "time" + "testing/synctest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -64,34 +64,35 @@ func (s *auditSFNExecutor) LastExecution() sfnExecution { func TestDelivery_StepFunctions_DeliversEvent(t *testing.T) { t.Parallel() - b := newBackend() - sfn := &auditSFNExecutor{} - smARN := "arn:aws:states:us-east-1:123456789012:stateMachine:my-sm" + synctest.Test(t, func(t *testing.T) { + b := newBackend() + sfn := &auditSFNExecutor{} + smARN := "arn:aws:states:us-east-1:123456789012:stateMachine:my-sm" - b.SetDeliveryTargets(&eventbridge.DeliveryTargets{StepFunctions: sfn}) + b.SetDeliveryTargets(&eventbridge.DeliveryTargets{StepFunctions: sfn}) - _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: "sfn-rule", - EventPattern: `{"source":["sfn-test"]}`, - }) - require.NoError(t, err) + _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: "sfn-rule", + EventPattern: `{"source":["sfn-test"]}`, + }) + require.NoError(t, err) - _, err = b.PutTargets(context.Background(), "sfn-rule", "", []eventbridge.Target{ - {ID: "t1", Arn: smARN}, - }) - require.NoError(t, err) + _, err = b.PutTargets(context.Background(), "sfn-rule", "", []eventbridge.Target{ + {ID: "t1", Arn: smARN}, + }) + require.NoError(t, err) - b.PutEvents(context.Background(), []eventbridge.EventEntry{ - {Source: "sfn-test", DetailType: "Order", Detail: `{"id":42}`}, - }) + b.PutEvents(context.Background(), []eventbridge.EventEntry{ + {Source: "sfn-test", DetailType: "Order", Detail: `{"id":42}`}, + }) + synctest.Wait() - require.Eventually(t, func() bool { - return sfn.Count() > 0 - }, 2*time.Second, 10*time.Millisecond, "Step Functions should have been invoked") + require.Positive(t, sfn.Count(), "Step Functions should have been invoked") - exec := sfn.LastExecution() - assert.Equal(t, smARN, exec.StateMachineARN) - assert.NotEmpty(t, exec.Input) + exec := sfn.LastExecution() + assert.Equal(t, smARN, exec.StateMachineARN) + assert.NotEmpty(t, exec.Input) + }) } func TestDelivery_StepFunctions_NilHandlerSkipsGracefully(t *testing.T) { @@ -122,44 +123,45 @@ func TestDelivery_StepFunctions_NilHandlerSkipsGracefully(t *testing.T) { func TestDelivery_StepFunctions_FailureSendsToDLQ(t *testing.T) { t.Parallel() - b := newBackend() + synctest.Test(t, func(t *testing.T) { + b := newBackend() - dlqSink := newMockSQSSender() - dlqARN := "arn:aws:sqs:us-east-1:123456789012:sfn-dlq" - smARN := "arn:aws:states:us-east-1:123456789012:stateMachine:failing-sm" + dlqSink := newMockSQSSender() + dlqARN := "arn:aws:sqs:us-east-1:123456789012:sfn-dlq" + smARN := "arn:aws:states:us-east-1:123456789012:stateMachine:failing-sm" - sfnSink := &auditSFNExecutor{returnErr: errExecutionLimitReached} + sfnSink := &auditSFNExecutor{returnErr: errExecutionLimitReached} - b.SetDeliveryTargets(&eventbridge.DeliveryTargets{ - StepFunctions: sfnSink, - SQS: dlqSink, - }) - - _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: "sfn-fail-rule", - EventPattern: `{"source":["sfn-fail"]}`, - }) - require.NoError(t, err) + b.SetDeliveryTargets(&eventbridge.DeliveryTargets{ + StepFunctions: sfnSink, + SQS: dlqSink, + }) - _, err = b.PutTargets(context.Background(), "sfn-fail-rule", "", []eventbridge.Target{ - { - ID: "t1", - Arn: smARN, - DeadLetterConfig: &eventbridge.DeadLetterConfig{ - Arn: dlqARN, + _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: "sfn-fail-rule", + EventPattern: `{"source":["sfn-fail"]}`, + }) + require.NoError(t, err) + + _, err = b.PutTargets(context.Background(), "sfn-fail-rule", "", []eventbridge.Target{ + { + ID: "t1", + Arn: smARN, + DeadLetterConfig: &eventbridge.DeadLetterConfig{ + Arn: dlqARN, + }, + RetryPolicy: &eventbridge.RetryPolicy{MaximumRetryAttempts: 0}, }, - RetryPolicy: &eventbridge.RetryPolicy{MaximumRetryAttempts: 0}, - }, - }) - require.NoError(t, err) + }) + require.NoError(t, err) - b.PutEvents(context.Background(), []eventbridge.EventEntry{ - {Source: "sfn-fail", DetailType: "T", Detail: `{}`}, - }) + b.PutEvents(context.Background(), []eventbridge.EventEntry{ + {Source: "sfn-fail", DetailType: "T", Detail: `{}`}, + }) + synctest.Wait() - require.Eventually(t, func() bool { - return len(dlqSink.MessagesFor(dlqARN)) > 0 - }, 2*time.Second, 10*time.Millisecond, "DLQ should receive failed SFN delivery") + assert.NotEmpty(t, dlqSink.MessagesFor(dlqARN), "DLQ should receive failed SFN delivery") + }) } func TestDelivery_IsStateMachineARN(t *testing.T) { @@ -180,32 +182,33 @@ func TestDelivery_IsStateMachineARN(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := newBackend() - sfn := &auditSFNExecutor{} - b.SetDeliveryTargets(&eventbridge.DeliveryTargets{StepFunctions: sfn}) - - _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ - Name: "arn-test-" + tt.name, - EventPattern: `{"source":["arn-probe-` + tt.name + `"]}`, - }) - require.NoError(t, err) - - _, err = b.PutTargets(context.Background(), "arn-test-"+tt.name, "", []eventbridge.Target{ - {ID: "t1", Arn: tt.arn}, + synctest.Test(t, func(t *testing.T) { + b := newBackend() + sfn := &auditSFNExecutor{} + b.SetDeliveryTargets(&eventbridge.DeliveryTargets{StepFunctions: sfn}) + + _, err := b.PutRule(context.Background(), eventbridge.PutRuleInput{ + Name: "arn-test-" + tt.name, + EventPattern: `{"source":["arn-probe-` + tt.name + `"]}`, + }) + require.NoError(t, err) + + _, err = b.PutTargets(context.Background(), "arn-test-"+tt.name, "", []eventbridge.Target{ + {ID: "t1", Arn: tt.arn}, + }) + require.NoError(t, err) + + b.PutEvents(context.Background(), []eventbridge.EventEntry{ + {Source: "arn-probe-" + tt.name, DetailType: "T", Detail: `{}`}, + }) + synctest.Wait() + + if tt.want { + assert.Positive(t, sfn.Count(), "expected SFN invocation for ARN %s", tt.arn) + } else { + assert.Equal(t, 0, sfn.Count(), "expected no SFN invocation for non-SM ARN %s", tt.arn) + } }) - require.NoError(t, err) - - b.PutEvents(context.Background(), []eventbridge.EventEntry{ - {Source: "arn-probe-" + tt.name, DetailType: "T", Detail: `{}`}, - }) - - time.Sleep(50 * time.Millisecond) - - if tt.want { - assert.Positive(t, sfn.Count(), "expected SFN invocation for ARN %s", tt.arn) - } else { - assert.Equal(t, 0, sfn.Count(), "expected no SFN invocation for non-SM ARN %s", tt.arn) - } }) } } diff --git a/services/firehose/kinesis_source_test.go b/services/firehose/kinesis_source_test.go index 76f05f1c90..5c964ba77d 100644 --- a/services/firehose/kinesis_source_test.go +++ b/services/firehose/kinesis_source_test.go @@ -7,7 +7,7 @@ import ( "log/slog" "sync" "testing" - "time" + "testing/synctest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -112,89 +112,93 @@ func totalRecords(t *testing.T, b *firehose.InMemoryBackend, streamName string) func TestFirehose_KinesisSource_PollerDeliversSingleRecord(t *testing.T) { t.Parallel() - b := newFirehoseBackend(t) - kinesis := &mockKinesisReader{} - kinesis.addRecords([]byte("record-1")) - - b.SetKinesisBackend(kinesis) - - streamARN := "arn:aws:kinesis:us-east-1:123456789012:stream/my-stream" - _, err := b.CreateDeliveryStream(context.TODO(), firehose.CreateDeliveryStreamInput{ - Name: "poll-stream", - DeliveryStreamType: "KinesisStreamAsSource", - Source: &firehose.SourceDescription{ - KinesisStreamSourceDescription: &firehose.KinesisStreamSourceDescription{ - KinesisStreamARN: streamARN, + synctest.Test(t, func(t *testing.T) { + b := newFirehoseBackend(t) + kinesis := &mockKinesisReader{} + kinesis.addRecords([]byte("record-1")) + + b.SetKinesisBackend(kinesis) + + streamARN := "arn:aws:kinesis:us-east-1:123456789012:stream/my-stream" + _, err := b.CreateDeliveryStream(context.TODO(), firehose.CreateDeliveryStreamInput{ + Name: "poll-stream", + DeliveryStreamType: "KinesisStreamAsSource", + Source: &firehose.SourceDescription{ + KinesisStreamSourceDescription: &firehose.KinesisStreamSourceDescription{ + KinesisStreamARN: streamARN, + }, }, - }, - }) - require.NoError(t, err) + }) + require.NoError(t, err) - // Wait for the poller to deliver the record. - require.Eventually(t, func() bool { - return totalRecords(t, b, "poll-stream") >= 1 - }, 3*time.Second, 50*time.Millisecond, "poller should deliver records from Kinesis to Firehose") + // The poller delivers the record on its first pass, then blocks on + // its next-poll timer once the shard is drained. + synctest.Wait() - assert.Equal(t, int64(1), totalRecords(t, b, "poll-stream")) + assert.Equal(t, int64(1), totalRecords(t, b, "poll-stream")) + }) } func TestFirehose_KinesisSource_PollerDeliversManyRecords(t *testing.T) { t.Parallel() - b := newFirehoseBackend(t) - kinesis := &mockKinesisReader{} - kinesis.addRecords([]byte("a"), []byte("b"), []byte("c")) - - b.SetKinesisBackend(kinesis) - - streamARN := "arn:aws:kinesis:us-east-1:123456789012:stream/multi-stream" - _, err := b.CreateDeliveryStream(context.TODO(), firehose.CreateDeliveryStreamInput{ - Name: "multi-poll-stream", - DeliveryStreamType: "KinesisStreamAsSource", - Source: &firehose.SourceDescription{ - KinesisStreamSourceDescription: &firehose.KinesisStreamSourceDescription{ - KinesisStreamARN: streamARN, + synctest.Test(t, func(t *testing.T) { + b := newFirehoseBackend(t) + kinesis := &mockKinesisReader{} + kinesis.addRecords([]byte("a"), []byte("b"), []byte("c")) + + b.SetKinesisBackend(kinesis) + + streamARN := "arn:aws:kinesis:us-east-1:123456789012:stream/multi-stream" + _, err := b.CreateDeliveryStream(context.TODO(), firehose.CreateDeliveryStreamInput{ + Name: "multi-poll-stream", + DeliveryStreamType: "KinesisStreamAsSource", + Source: &firehose.SourceDescription{ + KinesisStreamSourceDescription: &firehose.KinesisStreamSourceDescription{ + KinesisStreamARN: streamARN, + }, }, - }, - }) - require.NoError(t, err) + }) + require.NoError(t, err) - require.Eventually(t, func() bool { - return totalRecords(t, b, "multi-poll-stream") >= 3 - }, 3*time.Second, 50*time.Millisecond, "poller should deliver all 3 records") + synctest.Wait() - assert.Equal(t, int64(3), totalRecords(t, b, "multi-poll-stream")) + assert.Equal(t, int64(3), totalRecords(t, b, "multi-poll-stream")) + }) } func TestFirehose_KinesisSource_DeleteStopsPoller(t *testing.T) { t.Parallel() - b := newFirehoseBackend(t) - kinesis := &mockKinesisReader{} // no records, infinite polling + synctest.Test(t, func(t *testing.T) { + b := newFirehoseBackend(t) + kinesis := &mockKinesisReader{} // no records, infinite polling - b.SetKinesisBackend(kinesis) + b.SetKinesisBackend(kinesis) - streamARN := "arn:aws:kinesis:us-east-1:123456789012:stream/stop-stream" - _, err := b.CreateDeliveryStream(context.TODO(), firehose.CreateDeliveryStreamInput{ - Name: "stop-poll-stream", - DeliveryStreamType: "KinesisStreamAsSource", - Source: &firehose.SourceDescription{ - KinesisStreamSourceDescription: &firehose.KinesisStreamSourceDescription{ - KinesisStreamARN: streamARN, + streamARN := "arn:aws:kinesis:us-east-1:123456789012:stream/stop-stream" + _, err := b.CreateDeliveryStream(context.TODO(), firehose.CreateDeliveryStreamInput{ + Name: "stop-poll-stream", + DeliveryStreamType: "KinesisStreamAsSource", + Source: &firehose.SourceDescription{ + KinesisStreamSourceDescription: &firehose.KinesisStreamSourceDescription{ + KinesisStreamARN: streamARN, + }, }, - }, - }) - require.NoError(t, err) + }) + require.NoError(t, err) - // Wait a bit then delete. - time.Sleep(50 * time.Millisecond) + // Let the poller reach its idle wait before deleting. + synctest.Wait() - err = b.DeleteDeliveryStream(context.TODO(), "stop-poll-stream") - require.NoError(t, err) + err = b.DeleteDeliveryStream(context.TODO(), "stop-poll-stream") + require.NoError(t, err) + synctest.Wait() - // Verify stream is gone and no panic. - _, err = b.DescribeDeliveryStream(context.TODO(), "stop-poll-stream") - assert.Error(t, err) + // Verify stream is gone and no panic. + _, err = b.DescribeDeliveryStream(context.TODO(), "stop-poll-stream") + assert.Error(t, err) + }) } func TestFirehose_KinesisSource_NoBackendDoesNotStart(t *testing.T) { @@ -296,25 +300,28 @@ func TestFirehose_KinesisSource_DirectPutUnaffected(t *testing.T) { func TestFirehose_KinesisSource_ListShardsError_NoBlock(t *testing.T) { t.Parallel() - b := newFirehoseBackend(t) - kinesis := &mockKinesisReader{listErr: errAccessDenied} - b.SetKinesisBackend(kinesis) - - streamARN := "arn:aws:kinesis:us-east-1:123456789012:stream/error-stream" - _, err := b.CreateDeliveryStream(context.TODO(), firehose.CreateDeliveryStreamInput{ - Name: "error-stream", - DeliveryStreamType: "KinesisStreamAsSource", - Source: &firehose.SourceDescription{ - KinesisStreamSourceDescription: &firehose.KinesisStreamSourceDescription{ - KinesisStreamARN: streamARN, + synctest.Test(t, func(t *testing.T) { + b := newFirehoseBackend(t) + kinesis := &mockKinesisReader{listErr: errAccessDenied} + b.SetKinesisBackend(kinesis) + + streamARN := "arn:aws:kinesis:us-east-1:123456789012:stream/error-stream" + _, err := b.CreateDeliveryStream(context.TODO(), firehose.CreateDeliveryStreamInput{ + Name: "error-stream", + DeliveryStreamType: "KinesisStreamAsSource", + Source: &firehose.SourceDescription{ + KinesisStreamSourceDescription: &firehose.KinesisStreamSourceDescription{ + KinesisStreamARN: streamARN, + }, }, - }, - }) - require.NoError(t, err, "CreateDeliveryStream must succeed even when Kinesis polling will fail") + }) + require.NoError(t, err, "CreateDeliveryStream must succeed even when Kinesis polling will fail") - // Give the goroutine time to attempt and fail. - time.Sleep(100 * time.Millisecond) + // ListShards fails synchronously, so the poller goroutine exits + // immediately without ever polling for records. + synctest.Wait() - // No panic, no records. - assert.Equal(t, int64(0), totalRecords(t, b, "error-stream")) + // No panic, no records. + assert.Equal(t, int64(0), totalRecords(t, b, "error-stream")) + }) } diff --git a/services/fis/actions_test.go b/services/fis/actions_test.go index 099cdec8cc..9c230c8c77 100644 --- a/services/fis/actions_test.go +++ b/services/fis/actions_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -569,220 +570,226 @@ func TestBackend_ListActions_WithProviders(t *testing.T) { func TestFISHandler_StopExperiment_AlreadyStopped(t *testing.T) { t.Parallel() - h := newTestHandler(t) - templateID := createTestTemplate(t, h) - - // Start experiment. - rec := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ - "experimentTemplateId": templateID, - }) - require.Equal(t, http.StatusCreated, rec.Code) - - var expResp struct { - Experiment struct { - ID string `json:"id"` - } `json:"experiment"` - } - - mustJSON(t, rec, &expResp) - expID := expResp.Experiment.ID + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + templateID := createTestTemplate(t, h) - // Stop experiment. - rec2 := doRequest(t, h, http.MethodDelete, "/experiments/"+expID, nil) - assert.Equal(t, http.StatusOK, rec2.Code) + // Start experiment. + rec := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ + "experimentTemplateId": templateID, + }) + require.Equal(t, http.StatusCreated, rec.Code) - // Wait for it to actually stop. - require.Eventually(t, func() bool { - rec3 := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) - var resp struct { + var expResp struct { Experiment struct { - Status struct { - Status string `json:"status"` - } `json:"status"` + ID string `json:"id"` } `json:"experiment"` } - if err := json.Unmarshal(rec3.Body.Bytes(), &resp); err != nil { - return false - } + mustJSON(t, rec, &expResp) + expID := expResp.Experiment.ID - s := resp.Experiment.Status.Status + // Stop experiment. + rec2 := doRequest(t, h, http.MethodDelete, "/experiments/"+expID, nil) + assert.Equal(t, http.StatusOK, rec2.Code) // Stopping this fast after StartExperiment races the background // lifecycle goroutine: it may still be in "pending"/"initiating" when // the stop signal arrives, in which case real AWS FIS reports // "cancelled" rather than "stopped" (see runExperiment); it may also // have already reached "completed" if the template has no timed - // actions. All three are valid terminal outcomes of this race. - return s == "stopped" || s == "completed" || s == "cancelled" - }, 5*time.Second, 50*time.Millisecond) - - // Attempt to stop the now-terminal experiment — should fail with 400 - // ValidationException. StopExperiment's generated deserializer in - // aws-sdk-go-v2/service/fis only recognizes ResourceNotFoundException and - // ValidationException — it has no ConflictException case. - rec4 := doRequest(t, h, http.MethodDelete, "/experiments/"+expID, nil) - assert.Equal(t, http.StatusBadRequest, rec4.Code) + // actions. All three are valid terminal outcomes of this race, so the + // deliberate race itself is preserved — only the wall-clock cost of + // waiting it out is not, since Eventually's ticks run on the bubble's + // fake clock. + require.Eventually(t, func() bool { + rec3 := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) + var resp struct { + Experiment struct { + Status struct { + Status string `json:"status"` + } `json:"status"` + } `json:"experiment"` + } + + if err := json.Unmarshal(rec3.Body.Bytes(), &resp); err != nil { + return false + } + + s := resp.Experiment.Status.Status + + return s == "stopped" || s == "completed" || s == "cancelled" + }, 5*time.Second, 50*time.Millisecond) + + // Attempt to stop the now-terminal experiment — should fail with 400 + // ValidationException. StopExperiment's generated deserializer in + // aws-sdk-go-v2/service/fis only recognizes ResourceNotFoundException and + // ValidationException — it has no ConflictException case. + rec4 := doRequest(t, h, http.MethodDelete, "/experiments/"+expID, nil) + assert.Equal(t, http.StatusBadRequest, rec4.Code) + }) } func TestFISHandler_ExperimentFails_WhenActionProviderFails(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - // Register a mock provider that always fails. - mock := &fis.MockFISActionProvider{ - ExecErr: fis.ErrMockAction, - Definitions: []service.FISActionDefinition{ - {ActionID: "aws:test:fail-action", TargetType: "aws:ec2:instance"}, - }, - } - h.SetActionProviders([]service.FISActionProvider{mock}) - - body := map[string]any{ - "roleArn": "arn:aws:iam::000000000000:role/FISRole", - "stopConditions": []map[string]any{{"source": "none"}}, - "targets": map[string]any{ - "MyInstances": map[string]any{ - "resourceType": "aws:ec2:instance", - "selectionMode": "ALL", - "resourceArns": []string{"arn:aws:ec2:us-east-1:000:instance/i-abc123"}, + // Register a mock provider that always fails. + mock := &fis.MockFISActionProvider{ + ExecErr: fis.ErrMockAction, + Definitions: []service.FISActionDefinition{ + {ActionID: "aws:test:fail-action", TargetType: "aws:ec2:instance"}, }, - }, - "actions": map[string]any{ - "fail": map[string]any{ - "actionId": "aws:test:fail-action", - "targets": map[string]string{"Instances": "MyInstances"}, + } + h.SetActionProviders([]service.FISActionProvider{mock}) + + body := map[string]any{ + "roleArn": "arn:aws:iam::000000000000:role/FISRole", + "stopConditions": []map[string]any{{"source": "none"}}, + "targets": map[string]any{ + "MyInstances": map[string]any{ + "resourceType": "aws:ec2:instance", + "selectionMode": "ALL", + "resourceArns": []string{"arn:aws:ec2:us-east-1:000:instance/i-abc123"}, + }, }, - }, - } + "actions": map[string]any{ + "fail": map[string]any{ + "actionId": "aws:test:fail-action", + "targets": map[string]string{"Instances": "MyInstances"}, + }, + }, + } - rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) - require.Equal(t, http.StatusCreated, rec.Code) + rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) + require.Equal(t, http.StatusCreated, rec.Code) - var tplResp struct { - ExperimentTemplate struct { - ID string `json:"id"` - } `json:"experimentTemplate"` - } + var tplResp struct { + ExperimentTemplate struct { + ID string `json:"id"` + } `json:"experimentTemplate"` + } - mustJSON(t, rec, &tplResp) - templateID := tplResp.ExperimentTemplate.ID + mustJSON(t, rec, &tplResp) + templateID := tplResp.ExperimentTemplate.ID - rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ - "experimentTemplateId": templateID, - }) - require.Equal(t, http.StatusCreated, rec2.Code) + rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ + "experimentTemplateId": templateID, + }) + require.Equal(t, http.StatusCreated, rec2.Code) - var expResp struct { - Experiment struct { - ID string `json:"id"` - } `json:"experiment"` - } + var expResp struct { + Experiment struct { + ID string `json:"id"` + } `json:"experiment"` + } - mustJSON(t, rec2, &expResp) - expID := expResp.Experiment.ID - - var finalResp struct { - Experiment struct { - Status struct { - Error *struct { - Code string `json:"code"` - Location string `json:"location"` - AccountID string `json:"accountId"` - } `json:"error"` - Status string `json:"status"` - Reason string `json:"reason"` - } `json:"status"` - } `json:"experiment"` - } + mustJSON(t, rec2, &expResp) + expID := expResp.Experiment.ID - require.Eventually(t, func() bool { - rec3 := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) - if rec3.Code != http.StatusOK { - return false - } + // The mock action fails synchronously once running; only the + // pending -> initiating delay gates it. + time.Sleep(fis.LifecycleDelayForTest + time.Millisecond) + synctest.Wait() - if err := json.Unmarshal(rec3.Body.Bytes(), &finalResp); err != nil { - return false + var finalResp struct { + Experiment struct { + Status struct { + Error *struct { + Code string `json:"code"` + Location string `json:"location"` + AccountID string `json:"accountId"` + } `json:"error"` + Status string `json:"status"` + Reason string `json:"reason"` + } `json:"status"` + } `json:"experiment"` } - return finalResp.Experiment.Status.Status == "failed" - }, 5*time.Second, 50*time.Millisecond) - - // Regression test: cleanupActions used to unconditionally overwrite - // exp.Status right after markExperimentFailed set it, clobbering the - // structured ExperimentStatusError before any client could ever observe - // it. Verify Reason and the full structured error survive end-to-end. - assert.NotEmpty(t, finalResp.Experiment.Status.Reason, "failed experiment must retain its reason") - require.NotNil(t, finalResp.Experiment.Status.Error, "failed experiment must retain its structured error") - assert.Equal(t, "ActionExecutionFailed", finalResp.Experiment.Status.Error.Code) - assert.Equal(t, "fail", finalResp.Experiment.Status.Error.Location) - assert.Equal(t, "000000000000", finalResp.Experiment.Status.Error.AccountID) + rec3 := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) + require.Equal(t, http.StatusOK, rec3.Code) + require.NoError(t, json.Unmarshal(rec3.Body.Bytes(), &finalResp)) + require.Equal(t, "failed", finalResp.Experiment.Status.Status) + + // Regression test: cleanupActions used to unconditionally overwrite + // exp.Status right after markExperimentFailed set it, clobbering the + // structured ExperimentStatusError before any client could ever observe + // it. Verify Reason and the full structured error survive end-to-end. + assert.NotEmpty(t, finalResp.Experiment.Status.Reason, "failed experiment must retain its reason") + require.NotNil(t, finalResp.Experiment.Status.Error, "failed experiment must retain its structured error") + assert.Equal(t, "ActionExecutionFailed", finalResp.Experiment.Status.Error.Code) + assert.Equal(t, "fail", finalResp.Experiment.Status.Error.Location) + assert.Equal(t, "000000000000", finalResp.Experiment.Status.Error.AccountID) + }) } func TestFISHandler_ExperimentSucceeds_WithMockActionProvider(t *testing.T) { t.Parallel() - h := newTestHandler(t) - - // Register a mock provider that succeeds. - mock := &fis.MockFISActionProvider{ - Definitions: []service.FISActionDefinition{ - {ActionID: "aws:test:succeed-action", TargetType: "aws:ec2:instance"}, - }, - } - h.SetActionProviders([]service.FISActionProvider{mock}) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - body := map[string]any{ - "roleArn": "arn:aws:iam::000000000000:role/FISRole", - "stopConditions": []map[string]any{{"source": "none"}}, - "targets": map[string]any{ - "MyInstances": map[string]any{ - "resourceType": "aws:ec2:instance", - "selectionMode": "ALL", - "resourceArns": []string{"arn:aws:ec2:us-east-1:000:instance/i-abc123"}, + // Register a mock provider that succeeds. + mock := &fis.MockFISActionProvider{ + Definitions: []service.FISActionDefinition{ + {ActionID: "aws:test:succeed-action", TargetType: "aws:ec2:instance"}, }, - }, - "actions": map[string]any{ - "succeed": map[string]any{ - "actionId": "aws:test:succeed-action", - "targets": map[string]string{"Instances": "MyInstances"}, + } + h.SetActionProviders([]service.FISActionProvider{mock}) + + body := map[string]any{ + "roleArn": "arn:aws:iam::000000000000:role/FISRole", + "stopConditions": []map[string]any{{"source": "none"}}, + "targets": map[string]any{ + "MyInstances": map[string]any{ + "resourceType": "aws:ec2:instance", + "selectionMode": "ALL", + "resourceArns": []string{"arn:aws:ec2:us-east-1:000:instance/i-abc123"}, + }, }, - }, - } + "actions": map[string]any{ + "succeed": map[string]any{ + "actionId": "aws:test:succeed-action", + "targets": map[string]string{"Instances": "MyInstances"}, + }, + }, + } - rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) - require.Equal(t, http.StatusCreated, rec.Code) + rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) + require.Equal(t, http.StatusCreated, rec.Code) - var tplResp struct { - ExperimentTemplate struct { - ID string `json:"id"` - } `json:"experimentTemplate"` - } + var tplResp struct { + ExperimentTemplate struct { + ID string `json:"id"` + } `json:"experimentTemplate"` + } - mustJSON(t, rec, &tplResp) - templateID := tplResp.ExperimentTemplate.ID + mustJSON(t, rec, &tplResp) + templateID := tplResp.ExperimentTemplate.ID - rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ - "experimentTemplateId": templateID, - }) - require.Equal(t, http.StatusCreated, rec2.Code) + rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ + "experimentTemplateId": templateID, + }) + require.Equal(t, http.StatusCreated, rec2.Code) - var expResp struct { - Experiment struct { - ID string `json:"id"` - } `json:"experiment"` - } + var expResp struct { + Experiment struct { + ID string `json:"id"` + } `json:"experiment"` + } - mustJSON(t, rec2, &expResp) - expID := expResp.Experiment.ID + mustJSON(t, rec2, &expResp) + expID := expResp.Experiment.ID + + // Mock action succeeds synchronously; initiating delay plus the + // no-timed-action grace period gate completion. + time.Sleep(2*fis.LifecycleDelayForTest + time.Millisecond) + synctest.Wait() - require.Eventually(t, func() bool { rec3 := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) - if rec3.Code != http.StatusOK { - return false - } + require.Equal(t, http.StatusOK, rec3.Code) var resp struct { Experiment struct { @@ -792,10 +799,7 @@ func TestFISHandler_ExperimentSucceeds_WithMockActionProvider(t *testing.T) { } `json:"experiment"` } - if err := json.Unmarshal(rec3.Body.Bytes(), &resp); err != nil { - return false - } - - return resp.Experiment.Status.Status == "completed" - }, 5*time.Second, 50*time.Millisecond) + require.NoError(t, json.Unmarshal(rec3.Body.Bytes(), &resp)) + require.Equal(t, "completed", resp.Experiment.Status.Status) + }) } diff --git a/services/fis/experiment_actions_mode_test.go b/services/fis/experiment_actions_mode_test.go index f7cf84c48d..af73407e38 100644 --- a/services/fis/experiment_actions_mode_test.go +++ b/services/fis/experiment_actions_mode_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -81,112 +82,116 @@ func pollExperimentUntilTerminal(t *testing.T, h *fis.Handler, expID string) map func TestStartExperiment_ActionsMode_SkipAll_SkipsActionsAndProvider(t *testing.T) { t.Parallel() - h := newTestHandler(t) - - mock := &fis.MockFISActionProvider{ - Definitions: []service.FISActionDefinition{ - {ActionID: "aws:test:mode-action", TargetType: "aws:ec2:instance"}, - }, - } - h.SetActionProviders([]service.FISActionProvider{mock}) - - rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", actionsModeTemplateBody()) - require.Equal(t, http.StatusCreated, rec.Code) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - var tplResp struct { - ExperimentTemplate struct { - ID string `json:"id"` - } `json:"experimentTemplate"` - } + mock := &fis.MockFISActionProvider{ + Definitions: []service.FISActionDefinition{ + {ActionID: "aws:test:mode-action", TargetType: "aws:ec2:instance"}, + }, + } + h.SetActionProviders([]service.FISActionProvider{mock}) - mustJSON(t, rec, &tplResp) + rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", actionsModeTemplateBody()) + require.Equal(t, http.StatusCreated, rec.Code) - rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ - "experimentTemplateId": tplResp.ExperimentTemplate.ID, - "experimentOptions": map[string]any{"actionsMode": "skip-all"}, - }) - require.Equal(t, http.StatusCreated, rec2.Code) + var tplResp struct { + ExperimentTemplate struct { + ID string `json:"id"` + } `json:"experimentTemplate"` + } - var startResp struct { - Experiment struct { - ID string `json:"id"` - ExperimentOptions struct { - ActionsMode string `json:"actionsMode"` - } `json:"experimentOptions"` - } `json:"experiment"` - } + mustJSON(t, rec, &tplResp) + + rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ + "experimentTemplateId": tplResp.ExperimentTemplate.ID, + "experimentOptions": map[string]any{"actionsMode": "skip-all"}, + }) + require.Equal(t, http.StatusCreated, rec2.Code) + + var startResp struct { + Experiment struct { + ID string `json:"id"` + ExperimentOptions struct { + ActionsMode string `json:"actionsMode"` + } `json:"experimentOptions"` + } `json:"experiment"` + } - mustJSON(t, rec2, &startResp) - assert.Equal(t, "skip-all", startResp.Experiment.ExperimentOptions.ActionsMode) + mustJSON(t, rec2, &startResp) + assert.Equal(t, "skip-all", startResp.Experiment.ExperimentOptions.ActionsMode) - exp := pollExperimentUntilTerminal(t, h, startResp.Experiment.ID) + exp := pollExperimentUntilTerminal(t, h, startResp.Experiment.ID) - var status struct { - Status string `json:"status"` - } + var status struct { + Status string `json:"status"` + } - require.NoError(t, json.Unmarshal(exp["status"], &status)) - assert.Equal(t, "completed", status.Status) + require.NoError(t, json.Unmarshal(exp["status"], &status)) + assert.Equal(t, "completed", status.Status) - var actions map[string]struct { - Status struct { - Status string `json:"status"` - } `json:"status"` - } + var actions map[string]struct { + Status struct { + Status string `json:"status"` + } `json:"status"` + } - require.NoError(t, json.Unmarshal(exp["actions"], &actions)) - require.Contains(t, actions, "modeAction") - assert.Equal(t, "skipped", actions["modeAction"].Status.Status) + require.NoError(t, json.Unmarshal(exp["actions"], &actions)) + require.Contains(t, actions, "modeAction") + assert.Equal(t, "skipped", actions["modeAction"].Status.Status) - assert.Equal(t, 0, mock.Calls, "skip-all must not invoke the external action provider") + assert.Equal(t, 0, mock.Calls, "skip-all must not invoke the external action provider") + }) } func TestStartExperiment_ActionsMode_RunAll_InvokesProvider(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - mock := &fis.MockFISActionProvider{ - Definitions: []service.FISActionDefinition{ - {ActionID: "aws:test:mode-action", TargetType: "aws:ec2:instance"}, - }, - } - h.SetActionProviders([]service.FISActionProvider{mock}) + mock := &fis.MockFISActionProvider{ + Definitions: []service.FISActionDefinition{ + {ActionID: "aws:test:mode-action", TargetType: "aws:ec2:instance"}, + }, + } + h.SetActionProviders([]service.FISActionProvider{mock}) - rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", actionsModeTemplateBody()) - require.Equal(t, http.StatusCreated, rec.Code) + rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", actionsModeTemplateBody()) + require.Equal(t, http.StatusCreated, rec.Code) - var tplResp struct { - ExperimentTemplate struct { - ID string `json:"id"` - } `json:"experimentTemplate"` - } + var tplResp struct { + ExperimentTemplate struct { + ID string `json:"id"` + } `json:"experimentTemplate"` + } - mustJSON(t, rec, &tplResp) + mustJSON(t, rec, &tplResp) - rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ - "experimentTemplateId": tplResp.ExperimentTemplate.ID, - "experimentOptions": map[string]any{"actionsMode": "run-all"}, - }) - require.Equal(t, http.StatusCreated, rec2.Code) + rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ + "experimentTemplateId": tplResp.ExperimentTemplate.ID, + "experimentOptions": map[string]any{"actionsMode": "run-all"}, + }) + require.Equal(t, http.StatusCreated, rec2.Code) - var startResp struct { - Experiment struct { - ID string `json:"id"` - } `json:"experiment"` - } + var startResp struct { + Experiment struct { + ID string `json:"id"` + } `json:"experiment"` + } - mustJSON(t, rec2, &startResp) + mustJSON(t, rec2, &startResp) - exp := pollExperimentUntilTerminal(t, h, startResp.Experiment.ID) + exp := pollExperimentUntilTerminal(t, h, startResp.Experiment.ID) - var status struct { - Status string `json:"status"` - } + var status struct { + Status string `json:"status"` + } - require.NoError(t, json.Unmarshal(exp["status"], &status)) - assert.Equal(t, "completed", status.Status) - assert.Equal(t, 1, mock.Calls, "run-all must invoke the external action provider exactly once") + require.NoError(t, json.Unmarshal(exp["status"], &status)) + assert.Equal(t, "completed", status.Status) + assert.Equal(t, 1, mock.Calls, "run-all must invoke the external action provider exactly once") + }) } func TestStartExperiment_ActionsMode_DefaultsToRunAll(t *testing.T) { diff --git a/services/fis/experiment_execution_test.go b/services/fis/experiment_execution_test.go index 7bd8967c74..f868fdaf2d 100644 --- a/services/fis/experiment_execution_test.go +++ b/services/fis/experiment_execution_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -17,52 +18,54 @@ import ( func TestFISHandler_ExperimentCompletesAfterDuration(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - // Template with a very short wait action. - body := map[string]any{ - "roleArn": "arn:aws:iam::000000000000:role/FISRole", - "stopConditions": []map[string]any{{"source": "none"}}, - "targets": map[string]any{}, - "actions": map[string]any{ - "wait": map[string]any{ - "actionId": "aws:fis:wait", - "parameters": map[string]string{"duration": "PT0.1S"}, + // Template with a very short wait action. + const waitDuration = "PT0.1S" + + body := map[string]any{ + "roleArn": "arn:aws:iam::000000000000:role/FISRole", + "stopConditions": []map[string]any{{"source": "none"}}, + "targets": map[string]any{}, + "actions": map[string]any{ + "wait": map[string]any{ + "actionId": "aws:fis:wait", + "parameters": map[string]string{"duration": waitDuration}, + }, }, - }, - } + } - rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) - require.Equal(t, http.StatusCreated, rec.Code) + rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) + require.Equal(t, http.StatusCreated, rec.Code) - var tplResp struct { - ExperimentTemplate struct { - ID string `json:"id"` - } `json:"experimentTemplate"` - } + var tplResp struct { + ExperimentTemplate struct { + ID string `json:"id"` + } `json:"experimentTemplate"` + } - mustJSON(t, rec, &tplResp) + mustJSON(t, rec, &tplResp) - rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ - "experimentTemplateId": tplResp.ExperimentTemplate.ID, - }) - require.Equal(t, http.StatusCreated, rec2.Code) + rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ + "experimentTemplateId": tplResp.ExperimentTemplate.ID, + }) + require.Equal(t, http.StatusCreated, rec2.Code) - var expResp struct { - Experiment struct { - ID string `json:"id"` - } `json:"experiment"` - } + var expResp struct { + Experiment struct { + ID string `json:"id"` + } `json:"experiment"` + } - mustJSON(t, rec2, &expResp) - expID := expResp.Experiment.ID + mustJSON(t, rec2, &expResp) + expID := expResp.Experiment.ID + + time.Sleep(2*fis.LifecycleDelayForTest + fis.ParseISODurationForTest(waitDuration) + time.Millisecond) + synctest.Wait() - // Wait for the experiment to complete. - require.Eventually(t, func() bool { rec3 := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) - if rec3.Code != http.StatusOK { - return false - } + require.Equal(t, http.StatusOK, rec3.Code) var resp struct { Experiment struct { @@ -72,12 +75,9 @@ func TestFISHandler_ExperimentCompletesAfterDuration(t *testing.T) { } `json:"experiment"` } - if err := json.Unmarshal(rec3.Body.Bytes(), &resp); err != nil { - return false - } - - return resp.Experiment.Status.Status == "completed" - }, 5*time.Second, 100*time.Millisecond) + require.NoError(t, json.Unmarshal(rec3.Body.Bytes(), &resp)) + require.Equal(t, "completed", resp.Experiment.Status.Status) + }) } // ---------------------------------------- @@ -108,44 +108,45 @@ func TestFISHandler_SetFaultStore(t *testing.T) { func TestFISHandler_ExperimentCompletes_NoTimedActions(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - // Template with no actions → maxDuration is 0, should complete immediately. - rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", map[string]any{ - "roleArn": "arn:aws:iam::000000000000:role/FISRole", - "stopConditions": []map[string]any{{"source": "none"}}, - "targets": map[string]any{}, - "actions": map[string]any{}, - }) - require.Equal(t, http.StatusCreated, rec.Code) + // Template with no actions → maxDuration is 0, should complete immediately. + rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", map[string]any{ + "roleArn": "arn:aws:iam::000000000000:role/FISRole", + "stopConditions": []map[string]any{{"source": "none"}}, + "targets": map[string]any{}, + "actions": map[string]any{}, + }) + require.Equal(t, http.StatusCreated, rec.Code) - var tplResp struct { - ExperimentTemplate struct { - ID string `json:"id"` - } `json:"experimentTemplate"` - } + var tplResp struct { + ExperimentTemplate struct { + ID string `json:"id"` + } `json:"experimentTemplate"` + } - mustJSON(t, rec, &tplResp) + mustJSON(t, rec, &tplResp) - rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ - "experimentTemplateId": tplResp.ExperimentTemplate.ID, - }) - require.Equal(t, http.StatusCreated, rec2.Code) + rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ + "experimentTemplateId": tplResp.ExperimentTemplate.ID, + }) + require.Equal(t, http.StatusCreated, rec2.Code) - var expResp struct { - Experiment struct { - ID string `json:"id"` - } `json:"experiment"` - } + var expResp struct { + Experiment struct { + ID string `json:"id"` + } `json:"experiment"` + } - mustJSON(t, rec2, &expResp) - expID := expResp.Experiment.ID + mustJSON(t, rec2, &expResp) + expID := expResp.Experiment.ID + + time.Sleep(2*fis.LifecycleDelayForTest + time.Millisecond) + synctest.Wait() - require.Eventually(t, func() bool { rec3 := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) - if rec3.Code != http.StatusOK { - return false - } + require.Equal(t, http.StatusOK, rec3.Code) var resp struct { Experiment struct { @@ -155,12 +156,9 @@ func TestFISHandler_ExperimentCompletes_NoTimedActions(t *testing.T) { } `json:"experiment"` } - if err := json.Unmarshal(rec3.Body.Bytes(), &resp); err != nil { - return false - } - - return resp.Experiment.Status.Status == "completed" - }, 5*time.Second, 50*time.Millisecond) + require.NoError(t, json.Unmarshal(rec3.Body.Bytes(), &resp)) + require.Equal(t, "completed", resp.Experiment.Status.Status) + }) } // ---------------------------------------- diff --git a/services/fis/experiment_reports_test.go b/services/fis/experiment_reports_test.go index b6d02e7740..666af94159 100644 --- a/services/fis/experiment_reports_test.go +++ b/services/fis/experiment_reports_test.go @@ -5,6 +5,7 @@ import ( "net/http" "strings" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -247,61 +248,66 @@ func startExperimentAndPollTerminal(t *testing.T, h *fis.Handler, templateID str func TestStartExperiment_WithReportConfiguration_GeneratesReport(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - body := minimalTemplateBody() - body["experimentReportConfiguration"] = reportConfigBody() + body := minimalTemplateBody() + body["experimentReportConfiguration"] = reportConfigBody() - rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) - require.Equal(t, http.StatusCreated, rec.Code) + rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) + require.Equal(t, http.StatusCreated, rec.Code) - var tplResp struct { - ExperimentTemplate struct { - ID string `json:"id"` - } `json:"experimentTemplate"` - } + var tplResp struct { + ExperimentTemplate struct { + ID string `json:"id"` + } `json:"experimentTemplate"` + } - mustJSON(t, rec, &tplResp) + mustJSON(t, rec, &tplResp) - result := startExperimentAndPollTerminal(t, h, tplResp.ExperimentTemplate.ID) + result := startExperimentAndPollTerminal(t, h, tplResp.ExperimentTemplate.ID) - assert.Equal(t, "completed", result.ExperimentReport.State.Status) - require.Len(t, result.ExperimentReport.S3Reports, 1) - assert.Equal(t, "experiment-report", result.ExperimentReport.S3Reports[0].ReportType) - assert.True(t, strings.HasPrefix(result.ExperimentReport.S3Reports[0].Arn, "arn:aws:s3:::my-fis-reports/reports/")) + assert.Equal(t, "completed", result.ExperimentReport.State.Status) + require.Len(t, result.ExperimentReport.S3Reports, 1) + assert.Equal(t, "experiment-report", result.ExperimentReport.S3Reports[0].ReportType) + assert.True(t, + strings.HasPrefix(result.ExperimentReport.S3Reports[0].Arn, "arn:aws:s3:::my-fis-reports/reports/")) + }) } func TestStartExperiment_ReportConfiguration_MissingS3Output_ReportFails(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - body := minimalTemplateBody() - body["experimentReportConfiguration"] = map[string]any{ - "dataSources": map[string]any{ - "cloudWatchDashboards": []map[string]any{ - {"dashboardIdentifier": "arn:aws:cloudwatch::000000000000:dashboard/MyDashboard"}, + body := minimalTemplateBody() + body["experimentReportConfiguration"] = map[string]any{ + "dataSources": map[string]any{ + "cloudWatchDashboards": []map[string]any{ + {"dashboardIdentifier": "arn:aws:cloudwatch::000000000000:dashboard/MyDashboard"}, + }, }, - }, - } + } - rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) - require.Equal(t, http.StatusCreated, rec.Code) + rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) + require.Equal(t, http.StatusCreated, rec.Code) - var tplResp struct { - ExperimentTemplate struct { - ID string `json:"id"` - } `json:"experimentTemplate"` - } + var tplResp struct { + ExperimentTemplate struct { + ID string `json:"id"` + } `json:"experimentTemplate"` + } - mustJSON(t, rec, &tplResp) + mustJSON(t, rec, &tplResp) - result := startExperimentAndPollTerminal(t, h, tplResp.ExperimentTemplate.ID) + result := startExperimentAndPollTerminal(t, h, tplResp.ExperimentTemplate.ID) - assert.Equal(t, "failed", result.ExperimentReport.State.Status) - require.NotNil(t, result.ExperimentReport.State.Error) - assert.Equal(t, "MissingReportOutputConfiguration", result.ExperimentReport.State.Error.Code) - assert.Empty(t, result.ExperimentReport.S3Reports) + assert.Equal(t, "failed", result.ExperimentReport.State.Status) + require.NotNil(t, result.ExperimentReport.State.Error) + assert.Equal(t, "MissingReportOutputConfiguration", result.ExperimentReport.State.Error.Code) + assert.Empty(t, result.ExperimentReport.S3Reports) + }) } func TestGetExperiment_NoReportConfig_OmitsReportFields(t *testing.T) { diff --git a/services/fis/experiment_status_test.go b/services/fis/experiment_status_test.go index 9b80c83ccc..b74c153c6c 100644 --- a/services/fis/experiment_status_test.go +++ b/services/fis/experiment_status_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -12,6 +13,10 @@ import ( "github.com/blackbirdworks/gopherstack/services/fis" ) +// waitActionISODuration is the lifecycle tests' wait-action duration; the fake-clock +// advance is derived from it. +const waitActionISODuration = "PT0.05S" + func TestExperiment_EndTime_AbsentBeforeComplete(t *testing.T) { t.Parallel() @@ -60,85 +65,80 @@ func TestExperiment_EndTime_AbsentBeforeComplete(t *testing.T) { func TestExperiment_EndTime_PresentAfterComplete(t *testing.T) { t.Parallel() - h := newTestHandler(t) - - body := map[string]any{ - "roleArn": "arn:aws:iam::000000000000:role/FISRole", - "stopConditions": []map[string]any{{"source": "none"}}, - "targets": map[string]any{}, - "actions": map[string]any{ - "wait": map[string]any{ - "actionId": "aws:fis:wait", - "parameters": map[string]string{"duration": "PT0.05S"}, + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + + body := map[string]any{ + "roleArn": "arn:aws:iam::000000000000:role/FISRole", + "stopConditions": []map[string]any{{"source": "none"}}, + "targets": map[string]any{}, + "actions": map[string]any{ + "wait": map[string]any{ + "actionId": "aws:fis:wait", + "parameters": map[string]string{"duration": waitActionISODuration}, + }, }, - }, - } - - rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) - require.Equal(t, http.StatusCreated, rec.Code) - - var tplResp struct { - ExperimentTemplate struct { - ID string `json:"id"` - } `json:"experimentTemplate"` - } - - mustJSON(t, rec, &tplResp) + } - rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ - "experimentTemplateId": tplResp.ExperimentTemplate.ID, - }) - require.Equal(t, http.StatusCreated, rec2.Code) + rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) + require.Equal(t, http.StatusCreated, rec.Code) - var expResp struct { - Experiment struct { - ID string `json:"id"` - } `json:"experiment"` - } + var tplResp struct { + ExperimentTemplate struct { + ID string `json:"id"` + } `json:"experimentTemplate"` + } - mustJSON(t, rec2, &expResp) - expID := expResp.Experiment.ID + mustJSON(t, rec, &tplResp) - // Poll until completed. - require.Eventually(t, func() bool { - r := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) - if r.Code != http.StatusOK { - return false - } + rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ + "experimentTemplateId": tplResp.ExperimentTemplate.ID, + }) + require.Equal(t, http.StatusCreated, rec2.Code) - var gr struct { + var expResp struct { Experiment struct { - Status struct { - Status string `json:"status"` - } `json:"status"` + ID string `json:"id"` } `json:"experiment"` } - if err := json.Unmarshal(r.Body.Bytes(), &gr); err != nil { - return false - } + mustJSON(t, rec2, &expResp) + expID := expResp.Experiment.ID - return gr.Experiment.Status.Status == "completed" - }, 5*time.Second, 20*time.Millisecond) + // Advance the fake clock past initiating -> running -> wait action -> + // completing -> completed; margin avoids a same-instant timer race. + time.Sleep(2*fis.LifecycleDelayForTest + fis.ParseISODurationForTest(waitActionISODuration) + time.Millisecond) + synctest.Wait() - rec3 := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) - require.Equal(t, http.StatusOK, rec3.Code) + rec3 := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) + require.Equal(t, http.StatusOK, rec3.Code) - var raw map[string]json.RawMessage + var raw map[string]json.RawMessage - mustJSON(t, rec3, &raw) + mustJSON(t, rec3, &raw) - var expRaw map[string]json.RawMessage + var expRaw map[string]json.RawMessage - require.NoError(t, json.Unmarshal(raw["experiment"], &expRaw)) + require.NoError(t, json.Unmarshal(raw["experiment"], &expRaw)) - endTimeRaw, hasEndTime := expRaw["endTime"] - require.True(t, hasEndTime, "endTime must be present after completion") + statusRaw, hasStatus := expRaw["status"] + require.True(t, hasStatus) - var endTime float64 + var status struct { + Status string `json:"status"` + } + + require.NoError(t, json.Unmarshal(statusRaw, &status)) + require.Equal(t, "completed", status.Status) + + endTimeRaw, hasEndTime := expRaw["endTime"] + require.True(t, hasEndTime, "endTime must be present after completion") - require.NoError(t, json.Unmarshal(endTimeRaw, &endTime)) - assert.Greater(t, endTime, 0.0, "endTime must be a positive Unix timestamp") + var endTime float64 + + require.NoError(t, json.Unmarshal(endTimeRaw, &endTime)) + assert.Greater(t, endTime, 0.0, "endTime must be a positive Unix timestamp") + }) } // ---------------------------------------- @@ -148,77 +148,60 @@ func TestExperiment_EndTime_PresentAfterComplete(t *testing.T) { func TestStopExperiment_AlreadyStopped_Returns409(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - body := map[string]any{ - "roleArn": "arn:aws:iam::000000000000:role/FISRole", - "stopConditions": []map[string]any{{"source": "none"}}, - "targets": map[string]any{}, - "actions": map[string]any{}, - } - - rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) - require.Equal(t, http.StatusCreated, rec.Code) - - var tplResp struct { - ExperimentTemplate struct { - ID string `json:"id"` - } `json:"experimentTemplate"` - } - - mustJSON(t, rec, &tplResp) + body := map[string]any{ + "roleArn": "arn:aws:iam::000000000000:role/FISRole", + "stopConditions": []map[string]any{{"source": "none"}}, + "targets": map[string]any{}, + "actions": map[string]any{}, + } - rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ - "experimentTemplateId": tplResp.ExperimentTemplate.ID, - }) - require.Equal(t, http.StatusCreated, rec2.Code) + rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) + require.Equal(t, http.StatusCreated, rec.Code) - var expResp struct { - Experiment struct { - ID string `json:"id"` - } `json:"experiment"` - } + var tplResp struct { + ExperimentTemplate struct { + ID string `json:"id"` + } `json:"experimentTemplate"` + } - mustJSON(t, rec2, &expResp) - expID := expResp.Experiment.ID + mustJSON(t, rec, &tplResp) - // Poll until terminal. - require.Eventually(t, func() bool { - r := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) - if r.Code != http.StatusOK { - return false - } + rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ + "experimentTemplateId": tplResp.ExperimentTemplate.ID, + }) + require.Equal(t, http.StatusCreated, rec2.Code) - var gr struct { + var expResp struct { Experiment struct { - Status struct { - Status string `json:"status"` - } `json:"status"` + ID string `json:"id"` } `json:"experiment"` } - if err := json.Unmarshal(r.Body.Bytes(), &gr); err != nil { - return false - } - - s := gr.Experiment.Status.Status + mustJSON(t, rec2, &expResp) + expID := expResp.Experiment.ID - return s == "completed" || s == "failed" || s == "stopped" - }, 5*time.Second, 20*time.Millisecond) + // No actions: lifecycle is just initiating -> running -> completing -> + // completed, each gated by lifecycleDelay. + time.Sleep(2*fis.LifecycleDelayForTest + time.Millisecond) + synctest.Wait() - // Stop an already-terminal experiment → 400 ValidationException. StopExperiment's - // generated deserializer in aws-sdk-go-v2/service/fis only recognizes - // ResourceNotFoundException and ValidationException — it has no ConflictException - // case — so this must not be reported as a conflict. - rec3 := doRequest(t, h, http.MethodPost, "/experiments/"+expID+"/stop", nil) - assert.Equal(t, http.StatusBadRequest, rec3.Code) + // Stop an already-terminal experiment → 400 ValidationException. StopExperiment's + // generated deserializer in aws-sdk-go-v2/service/fis only recognizes + // ResourceNotFoundException and ValidationException — it has no ConflictException + // case — so this must not be reported as a conflict. + rec3 := doRequest(t, h, http.MethodPost, "/experiments/"+expID+"/stop", nil) + assert.Equal(t, http.StatusBadRequest, rec3.Code) - var errResp struct { - Type string `json:"__type"` - } + var errResp struct { + Type string `json:"__type"` + } - mustJSON(t, rec3, &errResp) - assert.Equal(t, "ValidationException", errResp.Type) + mustJSON(t, rec3, &errResp) + assert.Equal(t, "ValidationException", errResp.Type) + }) } // ---------------------------------------- @@ -285,93 +268,80 @@ func TestExperimentOptions_PassThrough(t *testing.T) { func TestExperiment_ActionStatus_AfterComplete(t *testing.T) { t.Parallel() - h := newTestHandler(t) - - body := map[string]any{ - "roleArn": "arn:aws:iam::000000000000:role/FISRole", - "stopConditions": []map[string]any{{"source": "none"}}, - "targets": map[string]any{}, - "actions": map[string]any{ - "myWait": map[string]any{ - "actionId": "aws:fis:wait", - "parameters": map[string]string{"duration": "PT0.05S"}, + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + + body := map[string]any{ + "roleArn": "arn:aws:iam::000000000000:role/FISRole", + "stopConditions": []map[string]any{{"source": "none"}}, + "targets": map[string]any{}, + "actions": map[string]any{ + "myWait": map[string]any{ + "actionId": "aws:fis:wait", + "parameters": map[string]string{"duration": waitActionISODuration}, + }, }, - }, - } - - rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) - require.Equal(t, http.StatusCreated, rec.Code) - - var tplResp struct { - ExperimentTemplate struct { - ID string `json:"id"` - } `json:"experimentTemplate"` - } + } - mustJSON(t, rec, &tplResp) + rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) + require.Equal(t, http.StatusCreated, rec.Code) - rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ - "experimentTemplateId": tplResp.ExperimentTemplate.ID, - }) - require.Equal(t, http.StatusCreated, rec2.Code) - - var expResp struct { - Experiment struct { - ID string `json:"id"` - } `json:"experiment"` - } + var tplResp struct { + ExperimentTemplate struct { + ID string `json:"id"` + } `json:"experimentTemplate"` + } - mustJSON(t, rec2, &expResp) - expID := expResp.Experiment.ID + mustJSON(t, rec, &tplResp) - // Poll until completed. - require.Eventually(t, func() bool { - r := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) - if r.Code != http.StatusOK { - return false - } + rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ + "experimentTemplateId": tplResp.ExperimentTemplate.ID, + }) + require.Equal(t, http.StatusCreated, rec2.Code) - var gr struct { + var expResp struct { Experiment struct { - Status struct { - Status string `json:"status"` - } `json:"status"` + ID string `json:"id"` } `json:"experiment"` } - if err := json.Unmarshal(r.Body.Bytes(), &gr); err != nil { - return false - } + mustJSON(t, rec2, &expResp) + expID := expResp.Experiment.ID - return gr.Experiment.Status.Status == "completed" - }, 5*time.Second, 20*time.Millisecond) + time.Sleep(2*fis.LifecycleDelayForTest + fis.ParseISODurationForTest(waitActionISODuration) + time.Millisecond) + synctest.Wait() - rec3 := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) - require.Equal(t, http.StatusOK, rec3.Code) + rec3 := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) + require.Equal(t, http.StatusOK, rec3.Code) - var resp struct { - Experiment struct { - Actions map[string]struct { - Status *struct { + var resp struct { + Experiment struct { + Actions map[string]struct { + Status *struct { + Status string `json:"status"` + } `json:"status"` + State *struct { + Status string `json:"status"` + } `json:"state"` + ActionID string `json:"actionId"` + } `json:"actions"` + Status struct { Status string `json:"status"` } `json:"status"` - State *struct { - Status string `json:"status"` - } `json:"state"` - ActionID string `json:"actionId"` - } `json:"actions"` - } `json:"experiment"` - } + } `json:"experiment"` + } - mustJSON(t, rec3, &resp) - action, ok := resp.Experiment.Actions["myWait"] - require.True(t, ok, "myWait action must be in experiment response") - assert.Equal(t, "aws:fis:wait", action.ActionID) - require.NotNil(t, action.Status, "action.status must not be nil") - assert.NotEmpty(t, action.Status.Status, "action.status.status must be set") - // Both status and state aliases must be present. - require.NotNil(t, action.State, "action.state must not be nil") - assert.Equal(t, action.Status.Status, action.State.Status, "action.status and action.state must agree") + mustJSON(t, rec3, &resp) + require.Equal(t, "completed", resp.Experiment.Status.Status) + action, ok := resp.Experiment.Actions["myWait"] + require.True(t, ok, "myWait action must be in experiment response") + assert.Equal(t, "aws:fis:wait", action.ActionID) + require.NotNil(t, action.Status, "action.status must not be nil") + assert.NotEmpty(t, action.Status.Status, "action.status.status must be set") + // Both status and state aliases must be present. + require.NotNil(t, action.State, "action.state must not be nil") + assert.Equal(t, action.Status.Status, action.State.Status, "action.status and action.state must agree") + }) } // ---------------------------------------- @@ -409,52 +379,53 @@ func TestExperiment_StatusAndState_BothPresent(t *testing.T) { func TestExperimentStatusLifecycle(t *testing.T) { t.Parallel() - h := newTestHandler(t) - - // Template with a very short wait to observe lifecycle transitions. - body := map[string]any{ - "roleArn": "arn:aws:iam::000000000000:role/FISRole", - "stopConditions": []map[string]any{{"source": "none"}}, - "targets": map[string]any{}, - "actions": map[string]any{ - "wait": map[string]any{ - "actionId": "aws:fis:wait", - "parameters": map[string]string{"duration": "PT0.05S"}, + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + + // Template with a very short wait to observe lifecycle transitions. + body := map[string]any{ + "roleArn": "arn:aws:iam::000000000000:role/FISRole", + "stopConditions": []map[string]any{{"source": "none"}}, + "targets": map[string]any{}, + "actions": map[string]any{ + "wait": map[string]any{ + "actionId": "aws:fis:wait", + "parameters": map[string]string{"duration": waitActionISODuration}, + }, }, - }, - } + } - rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) - require.Equal(t, http.StatusCreated, rec.Code) + rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) + require.Equal(t, http.StatusCreated, rec.Code) - var tplResp struct { - ExperimentTemplate struct { - ID string `json:"id"` - } `json:"experimentTemplate"` - } + var tplResp struct { + ExperimentTemplate struct { + ID string `json:"id"` + } `json:"experimentTemplate"` + } - mustJSON(t, rec, &tplResp) + mustJSON(t, rec, &tplResp) - rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ - "experimentTemplateId": tplResp.ExperimentTemplate.ID, - }) - require.Equal(t, http.StatusCreated, rec2.Code) + rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ + "experimentTemplateId": tplResp.ExperimentTemplate.ID, + }) + require.Equal(t, http.StatusCreated, rec2.Code) - var expResp struct { - Experiment struct { - ID string `json:"id"` - } `json:"experiment"` - } + var expResp struct { + Experiment struct { + ID string `json:"id"` + } `json:"experiment"` + } - mustJSON(t, rec2, &expResp) - expID := expResp.Experiment.ID + mustJSON(t, rec2, &expResp) + expID := expResp.Experiment.ID - // Poll for completed status — lifecycle goes pending→initiating→running→completing→completed. - require.Eventually(t, func() bool { - r := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) - if r.Code != http.StatusOK { - return false - } + // Lifecycle goes pending→initiating→running→completing→completed. + time.Sleep(2*fis.LifecycleDelayForTest + fis.ParseISODurationForTest(waitActionISODuration) + time.Millisecond) + synctest.Wait() + + rec3 := doRequest(t, h, http.MethodGet, "/experiments/"+expID, nil) + require.Equal(t, http.StatusOK, rec3.Code) var gr struct { Experiment struct { @@ -464,12 +435,9 @@ func TestExperimentStatusLifecycle(t *testing.T) { } `json:"experiment"` } - if err := json.Unmarshal(r.Body.Bytes(), &gr); err != nil { - return false - } - - return gr.Experiment.Status.Status == "completed" - }, 5*time.Second, 20*time.Millisecond) + mustJSON(t, rec3, &gr) + require.Equal(t, "completed", gr.Experiment.Status.Status) + }) } // ---------------------------------------- diff --git a/services/fis/experiment_templates_test.go b/services/fis/experiment_templates_test.go index d97e32da7c..6670e3e2d3 100644 --- a/services/fis/experiment_templates_test.go +++ b/services/fis/experiment_templates_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "strings" "testing" + "testing/synctest" "time" "github.com/labstack/echo/v5" @@ -293,39 +294,41 @@ func TestExperimentTemplateARN_Shape(t *testing.T) { func TestUpdateTemplate_LastUpdateTime_Changes(t *testing.T) { t.Parallel() - h := newTestHandler(t) - tplID := seedTemplate(t, h) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + tplID := seedTemplate(t, h) - rec := doRequest(t, h, http.MethodGet, "/experimentTemplates/"+tplID, nil) - require.Equal(t, http.StatusOK, rec.Code) + rec := doRequest(t, h, http.MethodGet, "/experimentTemplates/"+tplID, nil) + require.Equal(t, http.StatusOK, rec.Code) - var before struct { - ExperimentTemplate struct { - CreationTime float64 `json:"creationTime"` - LastUpdateTime float64 `json:"lastUpdateTime"` - } `json:"experimentTemplate"` - } + var before struct { + ExperimentTemplate struct { + CreationTime float64 `json:"creationTime"` + LastUpdateTime float64 `json:"lastUpdateTime"` + } `json:"experimentTemplate"` + } - mustJSON(t, rec, &before) + mustJSON(t, rec, &before) - time.Sleep(5 * time.Millisecond) + time.Sleep(5 * time.Millisecond) - rec2 := doRequest(t, h, http.MethodPatch, "/experimentTemplates/"+tplID, map[string]any{ - "description": "updated description", - }) - require.Equal(t, http.StatusOK, rec2.Code) + rec2 := doRequest(t, h, http.MethodPatch, "/experimentTemplates/"+tplID, map[string]any{ + "description": "updated description", + }) + require.Equal(t, http.StatusOK, rec2.Code) - var after struct { - ExperimentTemplate struct { - Description string `json:"description"` - LastUpdateTime float64 `json:"lastUpdateTime"` - } `json:"experimentTemplate"` - } + var after struct { + ExperimentTemplate struct { + Description string `json:"description"` + LastUpdateTime float64 `json:"lastUpdateTime"` + } `json:"experimentTemplate"` + } - mustJSON(t, rec2, &after) - assert.Equal(t, "updated description", after.ExperimentTemplate.Description) - assert.GreaterOrEqual(t, after.ExperimentTemplate.LastUpdateTime, before.ExperimentTemplate.LastUpdateTime, - "lastUpdateTime must not decrease after PATCH") + mustJSON(t, rec2, &after) + assert.Equal(t, "updated description", after.ExperimentTemplate.Description) + assert.GreaterOrEqual(t, after.ExperimentTemplate.LastUpdateTime, before.ExperimentTemplate.LastUpdateTime, + "lastUpdateTime must not decrease after PATCH") + }) } // ---------------------------------------- diff --git a/services/fis/experiments_test.go b/services/fis/experiments_test.go index 5f38446d9c..4c46837045 100644 --- a/services/fis/experiments_test.go +++ b/services/fis/experiments_test.go @@ -5,6 +5,7 @@ import ( "net/http" "strings" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -489,41 +490,41 @@ func TestListExperiments_Pagination(t *testing.T) { func TestListExperiments_FilterByStatus(t *testing.T) { t.Parallel() - h := newTestHandler(t) - tplID := seedTemplate(t, h) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + tplID := seedTemplate(t, h) - // Create one experiment that immediately completes (no timed actions). - body := map[string]any{ - "roleArn": "arn:aws:iam::000000000000:role/FISRole", - "stopConditions": []map[string]any{{"source": "none"}}, - "targets": map[string]any{}, - "actions": map[string]any{}, - } + // Create one experiment that immediately completes (no timed actions). + body := map[string]any{ + "roleArn": "arn:aws:iam::000000000000:role/FISRole", + "stopConditions": []map[string]any{{"source": "none"}}, + "targets": map[string]any{}, + "actions": map[string]any{}, + } - rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) - require.Equal(t, http.StatusCreated, rec.Code) + rec := doRequest(t, h, http.MethodPost, "/experimentTemplates", body) + require.Equal(t, http.StatusCreated, rec.Code) - var tplResp struct { - ExperimentTemplate struct { - ID string `json:"id"` - } `json:"experimentTemplate"` - } + var tplResp struct { + ExperimentTemplate struct { + ID string `json:"id"` + } `json:"experimentTemplate"` + } - mustJSON(t, rec, &tplResp) + mustJSON(t, rec, &tplResp) - rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ - "experimentTemplateId": tplResp.ExperimentTemplate.ID, - }) - require.Equal(t, http.StatusCreated, rec2.Code) + rec2 := doRequest(t, h, http.MethodPost, "/experiments", map[string]any{ + "experimentTemplateId": tplResp.ExperimentTemplate.ID, + }) + require.Equal(t, http.StatusCreated, rec2.Code) - _ = tplID // used above for experiment + _ = tplID // used above for experiment + + time.Sleep(2*fis.LifecycleDelayForTest + time.Millisecond) + synctest.Wait() - // Filter by status=pending or status=initiating (experiment is in early lifecycle). - require.Eventually(t, func() bool { r := doRequest(t, h, http.MethodGet, "/experiments?status=completed", nil) - if r.Code != http.StatusOK { - return false - } + require.Equal(t, http.StatusOK, r.Code) var gr struct { Experiments []struct { @@ -531,12 +532,9 @@ func TestListExperiments_FilterByStatus(t *testing.T) { } `json:"experiments"` } - if err := json.Unmarshal(r.Body.Bytes(), &gr); err != nil { - return false - } - - return len(gr.Experiments) > 0 - }, 5*time.Second, 50*time.Millisecond) + require.NoError(t, json.Unmarshal(r.Body.Bytes(), &gr)) + assert.NotEmpty(t, gr.Experiments) + }) } func TestListExperiments_FilterByTemplateID(t *testing.T) { diff --git a/services/fis/safety_levers.go b/services/fis/safety_levers.go index 6b6993762c..d97ecd736f 100644 --- a/services/fis/safety_levers.go +++ b/services/fis/safety_levers.go @@ -1,6 +1,9 @@ package fis -import "fmt" +import ( + "fmt" + "maps" +) // ---------------------------------------- // Phase 3 — Safety Lever @@ -26,6 +29,7 @@ func (b *InMemoryBackend) GetSafetyLever(id string) (*SafetyLever, error) { } cp := *b.safetyLever + cp.Tags = maps.Clone(b.safetyLever.Tags) return &cp, nil } @@ -60,6 +64,7 @@ func (b *InMemoryBackend) UpdateSafetyLeverState( } cp := *b.safetyLever + cp.Tags = maps.Clone(b.safetyLever.Tags) return &cp, nil } diff --git a/services/glacier/handler_jobs_test.go b/services/glacier/handler_jobs_test.go index 25140dadaf..4452779c4c 100644 --- a/services/glacier/handler_jobs_test.go +++ b/services/glacier/handler_jobs_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "strings" "testing" + "testing/synctest" "time" "github.com/labstack/echo/v5" @@ -663,24 +664,32 @@ func TestInitiateJob_SucceedsAfterDelay(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newDelayedHandler(tt.delay) - jobID := initiateJob(t, h, tt.vaultName, tt.jobType) - - // Immediately after initiation the job should be InProgress. - recEarly := doRequest(t, h, http.MethodGet, "/"+testAccountID+"/vaults/"+tt.vaultName+"/jobs/"+jobID, "") - require.Equal(t, http.StatusOK, recEarly.Code) - var earlyDesc map[string]any - require.NoError(t, json.Unmarshal(recEarly.Body.Bytes(), &earlyDesc)) - assert.Equal(t, "InProgress", earlyDesc["StatusCode"]) - - // After the delay elapses the job should be Succeeded. - time.Sleep(tt.wait) - - recLate := doRequest(t, h, http.MethodGet, "/"+testAccountID+"/vaults/"+tt.vaultName+"/jobs/"+jobID, "") - require.Equal(t, http.StatusOK, recLate.Code) - var lateDesc map[string]any - require.NoError(t, json.Unmarshal(recLate.Body.Bytes(), &lateDesc)) - assert.Equal(t, "Succeeded", lateDesc["StatusCode"]) + synctest.Test(t, func(t *testing.T) { + h := newDelayedHandler(tt.delay) + jobID := initiateJob(t, h, tt.vaultName, tt.jobType) + + // Immediately after initiation the job should be InProgress. + recEarly := doRequest( + t, + h, + http.MethodGet, + "/"+testAccountID+"/vaults/"+tt.vaultName+"/jobs/"+jobID, + "", + ) + require.Equal(t, http.StatusOK, recEarly.Code) + var earlyDesc map[string]any + require.NoError(t, json.Unmarshal(recEarly.Body.Bytes(), &earlyDesc)) + assert.Equal(t, "InProgress", earlyDesc["StatusCode"]) + + // After the delay elapses the job should be Succeeded. + time.Sleep(tt.wait) + + recLate := doRequest(t, h, http.MethodGet, "/"+testAccountID+"/vaults/"+tt.vaultName+"/jobs/"+jobID, "") + require.Equal(t, http.StatusOK, recLate.Code) + var lateDesc map[string]any + require.NoError(t, json.Unmarshal(recLate.Body.Bytes(), &lateDesc)) + assert.Equal(t, "Succeeded", lateDesc["StatusCode"]) + }) }) } } diff --git a/services/glacier/jobs_test.go b/services/glacier/jobs_test.go index c41cb963cf..3d5952dbf1 100644 --- a/services/glacier/jobs_test.go +++ b/services/glacier/jobs_test.go @@ -2,6 +2,7 @@ package glacier_test import ( "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -118,28 +119,28 @@ func TestRetrievalJobAsyncLifecycle(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - bk := glacier.NewInMemoryBackend() - glacier.SetRetrievalDelay(bk, tt.delay) - - _, err := bk.CreateVault(testAccountID, testRegion, "vault") - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + bk := glacier.NewInMemoryBackend() + glacier.SetRetrievalDelay(bk, tt.delay) - j, err := bk.InitiateJob(testAccountID, testRegion, "vault", - &glacier.ExportedInitiateJobRequest{Type: "InventoryRetrieval"}) - require.NoError(t, err) + _, err := bk.CreateVault(testAccountID, testRegion, "vault") + require.NoError(t, err) - if tt.waitForReady { - require.Eventually(t, func() bool { - got, descErr := bk.DescribeJob(testAccountID, testRegion, "vault", j.JobID) + j, err := bk.InitiateJob(testAccountID, testRegion, "vault", + &glacier.ExportedInitiateJobRequest{Type: "InventoryRetrieval"}) + require.NoError(t, err) - return descErr == nil && got.Completed - }, time.Second, 2*time.Millisecond) - } + if tt.waitForReady { + // Completion is lazy-on-read (readyAt vs now): sleep past the + // window, then assert directly instead of polling. + time.Sleep(tt.delay + time.Millisecond) + } - got, err := bk.DescribeJob(testAccountID, testRegion, "vault", j.JobID) - require.NoError(t, err) - assert.Equal(t, tt.wantCompleted, got.Completed) - assert.Equal(t, tt.wantStatus, got.StatusCode) + got, err := bk.DescribeJob(testAccountID, testRegion, "vault", j.JobID) + require.NoError(t, err) + assert.Equal(t, tt.wantCompleted, got.Completed) + assert.Equal(t, tt.wantStatus, got.StatusCode) + }) }) } } diff --git a/services/guardduty/detectors_test.go b/services/guardduty/detectors_test.go index 0f53a854e1..da14bb60f2 100644 --- a/services/guardduty/detectors_test.go +++ b/services/guardduty/detectors_test.go @@ -6,6 +6,7 @@ import ( "net/http" "regexp" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -232,38 +233,40 @@ func TestDetector_Timestamps_Present(t *testing.T) { func TestDetector_UpdatedAt_Advances(t *testing.T) { t.Parallel() - h := newTestHandler(t) - id := createTestDetector(t, h) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + id := createTestDetector(t, h) - rec1 := doRequest(t, h, http.MethodGet, "/detector/"+id, nil) - require.Equal(t, http.StatusOK, rec1.Code) + rec1 := doRequest(t, h, http.MethodGet, "/detector/"+id, nil) + require.Equal(t, http.StatusOK, rec1.Code) - var before map[string]any - require.NoError(t, json.Unmarshal(rec1.Body.Bytes(), &before)) - createdAt := before["createdAt"].(string) - updatedAt1 := before["updatedAt"].(string) + var before map[string]any + require.NoError(t, json.Unmarshal(rec1.Body.Bytes(), &before)) + createdAt := before["createdAt"].(string) + updatedAt1 := before["updatedAt"].(string) - time.Sleep(2 * time.Millisecond) + time.Sleep(2 * time.Millisecond) - rec := doRequest(t, h, http.MethodPost, "/detector/"+id, map[string]any{ - "findingPublishingFrequency": "FIFTEEN_MINUTES", - }) - require.Equal(t, http.StatusOK, rec.Code) + rec := doRequest(t, h, http.MethodPost, "/detector/"+id, map[string]any{ + "findingPublishingFrequency": "FIFTEEN_MINUTES", + }) + require.Equal(t, http.StatusOK, rec.Code) - rec2 := doRequest(t, h, http.MethodGet, "/detector/"+id, nil) - require.Equal(t, http.StatusOK, rec2.Code) + rec2 := doRequest(t, h, http.MethodGet, "/detector/"+id, nil) + require.Equal(t, http.StatusOK, rec2.Code) - var after map[string]any - require.NoError(t, json.Unmarshal(rec2.Body.Bytes(), &after)) - createdAt2 := after["createdAt"].(string) - updatedAt2 := after["updatedAt"].(string) + var after map[string]any + require.NoError(t, json.Unmarshal(rec2.Body.Bytes(), &after)) + createdAt2 := after["createdAt"].(string) + updatedAt2 := after["updatedAt"].(string) - assert.Equal(t, createdAt, createdAt2, "createdAt must not change after UpdateDetector") + assert.Equal(t, createdAt, createdAt2, "createdAt must not change after UpdateDetector") - t1 := parseTS(t, "updatedAt before", updatedAt1) - t2 := parseTS(t, "updatedAt after", updatedAt2) - assert.True(t, t2.After(t1) || t2.Equal(t1), - "updatedAt must not regress: before=%s after=%s", updatedAt1, updatedAt2) + t1 := parseTS(t, "updatedAt before", updatedAt1) + t2 := parseTS(t, "updatedAt after", updatedAt2) + assert.True(t, t2.After(t1) || t2.Equal(t1), + "updatedAt must not regress: before=%s after=%s", updatedAt1, updatedAt2) + }) } func TestDetector_Tags_EmptyMap_Not_Null(t *testing.T) { diff --git a/services/iam/PARITY.md b/services/iam/PARITY.md index 584e699227..51838d5761 100644 --- a/services/iam/PARITY.md +++ b/services/iam/PARITY.md @@ -16,6 +16,23 @@ overall: A # parity-sweep (2026-09-19): implemented Role Manager (AcquireRole, # console-only-resource seam services/cloudwatchlogs's AddAnomalyInternal and # services/quicksight's AddAppInternal already establish. See ops.AcquireRole # et al and families.role_manager/account_properties below. + # sweep 14 (2026-09-26, items_still_open triage): confirmed the 2026-09-26 + # condition-operator/--enforce-iam fixes (condeval.ArnMatch, net.IP compare, + # aws:SecureTransport, epoch Date, NullIfExists rejection) were already + # live at HEAD with passing enforcement_integration_test.go coverage -- + # removed that items_still_open entry as already-fixed, no new change. + # Fixed ListGroupsForUser/ListServerCertificates/ListServiceSpecificCredentials: + # all 3 hardcoded IsTruncated=false (or, for the credentials op, had no + # IsTruncated/Marker fields at all) despite their real Inputs declaring + # Marker/MaxItems -- ListGroupsForUser/ListServerCertificates had been + # misfiled as a "disclosed structural gap" in sweep 13's note when they are + # the same mechanical pkgs/page gap already fixed elsewhere in this service. + # See the new ops entry and TestListGroupsForUser_ServerCertificates_ + # ServiceSpecificCredentials_Pagination (pagination_gap_whitebox_test.go). + # Consolidated items_still_open from 9 stale/overlapping historical entries + # down to 5 current ones (gopherstack-anjf: items_still_open is the only + # authoritative open list; several entries were pure sweep-6/10/11 history + # already captured by ops: entries above, not live gaps). # sweep 13 (wrapper-key sweep, uncommitted as of this note): fixed # ListAttached{User,Role,Group}Policies dropping PathPrefix/Marker/MaxItems entirely # (silent unfiltered, unpaginated full list) and policyNameFromARN's wrong-separator @@ -117,43 +134,38 @@ ops: GetRoleTemplateVersion: {wire: ok, errors: ok, state: ok, persist: ok, note: "NEW 2026-09-19. Declared errors InvalidInput/NoSuchEntity/ServiceFailure wired. Unspecified MinorVersion resolves to the stored version whose own MinorVersion equals its DefaultMinorVersion (falling back to the highest seeded MinorVersion if the seeded set has none matching -- deterministic, never fabricated, since it's still one of the caller's own seeded versions). Proven via TestRealClient_GetRoleTemplateVersion (found + not-found cases)."} GetAccountProperties: {wire: ok, errors: ok, state: ok, persist: ok, note: "NEW 2026-09-19. No input members; a fresh account returns an empty Properties map, matching real AWS (no documented default properties exist until PutAccountProperties is called -- not fabricating a pre-populated RoleManager entry). Properties>entry>key/value confirmed lowercase against iam@v1.63.0's AccountPropertiesMapType (value.Map(\"key\",\"value\")), distinct from this service's usual PascalCase members. Proven via TestRealClient_AccountProperties."} PutAccountProperties: {wire: fixed, errors: ok, state: ok, persist: ok, note: "NEW 2026-09-19. Real AWS's own two documented structural constraints -- 'the key must contain exactly one / ... and cannot start or end with /' and 'all properties in a single request must belong to the same namespace' -- are both validated for real (InvalidInput on violation); per-property value typing ('boolean properties expect true or false') is NOT enforced since AWS doesn't publish the namespace/property/type registry this backend would need to check it honestly -- disclosed in families.account_properties, not silently accepted as a loosened check (the two constraints AWS DOES document are fully enforced). wire: fixed because PutAccountPropertiesOutput's response requires an empty element the real SDK client's deserializer unconditionally looks for even though the output carries no members -- confirmed against deserializers.go's GetElement(\"PutAccountPropertiesResult\") call, which errors if absent; found via a failing real-client round trip before this element was added. Proven via TestRealClient_AccountProperties, TestRealClient_PutAccountProperties_MixedNamespaceRejected, TestRealClient_PutAccountProperties_MalformedKeyRejected."} + ListGroupsForUser/ListServerCertificates/ListServiceSpecificCredentials: {wire: fixed, errors: ok, state: ok, persist: n/a, note: "FIXED (2026-09-26 parity sweep). All 3 real Inputs (api_op_ListGroupsForUser.go, api_op_ListServerCertificates.go, api_op_ListServiceSpecificCredentials.go) declare Marker/MaxItems; ListGroupsForUser/ListServerCertificates hardcoded IsTruncated=false with no Marker field on the wire at all (misclassified as a structural gap in sweep 13's note -- it is the same mechanical pkgs/page gap already fixed for ListAccessKeys/ListSigningCertificates/ListSSHPublicKeys, not a real structural limit), and ListServiceSpecificCredentials's Result struct had no IsTruncated/Marker fields either. All 3 now return page.Page[T] from the backend (StorageBackend signatures gained marker/maxItems params) and echo Marker/IsTruncated in the response, same page.New(items, marker, maxItems, iamDefaultMaxItems) template as every other paginated List op here. ListServiceSpecificCredentials's AllUsers filter and UserName-optional (defaults to caller identity) remain unimplemented -- gopherstack has no caller-identity plumbing, the same disclosed gap named for AssociateDelegationRequest/ListDelegationRequests's OwnerId filter. Proven via TestListGroupsForUser_ServerCertificates_ServiceSpecificCredentials_Pagination (pagination_gap_whitebox_test.go), a real-SDK-client table test: 3 items each, MaxItems=2 returns 2 with IsTruncated=true and a non-nil Marker, a second call with that Marker returns the remaining 1 with IsTruncated=false/Marker=nil."} invented_ops_removed: - "GetUserPermissionsBoundary / GetRolePermissionsBoundary: not real IAM actions (no api_op_Get{User,Role}PermissionsBoundary.go in the SDK) — permissions-boundary info is returned as a field on GetUser/GetRole (types.User.PermissionsBoundary / types.Role.PermissionsBoundary), which gopherstack already does correctly. Deleted the fabricated duplicate getters, their GetSupportedOperations entries, and updated the 2 tests that called them to assert via GetUser/GetRole instead." - "TagGroup / UntagGroup / ListGroupTags: not real IAM actions — Group is not a taggable resource type in real AWS (aws-sdk-go-v2/service/iam/types.Group has no Tags field, no api_op_{Tag,Untag,ListGroupTags}.go exist). Deleted the fabricated backend methods (InMemoryBackend.TagGroup/UntagGroup), the StorageBackend interface methods, the dispatch entries, the Group.Tags / GroupXML.Tags model fields, and the 4 tests that exercised them." gaps: [] leaks: {status: clean, note: "persistence leaks clean (unchanged); 2 leak classes found+fixed sweep 5 — see DeleteUser/DeleteRole/DeleteGroup/DeleteInstanceProfile ghost-row entries and the Handler-level tag leak entry above. go test -race passes."} items_still_open: - - "aws_iam_security_token_service_preferences (2026-09-24, iam-detective-and-s3-replication terraform - sweep): dropped from test/terraform/fixtures/iam-detective-and-s3-replication.tf after a real - attempt. terraform-provider-aws v5.100.0 fails apply with 'Provider produced - inconsistent result after apply ... root object was present, but now absent', - the identical symptom already recorded for aws_ecr_registry_scanning_configuration - /aws_ecr_replication_configuration in services/ecr/PARITY.md (gopherstack-101r, - 2026-09-19) -- a Put-then-immediate-Read singleton-settings resource pattern - that trips a legacy-SDK/plugin-framework state-consistency check in Terraform - Core itself, not this emulator: SetSecurityTokenServicePreferences and its - read path (GetAccountSummary's GlobalEndpointTokenVersion entry) are already - verified wire-correct (see the SetSecurityTokenServicePreferences ops entry - above). Left out rather than re-chased blind, same reasoning as the ECR entry." - - "2026-09-19 (parity-sweep): PutAccountProperties enforces both AWS-documented - structural key constraints (one '/' separator, no leading/trailing '/', single - namespace per request) but not per-property value typing (e.g. RoleManager's - boolean expectation) -- AWS does not publish the full namespace/property/type - registry needed to check that honestly. AcquireRole's List-type - (StringList/NumberList/ArnList) ReplacementValues join with ',' when substituted - into a string pattern -- AWS does not document the real join format, disclosed as - this backend's own choice. AcquireRole's 'role that matches the template' idempotency - check is by resolved role name only (real AWS doesn't document a finer-grained - match signal either). Role templates have no Create/Put/List/Delete/Enable/Disable - operation anywhere in the pinned SDK -- AddRoleTemplateVersionInternal is the only - way this backend's role-template state is ever populated, a structural (not - fixable) gap matching services/quicksight's AddAppInternal precedent." - - "2026-08-29 constraint-parameter sweep fixed PathPrefix+pagination truncation across ListUsers/ListRoles/ListGroups/ListInstanceProfiles/ListPolicies, and ListPolicies' OnlyAttached/PolicyUsageFilter (see the sweep's own section above for detail). Sweep 13 closed ListAttached{User,Role,Group}Policies' PathPrefix (see its own ops: entry). ListEntitiesForPolicy's EntityFilter/PathPrefix/PolicyUsageFilter/Marker/MaxItems (confirmed present sweep 13, deliberately left open pending a StorageBackend surface change) is now also closed (gopherstack-fjmw, see its own ops: entry -- new PermissionsBoundaryEntities method) -- still open: the pagination-only params on ListMFADevices/ListAccessKeys/ListSigningCertificates/ListSSHPublicKeys/ListServiceSpecificCredentials (not re-checked)." - - "Sweep 13 (wrapper-key sweep, iam+eventbridge scope): field-level enumeration via go/types selector-usage scan doesn't apply to IAM -- it's AWS Query/XML with no request struct types at all (handlers pull vals.Get(\"Key\") directly), unlike eventbridge's JSON *Input structs. Instead re-verified the known filter-after-pagination class (confirmed still fixed for the 5 ops sweep 12's PathPrefix-family header names) and found the same silent-full-list shape one layer over: ListAttached{User,Role,Group}Policies (fixed) and ListEntitiesForPolicy (confirmed, left open) both read PolicyArn/EntityType-only and ignore PathPrefix/PolicyUsageFilter/Marker/MaxItems entirely. Also fixed a wrong-Go-value bug found while writing the ListAttached* regression test: policyNameFromARN split on the wrong separator for any policy with a non-default Path. ListServerCertificates spot-checked clean (PathPrefix read and filtered correctly; no Marker/MaxItems support at all is a disclosed structural gap, not a filter-after-pagination bug -- there's no pagination to cut wrong). ListGroupsForUser spot-checked: hardcodes IsTruncated=false with no Marker/MaxItems read at all -- same disclosed structural gap, not fixed, not this sweep's named scope." - - "This sweep (6) closed both remaining gopherstack-gjp/2sz3 items: (1) comprehensiveBackend's private sync.Mutex is gone — its fields (sshPublicKeys, mfaUserLinks, accessAdvisorJobs, serviceLastAccessed, orgReportJobs) are now guarded by the same coarse b.mu as every other backend map, per the one-coarse-lock convention (.claude/memories/pkgs-catalog.md). Two call sites (GetCredentialReport, ListMFADevicesForUser) previously nested c.mu inside a held b.mu.RLock; DeleteUser's dependency check ran entirely BEFORE taking b.mu, a real TOCTOU window between the SSH-key/MFA-device check and the delete. All three are now single atomic critical sections under b.mu. Snapshot()/Restore() also now read/write comprehensiveBackend state inside the same b.mu section as the rest of backend state, instead of a separate before/after step — Snapshot() gets one consistent point-in-time view (previously the comprehensive-state read and the rest-of-backend read were NOT atomic with each other). Covered by TestComprehensiveBackend_NoDataRace (-race, concurrent workers hitting both comprehensiveBackend and regular backend ops) and TestDeleteUser_SSHKeyConflictIsAtomic. (2) GetAccountAuthorizationDetails now honors Marker/MaxItems/Filter — see the ops entry above." - - "NOT re-verified this sweep (no evidence of a bug found, but not field-diffed line-by-line either): policy simulation (SimulateCustomPolicy/SimulatePrincipalPolicy/evaluator.go), access advisor / service-last-accessed, credential report generation, account summary, condition-key evaluation (conditions.go), resource-policy evaluation (resource_arn.go). These were already marked ok/PROVEN by sweeps 1-4 and no new evidence surfaced against them. (SSH key / signing certificate CRUD -- the other family named in this line as of sweep 9 -- was field-diffed member-by-member in sweep 10: SSH key ops (Upload/Get/List/Update/DeleteSSHPublicKey) all read every serialized member correctly, no bug; signing certificates had a real ownership-bypass bug, now fixed, plus a disclosed pagination gap -- see ops entries above.)" - - "Sweep 10 also confirmed policy evaluation itself (evaluator.go, conditions.go, resource_arn.go) and SimulatePrincipalPolicy/SimulateCustomPolicy remain untouched and out of scope: gopherstack has no real IAM policy evaluator, and building one is explicitly outside this campaign's charter (modelling gap, not a bug)." - - "Sweep 11 closed 3 of this list's named items: ListSigningCertificates' disclosed pagination gap (now fixed, plus a second real gap found in the same area -- sibling ListSSHPublicKeys' response never echoed Marker despite genuinely paginating -- also fixed), and GetDelegationRequest/ListDelegationRequests (both now real, no longer disclosed stubs -- see ops entries above). Access advisor / credential report / account summary (named 'not re-verified since sweep 4' above) is now re-verified: GetCredentialReport/GenerateCredentialReport clean (no bug -- both real inputs are empty, output fields all correct); GetAccountSummary had a real bug, now fixed (fabricated 'SAMLProviders' key, OIDCProviders never surfaced -- see ops entry); GenerateServiceLastAccessedDetails/GetServiceLastAccessedDetails have 2 shadowed-dead-code duplicates now documented (no behavior change, see ops entry) plus a genuine, NOT-fixed disclosed gap: GenerateServiceLastAccessedDetailsInput's optional Granularity (SERVICE_LEVEL|ACTION_LEVEL) is not honored, and GetServiceLastAccessedDetailsInput's Marker/MaxItems are not paginated -- gopherstack's access-advisor backend tracks only per-service data with no per-action tracking and no pagination concept, so ACTION_LEVEL granularity would mean fabricating data gopherstack cannot honestly produce (same invented-capability-is-worse-than-absent line as GetHumanReadableSummary); Marker/MaxItems pagination is mechanical (same page.Page[T] template used everywhere else in this service) but was left out of this sweep's named scope to keep it focused. ListDelegationRequests' real OwnerId filter is also a disclosed, deliberately-unapplied gap (see its ops entry): gopherstack has no caller-identity plumbing to ever populate a stored request's owner identity, the same gap AssociateDelegationRequest already discloses. Condition-key evaluation (conditions.go) and resource-policy evaluation (resource_arn.go) remain NOT re-verified since sweep 4 -- out of this sweep's named scope, no evidence checked either way." + - "aws_iam_security_token_service_preferences (2026-09-24): dropped from the iam-detective-and-s3-replication + terraform fixture -- terraform-provider-aws v5.100.0's Put-then-immediate-Read singleton-settings pattern + trips a state-consistency check in Terraform Core itself (same symptom as services/ecr's + aws_ecr_registry_scanning_configuration, gopherstack-101r), not this emulator; the op itself is already + wire-verified (see SetSecurityTokenServicePreferences ops entry). External tooling issue, not re-chased." + - "Role manager/account properties (2026-09-19): PutAccountProperties enforces AWS's documented structural + key constraints but not per-property value typing (AWS publishes no namespace/property/type registry to + check against); AcquireRole's List-type ReplacementValues join with ',' (AWS doesn't document the real + join format) and its idempotency match is by resolved role name only; role templates have no + Create/Put/List/Delete/Enable/Disable op in the pinned SDK at all (AddRoleTemplateVersionInternal is the + only seam). All disclosed choices, not bugs -- see families.role_manager/account_properties." + - "Policy simulation (SimulateCustomPolicy/SimulatePrincipalPolicy, evaluator.go) has not been field-diffed + since sweep 4, and the top-of-file sdk_module note flags that its response shape changed in SDK v1.57 + (per-resource entries -> aggregated top-level results) with no re-verification since the version bump -- + building a real IAM policy evaluator is out of this campaign's charter regardless (modelling gap)." + - "resource_arn.go (resource-policy evaluation) has not been re-verified since sweep 4; conditions.go + (condition-key evaluation) WAS re-verified and fixed this sweep (2026-09-26, see condeval.ArnMatch/ + net.IP/aws:SecureTransport/epoch-Date/NullIfExists fixes, enforcement_integration_test.go)." + - "Access advisor: GenerateServiceLastAccessedDetailsInput's optional Granularity (SERVICE_LEVEL|ACTION_LEVEL) + is not honored and GetServiceLastAccessedDetailsInput's Marker/MaxItems are not paginated -- the backend + (access_advisor.go) tracks only per-service data with no per-action tracking or pagination concept, so + ACTION_LEVEL would mean fabricating data gopherstack cannot honestly produce (same line as + GetHumanReadableSummary's LLM-content gap); Marker/MaxItems pagination is mechanical but not yet done. + ListDelegationRequests' real OwnerId filter is the same class of gap: no caller-identity plumbing exists + to ever populate a stored request's owner, so the filter is deliberately left unapplied (see its ops entry)." --- ## Notes diff --git a/services/iam/README.md b/services/iam/README.md index 171a220212..e88e94c49e 100644 --- a/services/iam/README.md +++ b/services/iam/README.md @@ -7,22 +7,19 @@ | Metric | Value | | --- | --- | -| PARITY entries audited | 37 (37 ok) | +| PARITY entries audited | 38 (38 ok) | | Feature families | 6 (6 ok) | -| Known gaps | 8 | +| Known gaps | 5 | | Deferred items | 0 | | Resource leaks | clean | ### Known gaps -- "aws_iam_security_token_service_preferences (2026-09-24, iam-detective-and-s3-replication terraform sweep): dropped from test/terraform/fixtures/iam-detective-and-s3-replication.tf after a real attempt. terraform-provider-aws v5.100.0 fails apply with 'Provider produced inconsistent result after apply ... root object was present, but now absent', the identical symptom already recorded for aws_ecr_registry_scanning_configuration /aws_ecr_replication_configuration in services/ecr/PARITY.md (gopherstack-101r, 2026-09-19) -- a Put-then-immediate-Read singleton-settings resource pattern that trips a legacy-SDK/plugin-framework state-consistency check in Terraform Core itself, not this emulator: SetSecurityTokenServicePreferences and its read path (GetAccountSummary's GlobalEndpointTokenVersion entry) are already verified wire-correct (see the SetSecurityTokenServicePreferences ops entry above). Left out rather than re-chased blind, same reasoning as the ECR entry." -- "2026-09-19 (parity-sweep): PutAccountProperties enforces both AWS-documented structural key constraints (one '/' separator, no leading/trailing '/', single namespace per request) but not per-property value typing (e.g. RoleManager's boolean expectation) -- AWS does not publish the full namespace/property/type registry needed to check that honestly. AcquireRole's List-type (StringList/NumberList/ArnList) ReplacementValues join with ',' when substituted into a string pattern -- AWS does not document the real join format, disclosed as this backend's own choice. AcquireRole's 'role that matches the template' idempotency check is by resolved role name only (real AWS doesn't document a finer-grained match signal either). Role templates have no Create/Put/List/Delete/Enable/Disable operation anywhere in the pinned SDK -- AddRoleTemplateVersionInternal is the only way this backend's role-template state is ever populated, a structural (not fixable) gap matching services/quicksight's AddAppInternal precedent." -- 2026-08-29 constraint-parameter sweep fixed PathPrefix+pagination truncation across ListUsers/ListRoles/ListGroups/ListInstanceProfiles/ListPolicies, and ListPolicies' OnlyAttached/PolicyUsageFilter (see the sweep's own section above for detail). Sweep 13 closed ListAttached{User,Role,Group}Policies' PathPrefix (see its own ops: entry). ListEntitiesForPolicy's EntityFilter/PathPrefix/PolicyUsageFilter/Marker/MaxItems (confirmed present sweep 13, deliberately left open pending a StorageBackend surface change) is now also closed (gopherstack-fjmw, see its own ops: entry -- new PermissionsBoundaryEntities method) -- still open: the pagination-only params on ListMFADevices/ListAccessKeys/ListSigningCertificates/ListSSHPublicKeys/ListServiceSpecificCredentials (not re-checked). -- Sweep 13 (wrapper-key sweep, iam+eventbridge scope): field-level enumeration via go/types selector-usage scan doesn't apply to IAM -- it's AWS Query/XML with no request struct types at all (handlers pull vals.Get("Key") directly), unlike eventbridge's JSON *Input structs. Instead re-verified the known filter-after-pagination class (confirmed still fixed for the 5 ops sweep 12's PathPrefix-family header names) and found the same silent-full-list shape one layer over: ListAttached{User,Role,Group}Policies (fixed) and ListEntitiesForPolicy (confirmed, left open) both read PolicyArn/EntityType-only and ignore PathPrefix/PolicyUsageFilter/Marker/MaxItems entirely. Also fixed a wrong-Go-value bug found while writing the ListAttached* regression test: policyNameFromARN split on the wrong separator for any policy with a non-default Path. ListServerCertificates spot-checked clean (PathPrefix read and filtered correctly; no Marker/MaxItems support at all is a disclosed structural gap, not a filter-after-pagination bug -- there's no pagination to cut wrong). ListGroupsForUser spot-checked: hardcodes IsTruncated=false with no Marker/MaxItems read at all -- same disclosed structural gap, not fixed, not this sweep's named scope. -- This sweep (6) closed both remaining gopherstack-gjp/2sz3 items: (1) comprehensiveBackend's private sync.Mutex is gone — its fields (sshPublicKeys, mfaUserLinks, accessAdvisorJobs, serviceLastAccessed, orgReportJobs) are now guarded by the same coarse b.mu as every other backend map, per the one-coarse-lock convention (.claude/memories/pkgs-catalog.md). Two call sites (GetCredentialReport, ListMFADevicesForUser) previously nested c.mu inside a held b.mu.RLock; DeleteUser's dependency check ran entirely BEFORE taking b.mu, a real TOCTOU window between the SSH-key/MFA-device check and the delete. All three are now single atomic critical sections under b.mu. Snapshot()/Restore() also now read/write comprehensiveBackend state inside the same b.mu section as the rest of backend state, instead of a separate before/after step — Snapshot() gets one consistent point-in-time view (previously the comprehensive-state read and the rest-of-backend read were NOT atomic with each other). Covered by TestComprehensiveBackend_NoDataRace (-race, concurrent workers hitting both comprehensiveBackend and regular backend ops) and TestDeleteUser_SSHKeyConflictIsAtomic. (2) GetAccountAuthorizationDetails now honors Marker/MaxItems/Filter — see the ops entry above. -- NOT re-verified this sweep (no evidence of a bug found, but not field-diffed line-by-line either): policy simulation (SimulateCustomPolicy/SimulatePrincipalPolicy/evaluator.go), access advisor / service-last-accessed, credential report generation, account summary, condition-key evaluation (conditions.go), resource-policy evaluation (resource_arn.go). These were already marked ok/PROVEN by sweeps 1-4 and no new evidence surfaced against them. (SSH key / signing certificate CRUD -- the other family named in this line as of sweep 9 -- was field-diffed member-by-member in sweep 10: SSH key ops (Upload/Get/List/Update/DeleteSSHPublicKey) all read every serialized member correctly, no bug; signing certificates had a real ownership-bypass bug, now fixed, plus a disclosed pagination gap -- see ops entries above.) -- Sweep 10 also confirmed policy evaluation itself (evaluator.go, conditions.go, resource_arn.go) and SimulatePrincipalPolicy/SimulateCustomPolicy remain untouched and out of scope: gopherstack has no real IAM policy evaluator, and building one is explicitly outside this campaign's charter (modelling gap, not a bug). -- Sweep 11 closed 3 of this list's named items: ListSigningCertificates' disclosed pagination gap (now fixed, plus a second real gap found in the same area -- sibling ListSSHPublicKeys' response never echoed Marker despite genuinely paginating -- also fixed), and GetDelegationRequest/ListDelegationRequests (both now real, no longer disclosed stubs -- see ops entries above). Access advisor / credential report / account summary (named 'not re-verified since sweep 4' above) is now re-verified: GetCredentialReport/GenerateCredentialReport clean (no bug -- both real inputs are empty, output fields all correct); GetAccountSummary had a real bug, now fixed (fabricated 'SAMLProviders' key, OIDCProviders never surfaced -- see ops entry); GenerateServiceLastAccessedDetails/GetServiceLastAccessedDetails have 2 shadowed-dead-code duplicates now documented (no behavior change, see ops entry) plus a genuine, NOT-fixed disclosed gap: GenerateServiceLastAccessedDetailsInput's optional Granularity (SERVICE_LEVEL|ACTION_LEVEL) is not honored, and GetServiceLastAccessedDetailsInput's Marker/MaxItems are not paginated -- gopherstack's access-advisor backend tracks only per-service data with no per-action tracking and no pagination concept, so ACTION_LEVEL granularity would mean fabricating data gopherstack cannot honestly produce (same invented-capability-is-worse-than-absent line as GetHumanReadableSummary); Marker/MaxItems pagination is mechanical (same page.Page[T] template used everywhere else in this service) but was left out of this sweep's named scope to keep it focused. ListDelegationRequests' real OwnerId filter is also a disclosed, deliberately-unapplied gap (see its ops entry): gopherstack has no caller-identity plumbing to ever populate a stored request's owner identity, the same gap AssociateDelegationRequest already discloses. Condition-key evaluation (conditions.go) and resource-policy evaluation (resource_arn.go) remain NOT re-verified since sweep 4 -- out of this sweep's named scope, no evidence checked either way. +- "aws_iam_security_token_service_preferences (2026-09-24): dropped from the iam-detective-and-s3-replication terraform fixture -- terraform-provider-aws v5.100.0's Put-then-immediate-Read singleton-settings pattern trips a state-consistency check in Terraform Core itself (same symptom as services/ecr's aws_ecr_registry_scanning_configuration, gopherstack-101r), not this emulator; the op itself is already wire-verified (see SetSecurityTokenServicePreferences ops entry). External tooling issue, not re-chased." +- "Role manager/account properties (2026-09-19): PutAccountProperties enforces AWS's documented structural key constraints but not per-property value typing (AWS publishes no namespace/property/type registry to check against); AcquireRole's List-type ReplacementValues join with ',' (AWS doesn't document the real join format) and its idempotency match is by resolved role name only; role templates have no Create/Put/List/Delete/Enable/Disable op in the pinned SDK at all (AddRoleTemplateVersionInternal is the only seam). All disclosed choices, not bugs -- see families.role_manager/account_properties." +- "Policy simulation (SimulateCustomPolicy/SimulatePrincipalPolicy, evaluator.go) has not been field-diffed since sweep 4, and the top-of-file sdk_module note flags that its response shape changed in SDK v1.57 (per-resource entries -> aggregated top-level results) with no re-verification since the version bump -- building a real IAM policy evaluator is out of this campaign's charter regardless (modelling gap)." +- "resource_arn.go (resource-policy evaluation) has not been re-verified since sweep 4; conditions.go (condition-key evaluation) WAS re-verified and fixed this sweep (2026-09-26, see condeval.ArnMatch/ net.IP/aws:SecureTransport/epoch-Date/NullIfExists fixes, enforcement_integration_test.go)." +- "Access advisor: GenerateServiceLastAccessedDetailsInput's optional Granularity (SERVICE_LEVEL|ACTION_LEVEL) is not honored and GetServiceLastAccessedDetailsInput's Marker/MaxItems are not paginated -- the backend (access_advisor.go) tracks only per-service data with no per-action tracking or pagination concept, so ACTION_LEVEL would mean fabricating data gopherstack cannot honestly produce (same line as GetHumanReadableSummary's LLM-content gap); Marker/MaxItems pagination is mechanical but not yet done. ListDelegationRequests' real OwnerId filter is the same class of gap: no caller-identity plumbing exists to ever populate a stored request's owner, so the filter is deliberately left unapplied (see its ops entry)." ## More diff --git a/services/iam/conditions.go b/services/iam/conditions.go index 9eea31bcfc..88cb54387e 100644 --- a/services/iam/conditions.go +++ b/services/iam/conditions.go @@ -8,6 +8,8 @@ import ( "strconv" "strings" "time" + + "github.com/blackbirdworks/gopherstack/pkgs/condeval" ) // ctxKeySourceIP is the IAM condition key for the caller's source IP address. @@ -31,6 +33,9 @@ type ConditionContext struct { SourceIP string `json:"sourceIP,omitempty"` Username string `json:"username,omitempty"` UserID string `json:"userID,omitempty"` + // SecureTransport is "true"/"false", populated from whether the request + // arrived over TLS. Exposed as the aws:SecureTransport condition key. + SecureTransport string `json:"secureTransport,omitempty"` } // conditionMatches returns true if all condition operators in the map are satisfied @@ -95,6 +100,8 @@ func resolveAWSStandardKey(lower string, ctx ConditionContext) (string, bool) { return ctx.PrincipalAccount, true case "aws:requestedregion": return ctx.RequestedRegion, true + case "aws:securetransport": + return ctx.SecureTransport, true case "aws:currenttime": if ctx.CurrentTime != "" { return ctx.CurrentTime, true @@ -172,7 +179,15 @@ func lookupTag(tags map[string]string, key string) (string, bool) { // Returns true if the condition is satisfied. func evalSingleCondition(operator, ctxVal string, condVals []string) bool { // IfExists suffix: if the key is missing (empty), condition is always true. + // AWS docs: IfExists may suffix any operator except Null (Null already + // tests presence), so "nullifexists" is left as an unrecognized operator. + //nolint:lll // AWS doc URL, cannot be split + // https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html#Conditions_IfExists baseOp, ifExists := strings.CutSuffix(operator, "ifexists") + if ifExists && baseOp == "null" { + ifExists = false + baseOp = operator + } if ifExists && ctxVal == "" { return true } @@ -312,10 +327,10 @@ func evalIPARNCondition(baseOp, ctxVal string, condVals []string) (bool, bool) { return !anyIPMatch(ctxVal, condVals), true case "arnequals", "arnlike": - return anyStringLike(strings.ToLower(ctxVal), toLower(condVals)), true + return condeval.AnyArnMatch(condVals, ctxVal, wildcardMatch), true case "arnnotequals", "arnnotlike": - return !anyStringLike(strings.ToLower(ctxVal), toLower(condVals)), true + return !condeval.AnyArnMatch(condVals, ctxVal, wildcardMatch), true } return false, false @@ -337,18 +352,30 @@ func anyStringLike(ctxVal string, condVals []string) bool { return false } -// anyIPMatch returns true if ctxVal is an IP address that falls within any of -// the CIDR ranges (or equals any IP address literal) in condVals. +// anyIPMatch returns true if ctxVal is an IP address (IPv4 or IPv6) that +// falls within any of the CIDR ranges, or equals any bare IP address +// literal, in condVals. A bare literal is treated as its own /32 (or /128 +// for IPv6), per AWS's documented default routing prefix. +// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html#Conditions_IPAddress +// +//nolint:lll // AWS doc URL, cannot be split func anyIPMatch(ctxVal string, condVals []string) bool { ip := net.ParseIP(ctxVal) + if ip == nil { + return false + } for _, v := range condVals { if strings.Contains(v, "/") { _, ipNet, err := net.ParseCIDR(v) - if err == nil && ip != nil && ipNet.Contains(ip) { + if err == nil && ipNet.Contains(ip) { return true } - } else if v == ctxVal { + + continue + } + + if candidate := net.ParseIP(v); candidate != nil && candidate.Equal(ip) { return true } } @@ -419,31 +446,16 @@ func evalDateCondition(baseOp, ctxVal string, condVals []string) (bool, bool) { "datelessthanequals", "dategreaterthan", "dategreaterthanequals": - ctxTime, err := time.Parse(time.RFC3339, ctxVal) - if err != nil { + ctxTime, ok := condeval.ParseDate(ctxVal) + if !ok { return false, true } for _, v := range condVals { - condTime, errParse := time.Parse(time.RFC3339, v) - if errParse != nil { + condTime, okParse := condeval.ParseDate(v) + if !okParse { continue } - match := false - switch baseOp { - case "dateequals": - match = ctxTime.Equal(condTime) - case "datenotequals": - match = !ctxTime.Equal(condTime) - case "datelessthan": - match = ctxTime.Before(condTime) - case "datelessthanequals": - match = ctxTime.Before(condTime) || ctxTime.Equal(condTime) - case "dategreaterthan": - match = ctxTime.After(condTime) - case "dategreaterthanequals": - match = ctxTime.After(condTime) || ctxTime.Equal(condTime) - } - if match { + if condeval.CompareDate(baseOp, ctxTime, condTime) { return true, true } } diff --git a/services/iam/conditions_test.go b/services/iam/conditions_test.go index a479af523a..e9dd5387ba 100644 --- a/services/iam/conditions_test.go +++ b/services/iam/conditions_test.go @@ -304,6 +304,60 @@ func TestEvaluatePolicies_Conditions_NotIpAddress(t *testing.T) { } } +// TestEvaluatePolicies_Conditions_IpAddressIPv6 proves IpAddress supports +// IPv6 CIDR ranges and bare literals, matching AWS's documented IPv6 support. +// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html#Conditions_IPAddress +// +//nolint:lll // AWS doc URL, cannot be split +func TestEvaluatePolicies_Conditions_IpAddressIPv6(t *testing.T) { + t.Parallel() + + policy := `{"Version":"2012-10-17","Statement":[{ + "Effect":"Allow", + "Action":"s3:*", + "Resource":"*", + "Condition":{ + "IpAddress":{"aws:SourceIp":["2001:DB8:1234:5678::/64","203.0.113.7"]} + } + }]}` + + tests := []struct { + ctx iam.ConditionContext + name string + want iam.EvaluationResult + }{ + { + name: "ipv6_in_cidr", + ctx: iam.ConditionContext{SourceIP: "2001:db8:1234:5678::1"}, + want: iam.EvalAllow, + }, + { + name: "ipv6_outside_cidr", + ctx: iam.ConditionContext{SourceIP: "2001:db8:9999::1"}, + want: iam.EvalImplicitDeny, + }, + { + name: "ipv4_bare_literal_default_slash32", + ctx: iam.ConditionContext{SourceIP: "203.0.113.7"}, + want: iam.EvalAllow, + }, + { + name: "ipv4_bare_literal_mismatch", + ctx: iam.ConditionContext{SourceIP: "203.0.113.8"}, + want: iam.EvalImplicitDeny, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := iam.EvaluatePolicies([]string{policy}, "s3:GetObject", "*", tt.ctx) + assert.Equal(t, tt.want, got) + }) + } +} + func TestEvaluatePolicies_Conditions_ArnLike(t *testing.T) { t.Parallel() @@ -343,6 +397,73 @@ func TestEvaluatePolicies_Conditions_ArnLike(t *testing.T) { } } +// TestConditionArnSegmentWise proves ArnEquals/ArnLike compare each of the +// six colon-delimited ARN components separately (rather than one wildcard +// glob over the whole string), and that matching is case-sensitive. +// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html#Conditions_ARN +func TestConditionArnSegmentWise(t *testing.T) { + t.Parallel() + + const key = "aws:sourcearn" + + tests := []condCase{ + { + name: "wildcard_confined_to_last_segment", + operator: "ArnLike", + ctxVal: "arn:aws:sqs:us-east-1:123456789012:my-queue", + condVal: "arn:aws:sqs:us-east-1:123456789012:my-*", + want: true, + }, + { + // A malformed pattern missing a colon must not match by letting + // '*' span the region+account segment boundary, which a naive + // single-string glob (pre-fix) would have allowed. + name: "wildcard_does_not_span_segments", + operator: "ArnLike", + ctxVal: "arn:aws:s3:us-east-1:123456789012:mybucket", + condVal: "arn:aws:s3:*:mybucket", + want: false, + }, + { + name: "wildcard_confined_within_one_segment_still_matches", + operator: "ArnLike", + ctxVal: "arn:aws:iam::123456789012:role/prod/deploy", + condVal: "arn:aws:iam::123456789012:role*", + want: true, + }, + { + name: "region_segment_mismatch_no_match", + operator: "ArnEquals", + ctxVal: "arn:aws:iam::123456789012:role/prod", + condVal: "arn:aws:iam:us-east-1:123456789012:role/prod", + want: false, + }, + { + name: "differing_segment_count_no_match", + operator: "ArnEquals", + ctxVal: "not-an-arn-at-all", + condVal: "arn:aws:iam::123456789012:role/prod", + want: false, + }, + { + name: "case_sensitive_no_match", + operator: "ArnEquals", + ctxVal: "arn:aws:iam::123456789012:role/Prod", + condVal: "arn:aws:iam::123456789012:role/prod", + want: false, + }, + { + name: "arnnotlike_confined_to_segment", + operator: "ArnNotLike", + ctxVal: "arn:aws:sqs:us-east-1:123456789012:my-queue", + condVal: "arn:aws:sqs:us-east-1:123456789012:other-*", + want: true, + }, + } + + runCondCases(t, key, tests) +} + func TestEvaluatePolicies_Conditions_Bool(t *testing.T) { t.Parallel() @@ -374,6 +495,14 @@ func TestEvaluatePolicies_Conditions_Bool(t *testing.T) { }, want: iam.EvalImplicitDeny, }, + { + // SecureTransport is a dedicated ConditionContext field (populated + // by the enforcement middleware from the request's TLS state), + // not just an Extra entry. + name: "secure_transport_dedicated_field", + ctx: iam.ConditionContext{SecureTransport: "true"}, + want: iam.EvalAllow, + }, } for _, tt := range tests { @@ -446,6 +575,27 @@ func TestEvaluatePolicies_Conditions_Null(t *testing.T) { } } +// TestEvaluatePolicies_Conditions_NullIfExistsUnrecognized proves "NullIfExists" +// is not treated as a stripped-suffix alias for Null: AWS documents IfExists +// as valid on any operator except Null (Null already tests key presence), so +// gopherstack's evaluator must not silently accept the combination. +// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html#Conditions_IfExists +// +//nolint:lll // AWS doc URL, cannot be split +func TestEvaluatePolicies_Conditions_NullIfExistsUnrecognized(t *testing.T) { + t.Parallel() + + policy := `{"Version":"2012-10-17","Statement":[{ + "Effect":"Allow", + "Action":"s3:*", + "Resource":"*", + "Condition":{"NullIfExists":{"aws:username":"true"}} + }]}` + + got := iam.EvaluatePolicies([]string{policy}, "s3:GetObject", "*", iam.ConditionContext{}) + assert.Equal(t, iam.EvalImplicitDeny, got, "an unrecognized operator must not match") +} + func TestEvaluatePolicies_Conditions_IfExists(t *testing.T) { t.Parallel() @@ -692,6 +842,20 @@ func TestConditionDateOperators(t *testing.T) { condVal: []any{"bad", mid}, want: true, }, + // AWS accepts epoch (UNIX) seconds interchangeably with ISO 8601: + //nolint:lll // AWS doc URL, cannot be split + // https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html#Conditions_Date + {name: "epoch_ctx_matches_iso_cond", operator: "DateEquals", ctxVal: "1686830400", condVal: mid, want: true}, + {name: "iso_ctx_matches_epoch_cond", operator: "DateEquals", ctxVal: mid, condVal: "1686830400", want: true}, + {name: "epoch_both_sides", operator: "DateLessThan", ctxVal: "1672531200", condVal: "1686830400", want: true}, + { + name: "epoch_fractional_seconds", + operator: "DateEquals", + ctxVal: "1686830400.000", + condVal: mid, + want: true, + }, + {name: "date_only_iso", operator: "DateLessThan", ctxVal: "2023-01-01", condVal: mid, want: true}, } runCondCases(t, key, tests) diff --git a/services/iam/credentials.go b/services/iam/credentials.go index fb754799e2..a8f804bcc9 100644 --- a/services/iam/credentials.go +++ b/services/iam/credentials.go @@ -5,6 +5,7 @@ import ( "sort" "time" + "github.com/blackbirdworks/gopherstack/pkgs/page" "github.com/google/uuid" ) @@ -31,16 +32,16 @@ func (b *InMemoryBackend) ResetServiceSpecificCredentialFull( return cred, nil } -// ListServiceSpecificCredentials returns service-specific credentials for a user. -// If serviceName is non-empty, only credentials for that service are returned. +// ListServiceSpecificCredentials returns a page of a user's service-specific credentials, +// filtered to serviceName when non-empty. func (b *InMemoryBackend) ListServiceSpecificCredentials( - userName, serviceName string, -) ([]ServiceSpecificCredential, error) { + userName, serviceName, marker string, maxItems int, +) (page.Page[ServiceSpecificCredential], error) { b.mu.RLock("ListServiceSpecificCredentials") defer b.mu.RUnlock() if _, exists := b.users.Get(userName); !exists { - return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) + return page.Page[ServiceSpecificCredential]{}, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } result := make([]ServiceSpecificCredential, 0, b.serviceSpecificCreds.Len()) @@ -60,7 +61,7 @@ func (b *InMemoryBackend) ListServiceSpecificCredentials( return result[i].ServiceSpecificCredentialID < result[j].ServiceSpecificCredentialID }) - return result, nil + return page.New(result, marker, maxItems, iamDefaultMaxItems), nil } // DeleteServiceSpecificCredential deletes a service-specific credential. diff --git a/services/iam/credentials_test.go b/services/iam/credentials_test.go index 271aa4cc3f..2da2cae02e 100644 --- a/services/iam/credentials_test.go +++ b/services/iam/credentials_test.go @@ -86,10 +86,10 @@ func TestServiceSpecificCredential_UpdateToInactive(t *testing.T) { "ssc-update-user", cred.ServiceSpecificCredentialID, "Inactive", )) - creds, err := b.ListServiceSpecificCredentials("ssc-update-user", "") + p, err := b.ListServiceSpecificCredentials("ssc-update-user", "", "", 0) require.NoError(t, err) - require.Len(t, creds, 1) - assert.Equal(t, "Inactive", creds[0].Status) + require.Len(t, p.Data, 1) + assert.Equal(t, "Inactive", p.Data[0].Status) } func TestServiceSpecificCredential_DeleteRemoves(t *testing.T) { @@ -103,9 +103,9 @@ func TestServiceSpecificCredential_DeleteRemoves(t *testing.T) { require.NoError(t, b.DeleteServiceSpecificCredential("ssc-del-user", cred.ServiceSpecificCredentialID)) - creds, err := b.ListServiceSpecificCredentials("ssc-del-user", "") + p, err := b.ListServiceSpecificCredentials("ssc-del-user", "", "", 0) require.NoError(t, err) - assert.Empty(t, creds) + assert.Empty(t, p.Data) } func TestServiceSpecificCredential_UniqueIDs(t *testing.T) { diff --git a/services/iam/enforcement_integration_test.go b/services/iam/enforcement_integration_test.go index 9885a96b43..b3e00d0241 100644 --- a/services/iam/enforcement_integration_test.go +++ b/services/iam/enforcement_integration_test.go @@ -25,7 +25,7 @@ import ( func setupEnforcementTestServer( t *testing.T, backend *mockEnforcementBackend, -) (*httptest.Server, *mockEnforcementBackend) { +) *httptest.Server { t.Helper() e := echo.New() @@ -75,7 +75,7 @@ func setupEnforcementTestServer( srv := httptest.NewServer(e) t.Cleanup(srv.Close) - return srv, backend + return srv } func createTestS3Client( @@ -119,6 +119,124 @@ func createTestDynamoDBClient( }), nil } +// TestEnforcement_ConditionOperators_SDKIntegration drives the real +// EnforcementMiddleware with a typed S3 client to prove the IpAddress and +// Date condition operators are enforced end to end. httptest.Server +// connections originate from loopback, so aws:SourceIp always resolves to +// 127.0.0.1 here; the date conditions use a fixed reference far in the past +// so the outcome does not depend on when the test runs (no time.Sleep, no +// clock injection needed). +func TestEnforcement_ConditionOperators_SDKIntegration(t *testing.T) { + t.Parallel() + + const farPast = "2020-01-01T00:00:00Z" + + tests := []struct { + policy map[string]any + name string + expectAllowed bool + }{ + { + name: "allowed_when_source_ip_in_loopback_cidr", + policy: map[string]any{ + "Version": "2012-10-17", + "Statement": []map[string]any{{ + "Effect": "Allow", + "Action": []string{"s3:PutObject"}, + "Resource": []string{"arn:aws:s3:::allowed-bucket/*"}, + "Condition": map[string]any{ + "IpAddress": map[string]any{"aws:SourceIp": "127.0.0.1/32"}, + }, + }}, + }, + expectAllowed: true, + }, + { + name: "denied_when_source_ip_outside_cidr", + policy: map[string]any{ + "Version": "2012-10-17", + "Statement": []map[string]any{{ + "Effect": "Allow", + "Action": []string{"s3:PutObject"}, + "Resource": []string{"arn:aws:s3:::allowed-bucket/*"}, + "Condition": map[string]any{ + "IpAddress": map[string]any{"aws:SourceIp": "10.0.0.0/8"}, + }, + }}, + }, + expectAllowed: false, + }, + { + name: "allowed_when_date_less_than_still_in_future", + policy: map[string]any{ + "Version": "2012-10-17", + "Statement": []map[string]any{{ + "Effect": "Allow", + "Action": []string{"s3:PutObject"}, + "Resource": []string{"arn:aws:s3:::allowed-bucket/*"}, + "Condition": map[string]any{ + "DateGreaterThan": map[string]any{"aws:CurrentTime": farPast}, + }, + }}, + }, + expectAllowed: true, + }, + { + name: "denied_when_date_less_than_condition_cannot_hold", + policy: map[string]any{ + "Version": "2012-10-17", + "Statement": []map[string]any{{ + "Effect": "Allow", + "Action": []string{"s3:PutObject"}, + "Resource": []string{"arn:aws:s3:::allowed-bucket/*"}, + "Condition": map[string]any{ + "DateLessThan": map[string]any{"aws:CurrentTime": farPast}, + }, + }}, + }, + expectAllowed: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + policyBytes, marshalErr := json.Marshal(tt.policy) + require.NoError(t, marshalErr) + + const ( + accessKeyID = "AKIACONDITIONUSER" + userName = "condition-user" + ) + + backend := newMockEnforcementBackend() + backend.users[userName] = &iam.User{ + UserName: userName, + Arn: "arn:aws:iam::000000000000:user/" + userName, + } + backend.keyMap[accessKeyID] = userName + backend.policies[userName] = []string{string(policyBytes)} + + srv := setupEnforcementTestServer(t, backend) + + client, err := createTestS3Client(t.Context(), srv.URL, accessKeyID, "secret", "") + require.NoError(t, err) + + _, err = client.PutObject(t.Context(), &s3sdk.PutObjectInput{ + Bucket: aws.String("allowed-bucket"), + Key: aws.String("data.json"), + }) + + if tt.expectAllowed { + assert.NoError(t, err) + } else { + assert.Error(t, err) + } + }) + } +} + func TestEnforcement_MultiServiceSDKIntegration(t *testing.T) { t.Parallel() @@ -240,7 +358,7 @@ func TestEnforcement_MultiServiceSDKIntegration(t *testing.T) { backend.keyMap[tt.accessKeyID] = tt.userName backend.policies[tt.userName] = tt.policies - srv, _ := setupEnforcementTestServer(t, backend) + srv := setupEnforcementTestServer(t, backend) tt.runTest(t.Context(), t, srv.URL) }) diff --git a/services/iam/groups.go b/services/iam/groups.go index 092b242c9f..f65f13e2af 100644 --- a/services/iam/groups.go +++ b/services/iam/groups.go @@ -341,13 +341,13 @@ func (b *InMemoryBackend) purgeGroupsLocked(cutoff time.Time) { } } -// ListGroupsForUser returns all groups that the specified user belongs to. -func (b *InMemoryBackend) ListGroupsForUser(userName string) ([]Group, error) { +// ListGroupsForUser returns a paginated list of groups the specified user belongs to. +func (b *InMemoryBackend) ListGroupsForUser(userName, marker string, maxItems int) (page.Page[Group], error) { b.mu.RLock("ListGroupsForUser") defer b.mu.RUnlock() if _, exists := b.users.Get(userName); !exists { - return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) + return page.Page[Group]{}, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } result := make([]Group, 0, len(b.groupMembers)) @@ -361,7 +361,7 @@ func (b *InMemoryBackend) ListGroupsForUser(userName string) ([]Group, error) { sort.Slice(result, func(i, j int) bool { return result[i].GroupName < result[j].GroupName }) - return result, nil + return page.New(result, marker, maxItems, iamDefaultMaxItems), nil } // UpdateGroup renames a group and/or updates its path. diff --git a/services/iam/groups_test.go b/services/iam/groups_test.go index a94d906cae..b22c33a7e4 100644 --- a/services/iam/groups_test.go +++ b/services/iam/groups_test.go @@ -362,7 +362,7 @@ func TestListGroupsForUser(t *testing.T) { b := iam.NewInMemoryBackend() tt.setup(b) - groups, err := b.ListGroupsForUser(tt.userName) + p, err := b.ListGroupsForUser(tt.userName, "", 0) if tt.wantErr { require.Error(t, err) @@ -370,7 +370,7 @@ func TestListGroupsForUser(t *testing.T) { } require.NoError(t, err) - assert.Len(t, groups, tt.wantGroups) + assert.Len(t, p.Data, tt.wantGroups) }) } } diff --git a/services/iam/handler_credentials.go b/services/iam/handler_credentials.go index 0ab2cab800..c355c5d409 100644 --- a/services/iam/handler_credentials.go +++ b/services/iam/handler_credentials.go @@ -53,16 +53,17 @@ func (h *Handler) iamSSCResetDispatch() map[string]iamActionFn { func (h *Handler) iamServiceSpecificCredDispatch() map[string]iamActionFn { return map[string]iamActionFn{ "ListServiceSpecificCredentials": func(vals url.Values, reqID string) (any, error) { - creds, err := h.Backend.ListServiceSpecificCredentials( + p, err := h.Backend.ListServiceSpecificCredentials( vals.Get("UserName"), vals.Get("ServiceName"), + vals.Get("Marker"), parseMaxItems(vals.Get("MaxItems")), ) if err != nil { return nil, err } - xmlCreds := make([]ServiceSpecificCredentialMetadataXML, 0, len(creds)) - for i := range creds { - c := &creds[i] + xmlCreds := make([]ServiceSpecificCredentialMetadataXML, 0, len(p.Data)) + for i := range p.Data { + c := &p.Data[i] xmlCreds = append(xmlCreds, ServiceSpecificCredentialMetadataXML{ UserName: c.UserName, ServiceName: c.ServiceName, @@ -77,6 +78,8 @@ func (h *Handler) iamServiceSpecificCredDispatch() map[string]iamActionFn { Xmlns: iamXMLNS, Result: ListServiceSpecificCredentialsResult{ ServiceSpecificCredentials: xmlCreds, + IsTruncated: p.Next != "", + Marker: p.Next, }, Meta: ResponseMetadata{RequestID: reqID}, }, nil diff --git a/services/iam/handler_credentials_test.go b/services/iam/handler_credentials_test.go index d74ac5019e..10b0dd229c 100644 --- a/services/iam/handler_credentials_test.go +++ b/services/iam/handler_credentials_test.go @@ -93,9 +93,9 @@ func TestHandler_ServiceSpecificCredential_UpdateStatus(t *testing.T) { require.NoError(t, h.Handler()(e.NewContext(req, rec))) assert.Equal(t, http.StatusOK, rec.Code) - creds, _ := b.ListServiceSpecificCredentials("ssc-update-status-user", "") - require.Len(t, creds, 1) - assert.Equal(t, "Inactive", creds[0].Status) + p, _ := b.ListServiceSpecificCredentials("ssc-update-status-user", "", "", 0) + require.Len(t, p.Data, 1) + assert.Equal(t, "Inactive", p.Data[0].Status) } func TestHandler_ResetServiceSpecificCredential_ChangesPassword(t *testing.T) { diff --git a/services/iam/handler_groups.go b/services/iam/handler_groups.go index 134a877717..c872641d89 100644 --- a/services/iam/handler_groups.go +++ b/services/iam/handler_groups.go @@ -162,14 +162,16 @@ func toGroupDetailXML(g GroupDetail) GroupDetailXML { func (h *Handler) iamGroupRefinementDispatch() map[string]iamActionFn { return map[string]iamActionFn{ "ListGroupsForUser": func(vals url.Values, reqID string) (any, error) { - groups, err := h.Backend.ListGroupsForUser(vals.Get("UserName")) + p, err := h.Backend.ListGroupsForUser( + vals.Get("UserName"), vals.Get("Marker"), parseMaxItems(vals.Get("MaxItems")), + ) if err != nil { return nil, err } - xmlGroups := make([]ListGroupsForUserXML, 0, len(groups)) - for i := range groups { - g := &groups[i] + xmlGroups := make([]ListGroupsForUserXML, 0, len(p.Data)) + for i := range p.Data { + g := &p.Data[i] xmlGroups = append(xmlGroups, ListGroupsForUserXML{ GroupName: g.GroupName, GroupID: g.GroupID, @@ -183,7 +185,8 @@ func (h *Handler) iamGroupRefinementDispatch() map[string]iamActionFn { Xmlns: iamXMLNS, ListGroupsForUserResult: ListGroupsForUserResult{ Groups: xmlGroups, - IsTruncated: false, + IsTruncated: p.Next != "", + Marker: p.Next, }, ResponseMetadata: ResponseMetadata{RequestID: reqID}, }, nil diff --git a/services/iam/handler_server_certificates.go b/services/iam/handler_server_certificates.go index c7fd03005c..c37e4629b3 100644 --- a/services/iam/handler_server_certificates.go +++ b/services/iam/handler_server_certificates.go @@ -23,13 +23,15 @@ func looksLikePEMPrivateKey(s string) bool { func (h *Handler) iamServerCertReadDispatch() map[string]iamActionFn { return map[string]iamActionFn{ "ListServerCertificates": func(vals url.Values, reqID string) (any, error) { - certs, err := h.Backend.ListServerCertificates(vals.Get("PathPrefix")) + p, err := h.Backend.ListServerCertificates( + vals.Get("PathPrefix"), vals.Get("Marker"), parseMaxItems(vals.Get("MaxItems")), + ) if err != nil { return nil, err } - members := make([]serverCertMetaXML, 0, len(certs)) - for _, c := range certs { + members := make([]serverCertMetaXML, 0, len(p.Data)) + for _, c := range p.Data { members = append(members, serverCertMetaXML{ ServerCertificateName: c.ServerCertificateName, ServerCertificateID: c.ServerCertificateID, @@ -44,7 +46,8 @@ func (h *Handler) iamServerCertReadDispatch() map[string]iamActionFn { Xmlns: iamXMLNS, ListServerCertificatesResult: listServerCertificatesResult{ ServerCertificateMetadataList: members, - IsTruncated: false, + IsTruncated: p.Next != "", + Marker: p.Next, }, ResponseMetadata: ResponseMetadata{RequestID: reqID}, }, nil diff --git a/services/iam/middleware.go b/services/iam/middleware.go index cd20e48484..56d18f1065 100644 --- a/services/iam/middleware.go +++ b/services/iam/middleware.go @@ -167,6 +167,7 @@ func buildPrincipalConditionContext( Username: principal.SessionName, UserID: principal.UserID, SourceIP: extractClientIP(r), + SecureTransport: secureTransportValue(r), } } @@ -467,9 +468,27 @@ func buildConditionContext(r *http.Request, user *User) ConditionContext { Username: user.UserName, UserID: user.UserID, PrincipalTags: user.Tags, + SecureTransport: secureTransportValue(r), } } +// secureTransportValue reports whether the request arrived over TLS, as the +// aws:SecureTransport condition key expects ("true"/"false"). Checks r.TLS +// directly (gopherstack terminating TLS itself) and X-Forwarded-Proto (behind +// a reverse proxy), the same two signals services/sqs's CreateQueue already +// uses to pick a scheme for QueueURL. +func secureTransportValue(r *http.Request) string { + if r.TLS != nil { + return credTrue + } + + if strings.EqualFold(r.Header.Get("X-Forwarded-Proto"), "https") { + return credTrue + } + + return "false" +} + // extractClientIP returns the IP address of the client without the port. func extractClientIP(r *http.Request) string { // Prefer X-Forwarded-For when behind a proxy. diff --git a/services/iam/middleware_test.go b/services/iam/middleware_test.go index f38c5a4f5f..7cdae040fd 100644 --- a/services/iam/middleware_test.go +++ b/services/iam/middleware_test.go @@ -257,6 +257,41 @@ func TestEnforcementMiddleware(t *testing.T) { }, wantStatus: http.StatusOK, }, + { + // httptest.NewRequest never sets req.TLS, so aws:SecureTransport + // can only resolve true here via the X-Forwarded-Proto signal. + name: "condition_secure_transport_via_forwarded_proto", + setupBackend: func(b *mockEnforcementBackend) { + policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*",` + + `"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}}]}` + b.users["alice"] = &iam.User{UserName: "alice"} + b.keyMap["AKIATLS1"] = "alice" + b.policies["alice"] = []string{policy} + }, + requestPath: "/my-bucket/key", + requestMethod: http.MethodGet, + headers: map[string]string{ + "Authorization": "AWS4-HMAC-SHA256 Credential=AKIATLS1/20230101/us-east-1/s3/aws4_request", + "X-Forwarded-Proto": "https", + }, + wantStatus: http.StatusOK, + }, + { + name: "condition_secure_transport_plain_http_denied", + setupBackend: func(b *mockEnforcementBackend) { + policy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:*",` + + `"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}}]}` + b.users["alice"] = &iam.User{UserName: "alice"} + b.keyMap["AKIATLS2"] = "alice" + b.policies["alice"] = []string{policy} + }, + requestPath: "/my-bucket/key", + requestMethod: http.MethodGet, + headers: map[string]string{ + "Authorization": "AWS4-HMAC-SHA256 Credential=AKIATLS2/20230101/us-east-1/s3/aws4_request", + }, + wantStatus: http.StatusForbidden, + }, } for _, tt := range tests { diff --git a/services/iam/models_credentials.go b/services/iam/models_credentials.go index 204311744f..dc0726c52c 100644 --- a/services/iam/models_credentials.go +++ b/services/iam/models_credentials.go @@ -54,7 +54,9 @@ type ServiceSpecificCredentialMetadataXML struct { // ListServiceSpecificCredentialsResult contains the list of credentials. type ListServiceSpecificCredentialsResult struct { + Marker string `xml:"Marker,omitempty"` ServiceSpecificCredentials []ServiceSpecificCredentialMetadataXML `xml:"ServiceSpecificCredentials>member"` + IsTruncated bool `xml:"IsTruncated"` } // ListServiceSpecificCredentialsResponse is the XML response for ListServiceSpecificCredentials. diff --git a/services/iam/models_groups.go b/services/iam/models_groups.go index eb160573cc..c0e1066d7d 100644 --- a/services/iam/models_groups.go +++ b/services/iam/models_groups.go @@ -15,6 +15,7 @@ type ListGroupsForUserXML struct { // ListGroupsForUserResult contains the list of groups. type ListGroupsForUserResult struct { + Marker string `xml:"Marker,omitempty"` Groups []ListGroupsForUserXML `xml:"Groups>member"` IsTruncated bool `xml:"IsTruncated"` } diff --git a/services/iam/models_server_certificates.go b/services/iam/models_server_certificates.go index c4e6c0bb1d..cf5468a299 100644 --- a/services/iam/models_server_certificates.go +++ b/services/iam/models_server_certificates.go @@ -21,6 +21,7 @@ type serverCertXML struct { // listServerCertificatesResult contains the list of server certificates. type listServerCertificatesResult struct { + Marker string `xml:"Marker,omitempty"` ServerCertificateMetadataList []serverCertMetaXML `xml:"ServerCertificateMetadataList>member"` IsTruncated bool `xml:"IsTruncated"` } diff --git a/services/iam/pagination_gap_whitebox_test.go b/services/iam/pagination_gap_whitebox_test.go new file mode 100644 index 0000000000..4f81e93351 --- /dev/null +++ b/services/iam/pagination_gap_whitebox_test.go @@ -0,0 +1,124 @@ +package iam + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + iamsdk "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// paginationPage is one page's worth of assertable state, shared by the +// table below so each op's run func doesn't need its own huge return tuple. +type paginationPage struct { + marker *string + length int + isTruncated bool +} + +// TestListGroupsForUser_ServerCertificates_ServiceSpecificCredentials_Pagination checks +// Marker/MaxItems paging for three ops that previously returned every item. +func TestListGroupsForUser_ServerCertificates_ServiceSpecificCredentials_Pagination(t *testing.T) { + t.Parallel() + + tests := []struct { + setup func(*testing.T, *InMemoryBackend) + run func(*testing.T, *iamsdk.Client, *string) paginationPage + name string + }{ + { + name: "ListGroupsForUser", + setup: func(t *testing.T, b *InMemoryBackend) { + t.Helper() + _, _ = b.CreateUser("carol", "/", "") + for _, name := range []string{"g1", "g2", "g3"} { + _, err := b.CreateGroup(name, "/") + require.NoError(t, err) + require.NoError(t, b.AddUserToGroup(name, "carol")) + } + }, + run: func(t *testing.T, client *iamsdk.Client, marker *string) paginationPage { + t.Helper() + out, err := client.ListGroupsForUser(t.Context(), &iamsdk.ListGroupsForUserInput{ + UserName: aws.String("carol"), MaxItems: aws.Int32(2), Marker: marker, + }) + require.NoError(t, err) + + return paginationPage{length: len(out.Groups), isTruncated: out.IsTruncated, marker: out.Marker} + }, + }, + { + name: "ListServerCertificates", + setup: func(t *testing.T, b *InMemoryBackend) { + t.Helper() + for _, name := range []string{"cert1", "cert2", "cert3"} { + _, err := b.UploadServerCertificate(name, "/", "body", "") + require.NoError(t, err) + } + }, + run: func(t *testing.T, client *iamsdk.Client, marker *string) paginationPage { + t.Helper() + out, err := client.ListServerCertificates(t.Context(), &iamsdk.ListServerCertificatesInput{ + MaxItems: aws.Int32(2), Marker: marker, + }) + require.NoError(t, err) + + return paginationPage{ + length: len(out.ServerCertificateMetadataList), + isTruncated: out.IsTruncated, + marker: out.Marker, + } + }, + }, + { + name: "ListServiceSpecificCredentials", + setup: func(t *testing.T, b *InMemoryBackend) { + t.Helper() + _, _ = b.CreateUser("dave-ssc", "/", "") + for range 3 { + _, err := b.CreateServiceSpecificCredential("dave-ssc", "codecommit.amazonaws.com") + require.NoError(t, err) + } + }, + run: func(t *testing.T, client *iamsdk.Client, marker *string) paginationPage { + t.Helper() + out, err := client.ListServiceSpecificCredentials( + t.Context(), + &iamsdk.ListServiceSpecificCredentialsInput{ + UserName: aws.String("dave-ssc"), MaxItems: aws.Int32(2), Marker: marker, + }, + ) + require.NoError(t, err) + + return paginationPage{ + length: len(out.ServiceSpecificCredentials), + isTruncated: out.IsTruncated, + marker: out.Marker, + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := NewInMemoryBackend() + h := NewHandler(b) + client := newSigningCertTestClient(t, h) + tt.setup(t, b) + + page1 := tt.run(t, client, nil) + assert.Equal(t, 2, page1.length) + assert.True(t, page1.isTruncated) + require.NotNil(t, page1.marker) + assert.NotEmpty(t, *page1.marker) + + page2 := tt.run(t, client, page1.marker) + assert.Equal(t, 1, page2.length) + assert.False(t, page2.isTruncated) + assert.Nil(t, page2.marker) + }) + } +} diff --git a/services/iam/persistence_test.go b/services/iam/persistence_test.go index 2e73ae6229..e89711c833 100644 --- a/services/iam/persistence_test.go +++ b/services/iam/persistence_test.go @@ -226,10 +226,10 @@ func TestInMemoryBackend_FullStateSnapshotRestore(t *testing.T) { require.NoError(t, err) assert.Equal(t, "alice", gotLP.UserName) - creds, err := fresh.ListServiceSpecificCredentials("alice", "") + creds, err := fresh.ListServiceSpecificCredentials("alice", "", "", 0) require.NoError(t, err) - require.Len(t, creds, 1) - assert.Equal(t, cred.ServiceSpecificCredentialID, creds[0].ServiceSpecificCredentialID) + require.Len(t, creds.Data, 1) + assert.Equal(t, cred.ServiceSpecificCredentialID, creds.Data[0].ServiceSpecificCredentialID) mfaDevices, err := fresh.ListVirtualMFADevices("", 0) require.NoError(t, err) @@ -240,9 +240,9 @@ func TestInMemoryBackend_FullStateSnapshotRestore(t *testing.T) { require.NoError(t, err) assert.Len(t, signingCerts.Data, 1) - serverCerts, err := fresh.ListServerCertificates("") + serverCerts, err := fresh.ListServerCertificates("", "", 0) require.NoError(t, err) - assert.Len(t, serverCerts, 1) + assert.Len(t, serverCerts.Data, 1) require.NoError(t, fresh.AcceptDelegationRequest(delegation.DelegationID)) diff --git a/services/iam/policies.go b/services/iam/policies.go index 4eb002fa02..88b8b5a823 100644 --- a/services/iam/policies.go +++ b/services/iam/policies.go @@ -160,13 +160,27 @@ func (b *InMemoryBackend) ListPolicies(marker string, maxItems int) (page.Page[P return pageFromSortedNames( b.sortedPolicyNames, - b.policies.Get, + b.clonePolicyLocked, marker, maxItems, iamDefaultMaxItems, ), nil } +// clonePolicyLocked looks up a policy by name and returns a copy with its own +// Tags map, so ListPolicies cannot alias TagPolicy/UntagPolicy's in-place writes. +func (b *InMemoryBackend) clonePolicyLocked(policyName string) (*Policy, bool) { + p, exists := b.policies.Get(policyName) + if !exists { + return nil, false + } + + cp := *p + cp.Tags = maps.Clone(p.Tags) + + return &cp, true +} + // AttachUserPolicy attaches a policy to a user. func (b *InMemoryBackend) AttachUserPolicy(userName, policyArn string) error { b.mu.Lock("AttachUserPolicy") @@ -277,6 +291,8 @@ func (b *InMemoryBackend) GetPolicy(policyArn string) (*Policy, error) { return nil, fmt.Errorf("%w: policy %q not found", ErrPolicyNotFound, policyArn) } + pol.Tags = maps.Clone(pol.Tags) + return &pol, nil } diff --git a/services/iam/roles.go b/services/iam/roles.go index 5790a08cdd..562d5fa8bd 100644 --- a/services/iam/roles.go +++ b/services/iam/roles.go @@ -100,13 +100,27 @@ func (b *InMemoryBackend) ListRoles(marker string, maxItems int) (page.Page[Role return pageFromSortedNames( b.sortedRoleNames, - b.roles.Get, + b.cloneRoleLocked, marker, maxItems, iamDefaultMaxItems, ), nil } +// cloneRoleLocked looks up a role by name and returns a copy with its own +// Tags map, so ListRoles cannot alias TagRole/UntagRole's in-place writes. +func (b *InMemoryBackend) cloneRoleLocked(roleName string) (*Role, bool) { + r, exists := b.roles.Get(roleName) + if !exists { + return nil, false + } + + cp := *r + cp.Tags = maps.Clone(r.Tags) + + return &cp, true +} + // GetRole retrieves a single IAM role by name. func (b *InMemoryBackend) GetRole(roleName string) (*Role, error) { b.mu.RLock("GetRole") @@ -117,7 +131,10 @@ func (b *InMemoryBackend) GetRole(roleName string) (*Role, error) { return nil, fmt.Errorf("%w: role %q not found", ErrRoleNotFound, roleName) } - return r, nil + cp := *r + cp.Tags = maps.Clone(r.Tags) + + return &cp, nil } // GetRoleByArn retrieves a single IAM role by its full ARN. @@ -135,7 +152,10 @@ func (b *InMemoryBackend) GetRoleByArn(roleArn string) (*Role, error) { return nil, fmt.Errorf("%w: role with ARN %q not found", ErrRoleNotFound, roleArn) } - return role, nil + cp := *role + cp.Tags = maps.Clone(role.Tags) + + return &cp, nil } // UpdateRoleMaxSessionDuration sets the maximum session duration for a role. diff --git a/services/iam/server_cert_test.go b/services/iam/server_cert_test.go index 6d7744a4ac..84839e15a6 100644 --- a/services/iam/server_cert_test.go +++ b/services/iam/server_cert_test.go @@ -81,14 +81,14 @@ func TestServerCertificate_CRUD(t *testing.T) { _, err = b.UploadServerCertificate("list-cert-b", "/prod/", certBody, "") require.NoError(t, err) - all, err := b.ListServerCertificates("") + all, err := b.ListServerCertificates("", "", 0) require.NoError(t, err) - assert.Len(t, all, 2) + assert.Len(t, all.Data, 2) - prod, err := b.ListServerCertificates("/prod/") + prod, err := b.ListServerCertificates("/prod/", "", 0) require.NoError(t, err) - assert.Len(t, prod, 1) - assert.Equal(t, "list-cert-b", prod[0].ServerCertificateName) + assert.Len(t, prod.Data, 1) + assert.Equal(t, "list-cert-b", prod.Data[0].ServerCertificateName) }) t.Run("Update", func(t *testing.T) { @@ -270,9 +270,9 @@ func TestUploadServerCertificate_ListReflectsUpload(t *testing.T) { _, _ = b.UploadServerCertificate("cert-list-1", "/", "body", "") _, _ = b.UploadServerCertificate("cert-list-2", "/", "body", "") - certs, err := b.ListServerCertificates("/") + certs, err := b.ListServerCertificates("/", "", 0) require.NoError(t, err) - assert.Len(t, certs, 2) + assert.Len(t, certs.Data, 2) } func TestServerCertificate_CRUDRoundTrip(t *testing.T) { @@ -289,9 +289,9 @@ func TestServerCertificate_CRUDRoundTrip(t *testing.T) { require.NoError(t, err) assert.Equal(t, certBody, got.CertificateBody) - certs, err := b.ListServerCertificates("/") + certs, err := b.ListServerCertificates("/", "", 0) require.NoError(t, err) - assert.Len(t, certs, 1) + assert.Len(t, certs.Data, 1) require.NoError(t, b.UpdateServerCertificate("MyCert", "NewName", "/new/")) diff --git a/services/iam/server_certificates.go b/services/iam/server_certificates.go index caac14380a..06101585f8 100644 --- a/services/iam/server_certificates.go +++ b/services/iam/server_certificates.go @@ -7,6 +7,8 @@ import ( "sort" "strings" "time" + + "github.com/blackbirdworks/gopherstack/pkgs/page" ) // serverCertIDPrefix is the AWS-style prefix for server certificate IDs. @@ -23,7 +25,9 @@ func newServerCertID() string { } // UploadServerCertificate stores a new server certificate. -func (b *InMemoryBackend) UploadServerCertificate(name, path, certBody, certChain string) (*ServerCertificate, error) { +func (b *InMemoryBackend) UploadServerCertificate( + name, path, certBody, certChain string, +) (*ServerCertificate, error) { b.mu.Lock("UploadServerCertificate") defer b.mu.Unlock() @@ -36,7 +40,11 @@ func (b *InMemoryBackend) UploadServerCertificate(name, path, certBody, certChai } if _, exists := b.serverCertificates.Get(name); exists { - return nil, fmt.Errorf("%w: server certificate %q already exists", ErrUserAlreadyExists, name) + return nil, fmt.Errorf( + "%w: server certificate %q already exists", + ErrUserAlreadyExists, + name, + ) } normalizedPath := normPath(path) @@ -68,8 +76,11 @@ func (b *InMemoryBackend) GetServerCertificate(name string) (*ServerCertificate, return cert, nil } -// ListServerCertificates returns server certificates, filtered by path prefix if non-empty. -func (b *InMemoryBackend) ListServerCertificates(pathPrefix string) ([]ServerCertificate, error) { +// ListServerCertificates returns a paginated list of server certificates, filtered by path prefix if non-empty. +func (b *InMemoryBackend) ListServerCertificates( + pathPrefix, marker string, + maxItems int, +) (page.Page[ServerCertificate], error) { b.mu.RLock("ListServerCertificates") defer b.mu.RUnlock() @@ -85,7 +96,7 @@ func (b *InMemoryBackend) ListServerCertificates(pathPrefix string) ([]ServerCer return result[i].ServerCertificateName < result[j].ServerCertificateName }) - return result, nil + return page.New(result, marker, maxItems, iamDefaultMaxItems), nil } // UpdateServerCertificate renames a server certificate and/or changes its path. @@ -100,7 +111,11 @@ func (b *InMemoryBackend) UpdateServerCertificate(name, newName, newPath string) if newName != "" && newName != name { if _, nameExists := b.serverCertificates.Get(newName); nameExists { - return fmt.Errorf("%w: server certificate %q already exists", ErrUserAlreadyExists, newName) + return fmt.Errorf( + "%w: server certificate %q already exists", + ErrUserAlreadyExists, + newName, + ) } // serverCertificates is keyed by ServerCertificateName, which is diff --git a/services/iam/store.go b/services/iam/store.go index 075ad69460..869567561e 100644 --- a/services/iam/store.go +++ b/services/iam/store.go @@ -174,8 +174,8 @@ type StorageBackend interface { userName, serviceName string, ) (*ServiceSpecificCredential, error) ListServiceSpecificCredentials( - userName, serviceName string, - ) ([]ServiceSpecificCredential, error) + userName, serviceName, marker string, maxItems int, + ) (page.Page[ServiceSpecificCredential], error) DeleteServiceSpecificCredential(userName, credentialID string) error UpdateServiceSpecificCredential(userName, credentialID, status string) error @@ -263,12 +263,12 @@ type StorageBackend interface { // Server Certificates UploadServerCertificate(name, path, certBody, certChain string) (*ServerCertificate, error) GetServerCertificate(name string) (*ServerCertificate, error) - ListServerCertificates(pathPrefix string) ([]ServerCertificate, error) + ListServerCertificates(pathPrefix, marker string, maxItems int) (page.Page[ServerCertificate], error) UpdateServerCertificate(name, newName, newPath string) error DeleteServerCertificate(name string) error // Group membership queries - ListGroupsForUser(userName string) ([]Group, error) + ListGroupsForUser(userName, marker string, maxItems int) (page.Page[Group], error) // Account Password Policy GetAccountPasswordPolicy() *PasswordPolicy @@ -590,7 +590,9 @@ func sortedUsers(t *store.Table[User]) []User { users := make([]User, 0, len(items)) for _, u := range items { - users = append(users, *u) + cp := *u + cp.Tags = maps.Clone(u.Tags) + users = append(users, cp) } return users diff --git a/services/iam/tags_race_test.go b/services/iam/tags_race_test.go new file mode 100644 index 0000000000..52e317612d --- /dev/null +++ b/services/iam/tags_race_test.go @@ -0,0 +1,135 @@ +package iam_test + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/iam" +) + +// TestGetResourceConcurrentWithUntag proves Get/List for roles, users, and +// policies must not hand back a Tags map Untag mutates in place. +func TestGetResourceConcurrentWithUntag(t *testing.T) { + t.Parallel() + + tests := []struct { + setup func(t *testing.T, b *iam.InMemoryBackend) (id string) + reader func(b *iam.InMemoryBackend, id string) + mutate func(b *iam.InMemoryBackend, id string) + name string + }{ + { + name: "GetRole races UntagRole", + setup: func(t *testing.T, b *iam.InMemoryBackend) string { + t.Helper() + + r, err := b.CreateRole("race-role", "/", "", "") + require.NoError(t, err) + require.NoError(t, b.TagRole(r.RoleName, map[string]string{"env": "prod"})) + + return r.RoleName + }, + reader: func(b *iam.InMemoryBackend, id string) { + r, err := b.GetRole(id) + if err != nil { + return + } + + for k := range r.Tags { + _ = k + } + }, + mutate: func(b *iam.InMemoryBackend, id string) { + _ = b.TagRole(id, map[string]string{"env": "prod"}) + _ = b.UntagRole(id, []string{"env"}) + }, + }, + { + name: "GetUser races UntagUser", + setup: func(t *testing.T, b *iam.InMemoryBackend) string { + t.Helper() + + u, err := b.CreateUser("race-user", "/", "") + require.NoError(t, err) + require.NoError(t, b.TagUser(u.UserName, map[string]string{"env": "prod"})) + + return u.UserName + }, + reader: func(b *iam.InMemoryBackend, id string) { + u, err := b.GetUser(id) + if err != nil { + return + } + + for k := range u.Tags { + _ = k + } + }, + mutate: func(b *iam.InMemoryBackend, id string) { + _ = b.TagUser(id, map[string]string{"env": "prod"}) + _ = b.UntagUser(id, []string{"env"}) + }, + }, + { + name: "GetPolicy races UntagPolicy", + setup: func(t *testing.T, b *iam.InMemoryBackend) string { + t.Helper() + + p, err := b.CreatePolicy("race-policy", "/", + `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"*","Resource":"*"}]}`) + require.NoError(t, err) + require.NoError(t, b.TagPolicy(p.Arn, map[string]string{"env": "prod"})) + + return p.Arn + }, + reader: func(b *iam.InMemoryBackend, id string) { + p, err := b.GetPolicy(id) + if err != nil { + return + } + + for k := range p.Tags { + _ = k + } + }, + mutate: func(b *iam.InMemoryBackend, id string) { + _ = b.TagPolicy(id, map[string]string{"env": "prod"}) + _ = b.UntagPolicy(id, []string{"env"}) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := iam.NewInMemoryBackend() + id := tt.setup(t, b) + + const iterations = 500 + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + for range iterations { + tt.reader(b, id) + } + }() + + go func() { + defer wg.Done() + + for range iterations { + tt.mutate(b, id) + } + }() + + wg.Wait() + }) + } +} diff --git a/services/iam/users.go b/services/iam/users.go index 8f29449247..d4810d59fe 100644 --- a/services/iam/users.go +++ b/services/iam/users.go @@ -152,13 +152,27 @@ func (b *InMemoryBackend) ListUsers(marker string, maxItems int) (page.Page[User return pageFromSortedNames( b.sortedUserNames, - b.users.Get, + b.cloneUserLocked, marker, maxItems, iamDefaultMaxItems, ), nil } +// cloneUserLocked looks up a user by name and returns a copy with its own +// Tags map, so ListUsers cannot alias TagUser/UntagUser's in-place writes. +func (b *InMemoryBackend) cloneUserLocked(userName string) (*User, bool) { + u, exists := b.users.Get(userName) + if !exists { + return nil, false + } + + cp := *u + cp.Tags = maps.Clone(u.Tags) + + return &cp, true +} + // GetUser retrieves a single IAM user by name. func (b *InMemoryBackend) GetUser(userName string) (*User, error) { b.mu.RLock("GetUser") @@ -169,7 +183,10 @@ func (b *InMemoryBackend) GetUser(userName string) (*User, error) { return nil, fmt.Errorf("%w: user %q not found", ErrUserNotFound, userName) } - return u, nil + cp := *u + cp.Tags = maps.Clone(u.Tags) + + return &cp, nil } // ListAllUsers returns all users (for dashboard). diff --git a/services/inspector2/filters.go b/services/inspector2/filters.go index 534456f41f..0b076af0c4 100644 --- a/services/inspector2/filters.go +++ b/services/inspector2/filters.go @@ -271,6 +271,7 @@ func (b *InMemoryBackend) ListFilters( } clone := *f + clone.Tags = maps.Clone(f.Tags) matched = append(matched, &clone) } diff --git a/services/inspector2/handler.go b/services/inspector2/handler.go index 9270206ef7..280f752c2b 100644 --- a/services/inspector2/handler.go +++ b/services/inspector2/handler.go @@ -165,6 +165,10 @@ var ambiguousRouteMatchPrefixes = map[string]bool{ //nolint:gochecknoglobals // "/findings/": true, "/members/": true, "/configuration/": true, + // "/cluster/": DSQL's cluster resource paths (/cluster/{id}, + // /cluster/{id}/policy) share this prefix; Inspector2's only real + // operation here is the exact POST /cluster/get (gopherstack-7r6bz). + "/cluster/": true, } // RouteMatcher returns a matcher that accepts Inspector2 REST paths. diff --git a/services/iot/broker.go b/services/iot/broker.go index 8114838d1f..8573357c30 100644 --- a/services/iot/broker.go +++ b/services/iot/broker.go @@ -70,23 +70,14 @@ func (b *Broker) Start(ctx context.Context) error { // Store the server atomically before Serve() so Publish() can access it concurrently. b.server.Store(s) - done := make(chan struct{}) - defer close(done) - - go func() { - select { - case <-ctx.Done(): - _ = s.Close() - case <-done: - // Serve() returned; goroutine exits cleanly. - } - }() - + // mochi's Serve starts its listeners and event loop in goroutines and returns at once. if err := s.Serve(); err != nil { return fmt.Errorf("iot broker: serve: %w", err) } - return nil + <-ctx.Done() + + return s.Close() } // Run implements worker.Runner, adapting Start's blocking-with-error shape to diff --git a/services/iot/broker_test.go b/services/iot/broker_test.go index 5dffca2e98..e510951f48 100644 --- a/services/iot/broker_test.go +++ b/services/iot/broker_test.go @@ -118,10 +118,9 @@ func TestHandlerShutdownDrainsBrokerGoroutine(t *testing.T) { require.NoError(t, h.StartWorker(runCtx)) - // Give the broker goroutine a moment to actually start listening before - // asking it to stop. - time.Sleep(20 * time.Millisecond) - + // No readiness wait needed: worker.SingleRun.Stop cancels and blocks on + // the run's done channel, which is registered synchronously by Start + // before its goroutine runs, so Stop can't race a not-yet-started run. done := make(chan struct{}) go func() { h.Shutdown(t.Context()) diff --git a/services/kafkaconnect/PARITY.md b/services/kafkaconnect/PARITY.md new file mode 100644 index 0000000000..74a679a3be --- /dev/null +++ b/services/kafkaconnect/PARITY.md @@ -0,0 +1,67 @@ +--- +service: kafkaconnect +sdk_module: aws-sdk-go-v2/service/kafkaconnect@v1.39.1 +last_audit_commit: 709187947 # HEAD at audit time, pre-commit +last_audit_date: 2026-09-25 +overall: B # new service, control plane only, unit-tested against the real SDK client +ops: + CreateConnector: {wire: ok, errors: ok, state: ok, persist: ok, note: "returns RUNNING immediately -- see items_still_open"} + DescribeConnector: {wire: ok, errors: ok, state: ok, persist: ok} + ListConnectors: {wire: ok, errors: ok, state: ok, persist: ok, note: "connectorNamePrefix filter; opaque nextToken via pkgs/page"} + UpdateConnector: {wire: ok, errors: ok, state: ok, persist: ok, note: "currentVersion optimistic lock; exactly one of capacity/connectorConfiguration; records a real ConnectorOperation"} + DeleteConnector: {wire: ok, errors: ok, state: ok, persist: ok, note: "removes the connector immediately; response echoes DELETING per AWS's synchronous delete contract"} + RestartConnector: {wire: ok, errors: ok, state: ok, persist: ok, note: "records a real ConnectorOperation (RESTART_CONNECTOR, immediately RESTART_COMPLETE)"} + DescribeConnectorOperation: {wire: ok, errors: ok, state: ok, persist: ok} + ListConnectorOperations: {wire: ok, errors: ok, state: ok, persist: ok} + CreateCustomPlugin: {wire: ok, errors: ok, state: ok, persist: ok, note: "ACTIVE immediately -- see items_still_open"} + DescribeCustomPlugin: {wire: ok, errors: ok, state: ok, persist: ok} + ListCustomPlugins: {wire: ok, errors: ok, state: ok, persist: ok, note: "namePrefix filter"} + DeleteCustomPlugin: {wire: ok, errors: ok, state: ok, persist: ok} + CreateWorkerConfiguration: {wire: ok, errors: ok, state: ok, persist: ok, note: "propertiesFileContent stored/echoed as given (base64), always revision 1"} + DescribeWorkerConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + ListWorkerConfigurations: {wire: ok, errors: ok, state: ok, persist: ok, note: "namePrefix filter"} + DeleteWorkerConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + TagResource: {wire: ok, errors: ok, state: ok, persist: ok, note: "connector, custom plugin, or worker configuration by ARN"} + UntagResource: {wire: ok, errors: ok, state: ok, persist: ok} + ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok} +families: + Connector: {status: ok, note: "Create/Describe/List/Update/Delete/Restart verified end-to-end against the real aws-sdk-go-v2 client over an httptest server -- wire shapes, ISO8601 timestamps, ARN format, CurrentVersion optimistic locking, and error deserialization (BadRequestException/ConflictException/NotFoundException) all round-trip cleanly."} + ConnectorOperation: {status: ok, note: "UpdateConnector and RestartConnector each record a real ConnectorOperation (UPDATE_CONNECTOR_CONFIGURATION/UPDATE_WORKER_SETTING/RESTART_CONNECTOR, immediately *_COMPLETE) retrievable via DescribeConnectorOperation/ListConnectorOperations."} + CustomPlugin: {status: ok, note: "Create/Describe/List/Delete round-trip contentType, S3 location, and a derived (not real) file checksum/size -- see items_still_open. Plugin revisions beyond 1 and UpdateCustomPlugin are not part of the surface this pass implements."} + WorkerConfiguration: {status: ok, note: "Create/Describe/List/Delete round-trip name/description/propertiesFileContent. Always revision 1: UpdateWorkerConfiguration (which would create later revisions) is not part of the AWS API."} + Tags: {status: ok, note: "One generic tag family keyed by ARN across all three resource kinds, matching real AWS."} +gaps: [] +items_still_open: + - "CREATING/UPDATING/DELETING/RESTARTING transient connector states, and CREATING/DELETING + transient custom-plugin and worker-configuration states, are not modeled: every create + returns RUNNING/ACTIVE immediately and every delete removes the resource immediately + (the delete response itself still echoes the real API's synchronous DELETING state). + This is a deliberate, task-authorized simplification -- terraform-provider-aws's waiters + (waitConnectorCreated/Updated/Deleted, waitCustomPluginCreated/Deleted) poll for exactly + these target states, and this backend reaches them on the very first read." + - "CustomPlugin's S3 location (bucketArn/fileKey/objectVersion) is stored and echoed back + exactly as given but never read from the real services/s3 backend -- FileMd5 and FileSize + in DescribeCustomPlugin/ListCustomPlugins are derived from the location string, not the + actual object bytes. Backends are intentionally not coupled (services/kafka's MSK cluster + bootstrap-broker strings are similarly taken as opaque input, never generated by calling + into services/kafka)." + - "Custom plugin and worker configuration revisions beyond 1 (UpdateCustomPlugin, + UpdateWorkerConfiguration equivalents) are not part of the MSK Connect AWS API surface + this backend implements; AWS itself does not expose an UpdateWorkerConfiguration API." +--- + +## Notes + +Initial implementation (2026-09-25): control-plane REST-JSON API modeled after +services/kinesisvideo. Every operation mutates/reads real in-memory state via +pkgs/store.Table + pkgs/lockmetrics.RWMutex, with JSON snapshot/restore wired +into pkgs/persistence. Wire shapes, HTTP methods/paths (path-parameter REST, +not action-per-path like kinesisvideo), and error codes were verified against +the pinned aws-sdk-go-v2/service/kafkaconnect v1.39.1 serializers.go/ +deserializers.go. Timestamps are ISO8601 strings (smithy date-time), unlike +kinesisvideo's epoch-seconds numbers -- confirmed via deserializers.go's +smithytime.ParseDateTime calls. The shared /v1/tags/{resourceArn} path (also +claimed by services/kafka, batch, appsync, mq, codeartifact, pinpoint) is +guarded by decoding the ARN's own service field, the same pattern +services/kafka uses for its own /v1/tags/{arn} claim, per +.claude/memories -- route-matcher-prefix-collision. diff --git a/services/kafkaconnect/README.md b/services/kafkaconnect/README.md new file mode 100644 index 0000000000..2bf58140a8 --- /dev/null +++ b/services/kafkaconnect/README.md @@ -0,0 +1,25 @@ + +# Kafkaconnect + +**Parity grade: B** · SDK `aws-sdk-go-v2/service/kafkaconnect@v1.39.1` · last audited 2026-09-25 (`709187947`) + +## Coverage + +| Metric | Value | +| --- | --- | +| PARITY entries audited | 19 (19 ok) | +| Feature families | 5 (5 ok) | +| Known gaps | 3 | +| Deferred items | 0 | +| Resource leaks | unknown | + +### Known gaps + +- "CREATING/UPDATING/DELETING/RESTARTING transient connector states, and CREATING/DELETING transient custom-plugin and worker-configuration states, are not modeled: every create returns RUNNING/ACTIVE immediately and every delete removes the resource immediately (the delete response itself still echoes the real API's synchronous DELETING state). This is a deliberate, task-authorized simplification -- terraform-provider-aws's waiters (waitConnectorCreated/Updated/Deleted, waitCustomPluginCreated/Deleted) poll for exactly these target states, and this backend reaches them on the very first read." +- "CustomPlugin's S3 location (bucketArn/fileKey/objectVersion) is stored and echoed back exactly as given but never read from the real services/s3 backend -- FileMd5 and FileSize in DescribeCustomPlugin/ListCustomPlugins are derived from the location string, not the actual object bytes. Backends are intentionally not coupled (services/kafka's MSK cluster bootstrap-broker strings are similarly taken as opaque input, never generated by calling into services/kafka)." +- "Custom plugin and worker configuration revisions beyond 1 (UpdateCustomPlugin, UpdateWorkerConfiguration equivalents) are not part of the MSK Connect AWS API surface this backend implements; AWS itself does not expose an UpdateWorkerConfiguration API." + +## More + +- [Full parity audit](PARITY.md) +- [All services](../../README.md#services) diff --git a/services/kafkaconnect/connectors.go b/services/kafkaconnect/connectors.go new file mode 100644 index 0000000000..1b329c12a2 --- /dev/null +++ b/services/kafkaconnect/connectors.go @@ -0,0 +1,260 @@ +package kafkaconnect + +import ( + "fmt" + "maps" + "sort" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/blackbirdworks/gopherstack/pkgs/arn" + "github.com/blackbirdworks/gopherstack/pkgs/page" +) + +func connectorARN(region, accountID, name string) string { + return arn.Build("kafkaconnect", region, accountID, fmt.Sprintf("connector/%s/%s", name, uuid.NewString())) +} + +func connectorOperationARN(connectorArn string) string { + return connectorArn + "/operation/" + uuid.NewString() +} + +// CreateConnector creates a connector. Connectors become RUNNING immediately +// -- see PARITY.md for the CREATING/UPDATING/DELETING transient states this +// backend deliberately does not model. +func (b *InMemoryBackend) CreateConnector(accountID, region string, spec ConnectorSpec) (*Connector, error) { + if spec.Name == "" { + return nil, ErrValidation + } + + b.mu.Lock("CreateConnector") + defer b.mu.Unlock() + + if _, ok := b.connectorByName(spec.Name); ok { + return nil, ErrConnectorNameInUse + } + + tags := make(map[string]string, len(spec.Tags)) + maps.Copy(tags, spec.Tags) + + cfg := make(map[string]string, len(spec.ConnectorConfiguration)) + maps.Copy(cfg, spec.ConnectorConfiguration) + + c := &Connector{ + Name: spec.Name, + ARN: connectorARN(region, accountID, spec.Name), + Description: spec.Description, + State: connectorStateRunning, + CurrentVersion: newVersion(), + CreationTime: time.Now().UTC(), + ConnectorConfiguration: cfg, + Capacity: spec.Capacity.clone(), + ApacheKafkaCluster: spec.ApacheKafkaCluster, + KafkaClusterClientAuthentication: spec.KafkaClusterClientAuthentication, + KafkaClusterEncryptionInTransit: spec.KafkaClusterEncryptionInTransit, + KafkaConnectVersion: spec.KafkaConnectVersion, + ServiceExecutionRoleArn: spec.ServiceExecutionRoleArn, + NetworkType: spec.NetworkType, + Plugins: append([]PluginRef(nil), spec.Plugins...), + WorkerConfiguration: spec.WorkerConfiguration, + WorkerLogDelivery: spec.WorkerLogDelivery.clone(), + Tags: tags, + } + + b.connectors.Put(c) + + return c.clone(), nil +} + +// DescribeConnector returns the current information about a connector. +func (b *InMemoryBackend) DescribeConnector(connectorArn string) (*Connector, error) { + b.mu.RLock("DescribeConnector") + defer b.mu.RUnlock() + + c, ok := b.connectors.Get(connectorArn) + if !ok { + return nil, ErrConnectorNotFound + } + + return c.clone(), nil +} + +// ListConnectors returns connectors matching namePrefix, paginated by nextToken/maxResults. +func (b *InMemoryBackend) ListConnectors(namePrefix, nextToken string, maxResults int) ([]*Connector, string, error) { + b.mu.RLock("ListConnectors") + defer b.mu.RUnlock() + + all := b.connectors.All() + + matched := make([]*Connector, 0, len(all)) + + for _, c := range all { + if namePrefix != "" && !strings.HasPrefix(c.Name, namePrefix) { + continue + } + + matched = append(matched, c.clone()) + } + + sort.Slice(matched, func(i, j int) bool { return matched[i].Name < matched[j].Name }) + + p := page.New(matched, nextToken, maxResults, defaultListLimit) + + return p.Data, p.Next, nil +} + +// UpdateConnector updates a connector's capacity or configuration under +// optimistic lock (currentVersion). Exactly one of update.Capacity or +// update.ConnectorConfiguration must be set. The returned ConnectorOperation +// completes immediately (UPDATE_COMPLETE) -- see PARITY.md. +func (b *InMemoryBackend) UpdateConnector( + connectorArn, currentVersion string, + update ConnectorUpdate, +) (*Connector, *ConnectorOperation, error) { + if (update.Capacity == nil) == (update.ConnectorConfiguration == nil) { + return nil, nil, ErrValidation + } + + b.mu.Lock("UpdateConnector") + defer b.mu.Unlock() + + c, ok := b.connectors.Get(connectorArn) + if !ok { + return nil, nil, ErrConnectorNotFound + } + + if c.CurrentVersion != currentVersion { + return nil, nil, ErrVersionMismatch + } + + op := &ConnectorOperation{ + ARN: connectorOperationARN(connectorArn), + ConnectorArn: connectorArn, + State: connectorOperationStateComplete, + CreationTime: time.Now().UTC(), + OriginConnectorConfiguration: maps.Clone(c.ConnectorConfiguration), + TargetConnectorConfiguration: maps.Clone(c.ConnectorConfiguration), + } + + if update.Capacity != nil { + op.Type = connectorOperationTypeWorkerSetting + op.Steps = []ConnectorOperationStep{ + {StepType: connectorOperationStepUpdateWorkerSetting, StepState: connectorOperationStepStateCompleted}, + } + + origin := c.Capacity.clone() + op.OriginCapacity = &origin + c.Capacity = update.Capacity.clone() + target := c.Capacity.clone() + op.TargetCapacity = &target + } else { + op.Type = connectorOperationTypeConfiguration + op.Steps = []ConnectorOperationStep{ + {StepType: connectorOperationStepUpdateConfiguration, StepState: connectorOperationStepStateCompleted}, + } + + op.TargetConnectorConfiguration = maps.Clone(update.ConnectorConfiguration) + c.ConnectorConfiguration = maps.Clone(update.ConnectorConfiguration) + } + + op.EndTime = time.Now().UTC() + c.CurrentVersion = newVersion() + + b.connectorOperations.Put(op) + + return c.clone(), op.clone(), nil +} + +// DeleteConnector deletes a connector under optimistic lock (currentVersion, +// when supplied), returning a snapshot with State set to DELETING to mirror +// AWS's synchronous delete response. +func (b *InMemoryBackend) DeleteConnector(connectorArn, currentVersion string) (*Connector, error) { + b.mu.Lock("DeleteConnector") + defer b.mu.Unlock() + + c, ok := b.connectors.Get(connectorArn) + if !ok { + return nil, ErrConnectorNotFound + } + + if currentVersion != "" && c.CurrentVersion != currentVersion { + return nil, ErrVersionMismatch + } + + out := c.clone() + out.State = deletingState + + b.connectors.Delete(connectorArn) + + return out, nil +} + +// RestartConnector restarts a connector, recording a ConnectorOperation that +// completes immediately (RESTART_COMPLETE) -- see PARITY.md for the +// transient RESTARTING connector state and per-task restart tracking this +// backend deliberately does not model. +func (b *InMemoryBackend) RestartConnector(connectorArn string, _ bool) (*Connector, *ConnectorOperation, error) { + b.mu.Lock("RestartConnector") + defer b.mu.Unlock() + + c, ok := b.connectors.Get(connectorArn) + if !ok { + return nil, nil, ErrConnectorNotFound + } + + now := time.Now().UTC() + op := &ConnectorOperation{ + ARN: connectorOperationARN(connectorArn), + ConnectorArn: connectorArn, + Type: connectorOperationTypeRestart, + State: connectorOperationStateRestartComplete, + CreationTime: now, + EndTime: now, + } + + b.connectorOperations.Put(op) + + return c.clone(), op.clone(), nil +} + +// DescribeConnectorOperation returns the details of a single connector operation. +func (b *InMemoryBackend) DescribeConnectorOperation(operationArn string) (*ConnectorOperation, error) { + b.mu.RLock("DescribeConnectorOperation") + defer b.mu.RUnlock() + + op, ok := b.connectorOperations.Get(operationArn) + if !ok { + return nil, ErrConnectorOperationNotFound + } + + return op.clone(), nil +} + +// ListConnectorOperations returns operations for a connector, paginated by nextToken/maxResults. +func (b *InMemoryBackend) ListConnectorOperations( + connectorArn, nextToken string, + maxResults int, +) ([]*ConnectorOperation, string, error) { + b.mu.RLock("ListConnectorOperations") + defer b.mu.RUnlock() + + all := b.connectorOperations.All() + + matched := make([]*ConnectorOperation, 0, len(all)) + + for _, op := range all { + if op.ConnectorArn != connectorArn { + continue + } + + matched = append(matched, op.clone()) + } + + sort.Slice(matched, func(i, j int) bool { return matched[i].CreationTime.Before(matched[j].CreationTime) }) + + p := page.New(matched, nextToken, maxResults, defaultListLimit) + + return p.Data, p.Next, nil +} diff --git a/services/kafkaconnect/connectors_test.go b/services/kafkaconnect/connectors_test.go new file mode 100644 index 0000000000..7328d4082e --- /dev/null +++ b/services/kafkaconnect/connectors_test.go @@ -0,0 +1,311 @@ +package kafkaconnect_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + kafkaconnectsdk "github.com/aws/aws-sdk-go-v2/service/kafkaconnect" + "github.com/aws/aws-sdk-go-v2/service/kafkaconnect/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func minimalCreateConnectorInput(name string) *kafkaconnectsdk.CreateConnectorInput { + return &kafkaconnectsdk.CreateConnectorInput{ + Capacity: &types.Capacity{ + ProvisionedCapacity: &types.ProvisionedCapacity{McuCount: 1, WorkerCount: 1}, + }, + ConnectorConfiguration: map[string]string{"connector.class": "com.example.Connector"}, + ConnectorName: aws.String(name), + KafkaCluster: &types.KafkaCluster{ + ApacheKafkaCluster: &types.ApacheKafkaCluster{ + BootstrapServers: aws.String("broker1:9092,broker2:9092"), + Vpc: &types.Vpc{ + SecurityGroups: []string{"sg-12345"}, + Subnets: []string{"subnet-1", "subnet-2"}, + }, + }, + }, + KafkaClusterClientAuthentication: &types.KafkaClusterClientAuthentication{ + AuthenticationType: types.KafkaClusterClientAuthenticationTypeNone, + }, + KafkaClusterEncryptionInTransit: &types.KafkaClusterEncryptionInTransit{ + EncryptionType: types.KafkaClusterEncryptionInTransitTypePlaintext, + }, + KafkaConnectVersion: aws.String("2.7.1"), + Plugins: []types.Plugin{ + { + CustomPlugin: &types.CustomPlugin{ + CustomPluginArn: aws.String("arn:aws:kafkaconnect:us-east-1:123456789012:custom-plugin/p/abc"), + Revision: 1, + }, + }, + }, + ServiceExecutionRoleArn: aws.String("arn:aws:iam::123456789012:role/connect-role"), + } +} + +func TestCreateConnector(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + }{ + {name: "minimal"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + out, err := client.CreateConnector(t.Context(), minimalCreateConnectorInput("conn-"+tt.name)) + require.NoError(t, err) + assert.Contains(t, aws.ToString(out.ConnectorArn), "connector/conn-"+tt.name+"/") + assert.Equal(t, types.ConnectorStateRunning, out.ConnectorState) + }) + } +} + +func TestCreateConnector_DuplicateNameReturnsConflict(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateConnector(ctx, minimalCreateConnectorInput("dup-connector")) + require.NoError(t, err) + + _, err = client.CreateConnector(ctx, minimalCreateConnectorInput("dup-connector")) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ConflictException", apiErr.ErrorCode()) +} + +func TestDescribeConnector(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateConnector(ctx, minimalCreateConnectorInput("describe-me")) + require.NoError(t, err) + + out, err := client.DescribeConnector( + ctx, + &kafkaconnectsdk.DescribeConnectorInput{ConnectorArn: created.ConnectorArn}, + ) + require.NoError(t, err) + assert.Equal(t, "describe-me", aws.ToString(out.ConnectorName)) + assert.Equal(t, types.ConnectorStateRunning, out.ConnectorState) + require.NotNil(t, out.Capacity) + require.NotNil(t, out.Capacity.ProvisionedCapacity) + assert.EqualValues(t, 1, out.Capacity.ProvisionedCapacity.WorkerCount) + require.NotNil(t, out.KafkaCluster) + require.NotNil(t, out.KafkaCluster.ApacheKafkaCluster) + assert.Equal(t, "broker1:9092,broker2:9092", aws.ToString(out.KafkaCluster.ApacheKafkaCluster.BootstrapServers)) + assert.NotEmpty(t, aws.ToString(out.CurrentVersion)) +} + +func TestDescribeConnector_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.DescribeConnector(t.Context(), &kafkaconnectsdk.DescribeConnectorInput{ + ConnectorArn: aws.String("arn:aws:kafkaconnect:us-east-1:123456789012:connector/nope/abc"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "NotFoundException", apiErr.ErrorCode()) +} + +func TestListConnectors(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateConnector(ctx, minimalCreateConnectorInput("list-a")) + require.NoError(t, err) + _, err = client.CreateConnector(ctx, minimalCreateConnectorInput("list-b")) + require.NoError(t, err) + + out, err := client.ListConnectors(ctx, &kafkaconnectsdk.ListConnectorsInput{}) + require.NoError(t, err) + assert.Len(t, out.Connectors, 2) +} + +func TestUpdateConnector_Capacity(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateConnector(ctx, minimalCreateConnectorInput("update-me")) + require.NoError(t, err) + + described, err := client.DescribeConnector( + ctx, + &kafkaconnectsdk.DescribeConnectorInput{ConnectorArn: created.ConnectorArn}, + ) + require.NoError(t, err) + + out, err := client.UpdateConnector(ctx, &kafkaconnectsdk.UpdateConnectorInput{ + ConnectorArn: created.ConnectorArn, + CurrentVersion: described.CurrentVersion, + Capacity: &types.CapacityUpdate{ + ProvisionedCapacity: &types.ProvisionedCapacityUpdate{McuCount: 2, WorkerCount: 2}, + }, + }) + require.NoError(t, err) + assert.Equal(t, types.ConnectorStateRunning, out.ConnectorState) + assert.NotEmpty(t, aws.ToString(out.ConnectorOperationArn)) + + describedAfter, err := client.DescribeConnector( + ctx, + &kafkaconnectsdk.DescribeConnectorInput{ConnectorArn: created.ConnectorArn}, + ) + require.NoError(t, err) + require.NotNil(t, describedAfter.Capacity.ProvisionedCapacity) + assert.EqualValues(t, 2, describedAfter.Capacity.ProvisionedCapacity.WorkerCount) + assert.NotEqual(t, aws.ToString(described.CurrentVersion), aws.ToString(describedAfter.CurrentVersion)) + + opOut, err := client.DescribeConnectorOperation(ctx, &kafkaconnectsdk.DescribeConnectorOperationInput{ + ConnectorOperationArn: out.ConnectorOperationArn, + }) + require.NoError(t, err) + assert.Equal(t, types.ConnectorOperationStateUpdateComplete, opOut.ConnectorOperationState) + assert.Equal(t, types.ConnectorOperationTypeUpdateWorkerSetting, opOut.ConnectorOperationType) + + listOpsOut, err := client.ListConnectorOperations(ctx, &kafkaconnectsdk.ListConnectorOperationsInput{ + ConnectorArn: created.ConnectorArn, + }) + require.NoError(t, err) + assert.Len(t, listOpsOut.ConnectorOperations, 1) +} + +func TestUpdateConnector_VersionMismatch(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateConnector(ctx, minimalCreateConnectorInput("stale-version")) + require.NoError(t, err) + + _, err = client.UpdateConnector(ctx, &kafkaconnectsdk.UpdateConnectorInput{ + ConnectorArn: created.ConnectorArn, + CurrentVersion: aws.String("stale"), + ConnectorConfiguration: map[string]string{"foo": "bar"}, + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ConflictException", apiErr.ErrorCode()) +} + +func TestDeleteConnector(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateConnector(ctx, minimalCreateConnectorInput("delete-me")) + require.NoError(t, err) + + out, err := client.DeleteConnector(ctx, &kafkaconnectsdk.DeleteConnectorInput{ConnectorArn: created.ConnectorArn}) + require.NoError(t, err) + assert.Equal(t, types.ConnectorStateDeleting, out.ConnectorState) + + _, err = client.DescribeConnector(ctx, &kafkaconnectsdk.DescribeConnectorInput{ConnectorArn: created.ConnectorArn}) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "NotFoundException", apiErr.ErrorCode()) +} + +func TestRestartConnector(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + onlyFailedTasks bool + }{ + {name: "full"}, + {name: "only_failed_tasks", onlyFailedTasks: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateConnector(ctx, minimalCreateConnectorInput("restart-"+tt.name)) + require.NoError(t, err) + + out, err := client.RestartConnector(ctx, &kafkaconnectsdk.RestartConnectorInput{ + ConnectorArn: created.ConnectorArn, + OnlyFailedTasks: tt.onlyFailedTasks, + }) + require.NoError(t, err) + assert.Equal(t, aws.ToString(created.ConnectorArn), aws.ToString(out.ConnectorArn)) + assert.NotEmpty(t, aws.ToString(out.ConnectorOperationArn)) + + opOut, err := client.DescribeConnectorOperation(ctx, &kafkaconnectsdk.DescribeConnectorOperationInput{ + ConnectorOperationArn: out.ConnectorOperationArn, + }) + require.NoError(t, err) + assert.Equal(t, types.ConnectorOperationStateRestartComplete, opOut.ConnectorOperationState) + assert.Equal(t, types.ConnectorOperationTypeRestartConnector, opOut.ConnectorOperationType) + + described, err := client.DescribeConnector(ctx, &kafkaconnectsdk.DescribeConnectorInput{ + ConnectorArn: created.ConnectorArn, + }) + require.NoError(t, err) + assert.Equal(t, types.ConnectorStateRunning, described.ConnectorState) + }) + } +} + +func TestRestartConnector_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.RestartConnector(t.Context(), &kafkaconnectsdk.RestartConnectorInput{ + ConnectorArn: aws.String("arn:aws:kafkaconnect:us-east-1:123456789012:connector/nope/abc"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "NotFoundException", apiErr.ErrorCode()) +} + +func TestDescribeConnectorOperation_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.DescribeConnectorOperation(t.Context(), &kafkaconnectsdk.DescribeConnectorOperationInput{ + ConnectorOperationArn: aws.String( + "arn:aws:kafkaconnect:us-east-1:123456789012:connector/nope/abc/operation/def", + ), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "NotFoundException", apiErr.ErrorCode()) +} diff --git a/services/kafkaconnect/customplugins.go b/services/kafkaconnect/customplugins.go new file mode 100644 index 0000000000..23d149d918 --- /dev/null +++ b/services/kafkaconnect/customplugins.go @@ -0,0 +1,123 @@ +package kafkaconnect + +import ( + "fmt" + "maps" + "sort" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/blackbirdworks/gopherstack/pkgs/arn" + "github.com/blackbirdworks/gopherstack/pkgs/page" +) + +const fakePluginFileSizeBytes = 1024 + +func customPluginARN(region, accountID, name string) string { + return arn.Build("kafkaconnect", region, accountID, fmt.Sprintf("custom-plugin/%s/%s", name, uuid.NewString())) +} + +// CreateCustomPlugin creates a custom plugin. Plugins become ACTIVE +// immediately -- see PARITY.md for the CREATING/UPDATING/DELETING transient +// states this backend deliberately does not model, and for the S3 object +// coupling this backend does not perform (BucketArn/FileKey/ObjectVersion +// are stored and echoed back as given, never read from S3). +func (b *InMemoryBackend) CreateCustomPlugin( + accountID, region, name, description, contentType, bucketArn, fileKey, objectVersion string, + tags map[string]string, +) (*CustomPlugin, error) { + if name == "" || contentType == "" || bucketArn == "" || fileKey == "" { + return nil, ErrValidation + } + + b.mu.Lock("CreateCustomPlugin") + defer b.mu.Unlock() + + if _, ok := b.customPluginByName(name); ok { + return nil, ErrCustomPluginNameInUse + } + + t := make(map[string]string, len(tags)) + maps.Copy(t, tags) + + p := &CustomPlugin{ + Name: name, + ARN: customPluginARN(region, accountID, name), + Description: description, + State: customPluginStateActive, + ContentType: contentType, + BucketArn: bucketArn, + FileKey: fileKey, + ObjectVersion: objectVersion, + FileMD5: fakeFileChecksum(bucketArn + "/" + fileKey + "/" + objectVersion), + FileSizeBytes: fakePluginFileSizeBytes, + Revision: 1, + CreationTime: time.Now().UTC(), + Tags: t, + } + + b.customPlugins.Put(p) + + return p.clone(), nil +} + +// DescribeCustomPlugin returns the current information about a custom plugin. +func (b *InMemoryBackend) DescribeCustomPlugin(customPluginArn string) (*CustomPlugin, error) { + b.mu.RLock("DescribeCustomPlugin") + defer b.mu.RUnlock() + + p, ok := b.customPlugins.Get(customPluginArn) + if !ok { + return nil, ErrCustomPluginNotFound + } + + return p.clone(), nil +} + +// ListCustomPlugins returns custom plugins matching namePrefix, paginated by nextToken/maxResults. +func (b *InMemoryBackend) ListCustomPlugins( + namePrefix, nextToken string, + maxResults int, +) ([]*CustomPlugin, string, error) { + b.mu.RLock("ListCustomPlugins") + defer b.mu.RUnlock() + + all := b.customPlugins.All() + + matched := make([]*CustomPlugin, 0, len(all)) + + for _, p := range all { + if namePrefix != "" && !strings.HasPrefix(p.Name, namePrefix) { + continue + } + + matched = append(matched, p.clone()) + } + + sort.Slice(matched, func(i, j int) bool { return matched[i].Name < matched[j].Name }) + + pg := page.New(matched, nextToken, maxResults, defaultListLimit) + + return pg.Data, pg.Next, nil +} + +// DeleteCustomPlugin deletes a custom plugin, returning a snapshot with State +// set to DELETING to mirror AWS's synchronous delete response. +func (b *InMemoryBackend) DeleteCustomPlugin(customPluginArn string) (*CustomPlugin, error) { + b.mu.Lock("DeleteCustomPlugin") + defer b.mu.Unlock() + + p, ok := b.customPlugins.Get(customPluginArn) + if !ok { + return nil, ErrCustomPluginNotFound + } + + out := p.clone() + out.State = deletingState + + b.customPlugins.Delete(customPluginArn) + + return out, nil +} diff --git a/services/kafkaconnect/customplugins_test.go b/services/kafkaconnect/customplugins_test.go new file mode 100644 index 0000000000..d0ed6a0c29 --- /dev/null +++ b/services/kafkaconnect/customplugins_test.go @@ -0,0 +1,137 @@ +package kafkaconnect_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + kafkaconnectsdk "github.com/aws/aws-sdk-go-v2/service/kafkaconnect" + "github.com/aws/aws-sdk-go-v2/service/kafkaconnect/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func minimalCreateCustomPluginInput(name string) *kafkaconnectsdk.CreateCustomPluginInput { + return &kafkaconnectsdk.CreateCustomPluginInput{ + ContentType: types.CustomPluginContentTypeZip, + Location: &types.CustomPluginLocation{ + S3Location: &types.S3Location{ + BucketArn: aws.String("arn:aws:s3:::my-plugin-bucket"), + FileKey: aws.String("plugins/my-plugin.zip"), + }, + }, + Name: aws.String(name), + } +} + +func TestCreateCustomPlugin(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + out, err := client.CreateCustomPlugin(t.Context(), minimalCreateCustomPluginInput("plugin-one")) + require.NoError(t, err) + assert.Contains(t, aws.ToString(out.CustomPluginArn), "custom-plugin/plugin-one/") + assert.Equal(t, types.CustomPluginStateActive, out.CustomPluginState) + assert.EqualValues(t, 1, out.Revision) +} + +func TestCreateCustomPlugin_DuplicateNameReturnsConflict(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateCustomPlugin(ctx, minimalCreateCustomPluginInput("dup-plugin")) + require.NoError(t, err) + + _, err = client.CreateCustomPlugin(ctx, minimalCreateCustomPluginInput("dup-plugin")) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ConflictException", apiErr.ErrorCode()) +} + +func TestDescribeCustomPlugin(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateCustomPlugin(ctx, minimalCreateCustomPluginInput("describe-plugin")) + require.NoError(t, err) + + out, err := client.DescribeCustomPlugin( + ctx, + &kafkaconnectsdk.DescribeCustomPluginInput{CustomPluginArn: created.CustomPluginArn}, + ) + require.NoError(t, err) + assert.Equal(t, "describe-plugin", aws.ToString(out.Name)) + assert.Equal(t, types.CustomPluginStateActive, out.CustomPluginState) + require.NotNil(t, out.LatestRevision) + assert.EqualValues(t, 1, out.LatestRevision.Revision) + require.NotNil(t, out.LatestRevision.Location) + require.NotNil(t, out.LatestRevision.Location.S3Location) + assert.Equal(t, "plugins/my-plugin.zip", aws.ToString(out.LatestRevision.Location.S3Location.FileKey)) + require.NotNil(t, out.LatestRevision.FileDescription) + assert.NotEmpty(t, aws.ToString(out.LatestRevision.FileDescription.FileMd5)) +} + +func TestDescribeCustomPlugin_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.DescribeCustomPlugin(t.Context(), &kafkaconnectsdk.DescribeCustomPluginInput{ + CustomPluginArn: aws.String("arn:aws:kafkaconnect:us-east-1:123456789012:custom-plugin/nope/abc"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "NotFoundException", apiErr.ErrorCode()) +} + +func TestListCustomPlugins(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateCustomPlugin(ctx, minimalCreateCustomPluginInput("list-plugin-a")) + require.NoError(t, err) + _, err = client.CreateCustomPlugin(ctx, minimalCreateCustomPluginInput("list-plugin-b")) + require.NoError(t, err) + + out, err := client.ListCustomPlugins(ctx, &kafkaconnectsdk.ListCustomPluginsInput{}) + require.NoError(t, err) + assert.Len(t, out.CustomPlugins, 2) +} + +func TestDeleteCustomPlugin(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateCustomPlugin(ctx, minimalCreateCustomPluginInput("delete-plugin")) + require.NoError(t, err) + + out, err := client.DeleteCustomPlugin( + ctx, + &kafkaconnectsdk.DeleteCustomPluginInput{CustomPluginArn: created.CustomPluginArn}, + ) + require.NoError(t, err) + assert.Equal(t, types.CustomPluginStateDeleting, out.CustomPluginState) + + _, err = client.DescribeCustomPlugin( + ctx, + &kafkaconnectsdk.DescribeCustomPluginInput{CustomPluginArn: created.CustomPluginArn}, + ) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "NotFoundException", apiErr.ErrorCode()) +} diff --git a/services/kafkaconnect/errors.go b/services/kafkaconnect/errors.go new file mode 100644 index 0000000000..c7a31db981 --- /dev/null +++ b/services/kafkaconnect/errors.go @@ -0,0 +1,32 @@ +package kafkaconnect + +import "github.com/blackbirdworks/gopherstack/pkgs/awserr" + +var ( + // ErrConnectorNotFound is returned when a connector ARN does not resolve. + ErrConnectorNotFound = awserr.New("connector not found", awserr.ErrNotFound) + // ErrConnectorNameInUse is returned when a connector name is already in use. + ErrConnectorNameInUse = awserr.New("a connector with this name already exists", awserr.ErrAlreadyExists) + // ErrCustomPluginNotFound is returned when a custom plugin ARN does not resolve. + ErrCustomPluginNotFound = awserr.New("custom plugin not found", awserr.ErrNotFound) + // ErrCustomPluginNameInUse is returned when a custom plugin name is already in use. + ErrCustomPluginNameInUse = awserr.New("a custom plugin with this name already exists", awserr.ErrAlreadyExists) + // ErrWorkerConfigNotFound is returned when a worker configuration ARN does not resolve. + ErrWorkerConfigNotFound = awserr.New("worker configuration not found", awserr.ErrNotFound) + // ErrWorkerConfigNameInUse is returned when a worker configuration name is already in use. + ErrWorkerConfigNameInUse = awserr.New( + "a worker configuration with this name already exists", + awserr.ErrAlreadyExists, + ) + // ErrConnectorOperationNotFound is returned when a connector operation ARN does not resolve. + ErrConnectorOperationNotFound = awserr.New("connector operation not found", awserr.ErrNotFound) + // ErrResourceNotFound is returned by the generic tag operations when resourceArn resolves to nothing. + ErrResourceNotFound = awserr.New("resource not found", awserr.ErrNotFound) + // ErrVersionMismatch is returned when currentVersion does not match a connector's actual version. + ErrVersionMismatch = awserr.New( + "the current version specified does not match the connector's actual current version", + awserr.ErrConflict, + ) + // ErrValidation is returned when request input fails validation. + ErrValidation = awserr.New("invalid request", awserr.ErrInvalidParameter) +) diff --git a/services/kafkaconnect/handler.go b/services/kafkaconnect/handler.go new file mode 100644 index 0000000000..e6dfe2c91c --- /dev/null +++ b/services/kafkaconnect/handler.go @@ -0,0 +1,251 @@ +package kafkaconnect + +import ( + "encoding/json" + "errors" + "maps" + "net/http" + "strings" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/pkgs/awserr" + "github.com/blackbirdworks/gopherstack/pkgs/httputils" + "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/service" +) + +const ( + kafkaConnectService = "kafkaconnect" + kafkaConnectMatchPriority = service.PriorityPathVersioned +) + +// Operation names, matching the AWS API exactly. +const ( + opCreateConnector = "CreateConnector" + opDescribeConnector = "DescribeConnector" + opListConnectors = "ListConnectors" + opUpdateConnector = "UpdateConnector" + opDeleteConnector = "DeleteConnector" + opRestartConnector = "RestartConnector" + opDescribeConnectorOperation = "DescribeConnectorOperation" + opListConnectorOperations = "ListConnectorOperations" + + opCreateCustomPlugin = "CreateCustomPlugin" + opDescribeCustomPlugin = "DescribeCustomPlugin" + opListCustomPlugins = "ListCustomPlugins" + opDeleteCustomPlugin = "DeleteCustomPlugin" + + opCreateWorkerConfiguration = "CreateWorkerConfiguration" + opDescribeWorkerConfiguration = "DescribeWorkerConfiguration" + opListWorkerConfigurations = "ListWorkerConfigurations" + opDeleteWorkerConfiguration = "DeleteWorkerConfiguration" + + opTagResource = "TagResource" + opUntagResource = "UntagResource" + opListTagsForResource = "ListTagsForResource" +) + +// opFunc is the uniform signature for all dispatch operations; resource is +// the ARN parsed from the path (empty when the operation has none), and body +// is the raw JSON request body (empty for bodyless GET/DELETE requests). +type opFunc func(c *echo.Context, resource string, body []byte) error + +// Handler is the HTTP handler for the MSK Connect REST API. +type Handler struct { + Backend StorageBackend + ops map[string]opFunc + AccountID string + DefaultRegion string +} + +// NewHandler creates a new MSK Connect handler. +func NewHandler(backend StorageBackend) *Handler { + h := &Handler{Backend: backend} + h.ops = h.buildOps() + + return h +} + +func (h *Handler) buildOps() map[string]opFunc { + ops := make(map[string]opFunc, len(h.GetSupportedOperations())) + + maps.Copy(ops, h.buildConnectorOps()) + maps.Copy(ops, h.buildCustomPluginOps()) + maps.Copy(ops, h.buildWorkerConfigurationOps()) + maps.Copy(ops, h.buildTagOps()) + + return ops +} + +// Reset clears all backend state. +func (h *Handler) Reset() { + h.Backend.Reset() +} + +// Name returns the service name. +func (h *Handler) Name() string { return "KafkaConnect" } + +// GetSupportedOperations returns the list of supported operations. +func (h *Handler) GetSupportedOperations() []string { + return []string{ + opCreateConnector, + opDescribeConnector, + opListConnectors, + opUpdateConnector, + opDeleteConnector, + opRestartConnector, + opDescribeConnectorOperation, + opListConnectorOperations, + opCreateCustomPlugin, + opDescribeCustomPlugin, + opListCustomPlugins, + opDeleteCustomPlugin, + opCreateWorkerConfiguration, + opDescribeWorkerConfiguration, + opListWorkerConfigurations, + opDeleteWorkerConfiguration, + opTagResource, + opUntagResource, + opListTagsForResource, + } +} + +// ChaosServiceName returns the lowercase AWS service name for fault rule matching. +func (h *Handler) ChaosServiceName() string { return kafkaConnectService } + +// ChaosOperations returns all operations that can be fault-injected. +func (h *Handler) ChaosOperations() []string { return h.GetSupportedOperations() } + +// ChaosRegions returns all regions this handler handles. +func (h *Handler) ChaosRegions() []string { return []string{h.DefaultRegion} } + +// RouteMatcher returns a function that matches MSK Connect REST API requests. +func (h *Handler) RouteMatcher() service.Matcher { + return func(c *echo.Context) bool { + path := c.Request().URL.Path + + switch { + case strings.HasPrefix(path, connectorsPath): + return true + case strings.HasPrefix(path, connectorOperationsPath): + return true + case strings.HasPrefix(path, customPluginsPath): + return true + case strings.HasPrefix(path, workerConfigurationsPath): + return true + case strings.HasPrefix(path, tagsPrefix): + return isKafkaConnectTagsPath(path) + } + + return false + } +} + +// isKafkaConnectTagsPath reports whether path is a /v1/tags/{arn} path for a +// kafkaconnect ARN. Several restjson1 services (kafka, batch, appsync, ...) +// claim this same "/v1/tags/{arn}" prefix, so this is guarded by the ARN's +// own service field rather than claimed unconditionally, per +// .claude/memories -- route-matcher-prefix-collision. +func isKafkaConnectTagsPath(path string) bool { + encodedARN := strings.TrimPrefix(path, tagsPrefix) + if encodedARN == "" { + return false + } + + decodedARN, err := decodeResourceARN(encodedARN) + if err != nil { + return false + } + + parts := strings.SplitN(decodedARN, ":", arnMaxParts) + + return len(parts) >= arnServiceFieldIndex+1 && parts[arnServiceFieldIndex] == kafkaConnectService +} + +// MatchPriority returns the routing priority. +func (h *Handler) MatchPriority() int { return kafkaConnectMatchPriority } + +// ExtractOperation extracts the MSK Connect operation name from the request. +func (h *Handler) ExtractOperation(c *echo.Context) string { + op, _ := parseKafkaConnectPath(c.Request().Method, c.Request().URL.Path) + + return op +} + +// ExtractResource extracts the resource ARN from the request path. +func (h *Handler) ExtractResource(c *echo.Context) string { + _, resource := parseKafkaConnectPath(c.Request().Method, c.Request().URL.Path) + + return resource +} + +// Handler returns the Echo handler function for MSK Connect requests. +func (h *Handler) Handler() echo.HandlerFunc { + return func(c *echo.Context) error { + ctx := c.Request().Context() + log := logger.Load(ctx) + + op, resource := parseKafkaConnectPath(c.Request().Method, c.Request().URL.Path) + if op == "" { + return h.writeError(c, http.StatusBadRequest, "BadRequestException", "unknown operation") + } + + body, err := httputils.ReadBody(c.Request()) + if err != nil { + log.ErrorContext(ctx, "kafkaconnect: failed to read request body", "error", err) + + return h.writeError( + c, + http.StatusInternalServerError, + "InternalServerErrorException", + "failed to read request body", + ) + } + + log.DebugContext(ctx, "kafkaconnect request", "op", op, "resource", resource) + + return h.dispatch(c, op, resource, body) + } +} + +func (h *Handler) dispatch(c *echo.Context, op, resource string, body []byte) error { + fn, ok := h.ops[op] + if !ok { + return h.writeError(c, http.StatusBadRequest, "BadRequestException", "unknown operation") + } + + return fn(c, resource, body) +} + +func decodeBody[T any](body []byte, out *T) error { + if len(body) == 0 { + return nil + } + + return json.Unmarshal(body, out) +} + +// writeJSON writes a 200 JSON response. +func (h *Handler) writeJSON(c *echo.Context, v any) error { + return c.JSON(http.StatusOK, v) +} + +// writeError writes a kafkaconnect JSON error response with the AWS __type field. +func (h *Handler) writeError(c *echo.Context, status int, errType, message string) error { + return c.JSON(status, errorResponse{Type: errType, Message: message}) +} + +// writeBackendError maps a backend error to an HTTP error response with the appropriate AWS error type. +func (h *Handler) writeBackendError(c *echo.Context, err error) error { + switch { + case errors.Is(err, awserr.ErrNotFound): + return h.writeError(c, http.StatusNotFound, "NotFoundException", err.Error()) + case errors.Is(err, awserr.ErrAlreadyExists), errors.Is(err, awserr.ErrConflict): + return h.writeError(c, http.StatusConflict, "ConflictException", err.Error()) + case errors.Is(err, awserr.ErrInvalidParameter): + return h.writeError(c, http.StatusBadRequest, "BadRequestException", err.Error()) + default: + return h.writeError(c, http.StatusInternalServerError, "InternalServerErrorException", err.Error()) + } +} diff --git a/services/kafkaconnect/handler_connectors.go b/services/kafkaconnect/handler_connectors.go new file mode 100644 index 0000000000..cc1da7236b --- /dev/null +++ b/services/kafkaconnect/handler_connectors.go @@ -0,0 +1,182 @@ +package kafkaconnect + +import ( + "net/http" + "strconv" + + "github.com/labstack/echo/v5" +) + +func (h *Handler) buildConnectorOps() map[string]opFunc { + return map[string]opFunc{ + opCreateConnector: func(c *echo.Context, _ string, body []byte) error { + return h.handleCreateConnector(c, body) + }, + opDescribeConnector: func(c *echo.Context, resource string, _ []byte) error { + return h.handleDescribeConnector(c, resource) + }, + opListConnectors: func(c *echo.Context, _ string, _ []byte) error { + return h.handleListConnectors(c) + }, + opUpdateConnector: func(c *echo.Context, resource string, body []byte) error { + return h.handleUpdateConnector(c, resource, body) + }, + opDeleteConnector: func(c *echo.Context, resource string, _ []byte) error { + return h.handleDeleteConnector(c, resource) + }, + opRestartConnector: func(c *echo.Context, resource string, _ []byte) error { + return h.handleRestartConnector(c, resource) + }, + opDescribeConnectorOperation: func(c *echo.Context, resource string, _ []byte) error { + return h.handleDescribeConnectorOperation(c, resource) + }, + opListConnectorOperations: func(c *echo.Context, resource string, _ []byte) error { + return h.handleListConnectorOperations(c, resource) + }, + } +} + +func (h *Handler) handleCreateConnector(c *echo.Context, body []byte) error { + var req createConnectorRequest + if err := decodeBody(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "BadRequestException", "invalid request body") + } + + if req.ConnectorName == "" || req.ServiceExecutionRoleArn == "" || req.KafkaConnectVersion == "" { + return h.writeError(c, http.StatusBadRequest, "BadRequestException", "missing required field") + } + + spec := ConnectorSpec{ + Name: req.ConnectorName, + Description: req.ConnectorDescription, + ConnectorConfiguration: req.ConnectorConfiguration, + Capacity: capacityFromDTO(req.Capacity), + ApacheKafkaCluster: apacheKafkaClusterFromDTO(req.KafkaCluster), + KafkaClusterClientAuthentication: req.KafkaClusterClientAuthentication.AuthenticationType, + KafkaClusterEncryptionInTransit: req.KafkaClusterEncryptionInTransit.EncryptionType, + KafkaConnectVersion: req.KafkaConnectVersion, + ServiceExecutionRoleArn: req.ServiceExecutionRoleArn, + NetworkType: req.NetworkType, + Plugins: pluginsFromDTO(req.Plugins), + WorkerLogDelivery: workerLogDeliveryFromDTO(req.LogDelivery), + Tags: req.Tags, + } + + if req.WorkerConfiguration != nil { + spec.WorkerConfiguration = &WorkerConfigRef{ + Arn: req.WorkerConfiguration.WorkerConfigurationArn, + Revision: req.WorkerConfiguration.Revision, + } + } + + connector, err := h.Backend.CreateConnector(h.AccountID, h.DefaultRegion, spec) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, createConnectorResponse{ + ConnectorArn: connector.ARN, + ConnectorName: connector.Name, + ConnectorState: connector.State, + }) +} + +func (h *Handler) handleDescribeConnector(c *echo.Context, connectorArn string) error { + connector, err := h.Backend.DescribeConnector(connectorArn) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, describeConnectorResponse{connectorSummaryDTO: connectorToDTO(connector)}) +} + +func (h *Handler) handleListConnectors(c *echo.Context) error { + q := c.Request().URL.Query() + maxResults, _ := strconv.Atoi(q.Get("maxResults")) + + connectors, next, err := h.Backend.ListConnectors(q.Get("connectorNamePrefix"), q.Get("nextToken"), maxResults) + if err != nil { + return h.writeBackendError(c, err) + } + + dtos := make([]connectorSummaryDTO, 0, len(connectors)) + for _, connector := range connectors { + dtos = append(dtos, connectorToDTO(connector)) + } + + return h.writeJSON(c, listConnectorsResponse{Connectors: dtos, NextToken: next}) +} + +func (h *Handler) handleUpdateConnector(c *echo.Context, connectorArn string, body []byte) error { + var req updateConnectorRequest + if err := decodeBody(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "BadRequestException", "invalid request body") + } + + currentVersion := c.Request().URL.Query().Get("currentVersion") + + update := ConnectorUpdate{ConnectorConfiguration: req.ConnectorConfiguration} + if req.Capacity != nil { + capUpdate := capacityFromDTO(*req.Capacity) + update.Capacity = &capUpdate + } + + connector, op, err := h.Backend.UpdateConnector(connectorArn, currentVersion, update) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, updateConnectorResponse{ + ConnectorArn: connector.ARN, + ConnectorOperationArn: op.ARN, + ConnectorState: connector.State, + }) +} + +func (h *Handler) handleDeleteConnector(c *echo.Context, connectorArn string) error { + currentVersion := c.Request().URL.Query().Get("currentVersion") + + connector, err := h.Backend.DeleteConnector(connectorArn, currentVersion) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, deleteConnectorResponse{ConnectorArn: connector.ARN, ConnectorState: connector.State}) +} + +func (h *Handler) handleRestartConnector(c *echo.Context, connectorArn string) error { + onlyFailedTasks := c.Request().URL.Query().Get("onlyFailedTasks") == "true" + + _, op, err := h.Backend.RestartConnector(connectorArn, onlyFailedTasks) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, restartConnectorResponse{ConnectorArn: op.ConnectorArn, ConnectorOperationArn: op.ARN}) +} + +func (h *Handler) handleDescribeConnectorOperation(c *echo.Context, operationArn string) error { + op, err := h.Backend.DescribeConnectorOperation(operationArn) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, connectorOperationToDescribeDTO(op)) +} + +func (h *Handler) handleListConnectorOperations(c *echo.Context, connectorArn string) error { + q := c.Request().URL.Query() + maxResults, _ := strconv.Atoi(q.Get("maxResults")) + + ops, next, err := h.Backend.ListConnectorOperations(connectorArn, q.Get("nextToken"), maxResults) + if err != nil { + return h.writeBackendError(c, err) + } + + dtos := make([]connectorOperationSummaryDTO, 0, len(ops)) + for _, op := range ops { + dtos = append(dtos, connectorOperationToSummaryDTO(op)) + } + + return h.writeJSON(c, listConnectorOperationsResponse{ConnectorOperations: dtos, NextToken: next}) +} diff --git a/services/kafkaconnect/handler_customplugins.go b/services/kafkaconnect/handler_customplugins.go new file mode 100644 index 0000000000..4d91b1319f --- /dev/null +++ b/services/kafkaconnect/handler_customplugins.go @@ -0,0 +1,94 @@ +package kafkaconnect + +import ( + "net/http" + "strconv" + + "github.com/labstack/echo/v5" +) + +func (h *Handler) buildCustomPluginOps() map[string]opFunc { + return map[string]opFunc{ + opCreateCustomPlugin: func(c *echo.Context, _ string, body []byte) error { + return h.handleCreateCustomPlugin(c, body) + }, + opDescribeCustomPlugin: func(c *echo.Context, resource string, _ []byte) error { + return h.handleDescribeCustomPlugin(c, resource) + }, + opListCustomPlugins: func(c *echo.Context, _ string, _ []byte) error { + return h.handleListCustomPlugins(c) + }, + opDeleteCustomPlugin: func(c *echo.Context, resource string, _ []byte) error { + return h.handleDeleteCustomPlugin(c, resource) + }, + } +} + +func (h *Handler) handleCreateCustomPlugin(c *echo.Context, body []byte) error { + var req createCustomPluginRequest + if err := decodeBody(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "BadRequestException", "invalid request body") + } + + if req.Name == "" || req.ContentType == "" { + return h.writeError(c, http.StatusBadRequest, "BadRequestException", "missing required field") + } + + plugin, err := h.Backend.CreateCustomPlugin( + h.AccountID, h.DefaultRegion, req.Name, req.Description, req.ContentType, + req.Location.S3Location.BucketArn, req.Location.S3Location.FileKey, req.Location.S3Location.ObjectVersion, + req.Tags, + ) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, createCustomPluginResponse{ + CustomPluginArn: plugin.ARN, + CustomPluginState: plugin.State, + Name: plugin.Name, + Revision: plugin.Revision, + }) +} + +func (h *Handler) handleDescribeCustomPlugin(c *echo.Context, customPluginArn string) error { + plugin, err := h.Backend.DescribeCustomPlugin(customPluginArn) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, describeCustomPluginResponse{ + CreationTime: formatTime(plugin.CreationTime), + CustomPluginArn: plugin.ARN, + CustomPluginState: plugin.State, + Description: plugin.Description, + Name: plugin.Name, + LatestRevision: customPluginToRevisionSummaryDTO(plugin), + }) +} + +func (h *Handler) handleListCustomPlugins(c *echo.Context) error { + q := c.Request().URL.Query() + maxResults, _ := strconv.Atoi(q.Get("maxResults")) + + plugins, next, err := h.Backend.ListCustomPlugins(q.Get("namePrefix"), q.Get("nextToken"), maxResults) + if err != nil { + return h.writeBackendError(c, err) + } + + dtos := make([]customPluginSummaryDTO, 0, len(plugins)) + for _, p := range plugins { + dtos = append(dtos, customPluginToSummaryDTO(p)) + } + + return h.writeJSON(c, listCustomPluginsResponse{CustomPlugins: dtos, NextToken: next}) +} + +func (h *Handler) handleDeleteCustomPlugin(c *echo.Context, customPluginArn string) error { + plugin, err := h.Backend.DeleteCustomPlugin(customPluginArn) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, deleteCustomPluginResponse{CustomPluginArn: plugin.ARN, CustomPluginState: plugin.State}) +} diff --git a/services/kafkaconnect/handler_tags.go b/services/kafkaconnect/handler_tags.go new file mode 100644 index 0000000000..1aade91493 --- /dev/null +++ b/services/kafkaconnect/handler_tags.go @@ -0,0 +1,53 @@ +package kafkaconnect + +import ( + "net/http" + + "github.com/labstack/echo/v5" +) + +func (h *Handler) buildTagOps() map[string]opFunc { + return map[string]opFunc{ + opTagResource: func(c *echo.Context, resource string, body []byte) error { + return h.handleTagResource(c, resource, body) + }, + opUntagResource: func(c *echo.Context, resource string, _ []byte) error { + return h.handleUntagResource(c, resource) + }, + opListTagsForResource: func(c *echo.Context, resource string, _ []byte) error { + return h.handleListTagsForResource(c, resource) + }, + } +} + +func (h *Handler) handleTagResource(c *echo.Context, resourceArn string, body []byte) error { + var req tagResourceRequest + if err := decodeBody(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "BadRequestException", "invalid request body") + } + + if err := h.Backend.TagResource(resourceArn, req.Tags); err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, struct{}{}) +} + +func (h *Handler) handleUntagResource(c *echo.Context, resourceArn string) error { + tagKeys := c.Request().URL.Query()["tagKeys"] + + if err := h.Backend.UntagResource(resourceArn, tagKeys); err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, struct{}{}) +} + +func (h *Handler) handleListTagsForResource(c *echo.Context, resourceArn string) error { + tags, err := h.Backend.ListTagsForResource(resourceArn) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, listTagsForResourceResponse{Tags: tags}) +} diff --git a/services/kafkaconnect/handler_test.go b/services/kafkaconnect/handler_test.go new file mode 100644 index 0000000000..522348aa12 --- /dev/null +++ b/services/kafkaconnect/handler_test.go @@ -0,0 +1,56 @@ +package kafkaconnect_test + +import ( + "net/http/httptest" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awscfg "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + kafkaconnectsdk "github.com/aws/aws-sdk-go-v2/service/kafkaconnect" + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/kafkaconnect" +) + +const testRegion = "us-east-1" +const testAccountID = "123456789012" + +// newTestClient stands up the real aws-sdk-go-v2 kafkaconnect client against +// an httptest server running this package's Handler, wired through the same +// pkgs/service registry/router used in production. +func newTestClient(t *testing.T, h *kafkaconnect.Handler) *kafkaconnectsdk.Client { + t.Helper() + + e := echo.New() + registry := service.NewRegistry() + require.NoError(t, registry.Register(h)) + e.Use(service.NewServiceRouter(registry).RouteHandler()) + + srv := httptest.NewServer(e) + t.Cleanup(srv.Close) + + cfg, err := awscfg.LoadDefaultConfig( + t.Context(), + awscfg.WithRegion(testRegion), + awscfg.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider("test", "test", ""), + ), + ) + require.NoError(t, err) + + return kafkaconnectsdk.NewFromConfig(cfg, func(o *kafkaconnectsdk.Options) { + o.BaseEndpoint = aws.String(srv.URL) + }) +} + +func newTestHandler() *kafkaconnect.Handler { + backend := kafkaconnect.NewInMemoryBackend() + h := kafkaconnect.NewHandler(backend) + h.AccountID = testAccountID + h.DefaultRegion = testRegion + + return h +} diff --git a/services/kafkaconnect/handler_workerconfigs.go b/services/kafkaconnect/handler_workerconfigs.go new file mode 100644 index 0000000000..911c3e9c91 --- /dev/null +++ b/services/kafkaconnect/handler_workerconfigs.go @@ -0,0 +1,101 @@ +package kafkaconnect + +import ( + "net/http" + "strconv" + + "github.com/labstack/echo/v5" +) + +func (h *Handler) buildWorkerConfigurationOps() map[string]opFunc { + return map[string]opFunc{ + opCreateWorkerConfiguration: func(c *echo.Context, _ string, body []byte) error { + return h.handleCreateWorkerConfiguration(c, body) + }, + opDescribeWorkerConfiguration: func(c *echo.Context, resource string, _ []byte) error { + return h.handleDescribeWorkerConfiguration(c, resource) + }, + opListWorkerConfigurations: func(c *echo.Context, _ string, _ []byte) error { + return h.handleListWorkerConfigurations(c) + }, + opDeleteWorkerConfiguration: func(c *echo.Context, resource string, _ []byte) error { + return h.handleDeleteWorkerConfiguration(c, resource) + }, + } +} + +func (h *Handler) handleCreateWorkerConfiguration(c *echo.Context, body []byte) error { + var req createWorkerConfigurationRequest + if err := decodeBody(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "BadRequestException", "invalid request body") + } + + if req.Name == "" || req.PropertiesFileContent == "" { + return h.writeError(c, http.StatusBadRequest, "BadRequestException", "missing required field") + } + + cfg, err := h.Backend.CreateWorkerConfiguration( + h.AccountID, h.DefaultRegion, req.Name, req.Description, req.PropertiesFileContent, req.Tags, + ) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, createWorkerConfigurationResponse{ + CreationTime: formatTime(cfg.CreationTime), + Name: cfg.Name, + WorkerConfigurationArn: cfg.ARN, + WorkerConfigurationState: cfg.State, + LatestRevision: workerConfigToRevisionSummaryDTO(cfg), + }) +} + +func (h *Handler) handleDescribeWorkerConfiguration(c *echo.Context, workerConfigurationArn string) error { + cfg, err := h.Backend.DescribeWorkerConfiguration(workerConfigurationArn) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, describeWorkerConfigurationResponse{ + CreationTime: formatTime(cfg.CreationTime), + Description: cfg.Description, + Name: cfg.Name, + WorkerConfigurationArn: cfg.ARN, + WorkerConfigurationState: cfg.State, + LatestRevision: &workerConfigurationRevisionDescriptionDTO{ + CreationTime: formatTime(cfg.LatestRevision.CreationTime), + Description: cfg.LatestRevision.Description, + PropertiesFileContent: cfg.LatestRevision.PropertiesFileContent, + Revision: cfg.LatestRevision.Revision, + }, + }) +} + +func (h *Handler) handleListWorkerConfigurations(c *echo.Context) error { + q := c.Request().URL.Query() + maxResults, _ := strconv.Atoi(q.Get("maxResults")) + + configs, next, err := h.Backend.ListWorkerConfigurations(q.Get("namePrefix"), q.Get("nextToken"), maxResults) + if err != nil { + return h.writeBackendError(c, err) + } + + dtos := make([]workerConfigurationSummaryDTO, 0, len(configs)) + for _, cfg := range configs { + dtos = append(dtos, workerConfigToSummaryDTO(cfg)) + } + + return h.writeJSON(c, listWorkerConfigurationsResponse{WorkerConfigurations: dtos, NextToken: next}) +} + +func (h *Handler) handleDeleteWorkerConfiguration(c *echo.Context, workerConfigurationArn string) error { + cfg, err := h.Backend.DeleteWorkerConfiguration(workerConfigurationArn) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, deleteWorkerConfigurationResponse{ + WorkerConfigurationArn: cfg.ARN, + WorkerConfigurationState: cfg.State, + }) +} diff --git a/services/kafkaconnect/helpers.go b/services/kafkaconnect/helpers.go new file mode 100644 index 0000000000..621e09fc7b --- /dev/null +++ b/services/kafkaconnect/helpers.go @@ -0,0 +1,38 @@ +package kafkaconnect + +import ( + "crypto/sha256" + "encoding/hex" + "net/url" + "strings" + + "github.com/google/uuid" +) + +const defaultListLimit = 100 + +// arnMaxParts is the number of ":"-separated ARN segments: +// arn:partition:service:region:account:resource. +const arnMaxParts = 6 + +// arnServiceFieldIndex is the zero-based index of the service field within +// an ARN split by arnMaxParts. +const arnServiceFieldIndex = 2 + +// decodeResourceARN percent-decodes a path segment carrying an ARN. +func decodeResourceARN(encoded string) (string, error) { + return url.PathUnescape(encoded) +} + +func newVersion() string { + return strings.ReplaceAll(uuid.NewString(), "-", "")[:16] +} + +// fakeFileChecksum derives a stable, plausible-looking hex digest for a +// custom plugin's S3 object, since this backend does not read S3 object +// bytes -- see PARITY.md. +func fakeFileChecksum(s string) string { + sum := sha256.Sum256([]byte(s)) + + return hex.EncodeToString(sum[:])[:32] +} diff --git a/services/kafkaconnect/interfaces.go b/services/kafkaconnect/interfaces.go new file mode 100644 index 0000000000..d69e3fb556 --- /dev/null +++ b/services/kafkaconnect/interfaces.go @@ -0,0 +1,67 @@ +package kafkaconnect + +// ConnectorSpec carries every CreateConnector input field except the +// account/region needed to mint the ARN. +type ConnectorSpec struct { + Capacity Capacity + ConnectorConfiguration map[string]string + Tags map[string]string + WorkerConfiguration *WorkerConfigRef + WorkerLogDelivery *WorkerLogDelivery + Description string + Name string + KafkaConnectVersion string + KafkaClusterClientAuthentication string + KafkaClusterEncryptionInTransit string + ServiceExecutionRoleArn string + NetworkType string + ApacheKafkaCluster ApacheKafkaCluster + Plugins []PluginRef +} + +// ConnectorUpdate carries UpdateConnector's mutually exclusive target fields; +// exactly one of Capacity or ConnectorConfiguration is non-nil. +type ConnectorUpdate struct { + Capacity *Capacity + ConnectorConfiguration map[string]string +} + +// StorageBackend is the interface for the MSK Connect backend. +type StorageBackend interface { + CreateConnector(accountID, region string, spec ConnectorSpec) (*Connector, error) + DescribeConnector(connectorArn string) (*Connector, error) + ListConnectors(namePrefix, nextToken string, maxResults int) ([]*Connector, string, error) + UpdateConnector( + connectorArn, currentVersion string, + update ConnectorUpdate, + ) (*Connector, *ConnectorOperation, error) + DeleteConnector(connectorArn, currentVersion string) (*Connector, error) + RestartConnector(connectorArn string, onlyFailedTasks bool) (*Connector, *ConnectorOperation, error) + DescribeConnectorOperation(operationArn string) (*ConnectorOperation, error) + ListConnectorOperations(connectorArn, nextToken string, maxResults int) ([]*ConnectorOperation, string, error) + + CreateCustomPlugin( + accountID, region, name, description, contentType, bucketArn, fileKey, objectVersion string, + tags map[string]string, + ) (*CustomPlugin, error) + DescribeCustomPlugin(customPluginArn string) (*CustomPlugin, error) + ListCustomPlugins(namePrefix, nextToken string, maxResults int) ([]*CustomPlugin, string, error) + DeleteCustomPlugin(customPluginArn string) (*CustomPlugin, error) + + CreateWorkerConfiguration( + accountID, region, name, description, propertiesFileContent string, + tags map[string]string, + ) (*WorkerConfiguration, error) + DescribeWorkerConfiguration(workerConfigurationArn string) (*WorkerConfiguration, error) + ListWorkerConfigurations(namePrefix, nextToken string, maxResults int) ([]*WorkerConfiguration, string, error) + DeleteWorkerConfiguration(workerConfigurationArn string) (*WorkerConfiguration, error) + + TagResource(resourceArn string, tags map[string]string) error + UntagResource(resourceArn string, tagKeys []string) error + ListTagsForResource(resourceArn string) (map[string]string, error) + + Reset() +} + +// Compile-time assertion that InMemoryBackend implements StorageBackend. +var _ StorageBackend = (*InMemoryBackend)(nil) diff --git a/services/kafkaconnect/models.go b/services/kafkaconnect/models.go new file mode 100644 index 0000000000..56a8c5b632 --- /dev/null +++ b/services/kafkaconnect/models.go @@ -0,0 +1,298 @@ +package kafkaconnect + +import ( + "maps" + "slices" + "time" +) + +// Connector state values (types.ConnectorState). This backend only ever +// produces RUNNING -- see PARITY.md for the CREATING/UPDATING/DELETING +// transient states it deliberately does not model. +const ( + connectorStateRunning = "RUNNING" + deletingState = "DELETING" +) + +// CustomPluginState and WorkerConfigurationState values this backend produces. +const ( + customPluginStateActive = "ACTIVE" + workerConfigurationStateActive = "ACTIVE" +) + +// ConnectorOperationState/Type values this backend produces for UpdateConnector. +const ( + connectorOperationStateComplete = "UPDATE_COMPLETE" + connectorOperationTypeConfiguration = "UPDATE_CONNECTOR_CONFIGURATION" + connectorOperationTypeWorkerSetting = "UPDATE_WORKER_SETTING" + connectorOperationStepUpdateConfiguration = "UPDATE_CONNECTOR_CONFIGURATION" + connectorOperationStepUpdateWorkerSetting = "UPDATE_WORKER_SETTING" + connectorOperationStepStateCompleted = "COMPLETED" +) + +// ConnectorOperationState/Type values this backend produces for RestartConnector. +const ( + connectorOperationStateRestartComplete = "RESTART_COMPLETE" + connectorOperationTypeRestart = "RESTART_CONNECTOR" +) + +// AutoScaling mirrors types.AutoScalingDescription. +type AutoScaling struct { + MinWorkerCount int32 + MaxWorkerCount int32 + McuCount int32 + MaxAutoscalingTaskCount int32 + ScaleInCPUPercent int32 + ScaleOutCPUPercent int32 +} + +// ProvisionedCapacity mirrors types.ProvisionedCapacityDescription. +type ProvisionedCapacity struct { + McuCount int32 + WorkerCount int32 +} + +// Capacity mirrors types.CapacityDescription: exactly one of the two is set. +type Capacity struct { + AutoScaling *AutoScaling + Provisioned *ProvisionedCapacity +} + +func (c Capacity) clone() Capacity { + cp := c + if c.AutoScaling != nil { + as := *c.AutoScaling + cp.AutoScaling = &as + } + + if c.Provisioned != nil { + pc := *c.Provisioned + cp.Provisioned = &pc + } + + return cp +} + +// Vpc mirrors types.VpcDescription. +type Vpc struct { + SecurityGroups []string + Subnets []string +} + +// ApacheKafkaCluster mirrors types.ApacheKafkaClusterDescription. +type ApacheKafkaCluster struct { + BootstrapServers string + Vpc Vpc +} + +// PluginRef mirrors types.PluginDescription.CustomPlugin. +type PluginRef struct { + CustomPluginArn string + Revision int64 +} + +// WorkerConfigRef mirrors types.WorkerConfigurationDescription. +type WorkerConfigRef struct { + Arn string + Revision int64 +} + +// CloudWatchLogsDelivery mirrors types.CloudWatchLogsLogDeliveryDescription. +type CloudWatchLogsDelivery struct { + LogGroup string + Enabled bool +} + +// FirehoseDelivery mirrors types.FirehoseLogDeliveryDescription. +type FirehoseDelivery struct { + DeliveryStream string + Enabled bool +} + +// S3LogDelivery mirrors types.S3LogDeliveryDescription. +type S3LogDelivery struct { + Bucket string + Prefix string + Enabled bool +} + +// WorkerLogDelivery mirrors types.WorkerLogDeliveryDescription. +type WorkerLogDelivery struct { + CloudWatchLogs *CloudWatchLogsDelivery + Firehose *FirehoseDelivery + S3 *S3LogDelivery +} + +func (w *WorkerLogDelivery) clone() *WorkerLogDelivery { + if w == nil { + return nil + } + + cp := *w + if w.CloudWatchLogs != nil { + cw := *w.CloudWatchLogs + cp.CloudWatchLogs = &cw + } + + if w.Firehose != nil { + f := *w.Firehose + cp.Firehose = &f + } + + if w.S3 != nil { + s := *w.S3 + cp.S3 = &s + } + + return &cp +} + +// Connector is the persisted representation of an MSK Connect connector. +type Connector struct { + CreationTime time.Time + Capacity Capacity + WorkerConfiguration *WorkerConfigRef + WorkerLogDelivery *WorkerLogDelivery + ConnectorConfiguration map[string]string + Tags map[string]string + State string + Description string + ARN string + CurrentVersion string + KafkaConnectVersion string + KafkaClusterClientAuthentication string + KafkaClusterEncryptionInTransit string + ServiceExecutionRoleArn string + NetworkType string + Name string + ApacheKafkaCluster ApacheKafkaCluster + Plugins []PluginRef +} + +func (c *Connector) clone() *Connector { + if c == nil { + return nil + } + + cp := *c + cp.ConnectorConfiguration = maps.Clone(c.ConnectorConfiguration) + cp.Tags = maps.Clone(c.Tags) + cp.Plugins = slices.Clone(c.Plugins) + cp.Capacity = c.Capacity.clone() + cp.ApacheKafkaCluster.Vpc.SecurityGroups = slices.Clone(c.ApacheKafkaCluster.Vpc.SecurityGroups) + cp.ApacheKafkaCluster.Vpc.Subnets = slices.Clone(c.ApacheKafkaCluster.Vpc.Subnets) + cp.WorkerLogDelivery = c.WorkerLogDelivery.clone() + + if c.WorkerConfiguration != nil { + wc := *c.WorkerConfiguration + cp.WorkerConfiguration = &wc + } + + return &cp +} + +// CustomPlugin is the persisted representation of an MSK Connect custom plugin. +type CustomPlugin struct { + CreationTime time.Time + Tags map[string]string + Name string + ARN string + Description string + State string + ContentType string + BucketArn string + FileKey string + ObjectVersion string + FileMD5 string + FileSizeBytes int64 + Revision int64 +} + +func (p *CustomPlugin) clone() *CustomPlugin { + if p == nil { + return nil + } + + cp := *p + cp.Tags = maps.Clone(p.Tags) + + return &cp +} + +// WorkerConfigRevision is the persisted representation of a worker +// configuration revision. This backend only ever creates revision 1 -- +// UpdateWorkerConfiguration (which would create new revisions) is not part +// of the AWS API surface this backend implements. +type WorkerConfigRevision struct { + CreationTime time.Time + Description string + PropertiesFileContent string + Revision int64 +} + +// WorkerConfiguration is the persisted representation of an MSK Connect worker configuration. +type WorkerConfiguration struct { + CreationTime time.Time + Tags map[string]string + Name string + ARN string + Description string + State string + LatestRevision WorkerConfigRevision +} + +func (w *WorkerConfiguration) clone() *WorkerConfiguration { + if w == nil { + return nil + } + + cp := *w + cp.Tags = maps.Clone(w.Tags) + + return &cp +} + +// ConnectorOperationStep mirrors types.ConnectorOperationStep. +type ConnectorOperationStep struct { + StepType string + StepState string +} + +// ConnectorOperation is the persisted representation of an UpdateConnector +// operation, returned by DescribeConnectorOperation/ListConnectorOperations. +type ConnectorOperation struct { + CreationTime time.Time + EndTime time.Time + OriginConnectorConfiguration map[string]string + TargetConnectorConfiguration map[string]string + OriginCapacity *Capacity + TargetCapacity *Capacity + ARN string + ConnectorArn string + State string + Type string + Steps []ConnectorOperationStep +} + +func (o *ConnectorOperation) clone() *ConnectorOperation { + if o == nil { + return nil + } + + cp := *o + cp.OriginConnectorConfiguration = maps.Clone(o.OriginConnectorConfiguration) + cp.TargetConnectorConfiguration = maps.Clone(o.TargetConnectorConfiguration) + cp.Steps = slices.Clone(o.Steps) + + if o.OriginCapacity != nil { + c := o.OriginCapacity.clone() + cp.OriginCapacity = &c + } + + if o.TargetCapacity != nil { + c := o.TargetCapacity.clone() + cp.TargetCapacity = &c + } + + return &cp +} diff --git a/services/kafkaconnect/persistence.go b/services/kafkaconnect/persistence.go new file mode 100644 index 0000000000..5c03948bb7 --- /dev/null +++ b/services/kafkaconnect/persistence.go @@ -0,0 +1,103 @@ +package kafkaconnect + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/persistence" +) + +// ErrNoSnapshot is returned when a backend does not support snapshot/restore. +var ErrNoSnapshot = errors.New("backend does not support restore") + +// Snapshottable is an optional interface a StorageBackend may implement to +// support snapshot/restore for persistence or test isolation. +type Snapshottable interface { + Snapshot(ctx context.Context) []byte + Restore(context.Context, []byte) error +} + +// kafkaConnectSnapshotVersion identifies the shape of [backendSnapshot]. Bump +// it whenever a change would make an older snapshot unsafe to decode as the +// current shape; Restore discards (rather than partially decodes) any mismatch. +const kafkaConnectSnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the backend. Tables holds +// one JSON-encoded array per registered table name (connectors, customPlugins, +// workerConfigurations, connectorOperations -- see store_setup.go), produced +// by b.registry.SnapshotAll(). +type backendSnapshot struct { + Tables map[string]json.RawMessage `json:"tables"` + Version int `json:"version"` +} + +// Snapshot serializes backend state to JSON. +func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { + b.mu.RLock("Snapshot") + defer b.mu.RUnlock() + + tables, err := b.registry.SnapshotAll() + if err != nil { + logger.Load(ctx).WarnContext(ctx, "kafkaconnect: snapshot table marshal failed", "error", err) + + return nil + } + + snap := backendSnapshot{ + Version: kafkaConnectSnapshotVersion, + Tables: tables, + } + + return persistence.MarshalSnapshot(ctx, "kafkaconnect", &snap) +} + +// Restore deserializes backend state from a JSON snapshot. +func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { + var snap backendSnapshot + + if err := persistence.UnmarshalSnapshot(ctx, "kafkaconnect", data, &snap); err != nil { + return err + } + + b.mu.Lock("Restore") + defer b.mu.Unlock() + + if snap.Version != kafkaConnectSnapshotVersion { + logger.Load(ctx).WarnContext(ctx, + "kafkaconnect: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", kafkaConnectSnapshotVersion) + + b.registry.ResetAll() + + return nil + } + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("kafkaconnect: restore snapshot tables: %w", err) + } + + return nil +} + +// Snapshot implements persistence by delegating to the backend if it supports it. +func (h *Handler) Snapshot(ctx context.Context) []byte { + s, ok := h.Backend.(Snapshottable) + if !ok { + return nil + } + + return s.Snapshot(ctx) +} + +// Restore implements persistence by delegating to the backend if it supports it. +func (h *Handler) Restore(ctx context.Context, data []byte) error { + s, ok := h.Backend.(Snapshottable) + if !ok { + return ErrNoSnapshot + } + + return s.Restore(ctx, data) +} diff --git a/services/kafkaconnect/provider.go b/services/kafkaconnect/provider.go new file mode 100644 index 0000000000..1bbbc7e57b --- /dev/null +++ b/services/kafkaconnect/provider.go @@ -0,0 +1,23 @@ +package kafkaconnect + +import "github.com/blackbirdworks/gopherstack/pkgs/service" + +// Provider implements service.Provider for Amazon MSK Connect. +type Provider struct{} + +// Name returns the provider name. +func (p *Provider) Name() string { return "KafkaConnect" } + +// Init initializes the MSK Connect service backend and handler. +// +//nolint:ireturn,nolintlint // architecturally required to return interface +func (p *Provider) Init(ctx *service.AppContext) (service.Registerable, error) { + accountID, region := service.AccountRegionOrDefault(ctx) + + backend := NewInMemoryBackend() + handler := NewHandler(backend) + handler.AccountID = accountID + handler.DefaultRegion = region + + return handler, nil +} diff --git a/services/kafkaconnect/routes.go b/services/kafkaconnect/routes.go new file mode 100644 index 0000000000..217450dfda --- /dev/null +++ b/services/kafkaconnect/routes.go @@ -0,0 +1,162 @@ +package kafkaconnect + +import ( + "net/http" + "net/url" + "strings" +) + +const ( + connectorsPath = "/v1/connectors" + connectorsPrefix = "/v1/connectors/" + connectorOperationsPath = "/v1/connectorOperations/" + customPluginsPath = "/v1/custom-plugins" + customPluginsPrefix = "/v1/custom-plugins/" + workerConfigurationsPath = "/v1/worker-configurations" + workerConfigurationsPre = "/v1/worker-configurations/" + tagsPrefix = "/v1/tags/" + + operationsSuffix = "/operations" + restartSuffix = "/restart" +) + +// parseKafkaConnectPath parses an HTTP method + path into an operation name +// and its resource ARN (the empty string for operations with no resource in +// the path, e.g. CreateConnector). +func parseKafkaConnectPath(method, path string) (string, string) { + switch { + case path == connectorsPath: + return parseConnectorsRoot(method) + case strings.HasPrefix(path, connectorsPrefix): + return parseConnectorResource(method, path[len(connectorsPrefix):]) + case strings.HasPrefix(path, connectorOperationsPath): + return parseConnectorOperationResource(method, path[len(connectorOperationsPath):]) + case path == customPluginsPath: + return parseCustomPluginsRoot(method) + case strings.HasPrefix(path, customPluginsPrefix): + return parseCustomPluginResource(method, path[len(customPluginsPrefix):]) + case path == workerConfigurationsPath: + return parseWorkerConfigurationsRoot(method) + case strings.HasPrefix(path, workerConfigurationsPre): + return parseWorkerConfigurationResource(method, path[len(workerConfigurationsPre):]) + case strings.HasPrefix(path, tagsPrefix): + return parseTagsResource(method, path[len(tagsPrefix):]) + } + + return "", "" +} + +func parseConnectorsRoot(method string) (string, string) { + switch method { + case http.MethodPost: + return opCreateConnector, "" + case http.MethodGet: + return opListConnectors, "" + } + + return "", "" +} + +func parseConnectorResource(method, remainder string) (string, string) { + decoded, _ := url.PathUnescape(remainder) + + if connectorArn, ok := strings.CutSuffix(decoded, operationsSuffix); ok { + if method == http.MethodGet { + return opListConnectorOperations, connectorArn + } + + return "", "" + } + + if connectorArn, ok := strings.CutSuffix(decoded, restartSuffix); ok { + if method == http.MethodPost { + return opRestartConnector, connectorArn + } + + return "", "" + } + + switch method { + case http.MethodGet: + return opDescribeConnector, decoded + case http.MethodPut: + return opUpdateConnector, decoded + case http.MethodDelete: + return opDeleteConnector, decoded + } + + return "", "" +} + +func parseConnectorOperationResource(method, remainder string) (string, string) { + decoded, _ := url.PathUnescape(remainder) + + if method == http.MethodGet { + return opDescribeConnectorOperation, decoded + } + + return "", "" +} + +func parseCustomPluginsRoot(method string) (string, string) { + switch method { + case http.MethodPost: + return opCreateCustomPlugin, "" + case http.MethodGet: + return opListCustomPlugins, "" + } + + return "", "" +} + +func parseCustomPluginResource(method, remainder string) (string, string) { + decoded, _ := url.PathUnescape(remainder) + + switch method { + case http.MethodGet: + return opDescribeCustomPlugin, decoded + case http.MethodDelete: + return opDeleteCustomPlugin, decoded + } + + return "", "" +} + +func parseWorkerConfigurationsRoot(method string) (string, string) { + switch method { + case http.MethodPost: + return opCreateWorkerConfiguration, "" + case http.MethodGet: + return opListWorkerConfigurations, "" + } + + return "", "" +} + +func parseWorkerConfigurationResource(method, remainder string) (string, string) { + decoded, _ := url.PathUnescape(remainder) + + switch method { + case http.MethodGet: + return opDescribeWorkerConfiguration, decoded + case http.MethodDelete: + return opDeleteWorkerConfiguration, decoded + } + + return "", "" +} + +func parseTagsResource(method, remainder string) (string, string) { + decoded, _ := url.PathUnescape(remainder) + + switch method { + case http.MethodGet: + return opListTagsForResource, decoded + case http.MethodPost: + return opTagResource, decoded + case http.MethodDelete: + return opUntagResource, decoded + } + + return "", "" +} diff --git a/services/kafkaconnect/routes_whitebox_test.go b/services/kafkaconnect/routes_whitebox_test.go new file mode 100644 index 0000000000..3d9af7c4ce --- /dev/null +++ b/services/kafkaconnect/routes_whitebox_test.go @@ -0,0 +1,138 @@ +package kafkaconnect + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseKafkaConnectPath(t *testing.T) { + t.Parallel() + + const connArn = "arn:aws:kafkaconnect:us-east-1:000000000000:connector/test/uuid-1" + + tests := []struct { + name string + method string + path string + wantOp string + wantResource string + }{ + {name: "create_connector", method: http.MethodPost, path: "/v1/connectors", wantOp: opCreateConnector}, + {name: "list_connectors", method: http.MethodGet, path: "/v1/connectors", wantOp: opListConnectors}, + { + name: "describe_connector", method: http.MethodGet, path: "/v1/connectors/" + connArn, + wantOp: opDescribeConnector, wantResource: connArn, + }, + { + name: "update_connector", method: http.MethodPut, path: "/v1/connectors/" + connArn, + wantOp: opUpdateConnector, wantResource: connArn, + }, + { + name: "delete_connector", method: http.MethodDelete, path: "/v1/connectors/" + connArn, + wantOp: opDeleteConnector, wantResource: connArn, + }, + { + name: "restart_connector", method: http.MethodPost, path: "/v1/connectors/" + connArn + "/restart", + wantOp: opRestartConnector, wantResource: connArn, + }, + { + name: "list_connector_operations", + method: http.MethodGet, + path: "/v1/connectors/" + connArn + "/operations", + wantOp: opListConnectorOperations, + wantResource: connArn, + }, + { + name: "describe_connector_operation", method: http.MethodGet, + path: "/v1/connectorOperations/" + connArn + "/operation/op-1", + wantOp: opDescribeConnectorOperation, wantResource: connArn + "/operation/op-1", + }, + { + name: "create_custom_plugin", + method: http.MethodPost, + path: "/v1/custom-plugins", + wantOp: opCreateCustomPlugin, + }, + {name: "list_custom_plugins", method: http.MethodGet, path: "/v1/custom-plugins", wantOp: opListCustomPlugins}, + { + name: "describe_custom_plugin", method: http.MethodGet, path: "/v1/custom-plugins/plugin-arn", + wantOp: opDescribeCustomPlugin, wantResource: "plugin-arn", + }, + { + name: "delete_custom_plugin", method: http.MethodDelete, path: "/v1/custom-plugins/plugin-arn", + wantOp: opDeleteCustomPlugin, wantResource: "plugin-arn", + }, + { + name: "create_worker_configuration", method: http.MethodPost, path: "/v1/worker-configurations", + wantOp: opCreateWorkerConfiguration, + }, + { + name: "list_worker_configurations", method: http.MethodGet, path: "/v1/worker-configurations", + wantOp: opListWorkerConfigurations, + }, + { + name: "describe_worker_configuration", method: http.MethodGet, path: "/v1/worker-configurations/wc-arn", + wantOp: opDescribeWorkerConfiguration, wantResource: "wc-arn", + }, + { + name: "delete_worker_configuration", method: http.MethodDelete, path: "/v1/worker-configurations/wc-arn", + wantOp: opDeleteWorkerConfiguration, wantResource: "wc-arn", + }, + { + name: "tag_resource", method: http.MethodPost, path: "/v1/tags/" + connArn, + wantOp: opTagResource, wantResource: connArn, + }, + { + name: "untag_resource", method: http.MethodDelete, path: "/v1/tags/" + connArn, + wantOp: opUntagResource, wantResource: connArn, + }, + { + name: "list_tags_for_resource", method: http.MethodGet, path: "/v1/tags/" + connArn, + wantOp: opListTagsForResource, wantResource: connArn, + }, + {name: "unknown_path", method: http.MethodGet, path: "/v1/unknown", wantOp: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + op, resource := parseKafkaConnectPath(tt.method, tt.path) + assert.Equal(t, tt.wantOp, op) + assert.Equal(t, tt.wantResource, resource) + }) + } +} + +func TestIsKafkaConnectTagsPath(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + want bool + }{ + { + name: "kafkaconnect arn matches", + path: "/v1/tags/arn:aws:kafkaconnect:us-east-1:000000000000:connector/test/uuid-1", + want: true, + }, + { + name: "kafka msk arn does not match", + path: "/v1/tags/arn:aws:kafka:us-east-1:000000000000:cluster/test/uuid-1", + want: false, + }, + {name: "empty resource does not match", path: "/v1/tags/", want: false}, + {name: "non-arn resource does not match", path: "/v1/tags/not-an-arn", want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + assert.Equal(t, tt.want, isKafkaConnectTagsPath(tt.path)) + }) + } +} diff --git a/services/kafkaconnect/store.go b/services/kafkaconnect/store.go new file mode 100644 index 0000000000..fc8be3d77b --- /dev/null +++ b/services/kafkaconnect/store.go @@ -0,0 +1,66 @@ +package kafkaconnect + +import ( + "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +// InMemoryBackend is the in-memory implementation of StorageBackend. +type InMemoryBackend struct { + connectors *store.Table[Connector] + customPlugins *store.Table[CustomPlugin] + workerConfigurations *store.Table[WorkerConfiguration] + connectorOperations *store.Table[ConnectorOperation] + registry *store.Registry + mu *lockmetrics.RWMutex +} + +// NewInMemoryBackend creates a new in-memory MSK Connect backend. +func NewInMemoryBackend() *InMemoryBackend { + b := &InMemoryBackend{ + registry: store.NewRegistry(), + mu: lockmetrics.New("kafkaconnect"), + } + + registerAllTables(b) + + return b +} + +// Reset clears all backend state. +func (b *InMemoryBackend) Reset() { + b.mu.Lock("Reset") + defer b.mu.Unlock() + + b.registry.ResetAll() +} + +func (b *InMemoryBackend) connectorByName(name string) (*Connector, bool) { + for _, c := range b.connectors.All() { + if c.Name == name { + return c, true + } + } + + return nil, false +} + +func (b *InMemoryBackend) customPluginByName(name string) (*CustomPlugin, bool) { + for _, p := range b.customPlugins.All() { + if p.Name == name { + return p, true + } + } + + return nil, false +} + +func (b *InMemoryBackend) workerConfigurationByName(name string) (*WorkerConfiguration, bool) { + for _, w := range b.workerConfigurations.All() { + if w.Name == name { + return w, true + } + } + + return nil, false +} diff --git a/services/kafkaconnect/store_setup.go b/services/kafkaconnect/store_setup.go new file mode 100644 index 0000000000..4b163f5d36 --- /dev/null +++ b/services/kafkaconnect/store_setup.go @@ -0,0 +1,23 @@ +package kafkaconnect + +import "github.com/blackbirdworks/gopherstack/pkgs/store" + +func connectorKeyFn(v *Connector) string { return v.ARN } + +func customPluginKeyFn(v *CustomPlugin) string { return v.ARN } + +func workerConfigurationKeyFn(v *WorkerConfiguration) string { return v.ARN } + +func connectorOperationKeyFn(v *ConnectorOperation) string { return v.ARN } + +// registerAllTables registers every backend resource table exactly once. +// Must be called during construction only -- store.Register panics on a +// duplicate name. +func registerAllTables(b *InMemoryBackend) { + b.connectors = store.Register(b.registry, "connectors", store.New(connectorKeyFn)) + b.customPlugins = store.Register(b.registry, "customPlugins", store.New(customPluginKeyFn)) + b.workerConfigurations = store.Register( + b.registry, "workerConfigurations", store.New(workerConfigurationKeyFn), + ) + b.connectorOperations = store.Register(b.registry, "connectorOperations", store.New(connectorOperationKeyFn)) +} diff --git a/services/kafkaconnect/tags.go b/services/kafkaconnect/tags.go new file mode 100644 index 0000000000..080edea2ea --- /dev/null +++ b/services/kafkaconnect/tags.go @@ -0,0 +1,81 @@ +package kafkaconnect + +import "maps" + +// TagResource adds or replaces tags on a connector, custom plugin, or worker configuration by ARN. +func (b *InMemoryBackend) TagResource(resourceArn string, tags map[string]string) error { + b.mu.Lock("TagResource") + defer b.mu.Unlock() + + if c, ok := b.connectors.Get(resourceArn); ok { + maps.Copy(c.Tags, tags) + + return nil + } + + if p, ok := b.customPlugins.Get(resourceArn); ok { + maps.Copy(p.Tags, tags) + + return nil + } + + if w, ok := b.workerConfigurations.Get(resourceArn); ok { + maps.Copy(w.Tags, tags) + + return nil + } + + return ErrResourceNotFound +} + +// UntagResource removes tags from a connector, custom plugin, or worker configuration by ARN. +func (b *InMemoryBackend) UntagResource(resourceArn string, tagKeys []string) error { + b.mu.Lock("UntagResource") + defer b.mu.Unlock() + + if c, ok := b.connectors.Get(resourceArn); ok { + for _, k := range tagKeys { + delete(c.Tags, k) + } + + return nil + } + + if p, ok := b.customPlugins.Get(resourceArn); ok { + for _, k := range tagKeys { + delete(p.Tags, k) + } + + return nil + } + + if w, ok := b.workerConfigurations.Get(resourceArn); ok { + for _, k := range tagKeys { + delete(w.Tags, k) + } + + return nil + } + + return ErrResourceNotFound +} + +// ListTagsForResource returns all tags on a connector, custom plugin, or worker configuration by ARN. +func (b *InMemoryBackend) ListTagsForResource(resourceArn string) (map[string]string, error) { + b.mu.RLock("ListTagsForResource") + defer b.mu.RUnlock() + + if c, ok := b.connectors.Get(resourceArn); ok { + return maps.Clone(c.Tags), nil + } + + if p, ok := b.customPlugins.Get(resourceArn); ok { + return maps.Clone(p.Tags), nil + } + + if w, ok := b.workerConfigurations.Get(resourceArn); ok { + return maps.Clone(w.Tags), nil + } + + return nil, ErrResourceNotFound +} diff --git a/services/kafkaconnect/tags_test.go b/services/kafkaconnect/tags_test.go new file mode 100644 index 0000000000..2068ac3857 --- /dev/null +++ b/services/kafkaconnect/tags_test.go @@ -0,0 +1,83 @@ +package kafkaconnect_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + kafkaconnectsdk "github.com/aws/aws-sdk-go-v2/service/kafkaconnect" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTagUntagListTagsForResource(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateConnector(ctx, minimalCreateConnectorInput("tag-me")) + require.NoError(t, err) + + _, err = client.TagResource(ctx, &kafkaconnectsdk.TagResourceInput{ + ResourceArn: created.ConnectorArn, + Tags: map[string]string{"env": "test", "owner": "terraform"}, + }) + require.NoError(t, err) + + listOut, err := client.ListTagsForResource( + ctx, + &kafkaconnectsdk.ListTagsForResourceInput{ResourceArn: created.ConnectorArn}, + ) + require.NoError(t, err) + assert.Equal(t, map[string]string{"env": "test", "owner": "terraform"}, listOut.Tags) + + _, err = client.UntagResource(ctx, &kafkaconnectsdk.UntagResourceInput{ + ResourceArn: created.ConnectorArn, + TagKeys: []string{"env"}, + }) + require.NoError(t, err) + + listOut, err = client.ListTagsForResource( + ctx, + &kafkaconnectsdk.ListTagsForResourceInput{ResourceArn: created.ConnectorArn}, + ) + require.NoError(t, err) + assert.Equal(t, map[string]string{"owner": "terraform"}, listOut.Tags) +} + +func TestTagResource_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.TagResource(t.Context(), &kafkaconnectsdk.TagResourceInput{ + ResourceArn: aws.String("arn:aws:kafkaconnect:us-east-1:123456789012:connector/nope/abc"), + Tags: map[string]string{"a": "b"}, + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "NotFoundException", apiErr.ErrorCode()) +} + +func TestCreateConnectorWithTags(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + input := minimalCreateConnectorInput("tagged-on-create") + input.Tags = map[string]string{"Environment": "test"} + + created, err := client.CreateConnector(ctx, input) + require.NoError(t, err) + + listOut, err := client.ListTagsForResource( + ctx, + &kafkaconnectsdk.ListTagsForResourceInput{ResourceArn: created.ConnectorArn}, + ) + require.NoError(t, err) + assert.Equal(t, map[string]string{"Environment": "test"}, listOut.Tags) +} diff --git a/services/kafkaconnect/wire.go b/services/kafkaconnect/wire.go new file mode 100644 index 0000000000..8c3d504859 --- /dev/null +++ b/services/kafkaconnect/wire.go @@ -0,0 +1,626 @@ +package kafkaconnect + +import "time" + +// Wire DTOs for the MSK Connect REST-JSON control plane. Field names match +// the AWS smithy model exactly (aws-sdk-go-v2/service/kafkaconnect@v1.39.1 +// serializers.go/deserializers.go emit JSON keys equal to the Go struct +// field names with the first letter lowercased, no @jsonName overrides). +// Timestamps are ISO8601 strings (smithy "date-time"), not epoch numbers -- +// confirmed via deserializers.go's smithytime.ParseDateTime calls. + +func formatTime(t time.Time) string { + if t.IsZero() { + return "" + } + + return t.Format(time.RFC3339) +} + +type scaleInPolicyDTO struct { + CPUUtilizationPercentage int32 `json:"cpuUtilizationPercentage"` +} + +type scaleOutPolicyDTO struct { + CPUUtilizationPercentage int32 `json:"cpuUtilizationPercentage"` +} + +type autoScalingDTO struct { + ScaleInPolicy *scaleInPolicyDTO `json:"scaleInPolicy,omitempty"` + ScaleOutPolicy *scaleOutPolicyDTO `json:"scaleOutPolicy,omitempty"` + MaxAutoscalingTaskCount int32 `json:"maxAutoscalingTaskCount,omitempty"` + MaxWorkerCount int32 `json:"maxWorkerCount"` + McuCount int32 `json:"mcuCount"` + MinWorkerCount int32 `json:"minWorkerCount"` +} + +type provisionedCapacityDTO struct { + McuCount int32 `json:"mcuCount"` + WorkerCount int32 `json:"workerCount"` +} + +type capacityDTO struct { + AutoScaling *autoScalingDTO `json:"autoScaling,omitempty"` + ProvisionedCapacity *provisionedCapacityDTO `json:"provisionedCapacity,omitempty"` +} + +func capacityToDTO(c Capacity) capacityDTO { + var dto capacityDTO + + if c.AutoScaling != nil { + dto.AutoScaling = &autoScalingDTO{ + MinWorkerCount: c.AutoScaling.MinWorkerCount, + MaxWorkerCount: c.AutoScaling.MaxWorkerCount, + McuCount: c.AutoScaling.McuCount, + MaxAutoscalingTaskCount: c.AutoScaling.MaxAutoscalingTaskCount, + ScaleInPolicy: &scaleInPolicyDTO{CPUUtilizationPercentage: c.AutoScaling.ScaleInCPUPercent}, + ScaleOutPolicy: &scaleOutPolicyDTO{CPUUtilizationPercentage: c.AutoScaling.ScaleOutCPUPercent}, + } + } + + if c.Provisioned != nil { + dto.ProvisionedCapacity = &provisionedCapacityDTO{ + McuCount: c.Provisioned.McuCount, + WorkerCount: c.Provisioned.WorkerCount, + } + } + + return dto +} + +func capacityFromDTO(dto capacityDTO) Capacity { + var c Capacity + + if dto.AutoScaling != nil { + a := &AutoScaling{ + MinWorkerCount: dto.AutoScaling.MinWorkerCount, + MaxWorkerCount: dto.AutoScaling.MaxWorkerCount, + McuCount: dto.AutoScaling.McuCount, + MaxAutoscalingTaskCount: dto.AutoScaling.MaxAutoscalingTaskCount, + } + if dto.AutoScaling.ScaleInPolicy != nil { + a.ScaleInCPUPercent = dto.AutoScaling.ScaleInPolicy.CPUUtilizationPercentage + } + + if dto.AutoScaling.ScaleOutPolicy != nil { + a.ScaleOutCPUPercent = dto.AutoScaling.ScaleOutPolicy.CPUUtilizationPercentage + } + + c.AutoScaling = a + } + + if dto.ProvisionedCapacity != nil { + c.Provisioned = &ProvisionedCapacity{ + McuCount: dto.ProvisionedCapacity.McuCount, + WorkerCount: dto.ProvisionedCapacity.WorkerCount, + } + } + + return c +} + +type vpcDTO struct { + SecurityGroups []string `json:"securityGroups,omitempty"` + Subnets []string `json:"subnets,omitempty"` +} + +type apacheKafkaClusterDTO struct { + BootstrapServers string `json:"bootstrapServers,omitempty"` + Vpc vpcDTO `json:"vpc"` +} + +type kafkaClusterDTO struct { + ApacheKafkaCluster apacheKafkaClusterDTO `json:"apacheKafkaCluster"` +} + +func apacheKafkaClusterToDTO(c ApacheKafkaCluster) kafkaClusterDTO { + return kafkaClusterDTO{ + ApacheKafkaCluster: apacheKafkaClusterDTO{ + BootstrapServers: c.BootstrapServers, + Vpc: vpcDTO{ + SecurityGroups: c.Vpc.SecurityGroups, + Subnets: c.Vpc.Subnets, + }, + }, + } +} + +func apacheKafkaClusterFromDTO(dto kafkaClusterDTO) ApacheKafkaCluster { + return ApacheKafkaCluster{ + BootstrapServers: dto.ApacheKafkaCluster.BootstrapServers, + Vpc: Vpc{ + SecurityGroups: dto.ApacheKafkaCluster.Vpc.SecurityGroups, + Subnets: dto.ApacheKafkaCluster.Vpc.Subnets, + }, + } +} + +type kafkaClusterClientAuthenticationDTO struct { + AuthenticationType string `json:"authenticationType,omitempty"` +} + +type kafkaClusterEncryptionInTransitDTO struct { + EncryptionType string `json:"encryptionType,omitempty"` +} + +type customPluginRefDTO struct { + CustomPluginArn string `json:"customPluginArn,omitempty"` + Revision int64 `json:"revision"` +} + +type pluginDTO struct { + CustomPlugin customPluginRefDTO `json:"customPlugin"` +} + +func pluginsToDTO(refs []PluginRef) []pluginDTO { + out := make([]pluginDTO, 0, len(refs)) + for _, r := range refs { + out = append(out, pluginDTO{CustomPlugin: customPluginRefDTO(r)}) + } + + return out +} + +func pluginsFromDTO(dtos []pluginDTO) []PluginRef { + out := make([]PluginRef, 0, len(dtos)) + for _, d := range dtos { + out = append(out, PluginRef{CustomPluginArn: d.CustomPlugin.CustomPluginArn, Revision: d.CustomPlugin.Revision}) + } + + return out +} + +type workerConfigurationRefDTO struct { + WorkerConfigurationArn string `json:"workerConfigurationArn,omitempty"` + Revision int64 `json:"revision"` +} + +type cloudWatchLogsLogDeliveryDTO struct { + LogGroup string `json:"logGroup,omitempty"` + Enabled bool `json:"enabled"` +} + +type firehoseLogDeliveryDTO struct { + DeliveryStream string `json:"deliveryStream,omitempty"` + Enabled bool `json:"enabled"` +} + +type s3LogDeliveryDTO struct { + Bucket string `json:"bucket,omitempty"` + Prefix string `json:"prefix,omitempty"` + Enabled bool `json:"enabled"` +} + +type workerLogDeliveryDTO struct { + CloudWatchLogs *cloudWatchLogsLogDeliveryDTO `json:"cloudWatchLogs,omitempty"` + Firehose *firehoseLogDeliveryDTO `json:"firehose,omitempty"` + S3 *s3LogDeliveryDTO `json:"s3,omitempty"` +} + +type logDeliveryDTO struct { + WorkerLogDelivery *workerLogDeliveryDTO `json:"workerLogDelivery,omitempty"` +} + +func workerLogDeliveryToDTO(w *WorkerLogDelivery) *logDeliveryDTO { + if w == nil { + return nil + } + + dto := &workerLogDeliveryDTO{} + if w.CloudWatchLogs != nil { + dto.CloudWatchLogs = &cloudWatchLogsLogDeliveryDTO{ + Enabled: w.CloudWatchLogs.Enabled, + LogGroup: w.CloudWatchLogs.LogGroup, + } + } + + if w.Firehose != nil { + dto.Firehose = &firehoseLogDeliveryDTO{Enabled: w.Firehose.Enabled, DeliveryStream: w.Firehose.DeliveryStream} + } + + if w.S3 != nil { + dto.S3 = &s3LogDeliveryDTO{Enabled: w.S3.Enabled, Bucket: w.S3.Bucket, Prefix: w.S3.Prefix} + } + + return &logDeliveryDTO{WorkerLogDelivery: dto} +} + +func workerLogDeliveryFromDTO(dto *logDeliveryDTO) *WorkerLogDelivery { + if dto == nil || dto.WorkerLogDelivery == nil { + return nil + } + + w := &WorkerLogDelivery{} + src := dto.WorkerLogDelivery + + if src.CloudWatchLogs != nil { + w.CloudWatchLogs = &CloudWatchLogsDelivery{ + Enabled: src.CloudWatchLogs.Enabled, + LogGroup: src.CloudWatchLogs.LogGroup, + } + } + + if src.Firehose != nil { + w.Firehose = &FirehoseDelivery{Enabled: src.Firehose.Enabled, DeliveryStream: src.Firehose.DeliveryStream} + } + + if src.S3 != nil { + w.S3 = &S3LogDelivery{Enabled: src.S3.Enabled, Bucket: src.S3.Bucket, Prefix: src.S3.Prefix} + } + + return w +} + +type connectorSummaryDTO struct { + Capacity capacityDTO `json:"capacity"` + KafkaCluster kafkaClusterDTO `json:"kafkaCluster"` + KafkaClusterClientAuthentication kafkaClusterClientAuthenticationDTO `json:"kafkaClusterClientAuthentication"` + KafkaClusterEncryptionInTransit kafkaClusterEncryptionInTransitDTO `json:"kafkaClusterEncryptionInTransit"` + LogDelivery *logDeliveryDTO `json:"logDelivery,omitempty"` + WorkerConfiguration *workerConfigurationRefDTO `json:"workerConfiguration,omitempty"` + ConnectorArn string `json:"connectorArn,omitempty"` + ConnectorDescription string `json:"connectorDescription,omitempty"` + ConnectorName string `json:"connectorName,omitempty"` + ConnectorState string `json:"connectorState,omitempty"` + CreationTime string `json:"creationTime,omitempty"` + CurrentVersion string `json:"currentVersion,omitempty"` + KafkaConnectVersion string `json:"kafkaConnectVersion,omitempty"` + NetworkType string `json:"networkType,omitempty"` + ServiceExecutionRoleArn string `json:"serviceExecutionRoleArn,omitempty"` + Plugins []pluginDTO `json:"plugins,omitempty"` +} + +func connectorToDTO(c *Connector) connectorSummaryDTO { + dto := connectorSummaryDTO{ + Capacity: capacityToDTO(c.Capacity), + ConnectorArn: c.ARN, + ConnectorDescription: c.Description, + ConnectorName: c.Name, + ConnectorState: c.State, + CreationTime: formatTime(c.CreationTime), + CurrentVersion: c.CurrentVersion, + KafkaCluster: apacheKafkaClusterToDTO(c.ApacheKafkaCluster), + KafkaClusterClientAuthentication: kafkaClusterClientAuthenticationDTO{ + AuthenticationType: c.KafkaClusterClientAuthentication, + }, + KafkaClusterEncryptionInTransit: kafkaClusterEncryptionInTransitDTO{ + EncryptionType: c.KafkaClusterEncryptionInTransit, + }, + KafkaConnectVersion: c.KafkaConnectVersion, + LogDelivery: workerLogDeliveryToDTO(c.WorkerLogDelivery), + NetworkType: c.NetworkType, + Plugins: pluginsToDTO(c.Plugins), + ServiceExecutionRoleArn: c.ServiceExecutionRoleArn, + } + + if c.WorkerConfiguration != nil { + dto.WorkerConfiguration = &workerConfigurationRefDTO{ + WorkerConfigurationArn: c.WorkerConfiguration.Arn, + Revision: c.WorkerConfiguration.Revision, + } + } + + return dto +} + +type createConnectorRequest struct { + Capacity capacityDTO `json:"capacity"` + WorkerConfiguration *workerConfigurationRefDTO `json:"workerConfiguration,omitempty"` + ConnectorConfiguration map[string]string `json:"connectorConfiguration"` + Tags map[string]string `json:"tags,omitempty"` + LogDelivery *logDeliveryDTO `json:"logDelivery,omitempty"` + KafkaCluster kafkaClusterDTO `json:"kafkaCluster"` + KafkaClusterClientAuthentication kafkaClusterClientAuthenticationDTO `json:"kafkaClusterClientAuthentication"` + ConnectorDescription string `json:"connectorDescription,omitempty"` + ConnectorName string `json:"connectorName"` + KafkaConnectVersion string `json:"kafkaConnectVersion"` + NetworkType string `json:"networkType,omitempty"` + ServiceExecutionRoleArn string `json:"serviceExecutionRoleArn"` + KafkaClusterEncryptionInTransit kafkaClusterEncryptionInTransitDTO `json:"kafkaClusterEncryptionInTransit"` + Plugins []pluginDTO `json:"plugins"` +} + +type createConnectorResponse struct { + ConnectorArn string `json:"connectorArn,omitempty"` + ConnectorName string `json:"connectorName,omitempty"` + ConnectorState string `json:"connectorState,omitempty"` +} + +type describeConnectorResponse struct { + connectorSummaryDTO +} + +type listConnectorsResponse struct { + NextToken string `json:"nextToken,omitempty"` + Connectors []connectorSummaryDTO `json:"connectors"` +} + +type updateConnectorRequest struct { + Capacity *capacityDTO `json:"capacity,omitempty"` + ConnectorConfiguration map[string]string `json:"connectorConfiguration,omitempty"` +} + +type updateConnectorResponse struct { + ConnectorArn string `json:"connectorArn,omitempty"` + ConnectorOperationArn string `json:"connectorOperationArn,omitempty"` + ConnectorState string `json:"connectorState,omitempty"` +} + +type deleteConnectorResponse struct { + ConnectorArn string `json:"connectorArn,omitempty"` + ConnectorState string `json:"connectorState,omitempty"` +} + +type restartConnectorResponse struct { + ConnectorArn string `json:"connectorArn,omitempty"` + ConnectorOperationArn string `json:"connectorOperationArn,omitempty"` +} + +type s3LocationDTO struct { + BucketArn string `json:"bucketArn,omitempty"` + FileKey string `json:"fileKey,omitempty"` + ObjectVersion string `json:"objectVersion,omitempty"` +} + +type customPluginLocationDTO struct { + S3Location s3LocationDTO `json:"s3Location"` +} + +type createCustomPluginRequest struct { + Location customPluginLocationDTO `json:"location"` + Tags map[string]string `json:"tags,omitempty"` + ContentType string `json:"contentType"` + Description string `json:"description,omitempty"` + Name string `json:"name"` +} + +type createCustomPluginResponse struct { + CustomPluginArn string `json:"customPluginArn,omitempty"` + CustomPluginState string `json:"customPluginState,omitempty"` + Name string `json:"name,omitempty"` + Revision int64 `json:"revision"` +} + +type customPluginFileDescriptionDTO struct { + FileMd5 string `json:"fileMd5,omitempty"` + FileSize int64 `json:"fileSize"` +} + +type customPluginLocationDescriptionDTO struct { + S3Location s3LocationDTO `json:"s3Location"` +} + +type customPluginRevisionSummaryDTO struct { + FileDescription *customPluginFileDescriptionDTO `json:"fileDescription,omitempty"` + Location *customPluginLocationDescriptionDTO `json:"location,omitempty"` + ContentType string `json:"contentType,omitempty"` + CreationTime string `json:"creationTime,omitempty"` + Description string `json:"description,omitempty"` + Revision int64 `json:"revision"` +} + +func customPluginToRevisionSummaryDTO(p *CustomPlugin) *customPluginRevisionSummaryDTO { + return &customPluginRevisionSummaryDTO{ + ContentType: p.ContentType, + CreationTime: formatTime(p.CreationTime), + Description: p.Description, + Revision: p.Revision, + FileDescription: &customPluginFileDescriptionDTO{ + FileMd5: p.FileMD5, + FileSize: p.FileSizeBytes, + }, + Location: &customPluginLocationDescriptionDTO{ + S3Location: s3LocationDTO{BucketArn: p.BucketArn, FileKey: p.FileKey, ObjectVersion: p.ObjectVersion}, + }, + } +} + +type customPluginSummaryDTO struct { + LatestRevision *customPluginRevisionSummaryDTO `json:"latestRevision,omitempty"` + CreationTime string `json:"creationTime,omitempty"` + CustomPluginArn string `json:"customPluginArn,omitempty"` + CustomPluginState string `json:"customPluginState,omitempty"` + Description string `json:"description,omitempty"` + Name string `json:"name,omitempty"` +} + +func customPluginToSummaryDTO(p *CustomPlugin) customPluginSummaryDTO { + return customPluginSummaryDTO{ + CreationTime: formatTime(p.CreationTime), + CustomPluginArn: p.ARN, + CustomPluginState: p.State, + Description: p.Description, + Name: p.Name, + LatestRevision: customPluginToRevisionSummaryDTO(p), + } +} + +type describeCustomPluginResponse struct { + LatestRevision *customPluginRevisionSummaryDTO `json:"latestRevision,omitempty"` + CreationTime string `json:"creationTime,omitempty"` + CustomPluginArn string `json:"customPluginArn,omitempty"` + CustomPluginState string `json:"customPluginState,omitempty"` + Description string `json:"description,omitempty"` + Name string `json:"name,omitempty"` +} + +type listCustomPluginsResponse struct { + NextToken string `json:"nextToken,omitempty"` + CustomPlugins []customPluginSummaryDTO `json:"customPlugins"` +} + +type deleteCustomPluginResponse struct { + CustomPluginArn string `json:"customPluginArn,omitempty"` + CustomPluginState string `json:"customPluginState,omitempty"` +} + +type createWorkerConfigurationRequest struct { + Tags map[string]string `json:"tags,omitempty"` + Description string `json:"description,omitempty"` + Name string `json:"name"` + PropertiesFileContent string `json:"propertiesFileContent"` +} + +type workerConfigurationRevisionSummaryDTO struct { + CreationTime string `json:"creationTime,omitempty"` + Description string `json:"description,omitempty"` + Revision int64 `json:"revision"` +} + +func workerConfigToRevisionSummaryDTO(w *WorkerConfiguration) *workerConfigurationRevisionSummaryDTO { + return &workerConfigurationRevisionSummaryDTO{ + CreationTime: formatTime(w.LatestRevision.CreationTime), + Description: w.LatestRevision.Description, + Revision: w.LatestRevision.Revision, + } +} + +type createWorkerConfigurationResponse struct { + LatestRevision *workerConfigurationRevisionSummaryDTO `json:"latestRevision,omitempty"` + CreationTime string `json:"creationTime,omitempty"` + Name string `json:"name,omitempty"` + WorkerConfigurationArn string `json:"workerConfigurationArn,omitempty"` + WorkerConfigurationState string `json:"workerConfigurationState,omitempty"` +} + +type workerConfigurationSummaryDTO struct { + LatestRevision *workerConfigurationRevisionSummaryDTO `json:"latestRevision,omitempty"` + CreationTime string `json:"creationTime,omitempty"` + Description string `json:"description,omitempty"` + Name string `json:"name,omitempty"` + WorkerConfigurationArn string `json:"workerConfigurationArn,omitempty"` + WorkerConfigurationState string `json:"workerConfigurationState,omitempty"` +} + +func workerConfigToSummaryDTO(w *WorkerConfiguration) workerConfigurationSummaryDTO { + return workerConfigurationSummaryDTO{ + CreationTime: formatTime(w.CreationTime), + Description: w.Description, + Name: w.Name, + WorkerConfigurationArn: w.ARN, + WorkerConfigurationState: w.State, + LatestRevision: workerConfigToRevisionSummaryDTO(w), + } +} + +type workerConfigurationRevisionDescriptionDTO struct { + CreationTime string `json:"creationTime,omitempty"` + Description string `json:"description,omitempty"` + PropertiesFileContent string `json:"propertiesFileContent,omitempty"` + Revision int64 `json:"revision"` +} + +type describeWorkerConfigurationResponse struct { + LatestRevision *workerConfigurationRevisionDescriptionDTO `json:"latestRevision,omitempty"` + CreationTime string `json:"creationTime,omitempty"` + Description string `json:"description,omitempty"` + Name string `json:"name,omitempty"` + WorkerConfigurationArn string `json:"workerConfigurationArn,omitempty"` + WorkerConfigurationState string `json:"workerConfigurationState,omitempty"` +} + +type listWorkerConfigurationsResponse struct { + NextToken string `json:"nextToken,omitempty"` + WorkerConfigurations []workerConfigurationSummaryDTO `json:"workerConfigurations"` +} + +type deleteWorkerConfigurationResponse struct { + WorkerConfigurationArn string `json:"workerConfigurationArn,omitempty"` + WorkerConfigurationState string `json:"workerConfigurationState,omitempty"` +} + +type tagResourceRequest struct { + Tags map[string]string `json:"tags"` +} + +type listTagsForResourceResponse struct { + Tags map[string]string `json:"tags,omitempty"` +} + +type connectorOperationStepDTO struct { + StepState string `json:"stepState,omitempty"` + StepType string `json:"stepType,omitempty"` +} + +type workerSettingDTO struct { + Capacity *capacityDTO `json:"capacity,omitempty"` +} + +type describeConnectorOperationResponse struct { + ErrorInfo *stateDescriptionDTO `json:"errorInfo,omitempty"` + OriginWorkerSetting *workerSettingDTO `json:"originWorkerSetting,omitempty"` + TargetWorkerSetting *workerSettingDTO `json:"targetWorkerSetting,omitempty"` + OriginConnectorConfiguration map[string]string `json:"originConnectorConfiguration,omitempty"` + TargetConnectorConfiguration map[string]string `json:"targetConnectorConfiguration,omitempty"` + ConnectorArn string `json:"connectorArn,omitempty"` + ConnectorOperationArn string `json:"connectorOperationArn,omitempty"` + ConnectorOperationState string `json:"connectorOperationState,omitempty"` + ConnectorOperationType string `json:"connectorOperationType,omitempty"` + CreationTime string `json:"creationTime,omitempty"` + EndTime string `json:"endTime,omitempty"` + OperationSteps []connectorOperationStepDTO `json:"operationSteps,omitempty"` +} + +type stateDescriptionDTO struct { + Code string `json:"code,omitempty"` + Message string `json:"message,omitempty"` +} + +func connectorOperationToDescribeDTO(op *ConnectorOperation) describeConnectorOperationResponse { + steps := make([]connectorOperationStepDTO, 0, len(op.Steps)) + for _, s := range op.Steps { + steps = append(steps, connectorOperationStepDTO{StepType: s.StepType, StepState: s.StepState}) + } + + resp := describeConnectorOperationResponse{ + ConnectorArn: op.ConnectorArn, + ConnectorOperationArn: op.ARN, + ConnectorOperationState: op.State, + ConnectorOperationType: op.Type, + CreationTime: formatTime(op.CreationTime), + EndTime: formatTime(op.EndTime), + OperationSteps: steps, + OriginConnectorConfiguration: op.OriginConnectorConfiguration, + TargetConnectorConfiguration: op.TargetConnectorConfiguration, + } + + if op.OriginCapacity != nil { + dto := capacityToDTO(*op.OriginCapacity) + resp.OriginWorkerSetting = &workerSettingDTO{Capacity: &dto} + } + + if op.TargetCapacity != nil { + dto := capacityToDTO(*op.TargetCapacity) + resp.TargetWorkerSetting = &workerSettingDTO{Capacity: &dto} + } + + return resp +} + +type connectorOperationSummaryDTO struct { + CreationTime string `json:"creationTime,omitempty"` + EndTime string `json:"endTime,omitempty"` + ConnectorOperationArn string `json:"connectorOperationArn,omitempty"` + ConnectorOperationState string `json:"connectorOperationState,omitempty"` + ConnectorOperationType string `json:"connectorOperationType,omitempty"` +} + +func connectorOperationToSummaryDTO(op *ConnectorOperation) connectorOperationSummaryDTO { + return connectorOperationSummaryDTO{ + ConnectorOperationArn: op.ARN, + ConnectorOperationState: op.State, + ConnectorOperationType: op.Type, + CreationTime: formatTime(op.CreationTime), + EndTime: formatTime(op.EndTime), + } +} + +type listConnectorOperationsResponse struct { + NextToken string `json:"nextToken,omitempty"` + ConnectorOperations []connectorOperationSummaryDTO `json:"connectorOperations"` +} + +type errorResponse struct { + Type string `json:"__type"` + Message string `json:"message,omitempty"` +} diff --git a/services/kafkaconnect/workerconfigs.go b/services/kafkaconnect/workerconfigs.go new file mode 100644 index 0000000000..6477bd075a --- /dev/null +++ b/services/kafkaconnect/workerconfigs.go @@ -0,0 +1,124 @@ +package kafkaconnect + +import ( + "fmt" + "maps" + "sort" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/blackbirdworks/gopherstack/pkgs/arn" + "github.com/blackbirdworks/gopherstack/pkgs/page" +) + +func workerConfigurationARN(region, accountID, name string) string { + return arn.Build( + "kafkaconnect", region, accountID, fmt.Sprintf("worker-configuration/%s/%s", name, uuid.NewString()), + ) +} + +// CreateWorkerConfiguration creates a worker configuration, always at +// revision 1 and ACTIVE immediately -- UpdateWorkerConfiguration (which +// would create later revisions) is not part of the AWS API surface this +// backend implements; see PARITY.md. +func (b *InMemoryBackend) CreateWorkerConfiguration( + accountID, region, name, description, propertiesFileContent string, + tags map[string]string, +) (*WorkerConfiguration, error) { + if name == "" || propertiesFileContent == "" { + return nil, ErrValidation + } + + b.mu.Lock("CreateWorkerConfiguration") + defer b.mu.Unlock() + + if _, ok := b.workerConfigurationByName(name); ok { + return nil, ErrWorkerConfigNameInUse + } + + t := make(map[string]string, len(tags)) + maps.Copy(t, tags) + + now := time.Now().UTC() + + w := &WorkerConfiguration{ + Name: name, + ARN: workerConfigurationARN(region, accountID, name), + Description: description, + State: workerConfigurationStateActive, + CreationTime: now, + Tags: t, + LatestRevision: WorkerConfigRevision{ + Revision: 1, + Description: description, + PropertiesFileContent: propertiesFileContent, + CreationTime: now, + }, + } + + b.workerConfigurations.Put(w) + + return w.clone(), nil +} + +// DescribeWorkerConfiguration returns the current information about a worker configuration. +func (b *InMemoryBackend) DescribeWorkerConfiguration(workerConfigurationArn string) (*WorkerConfiguration, error) { + b.mu.RLock("DescribeWorkerConfiguration") + defer b.mu.RUnlock() + + w, ok := b.workerConfigurations.Get(workerConfigurationArn) + if !ok { + return nil, ErrWorkerConfigNotFound + } + + return w.clone(), nil +} + +// ListWorkerConfigurations returns worker configurations matching namePrefix, +// paginated by nextToken/maxResults. +func (b *InMemoryBackend) ListWorkerConfigurations( + namePrefix, nextToken string, + maxResults int, +) ([]*WorkerConfiguration, string, error) { + b.mu.RLock("ListWorkerConfigurations") + defer b.mu.RUnlock() + + all := b.workerConfigurations.All() + + matched := make([]*WorkerConfiguration, 0, len(all)) + + for _, w := range all { + if namePrefix != "" && !strings.HasPrefix(w.Name, namePrefix) { + continue + } + + matched = append(matched, w.clone()) + } + + sort.Slice(matched, func(i, j int) bool { return matched[i].Name < matched[j].Name }) + + pg := page.New(matched, nextToken, maxResults, defaultListLimit) + + return pg.Data, pg.Next, nil +} + +// DeleteWorkerConfiguration deletes a worker configuration, returning a +// snapshot with State set to DELETING to mirror AWS's synchronous delete response. +func (b *InMemoryBackend) DeleteWorkerConfiguration(workerConfigurationArn string) (*WorkerConfiguration, error) { + b.mu.Lock("DeleteWorkerConfiguration") + defer b.mu.Unlock() + + w, ok := b.workerConfigurations.Get(workerConfigurationArn) + if !ok { + return nil, ErrWorkerConfigNotFound + } + + out := w.clone() + out.State = deletingState + + b.workerConfigurations.Delete(workerConfigurationArn) + + return out, nil +} diff --git a/services/kafkaconnect/workerconfigs_test.go b/services/kafkaconnect/workerconfigs_test.go new file mode 100644 index 0000000000..42c5700a76 --- /dev/null +++ b/services/kafkaconnect/workerconfigs_test.go @@ -0,0 +1,128 @@ +package kafkaconnect_test + +import ( + "encoding/base64" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + kafkaconnectsdk "github.com/aws/aws-sdk-go-v2/service/kafkaconnect" + "github.com/aws/aws-sdk-go-v2/service/kafkaconnect/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func minimalCreateWorkerConfigurationInput(name string) *kafkaconnectsdk.CreateWorkerConfigurationInput { + content := base64.StdEncoding.EncodeToString([]byte("key.converter=org.apache.kafka.connect.json.JsonConverter\n")) + + return &kafkaconnectsdk.CreateWorkerConfigurationInput{ + Name: aws.String(name), + PropertiesFileContent: aws.String(content), + } +} + +func TestCreateWorkerConfiguration(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + out, err := client.CreateWorkerConfiguration(t.Context(), minimalCreateWorkerConfigurationInput("wc-one")) + require.NoError(t, err) + assert.Contains(t, aws.ToString(out.WorkerConfigurationArn), "worker-configuration/wc-one/") + assert.Equal(t, types.WorkerConfigurationStateActive, out.WorkerConfigurationState) + require.NotNil(t, out.LatestRevision) + assert.EqualValues(t, 1, out.LatestRevision.Revision) +} + +func TestCreateWorkerConfiguration_DuplicateNameReturnsConflict(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateWorkerConfiguration(ctx, minimalCreateWorkerConfigurationInput("dup-wc")) + require.NoError(t, err) + + _, err = client.CreateWorkerConfiguration(ctx, minimalCreateWorkerConfigurationInput("dup-wc")) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ConflictException", apiErr.ErrorCode()) +} + +func TestDescribeWorkerConfiguration(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + input := minimalCreateWorkerConfigurationInput("describe-wc") + + created, err := client.CreateWorkerConfiguration(ctx, input) + require.NoError(t, err) + + out, err := client.DescribeWorkerConfiguration(ctx, &kafkaconnectsdk.DescribeWorkerConfigurationInput{ + WorkerConfigurationArn: created.WorkerConfigurationArn, + }) + require.NoError(t, err) + assert.Equal(t, "describe-wc", aws.ToString(out.Name)) + require.NotNil(t, out.LatestRevision) + assert.Equal(t, aws.ToString(input.PropertiesFileContent), aws.ToString(out.LatestRevision.PropertiesFileContent)) +} + +func TestDescribeWorkerConfiguration_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.DescribeWorkerConfiguration(t.Context(), &kafkaconnectsdk.DescribeWorkerConfigurationInput{ + WorkerConfigurationArn: aws.String("arn:aws:kafkaconnect:us-east-1:123456789012:worker-configuration/nope/abc"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "NotFoundException", apiErr.ErrorCode()) +} + +func TestListWorkerConfigurations(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateWorkerConfiguration(ctx, minimalCreateWorkerConfigurationInput("list-wc-a")) + require.NoError(t, err) + _, err = client.CreateWorkerConfiguration(ctx, minimalCreateWorkerConfigurationInput("list-wc-b")) + require.NoError(t, err) + + out, err := client.ListWorkerConfigurations(ctx, &kafkaconnectsdk.ListWorkerConfigurationsInput{}) + require.NoError(t, err) + assert.Len(t, out.WorkerConfigurations, 2) +} + +func TestDeleteWorkerConfiguration(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateWorkerConfiguration(ctx, minimalCreateWorkerConfigurationInput("delete-wc")) + require.NoError(t, err) + + out, err := client.DeleteWorkerConfiguration(ctx, &kafkaconnectsdk.DeleteWorkerConfigurationInput{ + WorkerConfigurationArn: created.WorkerConfigurationArn, + }) + require.NoError(t, err) + assert.Equal(t, types.WorkerConfigurationStateDeleting, out.WorkerConfigurationState) + + _, err = client.DescribeWorkerConfiguration(ctx, &kafkaconnectsdk.DescribeWorkerConfigurationInput{ + WorkerConfigurationArn: created.WorkerConfigurationArn, + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "NotFoundException", apiErr.ErrorCode()) +} diff --git a/services/kinesis/PARITY.md b/services/kinesis/PARITY.md index 343277cdb5..6aed1c051a 100644 --- a/services/kinesis/PARITY.md +++ b/services/kinesis/PARITY.md @@ -7,8 +7,8 @@ overall: A # this pass (gopherstack-nbg8): the 2026-07-23 audit's "wi ops: IncreaseStreamRetentionPeriod: {wire: fixed, errors: ok, state: ok, persist: ok, note: "reverted 2b2086c9: that commit made equal-to-current RetentionPeriodHours return InvalidArgumentException (a strict reading of the aws-sdk-go-v2 doc comment 'Must be more than the current retention period'), which broke TestTerraform_Kinesis in CI -- terraform's aws_kinesis_stream resource issues IncreaseStreamRetentionPeriod even when the requested value already equals the stream's current retention (confirmed live: CreateStream -> 24h default -> Increase(48) OK -> a second Increase(48) against the already-48h stream 400'd with InvalidArgumentException before this fix). Real AWS tolerates the equal case rather than erroring on every no-drift re-apply, so restored equal-value == no-op success. Strictly-lower and out-of-[24,8760] values are still rejected. gopherstack-enpq (2026-08-22): Input had no StreamARN member at all (api_op_IncreaseStreamRetentionPeriod.go:43-58 (StreamARN:52) -- 'you must use either the StreamARN or the StreamName parameter, or both'); an ARN-only caller silently resolved to an empty stream name and 400'd. Fixed via resolveStreamNameAndRegion."} DecreaseStreamRetentionPeriod: {wire: fixed, errors: ok, state: ok, persist: ok, note: "reverted 2b2086c9, mirrored: equal-to-current RetentionPeriodHours is a no-op success again (not InvalidArgumentException), matching real AWS/terraform tolerance. Strictly-greater and below-24h-min values are still rejected. gopherstack-enpq (2026-08-22): same missing-StreamARN bug as IncreaseStreamRetentionPeriod (api_op_DecreaseStreamRetentionPeriod.go:39-54 (StreamARN:48)), fixed the same way."} - CreateStream: {wire: fixed, errors: ok, state: ok, persist: ok, note: "fixed: ON_DEMAND now defaults to 4 shards (was 1); inline Tags now validated pre-mutation and persisted via TagResource instead of a lost handler-local map. 2026-08-23 (request-side accept-and-drop sweep): CreateStreamInput's own MaxRecordSizeInKiB/WarmThroughputMiBps members (api_op_CreateStream.go:101-121 -- distinct from the same-named UpdateMaxRecordSize/UpdateStreamWarmThroughput fields) had no Go field at all; the backend already tracks both (Stream.MaxRecordSizeBytes/WarmThroughputMiBps, read back by DescribeStreamSummary), so a caller specifying either at creation time silently got the 1 MiB default / zero throughput instead. Fixed via resolveCreateStreamMaxRecordSize plus the same range checks UpdateMaxRecordSize/UpdateStreamWarmThroughput already apply."} - DeleteStream: {wire: fixed, errors: fixed, state: fixed, persist: ok, note: "fixed (gopherstack-enpq, cmd/structfielddiff): EnforceConsumerDeletion (real DeleteStreamInput member) was not accepted at all, so this backend deleted a stream unconditionally regardless of registered enhanced fan-out consumers -- more permissive than AWS, whose own doc comment says 'If this parameter is unset (null) or if you set it to false, and the stream has registered consumers, the call to DeleteStream fails with a ResourceInUseException.' Now checked against stream.Consumers before any mutation; new ErrStreamHasConsumers sentinel (ResourceInUseException) wired through resourceErrorDetails. Consumers themselves need no separate deletion step -- they are already keyed off the parent Stream struct (stream.Consumers), not a standalone global table, so they vanish with the stream regardless of EnforceConsumerDeletion's value once the delete is allowed to proceed. FIXED (gopherstack-6kj0, b8484292f): also left b.resourcePolicies[region][streamARN] behind, inherited by a recreated stream of the same name via GetResourcePolicy; now cleared alongside the FIS fault-injection entry. Regression: TestDeleteStream_ClearsResourcePolicyOnRecreate."} + CreateStream: {wire: fixed, errors: ok, state: fixed, persist: ok, note: "fixed: ON_DEMAND now defaults to 4 shards (was 1); inline Tags now validated pre-mutation and persisted via TagResource instead of a lost handler-local map. 2026-08-23 (request-side accept-and-drop sweep): CreateStreamInput's own MaxRecordSizeInKiB/WarmThroughputMiBps members (api_op_CreateStream.go:101-121 -- distinct from the same-named UpdateMaxRecordSize/UpdateStreamWarmThroughput fields) had no Go field at all; the backend already tracks both (Stream.MaxRecordSizeBytes/WarmThroughputMiBps, read back by DescribeStreamSummary), so a caller specifying either at creation time silently got the 1 MiB default / zero throughput instead. Fixed via resolveCreateStreamMaxRecordSize plus the same range checks UpdateMaxRecordSize/UpdateStreamWarmThroughput already apply. 2026-09-26 (gopherstack-nbg8, transient-state pass): CreateStream/UpdateShardCount/MergeShards/SplitShard/StartStreamEncryption/StopStreamEncryption/UpdateStreamMode/DeleteStream now model real transient state (CREATING/UPDATING/DELETING via a lazy ReadyAt deadline resolved on the next read, no goroutines -- see services/dsql's resolveClusterLocked/services/dax's sweepClusterTransitionsLocked for the same pattern) instead of a stream always being ACTIVE. DescribeStream/DescribeStreamSummary/ListStreams now show the transient status; each of these ops now rejects a non-ACTIVE stream with ResourceInUseException, matching its declared SDK error set."} + DeleteStream: {wire: fixed, errors: fixed, state: fixed, persist: ok, note: "fixed (gopherstack-enpq, cmd/structfielddiff): EnforceConsumerDeletion (real DeleteStreamInput member) was not accepted at all, so this backend deleted a stream unconditionally regardless of registered enhanced fan-out consumers -- more permissive than AWS, whose own doc comment says 'If this parameter is unset (null) or if you set it to false, and the stream has registered consumers, the call to DeleteStream fails with a ResourceInUseException.' Now checked against stream.Consumers before any mutation; new ErrStreamHasConsumers sentinel (ResourceInUseException) wired through resourceErrorDetails. Consumers themselves need no separate deletion step -- they are already keyed off the parent Stream struct (stream.Consumers), not a standalone global table, so they vanish with the stream regardless of EnforceConsumerDeletion's value once the delete is allowed to proceed. FIXED (gopherstack-6kj0, b8484292f): also left b.resourcePolicies[region][streamARN] behind, inherited by a recreated stream of the same name via GetResourcePolicy; now cleared alongside the FIS fault-injection entry. Regression: TestDeleteStream_ClearsResourcePolicyOnRecreate. 2026-09-26 (gopherstack-nbg8, transient-state pass): CreateStream/UpdateShardCount/MergeShards/SplitShard/StartStreamEncryption/StopStreamEncryption/UpdateStreamMode/DeleteStream now model real transient state (CREATING/UPDATING/DELETING via a lazy ReadyAt deadline resolved on the next read, no goroutines -- see services/dsql's resolveClusterLocked/services/dax's sweepClusterTransitionsLocked for the same pattern) instead of a stream always being ACTIVE. DescribeStream/DescribeStreamSummary/ListStreams now show the transient status; each of these ops now rejects a non-ACTIVE stream with ResourceInUseException, matching its declared SDK error set."} DescribeStream: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: Shards list now paginates (Limit/ExclusiveStartShardId/HasMoreShards); previously returned every shard in one page with HasMoreShards hardcoded false"} DescribeStreamSummary: {wire: fixed, errors: ok, state: ok, persist: ok, note: "gopherstack-enpq (2026-08-22): MaxRecordSizeInKiB and WarmThroughput (both real, optional StreamDescriptionSummary members, types/types.go) had no Go field at all -- the backend already tracks the underlying Stream.MaxRecordSizeBytes/WarmThroughputMiBps (set by UpdateMaxRecordSize/UpdateStreamWarmThroughput) but never surfaced either back on describe, so a client had no way to read back settings it had itself just applied. Fixed by adding both to DescribeStreamOutput and the wire response (WarmThroughput.Current/Target both mirror the synchronous-apply model UpdateStreamWarmThroughput already documents)."} ListStreams: {wire: fixed, errors: ok, state: ok, persist: ok, note: "gopherstack-wksw (2026-08-29): Limit's documented default AND max of 100 was not applied -- fixed, both directions now resolve to 100 (TestListStreams_DefaultLimit). 2026-09-18 (gopherstack-ud2): StreamSummaries (optional, types.StreamSummary) is now populated alongside the required StreamNames -- ListStreams was reshaped to paginate over []*Stream instead of []string so ARN/Status/CreatedAt/StreamMode ride along for free; previously only StreamNames was ever returned. TestListStreams_StreamSummaries (real aws-sdk-go-v2 client)."} @@ -22,18 +22,18 @@ ops: ListStreamConsumers: {wire: fixed, errors: ok, state: ok, persist: ok, note: "2026-08-19: same fabricated Consumer.StreamARN key as RegisterStreamConsumer (real types.Consumer has no StreamARN), same fix (jsonConsumer). gopherstack-wksw (2026-08-29): MaxResults' documented default of 100 (api_op_ListStreamConsumers.go) is also only applied when explicitly set and smaller than the result (same pattern as ListShards, consumers.go:197) -- judged NOT to need fixing: RegisterStreamConsumer enforces maxConsumersPerStream=20 (models.go:93) as a hard cap with no deletion-then-recreation-past-the-cap path modeled, so the unbounded branch can never actually return more than 20 consumers, structurally under the 100 default. Left as-is per RESTRAINT (medialive ListOfferings precedent) rather than fixed defensively."} DeregisterStreamConsumer: {wire: ok, errors: ok, state: ok, persist: ok} SubscribeToShard: {wire: fixed, errors: ok, state: fixed, persist: n/a, note: "event-stream binary framing verified byte-for-byte (prelude/CRC/headers); polling goroutine bounded by a real 5-min deadline, no leak; fixed: AT_TIMESTAMP with a genuinely omitted Timestamp now rejected InvalidArgumentException (was previously ambiguous between omitted and explicit-zero, both silently read from position 0). 2026-08-19: prior byte-level framing checks never ran the real aws-sdk-go-v2 client's own event-stream reader end to end -- new TestSubscribeToShard_RoundTrip (subscribe_roundtrip_test.go) drives client.SubscribeToShard + out.GetStream().Events() for real and confirms the SDK decodes a SubscribeToShardEvent with the record; SubscribeToShardEvent field names (ContinuationSequenceNumber/MillisBehindLatest/Records, deserializers.go:5549-5605) re-confirmed against the per-field switch. ChildShards (optional member of the same event, deserializers.go:5570-5573) is not populated on SubscribeToShardEvent -- see gaps. 2026-09-11 (gopherstack-s0ju item 4): fixed a real cadence bug -- the emulator closed an idle stream after 3 empty polls (~600ms, subscribeToShardMaxIdlePolls, now removed), directly contradicting 'The connection remains open for up to 5 minutes' (docs.aws.amazon.com/streams/latest/dev/building-enhanced-consumers-api.html); a real client idle-polling for more than 600ms would see the stream close and have to resubscribe, far more often than the documented 5-minute cadence. Now the stream stays open for the full (Handler-configurable, WithSubscribeToShardTiming) deadline, sending a heartbeat SubscribeToShardEvent (empty Records, real ContinuationSequenceNumber, MillisBehindLatest=0) once subscribeToShardHeartbeatInterval has elapsed since the last frame instead of closing -- API_SubscribeToShardEvent.html documents ContinuationSequenceNumber as required even with no data ('captures your shard progress even when no data is written to the shard'), which only makes sense if heartbeats are sent; SubscribeToShard's own backend method previously left ContinuationSequenceNumber empty whenever it returned zero new records, which is also now fixed (subscribeToShardContinuationSeq, consumers.go), reusing the last delivered record's sequence number, or the shard's own last record if the subscriber never advanced past a real delivery, or '' only for a shard with literally zero records ever (disclosed approximation, see gaps). TRIM_HORIZON/AT_TIMESTAMP StartingPosition now honor the same retentionCutoff GetShardIterator does (see that op's note above) instead of assuming position 0 is always untrimmed. Neither building-enhanced-consumers-api.html nor API_SubscribeToShardEvent.html states an exact heartbeat interval, so defaultSubscribeToShardHeartbeatInterval (5s) is a disclosed inference, not a verified AWS constant -- see gaps. Tests: TestSubscribeToShard_IdleCloseIsGraceful (rewritten to assert deadline-close-with-heartbeats instead of the old idle-close, via WithSubscribeToShardTiming), TestSubscribeToShard_HonoursRetentionWindow (retention_iterator_test.go), plus the existing 5-min-window default is exercised via short per-test overrides everywhere newTestHandler/subscribe_idle_close_test.go build a Handler -- the previous idle-close self-terminated fast enough that no test needed this before. 2026-09-18 (gopherstack-j60e, gopherstack-i8q7): openSubscribeToShardStream now sends Connection: close on this response -- i8q7's root cause (a keep-alive connection from an earlier ordinary call on the same client getting torn down by net/http's writeLoop while the event-stream reader is still mid-read on it here, surfacing as 'use of closed network connection') is a property of connection REUSE, previously worked around only in the internal test client (DisableKeepAlives, newTestKinesisClient). Marking this specific long-lived response non-reusable fixes it server-side for every caller, not just callers that remember to disable keep-alives themselves -- including test/integration/kinesis_test.go's real HTTP client, which is outside this pass's services/kinesis-only scope to edit directly. j60e itself was never reproduced locally (documented as unreproducible in-process, container/NAT-layer suspected); this closes the one deterministic, in-process root cause found for the shared symptom class rather than claiming j60e itself is fixed."} - UpdateShardCount: {wire: fixed, errors: ok, state: ok, persist: ok, note: "double/half scaling window, parent/adjacent-parent lineage, old shards kept CLOSED verified. gopherstack-enpq (2026-08-22): Input had no StreamARN member (api_op_UpdateShardCount.go:77-108 (StreamARN:102)); fixed via resolveStreamNameAndRegion."} + UpdateShardCount: {wire: fixed, errors: ok, state: fixed, persist: ok, note: "double/half scaling window, parent/adjacent-parent lineage, old shards kept CLOSED verified. gopherstack-enpq (2026-08-22): Input had no StreamARN member (api_op_UpdateShardCount.go:77-108 (StreamARN:102)); fixed via resolveStreamNameAndRegion. 2026-09-26 (gopherstack-nbg8, transient-state pass): CreateStream/UpdateShardCount/MergeShards/SplitShard/StartStreamEncryption/StopStreamEncryption/UpdateStreamMode/DeleteStream now model real transient state (CREATING/UPDATING/DELETING via a lazy ReadyAt deadline resolved on the next read, no goroutines -- see services/dsql's resolveClusterLocked/services/dax's sweepClusterTransitionsLocked for the same pattern) instead of a stream always being ACTIVE. DescribeStream/DescribeStreamSummary/ListStreams now show the transient status; each of these ops now rejects a non-ACTIVE stream with ResourceInUseException, matching its declared SDK error set."} EnableEnhancedMonitoring: {wire: fixed, errors: ok, state: ok, persist: ok, note: "gopherstack-enpq (2026-08-22): Input had no StreamARN member (api_op_EnableEnhancedMonitoring.go:34-71 (StreamARN:65)); fixed via resolveStreamNameAndRegion."} DisableEnhancedMonitoring: {wire: fixed, errors: ok, state: ok, persist: ok, note: "gopherstack-enpq (2026-08-22): same missing-StreamARN bug as EnableEnhancedMonitoring (api_op_DisableEnhancedMonitoring.go:34-71 (StreamARN:65)), fixed the same way (shared jsonEnhancedMonitoringReq)."} DescribeLimits: {wire: fixed, errors: ok, state: ok, persist: n/a, note: "gopherstack-nbg8: OnDemandStreamCount/OnDemandStreamCountLimit are both required output members (api_op_DescribeLimits.go:34-51, alongside ShardLimit/OpenShardCount) that were silently dropped -- a real client decoded zero values for both regardless of backend state. Wired to new CountOnDemandStreams (region-scoped, mirrors CountOpenShards) and OnDemandStreamCountLimit (the account-level ON_DEMAND cap CreateStream already enforced -- previously only reachable, incorrectly, through UpdateAccountSettings' fabricated shape below)."} DescribeAccountSettings: {wire: fixed, errors: ok, state: ok, persist: ok, note: "gopherstack-nbg8: the prior wire: ok claim was false. The real Output has exactly one member, MinimumThroughputBillingCommitment (api_op_DescribeAccountSettings.go:34-45); ShardLimit/OnDemandStreamCount/OnDemandStreamCountLimit were never real members of this op -- they belong to DescribeLimits (see above), suggesting the original audit confused the two sibling ops. Rebuilt around the real MinimumThroughputBillingCommitmentOutput shape (Status/StartedAt/EndedAt/EarliestAllowedEndAt, all epoch-seconds timestamps via pkgs/awstime.Epoch). This backend has no billing engine: no billing behaviour follows from an ENABLED commitment, and EarliestAllowedEndAt is never populated (it needs a commitment-window model this backend doesn't have -- see gaps). Status/StartedAt/EndedAt now persist across snapshot/restore."} UpdateAccountSettings: {wire: fixed, errors: ok, state: ok, persist: ok, note: "gopherstack-nbg8: the prior wire: ok claim was false. The real Input has exactly one, required, member, MinimumThroughputBillingCommitment (api_op_UpdateAccountSettings.go:42-51); ShardLimit/OnDemandStreamCount/OnDemandStreamCountLimit were fabricated with no basis in the real shape, so every real client's request was silently ignored. Now decodes and validates the real Status enum (ENABLED/DISABLED -> InvalidArgumentException otherwise), stores it as account-level state, and returns the real Output shape. The on-demand-stream cap this field used to (mis)configure is real internal state (CreateStream's checkOnDemandLimit via b.onDemandStreamCountLimit) -- real AWS manages it as a Service Quota, not via this op, so it moved to a Go-level-only SetOnDemandStreamCountLimit config knob (no wire equivalent, mirroring WithKMSValidator's cross-service config pattern) rather than being deleted."} - UpdateMaxRecordSize: {wire: fixed, errors: ok, state: ok, persist: ok, note: "gopherstack-nbg8: the prior wire: ok claim was false. Decoded a JSON key MaxRecordSizeBytes; the real (and only) required field is MaxRecordSizeInKiB (api_op_UpdateMaxRecordSize.go:30-47), and the unit is KiB, not bytes -- a real request left the value at zero, which the existing bounds check then always rejected with InvalidArgumentException (every real call 400'd). Also found while reading the whole operation, not just the flagged field: the real Input has no StreamName member at all -- only StreamARN, plus StreamId (reserved for future use, not modeled) -- but gopherstack was additionally decoding and consuming a fabricated StreamName field. Now resolves the stream from StreamARN only and converts the requested KiB value to bytes via bytesPerKiB before applying it to Stream.MaxRecordSizeBytes."} - UpdateStreamWarmThroughput: {wire: fixed, errors: ok, state: ok, persist: ok, note: "gopherstack-nbg8: the prior wire: ok claim was false, and 'intentional no-op' was not: the op decoded fabricated WriteCapacityUnits/ReadCapacityUnits fields with no basis in the real shape, so every real client's request silently no-op'd. The real required field is WarmThroughputMiBps (api_op_UpdateStreamWarmThroughput.go:63-70). Also unmodeled: the real Output (StreamARN/StreamName/WarmThroughput{CurrentMiBps,TargetMiBps}, api_op_UpdateStreamWarmThroughput.go:76-88) -- the handler returned an empty struct{}. Now decodes/validates WarmThroughputMiBps (bounds-checked against AWS's documented 10 GiBps default cap, maxWarmThroughputMiBps), stores it on Stream.WarmThroughputMiBps, and returns the real Output shape. Applied synchronously since this backend has no UPDATING transient-state model (unlike real AWS, which returns the stream to ACTIVE asynchronously) -- Current/Target always match on read; see gaps."} - MergeShards: {wire: ok, errors: ok, state: ok, persist: ok, note: "adjacency check (either shard may be passed first), closed-parent lineage verified"} - SplitShard: {wire: ok, errors: ok, state: ok, persist: ok, note: "NewStartingHashKey must be strictly inside parent range, verified"} - StartStreamEncryption: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: KeyId is now required and format-validated (UUID/key ARN/alias ARN/alias name, matching the four shapes the SDK doc comment enumerates) -- InvalidArgumentException if malformed; optional KMSKeyValidator (WithKMSValidator, wired to the real kms backend by cli.go's wireKinesisKMS) additionally verifies the key exists and is usable, returning KMSNotFoundException/KMSDisabledException/KMSInvalidStateException -- all three are real types.KMSNotFoundException-class exceptions confirmed present in the SDK's StartStreamEncryption error set (deserializers.go), contradicting the previous audit's claim that no KMS-specific exception exists for this op. With no validator wired, only the format check applies (a well-formed but nonexistent KeyId is accepted, same permissive behavior as before)."} - StopStreamEncryption: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: KeyId is now required and format-validated like StartStreamEncryption (matches the SDK's required-field validator); never calls the KMS validator since disabling encryption must succeed even if the key was later disabled/deleted"} + UpdateMaxRecordSize: {wire: fixed, errors: ok, state: fixed, persist: ok, note: "gopherstack-nbg8: the prior wire: ok claim was false. Decoded a JSON key MaxRecordSizeBytes; the real (and only) required field is MaxRecordSizeInKiB (api_op_UpdateMaxRecordSize.go:30-47), and the unit is KiB, not bytes -- a real request left the value at zero, which the existing bounds check then always rejected with InvalidArgumentException (every real call 400'd). Also found while reading the whole operation, not just the flagged field: the real Input has no StreamName member at all -- only StreamARN, plus StreamId (reserved for future use, not modeled) -- but gopherstack was additionally decoding and consuming a fabricated StreamName field. Now resolves the stream from StreamARN only and converts the requested KiB value to bytes via bytesPerKiB before applying it to Stream.MaxRecordSizeBytes. 2026-09-26 (gopherstack-nbg8): now that CreateStream/UpdateShardCount/MergeShards/SplitShard/StartStreamEncryption/StopStreamEncryption/UpdateStreamMode/DeleteStream model real CREATING/UPDATING/DELETING transient state, this op's declared ResourceInUseException is reachable (rejects a non-ACTIVE stream); the op itself still applies synchronously (no separate UPDATING transition of its own)."} + UpdateStreamWarmThroughput: {wire: fixed, errors: ok, state: fixed, persist: ok, note: "gopherstack-nbg8: the prior wire: ok claim was false, and 'intentional no-op' was not: the op decoded fabricated WriteCapacityUnits/ReadCapacityUnits fields with no basis in the real shape, so every real client's request silently no-op'd. The real required field is WarmThroughputMiBps (api_op_UpdateStreamWarmThroughput.go:63-70). Also unmodeled: the real Output (StreamARN/StreamName/WarmThroughput{CurrentMiBps,TargetMiBps}, api_op_UpdateStreamWarmThroughput.go:76-88) -- the handler returned an empty struct{}. Now decodes/validates WarmThroughputMiBps (bounds-checked against AWS's documented 10 GiBps default cap, maxWarmThroughputMiBps), stores it on Stream.WarmThroughputMiBps, and returns the real Output shape. Applied synchronously since this backend has no UPDATING transient-state model (unlike real AWS, which returns the stream to ACTIVE asynchronously) -- Current/Target always match on read; see gaps. 2026-09-26 (gopherstack-nbg8): now that CreateStream/UpdateShardCount/MergeShards/SplitShard/StartStreamEncryption/StopStreamEncryption/UpdateStreamMode/DeleteStream model real CREATING/UPDATING/DELETING transient state, this op's declared ResourceInUseException is reachable (rejects a non-ACTIVE stream); the op itself still applies synchronously (no separate UPDATING transition of its own)."} + MergeShards: {wire: ok, errors: ok, state: fixed, persist: ok, note: "adjacency check (either shard may be passed first), closed-parent lineage verified 2026-09-26 (gopherstack-nbg8, transient-state pass): CreateStream/UpdateShardCount/MergeShards/SplitShard/StartStreamEncryption/StopStreamEncryption/UpdateStreamMode/DeleteStream now model real transient state (CREATING/UPDATING/DELETING via a lazy ReadyAt deadline resolved on the next read, no goroutines -- see services/dsql's resolveClusterLocked/services/dax's sweepClusterTransitionsLocked for the same pattern) instead of a stream always being ACTIVE. DescribeStream/DescribeStreamSummary/ListStreams now show the transient status; each of these ops now rejects a non-ACTIVE stream with ResourceInUseException, matching its declared SDK error set."} + SplitShard: {wire: ok, errors: ok, state: fixed, persist: ok, note: "NewStartingHashKey must be strictly inside parent range, verified 2026-09-26 (gopherstack-nbg8, transient-state pass): CreateStream/UpdateShardCount/MergeShards/SplitShard/StartStreamEncryption/StopStreamEncryption/UpdateStreamMode/DeleteStream now model real transient state (CREATING/UPDATING/DELETING via a lazy ReadyAt deadline resolved on the next read, no goroutines -- see services/dsql's resolveClusterLocked/services/dax's sweepClusterTransitionsLocked for the same pattern) instead of a stream always being ACTIVE. DescribeStream/DescribeStreamSummary/ListStreams now show the transient status; each of these ops now rejects a non-ACTIVE stream with ResourceInUseException, matching its declared SDK error set."} + StartStreamEncryption: {wire: ok, errors: ok, state: fixed, persist: ok, note: "fixed: KeyId is now required and format-validated (UUID/key ARN/alias ARN/alias name, matching the four shapes the SDK doc comment enumerates) -- InvalidArgumentException if malformed; optional KMSKeyValidator (WithKMSValidator, wired to the real kms backend by cli.go's wireKinesisKMS) additionally verifies the key exists and is usable, returning KMSNotFoundException/KMSDisabledException/KMSInvalidStateException -- all three are real types.KMSNotFoundException-class exceptions confirmed present in the SDK's StartStreamEncryption error set (deserializers.go), contradicting the previous audit's claim that no KMS-specific exception exists for this op. With no validator wired, only the format check applies (a well-formed but nonexistent KeyId is accepted, same permissive behavior as before). 2026-09-26 (gopherstack-nbg8, transient-state pass): CreateStream/UpdateShardCount/MergeShards/SplitShard/StartStreamEncryption/StopStreamEncryption/UpdateStreamMode/DeleteStream now model real transient state (CREATING/UPDATING/DELETING via a lazy ReadyAt deadline resolved on the next read, no goroutines -- see services/dsql's resolveClusterLocked/services/dax's sweepClusterTransitionsLocked for the same pattern) instead of a stream always being ACTIVE. DescribeStream/DescribeStreamSummary/ListStreams now show the transient status; each of these ops now rejects a non-ACTIVE stream with ResourceInUseException, matching its declared SDK error set."} + StopStreamEncryption: {wire: ok, errors: ok, state: fixed, persist: ok, note: "fixed: KeyId is now required and format-validated like StartStreamEncryption (matches the SDK's required-field validator); never calls the KMS validator since disabling encryption must succeed even if the key was later disabled/deleted 2026-09-26 (gopherstack-nbg8, transient-state pass): CreateStream/UpdateShardCount/MergeShards/SplitShard/StartStreamEncryption/StopStreamEncryption/UpdateStreamMode/DeleteStream now model real transient state (CREATING/UPDATING/DELETING via a lazy ReadyAt deadline resolved on the next read, no goroutines -- see services/dsql's resolveClusterLocked/services/dax's sweepClusterTransitionsLocked for the same pattern) instead of a stream always being ACTIVE. DescribeStream/DescribeStreamSummary/ListStreams now show the transient status; each of these ops now rejects a non-ACTIVE stream with ResourceInUseException, matching its declared SDK error set."} DeleteResourcePolicy: {wire: ok, errors: ok, state: ok, persist: n/a, note: "resource policies not yet in backendSnapshot; see gaps"} GetResourcePolicy: {wire: ok, errors: ok, state: ok, persist: n/a} PutResourcePolicy: {wire: ok, errors: ok, state: ok, persist: n/a} @@ -43,7 +43,7 @@ ops: ListTagsForStream: {wire: fixed, errors: ok, state: ok, persist: ok, note: "fixed: now reads Backend.ListTagsForResource. gopherstack-enpq (2026-08-22): same missing-StreamARN bug (api_op_ListTagsForStream.go:35-53 (StreamARN:47)), fixed the same way."} TagResource: {wire: ok, errors: ok, state: ok, persist: ok, note: "fixed: now enforces the 50-tag cap consistently with AddTagsToStream (previously uncapped)"} UntagResource: {wire: ok, errors: ok, state: ok, persist: ok} - UpdateStreamMode: {wire: fixed, errors: ok, state: fixed, persist: ok, note: "2026-09-11 (gopherstack-s0ju item 2): CORRECTED this pass -- the prior 'fixed: PROVISIONED -> ON_DEMAND now auto-reshards up to defaultOnDemandShardCount (4)...when the stream is currently below that floor' behavior was itself wrong, contradicting the real doc it cited: 'When you switch from provisioned to on-demand capacity mode, your data stream initially retains whatever shard count it had before the transition, and from this point on, Kinesis Data Streams monitors your data traffic and scales the shard count' (docs.aws.amazon.com/streams/latest/dev/how-do-i-size-a-stream.html#switchingmodes). A 1-shard PROVISIONED stream switching to ON_DEMAND was being resharded up to 4 shards immediately, which real AWS never does -- it keeps 1. Removed the flooring reshard entirely; the transition now only flips StreamMode, matching the doc exactly for both directions (ON_DEMAND -> PROVISIONED already correctly kept the shard count, unchanged). CreateStream's own separate ON_DEMAND default (4 shards for a brand-new stream, 'A data stream in the on-demand mode accommodates up to double the peak write throughput observed in the previous 30 days,' same page's #ondemandmode section) is untouched by this fix -- that is a different, still-correct code path. Reactive scaling is now real, not absent: maybeAutoScaleOnDemand (new ondemand_scaling.go) tracks each ON_DEMAND stream's write-rate over a 60s sliding window (transient, in-memory only, InMemoryBackend.throughputTrackers -- never wired into backendSnapshot, so it does not appear in snapshot_inventory.json) and doubles the open shard count (capped at maxShardsPerStream, via the same reshardTo helper UpdateShardCount uses) once the aggregate rate exceeds the documented per-shard trigger: 'Kinesis Data Streams monitors traffic for each shard. When the incoming traffic exceeds 500 KB/s per shard, it splits the shard within 15 minutes' (same page, 'Handle read and write throughput exceptions'). Disclosed approximations, all in ondemand_scaling.go's own doc comments: (1) a 60-second window stands in for AWS's real 30-day peak-throughput history, which this emulator has no model for; (2) the whole stream's open shard count is doubled rather than splitting only the specific overloaded shard, since write-rate is tracked per-stream, not per-shard; (3) '500 KB' is read as 500 KiB (binary), matching this file's existing UpdateMaxRecordSize KiB convention, since AWS's own docs are not consistent about decimal vs. binary KB/MB. WarmThroughputMiBps handling (2026-08-23, request-side accept-and-drop sweep fix) is unchanged by this pass. Tests: TestUpdateStreamMode_OnDemandTransitionKeepsShardCount (replaces the old, now-incorrect TestUpdateStreamMode_OnDemandTransitionReshardsUpToFloor), TestUpdateStreamMode_ProvisionedToOnDemand_RealClientKeepsShardCount (real aws-sdk-go-v2 client round trip), TestUpdateStreamMode_OnDemandAutoScalesOnSustainedWrite, TestUpdateStreamMode_OnDemandAutoScaleIgnoresProvisioned (stream_modes_test.go)."} + UpdateStreamMode: {wire: fixed, errors: ok, state: fixed, persist: ok, note: "2026-09-11 (gopherstack-s0ju item 2): CORRECTED this pass -- the prior 'fixed: PROVISIONED -> ON_DEMAND now auto-reshards up to defaultOnDemandShardCount (4)...when the stream is currently below that floor' behavior was itself wrong, contradicting the real doc it cited: 'When you switch from provisioned to on-demand capacity mode, your data stream initially retains whatever shard count it had before the transition, and from this point on, Kinesis Data Streams monitors your data traffic and scales the shard count' (docs.aws.amazon.com/streams/latest/dev/how-do-i-size-a-stream.html#switchingmodes). A 1-shard PROVISIONED stream switching to ON_DEMAND was being resharded up to 4 shards immediately, which real AWS never does -- it keeps 1. Removed the flooring reshard entirely; the transition now only flips StreamMode, matching the doc exactly for both directions (ON_DEMAND -> PROVISIONED already correctly kept the shard count, unchanged). CreateStream's own separate ON_DEMAND default (4 shards for a brand-new stream, 'A data stream in the on-demand mode accommodates up to double the peak write throughput observed in the previous 30 days,' same page's #ondemandmode section) is untouched by this fix -- that is a different, still-correct code path. Reactive scaling is now real, not absent: maybeAutoScaleOnDemand (new ondemand_scaling.go) tracks each ON_DEMAND stream's write-rate over a 60s sliding window (transient, in-memory only, InMemoryBackend.throughputTrackers -- never wired into backendSnapshot, so it does not appear in snapshot_inventory.json) and doubles the open shard count (capped at maxShardsPerStream, via the same reshardTo helper UpdateShardCount uses) once the aggregate rate exceeds the documented per-shard trigger: 'Kinesis Data Streams monitors traffic for each shard. When the incoming traffic exceeds 500 KB/s per shard, it splits the shard within 15 minutes' (same page, 'Handle read and write throughput exceptions'). Disclosed approximations, all in ondemand_scaling.go's own doc comments: (1) a 60-second window stands in for AWS's real 30-day peak-throughput history, which this emulator has no model for; (2) the whole stream's open shard count is doubled rather than splitting only the specific overloaded shard, since write-rate is tracked per-stream, not per-shard; (3) '500 KB' is read as 500 KiB (binary), matching this file's existing UpdateMaxRecordSize KiB convention, since AWS's own docs are not consistent about decimal vs. binary KB/MB. WarmThroughputMiBps handling (2026-08-23, request-side accept-and-drop sweep fix) is unchanged by this pass. Tests: TestUpdateStreamMode_OnDemandTransitionKeepsShardCount (replaces the old, now-incorrect TestUpdateStreamMode_OnDemandTransitionReshardsUpToFloor), TestUpdateStreamMode_ProvisionedToOnDemand_RealClientKeepsShardCount (real aws-sdk-go-v2 client round trip), TestUpdateStreamMode_OnDemandAutoScalesOnSustainedWrite, TestUpdateStreamMode_OnDemandAutoScaleIgnoresProvisioned (stream_modes_test.go). 2026-09-26 (gopherstack-nbg8, transient-state pass): CreateStream/UpdateShardCount/MergeShards/SplitShard/StartStreamEncryption/StopStreamEncryption/UpdateStreamMode/DeleteStream now model real transient state (CREATING/UPDATING/DELETING via a lazy ReadyAt deadline resolved on the next read, no goroutines -- see services/dsql's resolveClusterLocked/services/dax's sweepClusterTransitionsLocked for the same pattern) instead of a stream always being ACTIVE. DescribeStream/DescribeStreamSummary/ListStreams now show the transient status; each of these ops now rejects a non-ACTIVE stream with ResourceInUseException, matching its declared SDK error set."} families: hash_key_routing: {status: ok, note: "MD5-based partition-key routing and explicit-hash-key routing verified against big.Int range math; shardForHashKey fallback-to-first-open-shard behavior documented"} sequence_numbers: {status: ok, note: "per-shard monotonic NextSeq counter, 49-prefixed AWS-shaped sequence string, persisted via Shard.NextSeq"} @@ -61,7 +61,7 @@ items_still_open: - "Buffered-but-unflushed channel records are not persisted across Snapshot/Restore (channelBuffers is in-memory-only). Handler.Shutdown/DeleteChannel/DeleteStream best-effort flush first, covering graceful shutdown and explicit deletion; only an ungraceful crash between an accepted PutRecord and the next flush loses that channel's currently-buffered records. No snapshot_inventory.json field exists for this by design. (gopherstack-s781r)" - "CreateChannel/DeleteChannel/DescribeChannel/ListChannels/UpdateChannel's documented 5 TPS-per-account throttle (LimitExceededException) is not modeled -- judged disproportionate to wire into this already-large file; not fabricated. ChannelDescription/ChannelSummary's S3TablesConfiguration.PartitionSpec round-trips but this backend performs no actual Iceberg partitioning to verify it against." - "No IAM policy evaluation engine exists anywhere in gopherstack, so three real, modeled error types have no honest trigger path: KMSAccessDeniedException (StartStreamEncryption/StopStreamEncryption) and AccessDeniedException (UpdateMaxRecordSize/UpdateStreamWarmThroughput). All three are wire-mapped for shape completeness but never fabricated with a fake denial rule. (gopherstack-ud2, gopherstack-nbg8)" - - "No stream-level transient-state model (CREATING/UPDATING/DELETING) exists anywhere in kinesis -- every stream is ACTIVE immediately and stays so. Two consequences, both honest: ResourceInUseException is declared but unreachable for UpdateMaxRecordSize/UpdateStreamWarmThroughput, and UpdateStreamWarmThroughput applies synchronously (Current/Target always match on read) where real AWS is asynchronous. (gopherstack-nbg8)" + - "UpdateMaxRecordSize and UpdateStreamWarmThroughput apply synchronously (Current/Target always match on read) where real AWS is asynchronous (sets UPDATING, then ACTIVE) -- unlike CreateStream/UpdateShardCount/MergeShards/SplitShard/StartStreamEncryption/StopStreamEncryption/UpdateStreamMode/DeleteStream, which now model that transient window via a lazy ReadyAt deadline. Both ops do correctly reject a non-ACTIVE stream with ResourceInUseException. (gopherstack-nbg8)" deferred: [] leaks: {status: clean, note: "stream.mu (lockmetrics) and stream.Tags always Close()'d on DeleteStream/Purge; SubscribeToShard polling goroutine bounded by a real 5-minute deadline (subscribeToShardMaxIdlePolls removed 2026-09-11, gopherstack-s0ju item 4 -- the stream now heartbeats instead of self-closing on idle, but the same deadline-bounded, ctx.Done()-exiting goroutine lifecycle applies), exits on ctx.Done(); FIS throughput-fault goroutines bound to experiment ctx or scheduled cleanup, lazily evict on read; janitor retention sweep is a single ticker goroutine stopped via context cancellation, no per-stream goroutines; this pass's reshardTo/closeShard/KMSKeyValidator additions introduce no goroutines, tickers, or new lock-acquisition orderings -- KMS validation is a synchronous in-process call into the kms package's own locked backend while kinesis holds stream.mu, safe because kms never calls back into kinesis. 2026-09-11 (gopherstack-s0ju items 2-4): the new InMemoryBackend.throughputMu (ondemand_scaling.go) is acquired only from putRecordLocked while the caller already holds that stream's mu (stream.mu -> throughputMu, a new but consistent ordering never reversed elsewhere) and is released before reshardTo/maybeAutoScaleOnDemand mutate shard state, so it never overlaps b.mu; introduces no goroutines or tickers."} --- diff --git a/services/kinesis/README.md b/services/kinesis/README.md index 440681b891..9f36296b1a 100644 --- a/services/kinesis/README.md +++ b/services/kinesis/README.md @@ -20,7 +20,7 @@ - Buffered-but-unflushed channel records are not persisted across Snapshot/Restore (channelBuffers is in-memory-only). Handler.Shutdown/DeleteChannel/DeleteStream best-effort flush first, covering graceful shutdown and explicit deletion; only an ungraceful crash between an accepted PutRecord and the next flush loses that channel's currently-buffered records. No snapshot_inventory.json field exists for this by design. (gopherstack-s781r) - CreateChannel/DeleteChannel/DescribeChannel/ListChannels/UpdateChannel's documented 5 TPS-per-account throttle (LimitExceededException) is not modeled -- judged disproportionate to wire into this already-large file; not fabricated. ChannelDescription/ChannelSummary's S3TablesConfiguration.PartitionSpec round-trips but this backend performs no actual Iceberg partitioning to verify it against. - No IAM policy evaluation engine exists anywhere in gopherstack, so three real, modeled error types have no honest trigger path: KMSAccessDeniedException (StartStreamEncryption/StopStreamEncryption) and AccessDeniedException (UpdateMaxRecordSize/UpdateStreamWarmThroughput). All three are wire-mapped for shape completeness but never fabricated with a fake denial rule. (gopherstack-ud2, gopherstack-nbg8) -- No stream-level transient-state model (CREATING/UPDATING/DELETING) exists anywhere in kinesis -- every stream is ACTIVE immediately and stays so. Two consequences, both honest: ResourceInUseException is declared but unreachable for UpdateMaxRecordSize/UpdateStreamWarmThroughput, and UpdateStreamWarmThroughput applies synchronously (Current/Target always match on read) where real AWS is asynchronous. (gopherstack-nbg8) +- UpdateMaxRecordSize and UpdateStreamWarmThroughput apply synchronously (Current/Target always match on read) where real AWS is asynchronous (sets UPDATING, then ACTIVE) -- unlike CreateStream/UpdateShardCount/MergeShards/SplitShard/StartStreamEncryption/StopStreamEncryption/UpdateStreamMode/DeleteStream, which now model that transient window via a lazy ReadyAt deadline. Both ops do correctly reject a non-ACTIVE stream with ResourceInUseException. (gopherstack-nbg8) ## More diff --git a/services/kinesis/account_settings.go b/services/kinesis/account_settings.go index 87cbba0935..19a6f5c0c0 100644 --- a/services/kinesis/account_settings.go +++ b/services/kinesis/account_settings.go @@ -118,18 +118,20 @@ func (b *InMemoryBackend) UpdateMaxRecordSize(ctx context.Context, input *Update region := regionFromARNOrCtx(ctx, input.StreamARN, b.region) streamName := streamNameFromARN(input.StreamARN) - b.mu.RLock("UpdateMaxRecordSize") - - stream, ok := b.streams.Get(streamKey(region, streamName)) - if !ok { - b.mu.RUnlock() + b.mu.Lock("UpdateMaxRecordSize") + defer b.mu.Unlock() - return ErrStreamNotFound + stream, err := b.resolveStreamTransitionLocked(region, streamName) + if err != nil { + return err } stream.mu.Lock("UpdateMaxRecordSize.stream") - b.mu.RUnlock() defer stream.mu.Unlock() + if stream.Status != streamStatusActive { + return ErrStreamNotActive + } + sizeBytes := input.MaxRecordSizeInKiB * bytesPerKiB if sizeBytes < defaultMaxRecordSizeBytes || sizeBytes > absoluteMaxRecordSizeBytes { return ErrInvalidArgument diff --git a/services/kinesis/cbor_test.go b/services/kinesis/cbor_test.go index d3b85f2ed6..b9b879bf7a 100644 --- a/services/kinesis/cbor_test.go +++ b/services/kinesis/cbor_test.go @@ -7,6 +7,8 @@ import ( "net/http" "net/http/httptest" "testing" + "testing/synctest" + "time" "github.com/aws/smithy-go/encoding/cbor" "github.com/labstack/echo/v5" @@ -97,26 +99,29 @@ func TestKinesisCBOR_PutRecord(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - h := newTestHandler(t) - createKinesisStream(t, h, "CborStream") - - putBody := cbor.Map{ - "StreamName": cbor.String("CborStream"), - "Data": cbor.Slice(tc.data), - "PartitionKey": cbor.String(tc.partKey), - } - - req := cborKinesisRequest(t, "PutRecord", putBody) - rr := serveCBOR(t, h, req) - assert.Equal(t, http.StatusOK, rr.Code, "PutRecord: %s", rr.Body.String()) - assert.Equal(t, service.ContentTypeCBOR, rr.Header().Get("Content-Type")) - - resp := decodeCBORKinesisResponse(t, rr) - - _, hasSeq := resp["SequenceNumber"] - assert.True(t, hasSeq, "response must contain SequenceNumber") - _, hasShard := resp["ShardId"] - assert.True(t, hasShard, "response must contain ShardId") + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + createKinesisStream(t, h, "CborStream") + time.Sleep(streamSettleWait) + + putBody := cbor.Map{ + "StreamName": cbor.String("CborStream"), + "Data": cbor.Slice(tc.data), + "PartitionKey": cbor.String(tc.partKey), + } + + req := cborKinesisRequest(t, "PutRecord", putBody) + rr := serveCBOR(t, h, req) + assert.Equal(t, http.StatusOK, rr.Code, "PutRecord: %s", rr.Body.String()) + assert.Equal(t, service.ContentTypeCBOR, rr.Header().Get("Content-Type")) + + resp := decodeCBORKinesisResponse(t, rr) + + _, hasSeq := resp["SequenceNumber"] + assert.True(t, hasSeq, "response must contain SequenceNumber") + _, hasShard := resp["ShardId"] + assert.True(t, hasShard, "response must contain ShardId") + }) }) } } @@ -163,67 +168,70 @@ func TestKinesisCBOR_PutRecords(t *testing.T) { func TestKinesisCBOR_GetRecords(t *testing.T) { t.Parallel() - h := newTestHandler(t) - createKinesisStream(t, h, "CborGetStream") - - payload := []byte("cbor-test-data") - - // Put a record via CBOR. - putBody := cbor.Map{ - "StreamName": cbor.String("CborGetStream"), - "Data": cbor.Slice(payload), - "PartitionKey": cbor.String("p1"), - } - putReq := cborKinesisRequest(t, "PutRecord", putBody) - putRR := serveCBOR(t, h, putReq) - require.Equal(t, http.StatusOK, putRR.Code) - - // Get shard iterator via JSON. - iterBody := map[string]any{ - "StreamName": "CborGetStream", - "ShardId": "shardId-000000000000", - "ShardIteratorType": "TRIM_HORIZON", - } - iterBytes, _ := json.Marshal(iterBody) - iterReq := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(iterBytes)) - iterReq.Header.Set("Content-Type", "application/x-amz-json-1.1") - iterReq.Header.Set("X-Amz-Target", kinesisPrefix+"GetShardIterator") - - e := echo.New() - iterRec := httptest.NewRecorder() - iterC := e.NewContext(iterReq, iterRec) - require.NoError(t, h.Handler()(iterC)) - require.Equal(t, http.StatusOK, iterRec.Code) - - var iterResp map[string]any - require.NoError(t, json.Unmarshal(iterRec.Body.Bytes(), &iterResp)) - shardIter, ok := iterResp["ShardIterator"].(string) - require.True(t, ok, "must have ShardIterator") - - // GetRecords via CBOR. - getBody := cbor.Map{ - "ShardIterator": cbor.String(shardIter), - "Limit": cbor.Uint(10), - } - getReq := cborKinesisRequest(t, "GetRecords", getBody) - getRR := serveCBOR(t, h, getReq) - require.Equal(t, http.StatusOK, getRR.Code) - - resp := decodeCBORKinesisResponse(t, getRR) - - recList, ok := resp["Records"].(cbor.List) - require.True(t, ok, "response must contain Records") - require.NotEmpty(t, recList, "should have at least one record") + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + createKinesisStream(t, h, "CborGetStream") + time.Sleep(streamSettleWait) - firstRec, ok := recList[0].(cbor.Map) - require.True(t, ok) + payload := []byte("cbor-test-data") - dataVal, ok := firstRec["Data"] - require.True(t, ok, "record must have Data field") - - sl, ok := dataVal.(cbor.Slice) - require.True(t, ok, "Data must be cbor.Slice over CBOR protocol, got %T", dataVal) - assert.Equal(t, payload, []byte(sl)) + // Put a record via CBOR. + putBody := cbor.Map{ + "StreamName": cbor.String("CborGetStream"), + "Data": cbor.Slice(payload), + "PartitionKey": cbor.String("p1"), + } + putReq := cborKinesisRequest(t, "PutRecord", putBody) + putRR := serveCBOR(t, h, putReq) + require.Equal(t, http.StatusOK, putRR.Code) + + // Get shard iterator via JSON. + iterBody := map[string]any{ + "StreamName": "CborGetStream", + "ShardId": "shardId-000000000000", + "ShardIteratorType": "TRIM_HORIZON", + } + iterBytes, _ := json.Marshal(iterBody) + iterReq := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(iterBytes)) + iterReq.Header.Set("Content-Type", "application/x-amz-json-1.1") + iterReq.Header.Set("X-Amz-Target", kinesisPrefix+"GetShardIterator") + + e := echo.New() + iterRec := httptest.NewRecorder() + iterC := e.NewContext(iterReq, iterRec) + require.NoError(t, h.Handler()(iterC)) + require.Equal(t, http.StatusOK, iterRec.Code) + + var iterResp map[string]any + require.NoError(t, json.Unmarshal(iterRec.Body.Bytes(), &iterResp)) + shardIter, ok := iterResp["ShardIterator"].(string) + require.True(t, ok, "must have ShardIterator") + + // GetRecords via CBOR. + getBody := cbor.Map{ + "ShardIterator": cbor.String(shardIter), + "Limit": cbor.Uint(10), + } + getReq := cborKinesisRequest(t, "GetRecords", getBody) + getRR := serveCBOR(t, h, getReq) + require.Equal(t, http.StatusOK, getRR.Code) + + resp := decodeCBORKinesisResponse(t, getRR) + + recList, ok := resp["Records"].(cbor.List) + require.True(t, ok, "response must contain Records") + require.NotEmpty(t, recList, "should have at least one record") + + firstRec, ok := recList[0].(cbor.Map) + require.True(t, ok) + + dataVal, ok := firstRec["Data"] + require.True(t, ok, "record must have Data field") + + sl, ok := dataVal.(cbor.Slice) + require.True(t, ok, "Data must be cbor.Slice over CBOR protocol, got %T", dataVal) + assert.Equal(t, payload, []byte(sl)) + }) } // TestKinesisCBOR_DataRoundTrip tests that binary written as base64 via JSON is @@ -231,61 +239,64 @@ func TestKinesisCBOR_GetRecords(t *testing.T) { func TestKinesisCBOR_DataRoundTrip(t *testing.T) { t.Parallel() - h := newTestHandler(t) - createKinesisStream(t, h, "CborB64Stream") + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + createKinesisStream(t, h, "CborB64Stream") + time.Sleep(streamSettleWait) + + rawData := []byte{0xDE, 0xAD, 0xBE, 0xEF, 0xCA, 0xFE} + b64Data := base64.StdEncoding.EncodeToString(rawData) + + // Write via JSON with base64 Data. + jsonPut := map[string]any{ + "StreamName": "CborB64Stream", + "Data": b64Data, + "PartitionKey": "pk1", + } + b, _ := json.Marshal(jsonPut) + putReq := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(b)) + putReq.Header.Set("Content-Type", "application/x-amz-json-1.1") + putReq.Header.Set("X-Amz-Target", kinesisPrefix+"PutRecord") + + e := echo.New() + putRec := httptest.NewRecorder() + putC := e.NewContext(putReq, putRec) + require.NoError(t, h.Handler()(putC)) + require.Equal(t, http.StatusOK, putRec.Code) + + // Get shard iterator. + iterReq := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(mustMarshalBytes(t, map[string]any{ + "StreamName": "CborB64Stream", + "ShardId": "shardId-000000000000", + "ShardIteratorType": "TRIM_HORIZON", + }))) + iterReq.Header.Set("Content-Type", "application/x-amz-json-1.1") + iterReq.Header.Set("X-Amz-Target", kinesisPrefix+"GetShardIterator") + iterRec := httptest.NewRecorder() + iterC := e.NewContext(iterReq, iterRec) + require.NoError(t, h.Handler()(iterC)) + var iterResp map[string]any + require.NoError(t, json.Unmarshal(iterRec.Body.Bytes(), &iterResp)) + shardIter := iterResp["ShardIterator"].(string) + + // Read via CBOR. + getReq := cborKinesisRequest(t, "GetRecords", cbor.Map{ + "ShardIterator": cbor.String(shardIter), + }) + getRR := serveCBOR(t, h, getReq) + require.Equal(t, http.StatusOK, getRR.Code) - rawData := []byte{0xDE, 0xAD, 0xBE, 0xEF, 0xCA, 0xFE} - b64Data := base64.StdEncoding.EncodeToString(rawData) + resp := decodeCBORKinesisResponse(t, getRR) + recs := resp["Records"].(cbor.List) + require.NotEmpty(t, recs) - // Write via JSON with base64 Data. - jsonPut := map[string]any{ - "StreamName": "CborB64Stream", - "Data": b64Data, - "PartitionKey": "pk1", - } - b, _ := json.Marshal(jsonPut) - putReq := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(b)) - putReq.Header.Set("Content-Type", "application/x-amz-json-1.1") - putReq.Header.Set("X-Amz-Target", kinesisPrefix+"PutRecord") + rec0 := recs[0].(cbor.Map) + dataVal := rec0["Data"] - e := echo.New() - putRec := httptest.NewRecorder() - putC := e.NewContext(putReq, putRec) - require.NoError(t, h.Handler()(putC)) - require.Equal(t, http.StatusOK, putRec.Code) - - // Get shard iterator. - iterReq := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(mustMarshalBytes(t, map[string]any{ - "StreamName": "CborB64Stream", - "ShardId": "shardId-000000000000", - "ShardIteratorType": "TRIM_HORIZON", - }))) - iterReq.Header.Set("Content-Type", "application/x-amz-json-1.1") - iterReq.Header.Set("X-Amz-Target", kinesisPrefix+"GetShardIterator") - iterRec := httptest.NewRecorder() - iterC := e.NewContext(iterReq, iterRec) - require.NoError(t, h.Handler()(iterC)) - var iterResp map[string]any - require.NoError(t, json.Unmarshal(iterRec.Body.Bytes(), &iterResp)) - shardIter := iterResp["ShardIterator"].(string) - - // Read via CBOR. - getReq := cborKinesisRequest(t, "GetRecords", cbor.Map{ - "ShardIterator": cbor.String(shardIter), + sl, ok := dataVal.(cbor.Slice) + require.True(t, ok, "Data must be cbor.Slice over CBOR protocol, got %T", dataVal) + assert.Equal(t, rawData, []byte(sl)) }) - getRR := serveCBOR(t, h, getReq) - require.Equal(t, http.StatusOK, getRR.Code) - - resp := decodeCBORKinesisResponse(t, getRR) - recs := resp["Records"].(cbor.List) - require.NotEmpty(t, recs) - - rec0 := recs[0].(cbor.Map) - dataVal := rec0["Data"] - - sl, ok := dataVal.(cbor.Slice) - require.True(t, ok, "Data must be cbor.Slice over CBOR protocol, got %T", dataVal) - assert.Equal(t, rawData, []byte(sl)) } // TestKinesisCBOR_InvalidBody verifies 400 for a malformed CBOR request body. @@ -341,68 +352,71 @@ func TestKinesisCBOR_ListStreams(t *testing.T) { func TestKinesisCBOR_JSONAndCBORCoexist(t *testing.T) { t.Parallel() - h := newTestHandler(t) - createKinesisStream(t, h, "CoexistStream") - - // Write via CBOR. - cborPayload := []byte("cbor-record") - putCBOR := cbor.Map{ - "StreamName": cbor.String("CoexistStream"), - "Data": cbor.Slice(cborPayload), - "PartitionKey": cbor.String("pk1"), - } - req1 := cborKinesisRequest(t, "PutRecord", putCBOR) - rr1 := serveCBOR(t, h, req1) - require.Equal(t, http.StatusOK, rr1.Code) - - // Write via JSON. - jsonPayload := []byte("json-record") - b64 := base64.StdEncoding.EncodeToString(jsonPayload) - putJSON := map[string]any{ - "StreamName": "CoexistStream", - "Data": b64, - "PartitionKey": "pk2", - } - rawJSON, _ := json.Marshal(putJSON) - req2 := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(rawJSON)) - req2.Header.Set("Content-Type", "application/x-amz-json-1.1") - req2.Header.Set("X-Amz-Target", kinesisPrefix+"PutRecord") - e := echo.New() - rec2 := httptest.NewRecorder() - c2 := e.NewContext(req2, rec2) - require.NoError(t, h.Handler()(c2)) - require.Equal(t, http.StatusOK, rec2.Code) - - // Read via JSON to confirm both records are present. - iterReq := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(mustMarshalBytes(t, map[string]any{ - "StreamName": "CoexistStream", - "ShardId": "shardId-000000000000", - "ShardIteratorType": "TRIM_HORIZON", - }))) - iterReq.Header.Set("Content-Type", "application/x-amz-json-1.1") - iterReq.Header.Set("X-Amz-Target", kinesisPrefix+"GetShardIterator") - iterRec := httptest.NewRecorder() - iterC := e.NewContext(iterReq, iterRec) - require.NoError(t, h.Handler()(iterC)) - var iterResp map[string]any - require.NoError(t, json.Unmarshal(iterRec.Body.Bytes(), &iterResp)) - shardIter := iterResp["ShardIterator"].(string) - - getReq := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(mustMarshalBytes(t, map[string]any{ - "ShardIterator": shardIter, - "Limit": 10, - }))) - getReq.Header.Set("Content-Type", "application/x-amz-json-1.1") - getReq.Header.Set("X-Amz-Target", kinesisPrefix+"GetRecords") - getRec := httptest.NewRecorder() - getC := e.NewContext(getReq, getRec) - require.NoError(t, h.Handler()(getC)) - require.Equal(t, http.StatusOK, getRec.Code) - - var getResp map[string]any - require.NoError(t, json.Unmarshal(getRec.Body.Bytes(), &getResp)) - recs := getResp["Records"].([]any) - assert.GreaterOrEqual(t, len(recs), 2, "both records must be visible via JSON") + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + createKinesisStream(t, h, "CoexistStream") + time.Sleep(streamSettleWait) + + // Write via CBOR. + cborPayload := []byte("cbor-record") + putCBOR := cbor.Map{ + "StreamName": cbor.String("CoexistStream"), + "Data": cbor.Slice(cborPayload), + "PartitionKey": cbor.String("pk1"), + } + req1 := cborKinesisRequest(t, "PutRecord", putCBOR) + rr1 := serveCBOR(t, h, req1) + require.Equal(t, http.StatusOK, rr1.Code) + + // Write via JSON. + jsonPayload := []byte("json-record") + b64 := base64.StdEncoding.EncodeToString(jsonPayload) + putJSON := map[string]any{ + "StreamName": "CoexistStream", + "Data": b64, + "PartitionKey": "pk2", + } + rawJSON, _ := json.Marshal(putJSON) + req2 := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(rawJSON)) + req2.Header.Set("Content-Type", "application/x-amz-json-1.1") + req2.Header.Set("X-Amz-Target", kinesisPrefix+"PutRecord") + e := echo.New() + rec2 := httptest.NewRecorder() + c2 := e.NewContext(req2, rec2) + require.NoError(t, h.Handler()(c2)) + require.Equal(t, http.StatusOK, rec2.Code) + + // Read via JSON to confirm both records are present. + iterReq := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(mustMarshalBytes(t, map[string]any{ + "StreamName": "CoexistStream", + "ShardId": "shardId-000000000000", + "ShardIteratorType": "TRIM_HORIZON", + }))) + iterReq.Header.Set("Content-Type", "application/x-amz-json-1.1") + iterReq.Header.Set("X-Amz-Target", kinesisPrefix+"GetShardIterator") + iterRec := httptest.NewRecorder() + iterC := e.NewContext(iterReq, iterRec) + require.NoError(t, h.Handler()(iterC)) + var iterResp map[string]any + require.NoError(t, json.Unmarshal(iterRec.Body.Bytes(), &iterResp)) + shardIter := iterResp["ShardIterator"].(string) + + getReq := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(mustMarshalBytes(t, map[string]any{ + "ShardIterator": shardIter, + "Limit": 10, + }))) + getReq.Header.Set("Content-Type", "application/x-amz-json-1.1") + getReq.Header.Set("X-Amz-Target", kinesisPrefix+"GetRecords") + getRec := httptest.NewRecorder() + getC := e.NewContext(getReq, getRec) + require.NoError(t, h.Handler()(getC)) + require.Equal(t, http.StatusOK, getRec.Code) + + var getResp map[string]any + require.NoError(t, json.Unmarshal(getRec.Body.Bytes(), &getResp)) + recs := getResp["Records"].([]any) + assert.GreaterOrEqual(t, len(recs), 2, "both records must be visible via JSON") + }) } func mustMarshalBytes(t *testing.T, v any) []byte { diff --git a/services/kinesis/channel_delivery_test.go b/services/kinesis/channel_delivery_test.go index 9f09dd0967..4fa1479416 100644 --- a/services/kinesis/channel_delivery_test.go +++ b/services/kinesis/channel_delivery_test.go @@ -8,6 +8,7 @@ import ( "strings" "sync" "testing" + "time" "github.com/aws/aws-sdk-go-v2/aws" kinesissdk "github.com/aws/aws-sdk-go-v2/service/kinesis" @@ -165,13 +166,15 @@ func TestChannelDelivery_PutRecordToS3(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) writer := &fakeChannelS3Writer{} backend.SetS3Writer(writer) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "delivery-stream-" + tt.name streamARN := createOnDemandStream(t, client, streamName) + clock.Advance(streamSettleWait) s3Dest := minimalS3DestinationConfig() s3Dest.StorageConfiguration.OutputKeyTemplate = aws.String(tt.outputKeyTmpl) @@ -206,12 +209,14 @@ func TestChannelDelivery_PutRecordToS3(t *testing.T) { func TestChannelDelivery_InvalidRecordGoesToDeadLetterQueue(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) writer := &fakeChannelS3Writer{} backend.SetS3Writer(writer) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamARN := createOnDemandStream(t, client, "dlq-stream") + clock.Advance(streamSettleWait) s3Dest := minimalS3DestinationConfig() streamCfg := []kinesissdktypes.ChannelStreamConfiguration{ @@ -254,12 +259,14 @@ func TestChannelDelivery_InvalidRecordGoesToDeadLetterQueue(t *testing.T) { func TestChannelDelivery_DeleteChannelFlushesBuffer(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) writer := &fakeChannelS3Writer{} backend.SetS3Writer(writer) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamARN := createOnDemandStream(t, client, "delete-flush-stream") + clock.Advance(streamSettleWait) created, err := client.CreateChannel(t.Context(), &kinesissdk.CreateChannelInput{ ChannelName: aws.String("delete-flush-chan"), @@ -293,10 +300,12 @@ func TestChannelDelivery_DeleteChannelFlushesBuffer(t *testing.T) { func TestChannelDelivery_NoWriterWiredIsNoop(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamARN := createOnDemandStream(t, client, "no-writer-stream") + clock.Advance(streamSettleWait) created, err := client.CreateChannel(t.Context(), &kinesissdk.CreateChannelInput{ ChannelName: aws.String("no-writer-chan"), diff --git a/services/kinesis/channels.go b/services/kinesis/channels.go index 43a1532a18..21f77e7885 100644 --- a/services/kinesis/channels.go +++ b/services/kinesis/channels.go @@ -2,6 +2,7 @@ package kinesis import ( "context" + "maps" "regexp" "sort" "strings" @@ -383,7 +384,10 @@ func (b *InMemoryBackend) DescribeChannel( return nil, ErrChannelNotFound } - return &DescribeChannelOutput{ChannelDescription: *channel}, nil + cd := *channel + cd.Tags = maps.Clone(channel.Tags) + + return &DescribeChannelOutput{ChannelDescription: cd}, nil } // channelMatchesStreamFilter reports whether c is associated with any of the @@ -413,7 +417,9 @@ func (b *InMemoryBackend) ListChannels(ctx context.Context, input *ListChannelsI matched := make([]Channel, 0, b.channelsByRegion.Len()) for _, c := range b.channelsByRegion.Get(region) { if channelMatchesStreamFilter(c, input.StreamFilter) { - matched = append(matched, *c) + cp := *c + cp.Tags = maps.Clone(c.Tags) + matched = append(matched, cp) } } b.mu.RUnlock() diff --git a/services/kinesis/consumers.go b/services/kinesis/consumers.go index aa18b4f95d..cf675231aa 100644 --- a/services/kinesis/consumers.go +++ b/services/kinesis/consumers.go @@ -149,7 +149,10 @@ func (b *InMemoryBackend) DescribeStreamConsumer( return nil, ErrConsumerNotFound } - return &DescribeStreamConsumerOutput{ConsumerDescription: *consumer}, nil + cd := *consumer + cd.Tags = maps.Clone(consumer.Tags) + + return &DescribeStreamConsumerOutput{ConsumerDescription: cd}, nil } // ListStreamConsumers lists all registered consumers for a stream. @@ -175,7 +178,9 @@ func (b *InMemoryBackend) ListStreamConsumers( consumers := make([]Consumer, 0, len(stream.Consumers)) for _, c := range stream.Consumers { - consumers = append(consumers, *c) + cp := *c + cp.Tags = maps.Clone(c.Tags) + consumers = append(consumers, cp) } // Sort for deterministic ordering. diff --git a/services/kinesis/consumers_test.go b/services/kinesis/consumers_test.go index acb84991f3..1c9f2e296c 100644 --- a/services/kinesis/consumers_test.go +++ b/services/kinesis/consumers_test.go @@ -9,6 +9,8 @@ import ( "strconv" "strings" "testing" + "testing/synctest" + "time" "github.com/labstack/echo/v5" "github.com/stretchr/testify/assert" @@ -110,11 +112,20 @@ func TestSubscribeToShard_StreamClosesAfterIdle(t *testing.T) { func TestSubscribeToShard_DeliversRecords(t *testing.T) { t.Parallel() - h := newTestHandler(t) + // Uses a real-time-based fakeClock (not synctest): SubscribeToShard reads + // records back via TRIM_HORIZON, which compares record timestamps against + // a retention cutoff computed from "now" -- mixing a synctest bubble's + // fake epoch (used while creating/putting) with real wall-clock time + // (used by subscribeAndCollect afterward) would make the just-written + // records look expired. clock starts at real time.Now() and only + // advances forward, keeping retention math consistent throughout. + clock := newFakeClock(time.Now()) + h := newTestHandlerWithBackend(t, kinesis.NewInMemoryBackend().WithClock(clock.Now)) streamName := "sub-records-stream" streamARN := createStreamAndGetARN(t, h, streamName) shardID := getFirstShardID(t, h, streamName) + clock.Advance(streamSettleWait) tests := []struct { label string @@ -149,11 +160,13 @@ func TestSubscribeToShard_DeliversRecords(t *testing.T) { func TestSubscribeToShard_MultipleSubscriptions(t *testing.T) { t.Parallel() - h := newTestHandler(t) + clock := newFakeClock(time.Now()) + h := newTestHandlerWithBackend(t, kinesis.NewInMemoryBackend().WithClock(clock.Now)) streamName := "sub-multi-stream" streamARN := createStreamAndGetARN(t, h, streamName) shardID := getFirstShardID(t, h, streamName) + clock.Advance(streamSettleWait) doRequest(t, h, "PutRecord", map[string]any{ "StreamName": streamName, @@ -484,46 +497,49 @@ func TestConsumerRegistrationAndList(t *testing.T) { func TestSubscribeToShard_ReturnsRecords(t *testing.T) { t.Parallel() - bk := kinesis.NewInMemoryBackend() - require.NoError( - t, - bk.CreateStream( - context.Background(), - &kinesis.CreateStreamInput{StreamName: "subscribe-stream", ShardCount: 1}, - ), - ) - - streamARN := "arn:aws:kinesis:us-east-1:123456789012:stream/subscribe-stream" - - regOut, err := bk.RegisterStreamConsumer(context.Background(), &kinesis.RegisterStreamConsumerInput{ - StreamARN: streamARN, - ConsumerName: "reader", - }) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + bk := kinesis.NewInMemoryBackend() + require.NoError( + t, + bk.CreateStream( + context.Background(), + &kinesis.CreateStreamInput{StreamName: "subscribe-stream", ShardCount: 1}, + ), + ) + time.Sleep(streamSettleWait) - // Put some records. - _, err = bk.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "subscribe-stream", - PartitionKey: "pk1", - Data: []byte("hello"), - }) - require.NoError(t, err) + streamARN := "arn:aws:kinesis:us-east-1:123456789012:stream/subscribe-stream" - shardOut, err := bk.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "subscribe-stream"}) - require.NoError(t, err) - require.Len(t, shardOut.Shards, 1) - shardID := shardOut.Shards[0].ShardID + regOut, err := bk.RegisterStreamConsumer(context.Background(), &kinesis.RegisterStreamConsumerInput{ + StreamARN: streamARN, + ConsumerName: "reader", + }) + require.NoError(t, err) - subOut, err := bk.SubscribeToShard(context.Background(), &kinesis.SubscribeToShardInput{ - ConsumerARN: regOut.Consumer.ConsumerARN, - ShardID: shardID, - StartingPosition: kinesis.StartingPosition{ - Type: "TRIM_HORIZON", - }, + // Put some records. + _, err = bk.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "subscribe-stream", + PartitionKey: "pk1", + Data: []byte("hello"), + }) + require.NoError(t, err) + + shardOut, err := bk.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "subscribe-stream"}) + require.NoError(t, err) + require.Len(t, shardOut.Shards, 1) + shardID := shardOut.Shards[0].ShardID + + subOut, err := bk.SubscribeToShard(context.Background(), &kinesis.SubscribeToShardInput{ + ConsumerARN: regOut.Consumer.ConsumerARN, + ShardID: shardID, + StartingPosition: kinesis.StartingPosition{ + Type: "TRIM_HORIZON", + }, + }) + require.NoError(t, err) + assert.Len(t, subOut.Event.Records, 1) + assert.Equal(t, []byte("hello"), subOut.Event.Records[0].Data) }) - require.NoError(t, err) - assert.Len(t, subOut.Event.Records, 1) - assert.Equal(t, []byte("hello"), subOut.Event.Records[0].Data) } // TestSubscribeToShard_AtTimestampRequiresTimestamp verifies AT_TIMESTAMP @@ -624,84 +640,87 @@ func TestDeregisterStreamConsumer_ByIdentifier(t *testing.T) { func TestConsumer_Lifecycle(t *testing.T) { t.Parallel() - b := newParityBackend(t) - ctx := context.Background() - - createParityStream(t, b, "consumer-test", 1) - - desc, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "consumer-test"}) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := newParityBackend(t) + ctx := context.Background() - streamARN := desc.StreamARN + createParityStream(t, b, "consumer-test", 1) + time.Sleep(streamSettleWait) - // Step 1: register. - regOut, err := b.RegisterStreamConsumer(ctx, &kinesis.RegisterStreamConsumerInput{ - StreamARN: streamARN, - ConsumerName: "my-consumer", - }) - require.NoError(t, err) - assert.Equal(t, "my-consumer", regOut.Consumer.ConsumerName) - assert.Equal(t, "ACTIVE", regOut.Consumer.ConsumerStatus) - assert.NotEmpty(t, regOut.Consumer.ConsumerARN) + desc, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "consumer-test"}) + require.NoError(t, err) - // Step 2: describe by name. - descOut, err := b.DescribeStreamConsumer(ctx, &kinesis.DescribeStreamConsumerInput{ - StreamARN: streamARN, - ConsumerName: "my-consumer", - }) - require.NoError(t, err) - assert.Equal(t, "my-consumer", descOut.ConsumerDescription.ConsumerName) + streamARN := desc.StreamARN - // Step 3: list. - listOut, err := b.ListStreamConsumers(ctx, &kinesis.ListStreamConsumersInput{StreamARN: streamARN}) - require.NoError(t, err) - require.Len(t, listOut.Consumers, 1) - assert.Equal(t, "my-consumer", listOut.Consumers[0].ConsumerName) - - // Step 4: subscribe delivers records. - _, err = b.PutRecord(ctx, &kinesis.PutRecordInput{ - StreamName: "consumer-test", - PartitionKey: "pk", - Data: []byte("fan-out"), - }) - require.NoError(t, err) + // Step 1: register. + regOut, err := b.RegisterStreamConsumer(ctx, &kinesis.RegisterStreamConsumerInput{ + StreamARN: streamARN, + ConsumerName: "my-consumer", + }) + require.NoError(t, err) + assert.Equal(t, "my-consumer", regOut.Consumer.ConsumerName) + assert.Equal(t, "ACTIVE", regOut.Consumer.ConsumerStatus) + assert.NotEmpty(t, regOut.Consumer.ConsumerARN) + + // Step 2: describe by name. + descOut, err := b.DescribeStreamConsumer(ctx, &kinesis.DescribeStreamConsumerInput{ + StreamARN: streamARN, + ConsumerName: "my-consumer", + }) + require.NoError(t, err) + assert.Equal(t, "my-consumer", descOut.ConsumerDescription.ConsumerName) + + // Step 3: list. + listOut, err := b.ListStreamConsumers(ctx, &kinesis.ListStreamConsumersInput{StreamARN: streamARN}) + require.NoError(t, err) + require.Len(t, listOut.Consumers, 1) + assert.Equal(t, "my-consumer", listOut.Consumers[0].ConsumerName) + + // Step 4: subscribe delivers records. + _, err = b.PutRecord(ctx, &kinesis.PutRecordInput{ + StreamName: "consumer-test", + PartitionKey: "pk", + Data: []byte("fan-out"), + }) + require.NoError(t, err) - consumerARN := descOut.ConsumerDescription.ConsumerARN + consumerARN := descOut.ConsumerDescription.ConsumerARN - subOut, err := b.SubscribeToShard(ctx, &kinesis.SubscribeToShardInput{ - ConsumerARN: consumerARN, - ShardID: "shardId-000000000000", - StartingPosition: kinesis.StartingPosition{ - Type: "TRIM_HORIZON", - }, - }) - require.NoError(t, err) - assert.Len(t, subOut.Event.Records, 1) - assert.Equal(t, []byte("fan-out"), subOut.Event.Records[0].Data) + subOut, err := b.SubscribeToShard(ctx, &kinesis.SubscribeToShardInput{ + ConsumerARN: consumerARN, + ShardID: "shardId-000000000000", + StartingPosition: kinesis.StartingPosition{ + Type: "TRIM_HORIZON", + }, + }) + require.NoError(t, err) + assert.Len(t, subOut.Event.Records, 1) + assert.Equal(t, []byte("fan-out"), subOut.Event.Records[0].Data) + + // Step 5: deregister. + err = b.DeregisterStreamConsumer(ctx, &kinesis.DeregisterStreamConsumerInput{ + StreamARN: streamARN, + ConsumerName: "my-consumer", + }) + require.NoError(t, err) - // Step 5: deregister. - err = b.DeregisterStreamConsumer(ctx, &kinesis.DeregisterStreamConsumerInput{ - StreamARN: streamARN, - ConsumerName: "my-consumer", - }) - require.NoError(t, err) + listOut2, err := b.ListStreamConsumers(ctx, &kinesis.ListStreamConsumersInput{StreamARN: streamARN}) + require.NoError(t, err) + assert.Empty(t, listOut2.Consumers) - listOut2, err := b.ListStreamConsumers(ctx, &kinesis.ListStreamConsumersInput{StreamARN: streamARN}) - require.NoError(t, err) - assert.Empty(t, listOut2.Consumers) - - // Step 6: duplicate registration rejected. - _, err = b.RegisterStreamConsumer(ctx, &kinesis.RegisterStreamConsumerInput{ - StreamARN: streamARN, - ConsumerName: "dup-consumer", - }) - require.NoError(t, err) + // Step 6: duplicate registration rejected. + _, err = b.RegisterStreamConsumer(ctx, &kinesis.RegisterStreamConsumerInput{ + StreamARN: streamARN, + ConsumerName: "dup-consumer", + }) + require.NoError(t, err) - _, err = b.RegisterStreamConsumer(ctx, &kinesis.RegisterStreamConsumerInput{ - StreamARN: streamARN, - ConsumerName: "dup-consumer", + _, err = b.RegisterStreamConsumer(ctx, &kinesis.RegisterStreamConsumerInput{ + StreamARN: streamARN, + ConsumerName: "dup-consumer", + }) + require.Error(t, err, "duplicate consumer registration must be rejected") }) - require.Error(t, err, "duplicate consumer registration must be rejected") } // createStreamAndGetARN is a helper that creates a stream with one shard and returns its ARN. diff --git a/services/kinesis/delete_stream_consumers_test.go b/services/kinesis/delete_stream_consumers_test.go index ba76f92a24..a6bf4dd224 100644 --- a/services/kinesis/delete_stream_consumers_test.go +++ b/services/kinesis/delete_stream_consumers_test.go @@ -2,6 +2,7 @@ package kinesis_test import ( "testing" + "time" "github.com/aws/aws-sdk-go-v2/aws" kinesissdk "github.com/aws/aws-sdk-go-v2/service/kinesis" @@ -23,7 +24,8 @@ import ( func TestDeleteStream_EnforceConsumerDeletion(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "consumer-guarded-stream" @@ -32,6 +34,7 @@ func TestDeleteStream_EnforceConsumerDeletion(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + clock.Advance(streamSettleWait) desc, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{ StreamName: aws.String(streamName), @@ -67,6 +70,7 @@ func TestDeleteStream_EnforceConsumerDeletion(t *testing.T) { EnforceConsumerDeletion: aws.Bool(true), }) require.NoError(t, err) + clock.Advance(streamSettleWait) _, err = client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{ StreamName: aws.String(streamName), diff --git a/services/kinesis/errors.go b/services/kinesis/errors.go index 9661cf7413..dbfd2da56e 100644 --- a/services/kinesis/errors.go +++ b/services/kinesis/errors.go @@ -24,7 +24,13 @@ var ( // registered enhanced fan-out consumers and EnforceConsumerDeletion is // unset or false (real DeleteStreamInput.EnforceConsumerDeletion doc // comment: "the call to DeleteStream fails with a ResourceInUseException"). - ErrStreamHasConsumers = awserr.New("ResourceInUseException", awserr.ErrConflict) + ErrStreamHasConsumers = awserr.New("ResourceInUseException", awserr.ErrConflict) + // ErrStreamNotActive is returned by control-plane mutations real AWS only + // accepts against an ACTIVE stream (DeleteStream, MergeShards, SplitShard, + // UpdateShardCount, StartStreamEncryption, StopStreamEncryption, + // UpdateStreamMode, UpdateMaxRecordSize, UpdateStreamWarmThroughput) when + // called while the stream is still CREATING/UPDATING/DELETING. + ErrStreamNotActive = awserr.New("ResourceInUseException", awserr.ErrConflict) ErrInvalidArgument = awserr.New("InvalidArgumentException", awserr.ErrInvalidParameter) ErrUnknownAction = errors.New("UnknownOperationException") ErrShardIteratorExpired = errors.New("ExpiredIteratorException") diff --git a/services/kinesis/faketime_test.go b/services/kinesis/faketime_test.go new file mode 100644 index 0000000000..d46af8695f --- /dev/null +++ b/services/kinesis/faketime_test.go @@ -0,0 +1,44 @@ +package kinesis_test + +import ( + "sync/atomic" + "time" +) + +// streamSettleWait safely exceeds Kinesis's internal transient-state +// transition delay (streamTransitionDelay, 250ms) so a single fakeClock.Advance +// call is guaranteed to move a CREATING/UPDATING/DELETING stream past its +// ReadyAt deadline. +const streamSettleWait = time.Second + +// fakeClock is a goroutine-safe, manually-advanced clock for driving +// Kinesis's lazy stream-transition deadlines deterministically in tests, +// via InMemoryBackend.WithClock -- no real waiting, no time.Sleep. Tests +// that only ever touch the backend from a single goroutine (direct backend +// calls, or the httptest.NewRecorder in-process handler pattern) can use a +// plain captured variable instead (see stream_modes_test.go's fakeNow), but +// a test driving a real httptest.NewServer + AWS SDK client needs this: the +// server's own request-handling goroutine reads the clock concurrently with +// the test goroutine advancing it. +type fakeClock struct { + now atomic.Pointer[time.Time] +} + +// newFakeClock creates a fakeClock starting at start. +func newFakeClock(start time.Time) *fakeClock { + c := &fakeClock{} + c.now.Store(&start) + + return c +} + +// Now returns the clock's current time. Suitable as InMemoryBackend.WithClock's argument. +func (c *fakeClock) Now() time.Time { + return *c.now.Load() +} + +// Advance moves the clock forward by d. +func (c *fakeClock) Advance(d time.Duration) { + next := c.Now().Add(d) + c.now.Store(&next) +} diff --git a/services/kinesis/fis_test.go b/services/kinesis/fis_test.go index 9e49702de8..6039ea4165 100644 --- a/services/kinesis/fis_test.go +++ b/services/kinesis/fis_test.go @@ -3,6 +3,7 @@ package kinesis_test import ( "context" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -18,6 +19,16 @@ func newFISKinesisHandler() *kinesis.Handler { return kinesis.NewHandler(backend) } +// newFISKinesisHandlerWithClock is newFISKinesisHandler with an injectable +// clock, for tests that need a stream's CREATING window to have already +// lazily elapsed (via clock.Advance) independent of real wall-clock time -- +// e.g. tests that also drive real FIS fault-duration timers. +func newFISKinesisHandlerWithClock(now func() time.Time) *kinesis.Handler { + backend := kinesis.NewInMemoryBackendWithConfig("000000000000", "us-east-1").WithClock(now) + + return kinesis.NewHandler(backend) +} + func TestKinesis_FISActions(t *testing.T) { t.Parallel() @@ -93,46 +104,50 @@ func TestKinesis_ExecuteFISAction_ThroughputException(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newFISKinesisHandler() - - // Create the stream if needed. - if tt.stream != "" { - err := h.Backend.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: tt.stream, - ShardCount: 1, - }) - require.NoError(t, err) - } - - err := h.ExecuteFISAction(t.Context(), service.FISActionExecution{ - ActionID: "aws:kinesis:stream-provisioned-throughput-exception", - Targets: tt.targets, - Duration: tt.duration, - }) + synctest.Test(t, func(t *testing.T) { + h := newFISKinesisHandler() - require.NoError(t, err) + // Create the stream if needed. + if tt.stream != "" { + err := h.Backend.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: tt.stream, + ShardCount: 1, + }) + require.NoError(t, err) + time.Sleep(streamSettleWait) + } - // Verify throughput exception is active on the stream. - if tt.stream != "" && len(tt.targets) > 0 { - _, putErr := h.Backend.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: tt.stream, - PartitionKey: "key", - Data: []byte("data"), + err := h.ExecuteFISAction(t.Context(), service.FISActionExecution{ + ActionID: "aws:kinesis:stream-provisioned-throughput-exception", + Targets: tt.targets, + Duration: tt.duration, }) - require.ErrorIs(t, putErr, kinesis.ErrProvisionedThroughputExceeded) - // After the duration, the fault should clear. - if tt.duration > 0 { - time.Sleep(tt.duration + 50*time.Millisecond) + require.NoError(t, err) - _, putAfter := h.Backend.PutRecord(context.Background(), &kinesis.PutRecordInput{ + // Verify throughput exception is active on the stream. + if tt.stream != "" && len(tt.targets) > 0 { + _, putErr := h.Backend.PutRecord(context.Background(), &kinesis.PutRecordInput{ StreamName: tt.stream, PartitionKey: "key", Data: []byte("data"), }) - assert.NoError(t, putAfter, "PutRecord should succeed after fault expires") + require.ErrorIs(t, putErr, kinesis.ErrProvisionedThroughputExceeded) + + // After the duration, the fault should clear. + if tt.duration > 0 { + time.Sleep(tt.duration + 50*time.Millisecond) + synctest.Wait() + + _, putAfter := h.Backend.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: tt.stream, + PartitionKey: "key", + Data: []byte("data"), + }) + assert.NoError(t, putAfter, "PutRecord should succeed after fault expires") + } } - } + }) }) } } @@ -140,7 +155,8 @@ func TestKinesis_ExecuteFISAction_ThroughputException(t *testing.T) { func TestKinesis_ExecuteFISAction_ThroughputException_ZeroPercentage(t *testing.T) { t.Parallel() - h := newFISKinesisHandler() + clock := newFakeClock(time.Now()) + h := newFISKinesisHandlerWithClock(clock.Now) const streamName = "zero-pct-stream" const sampleSize = 50 @@ -150,6 +166,7 @@ func TestKinesis_ExecuteFISAction_ThroughputException_ZeroPercentage(t *testing. ShardCount: 1, }) require.NoError(t, err) + clock.Advance(streamSettleWait) // Activate fault with 0% — no requests should ever be throttled. err = h.ExecuteFISAction(t.Context(), service.FISActionExecution{ @@ -258,7 +275,8 @@ func TestKinesis_ExecuteFISAction_ThroughputException_CtxCancel(t *testing.T) { func TestKinesis_ThroughputFault_ZeroPercentage_NoThrottle(t *testing.T) { t.Parallel() - h := newFISKinesisHandler() + clock := newFakeClock(time.Now()) + h := newFISKinesisHandlerWithClock(clock.Now) const streamName = "zero-pct-stream" @@ -267,6 +285,7 @@ func TestKinesis_ThroughputFault_ZeroPercentage_NoThrottle(t *testing.T) { ShardCount: 1, }) require.NoError(t, err) + clock.Advance(streamSettleWait) // Activate with 0% percentage — no requests should be throttled. err = h.ExecuteFISAction(t.Context(), service.FISActionExecution{ @@ -293,7 +312,8 @@ func TestKinesis_ThroughputFault_ZeroPercentage_NoThrottle(t *testing.T) { func TestKinesis_ThroughputFault_PartialPercentage(t *testing.T) { t.Parallel() - h := newFISKinesisHandler() + clock := newFakeClock(time.Now()) + h := newFISKinesisHandlerWithClock(clock.Now) const streamName = "partial-pct-stream" @@ -302,6 +322,7 @@ func TestKinesis_ThroughputFault_PartialPercentage(t *testing.T) { ShardCount: 1, }) require.NoError(t, err) + clock.Advance(streamSettleWait) // Activate with 50% percentage. err = h.ExecuteFISAction(t.Context(), service.FISActionExecution{ @@ -352,7 +373,8 @@ func TestKinesis_ExecuteFISAction_NonInMemoryBackend(t *testing.T) { func TestKinesis_ThroughputFaultActiveLocked_LazyEviction(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackendWithConfig("000000000000", "us-east-1").WithClock(clock.Now) const streamName = "lazy-evict-kinesis-stream" @@ -361,6 +383,7 @@ func TestKinesis_ThroughputFaultActiveLocked_LazyEviction(t *testing.T) { ShardCount: 1, }) require.NoError(t, err) + clock.Advance(streamSettleWait) // Inject an already-expired fault directly (no goroutine, guaranteed expired). backend.InjectExpiredThroughputFaultForTest(streamName) diff --git a/services/kinesis/get_records_child_shards_test.go b/services/kinesis/get_records_child_shards_test.go index 03af1552fb..0dd1ba4ff5 100644 --- a/services/kinesis/get_records_child_shards_test.go +++ b/services/kinesis/get_records_child_shards_test.go @@ -2,6 +2,7 @@ package kinesis_test import ( "testing" + "time" "github.com/aws/aws-sdk-go-v2/aws" kinesissdk "github.com/aws/aws-sdk-go-v2/service/kinesis" @@ -21,7 +22,8 @@ import ( func TestGetRecords_ChildShards(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "split-child-shards-stream" @@ -30,6 +32,7 @@ func TestGetRecords_ChildShards(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + clock.Advance(streamSettleWait) desc, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{ StreamName: aws.String(streamName), diff --git a/services/kinesis/handler.go b/services/kinesis/handler.go index ed3d67bed6..131e3e7584 100644 --- a/services/kinesis/handler.go +++ b/services/kinesis/handler.go @@ -417,6 +417,10 @@ func resourceErrorDetails(err error) (string, string, int, bool) { return errTypeResourceInUse, "The stream has registered consumers. Set EnforceConsumerDeletion to true to delete it anyway.", http.StatusBadRequest, true + case errors.Is(err, ErrStreamNotActive): + return errTypeResourceInUse, + "Stream is not in ACTIVE state.", + http.StatusBadRequest, true case errors.Is(err, ErrConsumerNotFound): return errTypeResourceNotFound, "Consumer not found.", diff --git a/services/kinesis/handler_max_record_size_test.go b/services/kinesis/handler_max_record_size_test.go index 8c27c30439..027c1d3c06 100644 --- a/services/kinesis/handler_max_record_size_test.go +++ b/services/kinesis/handler_max_record_size_test.go @@ -2,6 +2,7 @@ package kinesis_test import ( "testing" + "time" "github.com/aws/aws-sdk-go-v2/aws" kinesissdk "github.com/aws/aws-sdk-go-v2/service/kinesis" @@ -23,7 +24,8 @@ import ( func TestUpdateMaxRecordSize_RoundTrip(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "max-record-size-stream" @@ -34,6 +36,8 @@ func TestUpdateMaxRecordSize_RoundTrip(t *testing.T) { }) require.NoError(t, err) + clock.Advance(streamSettleWait) + desc, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{StreamName: aws.String(streamName)}) require.NoError(t, err) @@ -67,7 +71,8 @@ func TestUpdateMaxRecordSize_RoundTrip(t *testing.T) { func TestUpdateMaxRecordSize_OutOfRangeRejected(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "max-record-size-oor" @@ -78,6 +83,8 @@ func TestUpdateMaxRecordSize_OutOfRangeRejected(t *testing.T) { }) require.NoError(t, err) + clock.Advance(streamSettleWait) + desc, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{StreamName: aws.String(streamName)}) require.NoError(t, err) diff --git a/services/kinesis/handler_test.go b/services/kinesis/handler_test.go index 8a653ca0bd..5c1334b8ce 100644 --- a/services/kinesis/handler_test.go +++ b/services/kinesis/handler_test.go @@ -34,7 +34,14 @@ const ( func newTestHandler(t *testing.T) *kinesis.Handler { t.Helper() - backend := kinesis.NewInMemoryBackend() + return newTestHandlerWithBackend(t, kinesis.NewInMemoryBackend()) +} + +// newTestHandlerWithBackend builds a test Handler around a caller-supplied +// backend, so tests that need a controllable clock (see fakeClock) can wire +// it in before wrapping the backend in a Handler. +func newTestHandlerWithBackend(t *testing.T, backend *kinesis.InMemoryBackend) *kinesis.Handler { + t.Helper() return kinesis.NewHandler(backend).WithSubscribeToShardTiming( testSubscribeToShardStreamDuration, diff --git a/services/kinesis/internal_faketime_test.go b/services/kinesis/internal_faketime_test.go new file mode 100644 index 0000000000..cc13b4625e --- /dev/null +++ b/services/kinesis/internal_faketime_test.go @@ -0,0 +1,10 @@ +package kinesis //nolint:testpackage // shared const for this package's other internal (whitebox) tests. + +import "time" + +// streamSettleWaitInternal safely exceeds streamTransitionDelay, for the +// package-internal (whitebox) tests that need a stream's CREATING/UPDATING/ +// DELETING window to have lazily elapsed. Mirrors kinesis_test's +// streamSettleWait (faketime_test.go) -- kept separate since internal tests +// cannot import the external test package. +const streamSettleWaitInternal = time.Second diff --git a/services/kinesis/isolation_test.go b/services/kinesis/isolation_test.go index d914be3f20..0e0f57116d 100644 --- a/services/kinesis/isolation_test.go +++ b/services/kinesis/isolation_test.go @@ -3,6 +3,8 @@ package kinesis //nolint:testpackage // needs access to the unexported region co import ( "context" "testing" + "testing/synctest" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -20,6 +22,14 @@ func ctxRegion(region string) context.Context { func TestKinesisRegionIsolation(t *testing.T) { t.Parallel() + synctest.Test(t, func(t *testing.T) { + testKinesisRegionIsolation(t) + }) +} + +func testKinesisRegionIsolation(t *testing.T) { + t.Helper() + backend := NewInMemoryBackendWithConfig("000000000000", "us-east-1") ctxEast := ctxRegion("us-east-1") @@ -36,6 +46,7 @@ func TestKinesisRegionIsolation(t *testing.T) { StreamName: "shared", ShardCount: 2, })) + time.Sleep(streamSettleWaitInternal) // 3. Each region's stream carries its own ARN region and shard count. eastDesc, err := backend.DescribeStream(ctxEast, &DescribeStreamInput{StreamName: "shared"}) @@ -64,6 +75,7 @@ func TestKinesisRegionIsolation(t *testing.T) { // 5. Delete the stream in us-east-1; us-west-2 still has its stream. require.NoError(t, backend.DeleteStream(ctxEast, &DeleteStreamInput{StreamName: "shared"})) + time.Sleep(streamSettleWaitInternal) _, err = backend.DescribeStream(ctxEast, &DescribeStreamInput{StreamName: "shared"}) require.ErrorIs(t, err, ErrStreamNotFound) @@ -80,6 +92,14 @@ func TestKinesisRegionIsolation(t *testing.T) { func TestKinesisRecordRegionIsolation(t *testing.T) { t.Parallel() + synctest.Test(t, func(t *testing.T) { + testKinesisRecordRegionIsolation(t) + }) +} + +func testKinesisRecordRegionIsolation(t *testing.T) { + t.Helper() + backend := NewInMemoryBackendWithConfig("000000000000", "us-east-1") ctxEast := ctxRegion("us-east-1") @@ -91,6 +111,7 @@ func TestKinesisRecordRegionIsolation(t *testing.T) { ShardCount: 1, })) } + time.Sleep(streamSettleWaitInternal) // Write distinct records into each region's stream. _, err := backend.PutRecord(ctxEast, &PutRecordInput{ diff --git a/services/kinesis/janitor_test.go b/services/kinesis/janitor_test.go index e5de985a25..3a3a82b1f4 100644 --- a/services/kinesis/janitor_test.go +++ b/services/kinesis/janitor_test.go @@ -3,6 +3,7 @@ package kinesis_test import ( "context" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -134,16 +135,32 @@ func TestJanitor_Run_Cancel(t *testing.T) { func TestDeleteStream_CleansFaultEntry(t *testing.T) { t.Parallel() - bk := kinesis.NewInMemoryBackend() - require.NoError(t, bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "fault-stream"})) - - // Inject a fault for the stream. - bk.InjectFaultForTest("fault-stream") - assert.True(t, bk.HasFaultForTest("fault-stream"), "fault should be present before delete") - - require.NoError(t, bk.DeleteStream(context.Background(), &kinesis.DeleteStreamInput{StreamName: "fault-stream"})) - - assert.False(t, bk.HasFaultForTest("fault-stream"), "fault entry should be removed after delete") + synctest.Test(t, func(t *testing.T) { + bk := kinesis.NewInMemoryBackend() + require.NoError( + t, + bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "fault-stream"}), + ) + time.Sleep(streamSettleWait) + + // Inject a fault for the stream. + bk.InjectFaultForTest("fault-stream") + assert.True(t, bk.HasFaultForTest("fault-stream"), "fault should be present before delete") + + require.NoError( + t, + bk.DeleteStream(context.Background(), &kinesis.DeleteStreamInput{StreamName: "fault-stream"}), + ) + time.Sleep(streamSettleWait) + + // The fault entry is cleaned up lazily, once the DELETING deadline is + // physically resolved (any DescribeStream/ListStreams call) -- not at + // the DeleteStream call itself. + _, err := bk.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: "fault-stream"}) + require.ErrorIs(t, err, kinesis.ErrStreamNotFound) + + assert.False(t, bk.HasFaultForTest("fault-stream"), "fault entry should be removed after delete") + }) } // TestDeleteStream_ClearsResourcePolicyOnRecreate verifies that deleting a @@ -153,60 +170,73 @@ func TestDeleteStream_CleansFaultEntry(t *testing.T) { func TestDeleteStream_ClearsResourcePolicyOnRecreate(t *testing.T) { t.Parallel() - ctx := context.Background() - bk := kinesis.NewInMemoryBackend() - require.NoError(t, bk.CreateStream(ctx, &kinesis.CreateStreamInput{StreamName: "reused-stream"})) + synctest.Test(t, func(t *testing.T) { + ctx := context.Background() + bk := kinesis.NewInMemoryBackend() + require.NoError(t, bk.CreateStream(ctx, &kinesis.CreateStreamInput{StreamName: "reused-stream"})) + time.Sleep(streamSettleWait) - desc, err := bk.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "reused-stream"}) - require.NoError(t, err) + desc, err := bk.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "reused-stream"}) + require.NoError(t, err) - require.NoError(t, bk.PutResourcePolicy(ctx, &kinesis.PutResourcePolicyInput{ - ResourceARN: desc.StreamARN, - Policy: `{"Version":"2012-10-17"}`, - })) + require.NoError(t, bk.PutResourcePolicy(ctx, &kinesis.PutResourcePolicyInput{ + ResourceARN: desc.StreamARN, + Policy: `{"Version":"2012-10-17"}`, + })) - require.NoError(t, bk.DeleteStream(ctx, &kinesis.DeleteStreamInput{StreamName: "reused-stream"})) - require.NoError(t, bk.CreateStream(ctx, &kinesis.CreateStreamInput{StreamName: "reused-stream"})) + require.NoError(t, bk.DeleteStream(ctx, &kinesis.DeleteStreamInput{StreamName: "reused-stream"})) + time.Sleep(streamSettleWait) + require.NoError(t, bk.CreateStream(ctx, &kinesis.CreateStreamInput{StreamName: "reused-stream"})) - recreated, err := bk.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "reused-stream"}) - require.NoError(t, err) - require.Equal(t, desc.StreamARN, recreated.StreamARN) + recreated, err := bk.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "reused-stream"}) + require.NoError(t, err) + require.Equal(t, desc.StreamARN, recreated.StreamARN) - _, err = bk.GetResourcePolicy(ctx, &kinesis.GetResourcePolicyInput{ResourceARN: recreated.StreamARN}) - require.ErrorIs(t, err, kinesis.ErrResourcePolicyNotFound) + _, err = bk.GetResourcePolicy(ctx, &kinesis.GetResourcePolicyInput{ResourceARN: recreated.StreamARN}) + require.ErrorIs(t, err, kinesis.ErrResourcePolicyNotFound) + }) } func TestDeleteStream_LeavesOtherStreamResourcePolicyIntact(t *testing.T) { t.Parallel() - ctx := context.Background() - bk := kinesis.NewInMemoryBackend() - - require.NoError(t, bk.CreateStream(ctx, &kinesis.CreateStreamInput{StreamName: "gone-stream"})) - require.NoError(t, bk.CreateStream(ctx, &kinesis.CreateStreamInput{StreamName: "kept-stream"})) - - goneDesc, err := bk.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "gone-stream"}) - require.NoError(t, err) - keptDesc, err := bk.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "kept-stream"}) - require.NoError(t, err) - - require.NoError(t, bk.PutResourcePolicy(ctx, &kinesis.PutResourcePolicyInput{ - ResourceARN: goneDesc.StreamARN, - Policy: `{"Version":"2012-10-17","Statement":"gone"}`, - })) - require.NoError(t, bk.PutResourcePolicy(ctx, &kinesis.PutResourcePolicyInput{ - ResourceARN: keptDesc.StreamARN, - Policy: `{"Version":"2012-10-17","Statement":"kept"}`, - })) - - require.NoError(t, bk.DeleteStream(ctx, &kinesis.DeleteStreamInput{StreamName: "gone-stream"})) - - _, err = bk.GetResourcePolicy(ctx, &kinesis.GetResourcePolicyInput{ResourceARN: goneDesc.StreamARN}) - require.ErrorIs(t, err, kinesis.ErrResourcePolicyNotFound) - - kept, err := bk.GetResourcePolicy(ctx, &kinesis.GetResourcePolicyInput{ResourceARN: keptDesc.StreamARN}) - require.NoError(t, err) - assert.JSONEq(t, `{"Version":"2012-10-17","Statement":"kept"}`, kept.Policy) + synctest.Test(t, func(t *testing.T) { + ctx := context.Background() + bk := kinesis.NewInMemoryBackend() + + require.NoError(t, bk.CreateStream(ctx, &kinesis.CreateStreamInput{StreamName: "gone-stream"})) + require.NoError(t, bk.CreateStream(ctx, &kinesis.CreateStreamInput{StreamName: "kept-stream"})) + time.Sleep(streamSettleWait) + + goneDesc, err := bk.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "gone-stream"}) + require.NoError(t, err) + keptDesc, err := bk.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "kept-stream"}) + require.NoError(t, err) + + require.NoError(t, bk.PutResourcePolicy(ctx, &kinesis.PutResourcePolicyInput{ + ResourceARN: goneDesc.StreamARN, + Policy: `{"Version":"2012-10-17","Statement":"gone"}`, + })) + require.NoError(t, bk.PutResourcePolicy(ctx, &kinesis.PutResourcePolicyInput{ + ResourceARN: keptDesc.StreamARN, + Policy: `{"Version":"2012-10-17","Statement":"kept"}`, + })) + + require.NoError(t, bk.DeleteStream(ctx, &kinesis.DeleteStreamInput{StreamName: "gone-stream"})) + time.Sleep(streamSettleWait) + + // Force lazy physical removal of "gone-stream" (and its resource + // policy cleanup) via a resolving call. + _, err = bk.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "gone-stream"}) + require.ErrorIs(t, err, kinesis.ErrStreamNotFound) + + _, err = bk.GetResourcePolicy(ctx, &kinesis.GetResourcePolicyInput{ResourceARN: goneDesc.StreamARN}) + require.ErrorIs(t, err, kinesis.ErrResourcePolicyNotFound) + + kept, err := bk.GetResourcePolicy(ctx, &kinesis.GetResourcePolicyInput{ResourceARN: keptDesc.StreamARN}) + require.NoError(t, err) + assert.JSONEq(t, `{"Version":"2012-10-17","Statement":"kept"}`, kept.Policy) + }) } // TestRingBuffer_WrapAround checks that pushing more than maxRecordsPerShard records @@ -214,9 +244,18 @@ func TestDeleteStream_LeavesOtherStreamResourcePolicyIntact(t *testing.T) { func TestRingBuffer_WrapAround(t *testing.T) { t.Parallel() + synctest.Test(t, func(t *testing.T) { + testRingBufferWrapAround(t) + }) +} + +func testRingBufferWrapAround(t *testing.T) { + t.Helper() + const maxCap = 10000 bk := kinesis.NewInMemoryBackend() require.NoError(t, bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "ring-stream"})) + time.Sleep(streamSettleWait) desc, err := bk.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: "ring-stream"}) require.NoError(t, err) @@ -262,25 +301,37 @@ func TestRingBuffer_WrapAround(t *testing.T) { func TestBinarySearch_FindSequencePosition(t *testing.T) { t.Parallel() - bk := kinesis.NewInMemoryBackend() - require.NoError(t, bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "bsearch-stream"})) - - desc, err := bk.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: "bsearch-stream"}) - require.NoError(t, err) - shardID := desc.Shards[0].ShardID - - // Push 100 records. + var bk *kinesis.InMemoryBackend + var shardID string seqs := make([]string, 100) - for i := range 100 { - out, putErr := bk.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "bsearch-stream", - PartitionKey: "pk", - Data: []byte("data"), - }) - require.NoError(t, putErr) - seqs[i] = out.SequenceNumber - } + // Setup runs inside a synctest bubble so the 100 PutRecord calls land + // after the stream's CREATING->ACTIVE deadline lazily elapses, without a + // real time.Sleep -- see PARITY.md. Only setup needs this: the subtests + // below only call GetShardIterator/GetRecords, which do not gate on + // stream status. + synctest.Test(t, func(t *testing.T) { + bk = kinesis.NewInMemoryBackend() + require.NoError( + t, + bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "bsearch-stream"}), + ) + time.Sleep(streamSettleWait) + + desc, err := bk.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: "bsearch-stream"}) + require.NoError(t, err) + shardID = desc.Shards[0].ShardID + + for i := range 100 { + out, putErr := bk.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "bsearch-stream", + PartitionKey: "pk", + Data: []byte("data"), + }) + require.NoError(t, putErr) + seqs[i] = out.SequenceNumber + } + }) tests := []struct { name string @@ -438,10 +489,19 @@ func TestKinesisJanitor_DefaultInterval(t *testing.T) { func TestRetentionPeriod_JanitorEvictsOldRecords(t *testing.T) { t.Parallel() + synctest.Test(t, func(t *testing.T) { + testRetentionPeriodJanitorEvictsOldRecords(t) + }) +} + +func testRetentionPeriodJanitorEvictsOldRecords(t *testing.T) { + t.Helper() + b := newParityBackend(t) ctx := context.Background() createParityStream(t, b, "retention-test", 1) + time.Sleep(streamSettleWait) err := b.SetRetentionPeriodForTest("retention-test", 1) require.NoError(t, err) diff --git a/services/kinesis/models.go b/services/kinesis/models.go index 47f38a34af..45df8f82e3 100644 --- a/services/kinesis/models.go +++ b/services/kinesis/models.go @@ -9,9 +9,25 @@ import ( ) const ( + // streamStatusCreating is the status while a newly created stream is not + // yet ready for use. + streamStatusCreating = "CREATING" + + // streamStatusUpdating is the status while a shard/config change + // (UpdateShardCount, MergeShards, SplitShard, StartStreamEncryption, + // StopStreamEncryption, UpdateStreamMode) is in flight. + streamStatusUpdating = "UPDATING" + // streamStatusActive is the status when a stream is ready for use. streamStatusActive = "ACTIVE" + // streamTransitionDelay is how long a stream stays CREATING/UPDATING/ + // DELETING before the next resolving call lazily advances it to ACTIVE + // (or, for DELETING, removes it) -- see resolveStreamTransitionLocked. + // Kept short so Terraform/SDK waiters (StreamExistsWaiter etc.) converge + // quickly; mirrors services/rds's instanceTransitionDelay convention. + streamTransitionDelay = 250 * time.Millisecond + // encryptionTypeKMS is the KMS encryption type. encryptionTypeKMS = "KMS" @@ -137,6 +153,11 @@ const ( // Stream represents an in-memory Kinesis stream. type Stream struct { CreatedAt time.Time `json:"createdAt"` + // ReadyAt is the deadline at which a CREATING/UPDATING/DELETING stream + // lazily advances to its terminal state (ACTIVE, or removed for + // DELETING) -- see resolveStreamTransitionLocked. Zero when Status is + // not mid-transition. Additive persisted field. + ReadyAt time.Time `json:"readyAt"` mu *lockmetrics.RWMutex Tags *tags.Tags `json:"tags,omitempty"` Consumers map[string]*Consumer `json:"consumers,omitempty"` diff --git a/services/kinesis/persistence.go b/services/kinesis/persistence.go index da24a0514f..5e2a226754 100644 --- a/services/kinesis/persistence.go +++ b/services/kinesis/persistence.go @@ -44,6 +44,18 @@ type backendSnapshot struct { Version int `json:"version"` } +// streamAlias avoids infinite recursion from Stream.MarshalJSON. +type streamAlias Stream + +// MarshalJSON serialises the stream under stream.mu.RLock, since Snapshot must not race consumer/setting updates. +// Those mutate stream fields, including Consumers, under stream.mu without ever taking b.mu (gopherstack-fwd0g). +func (stream *Stream) MarshalJSON() ([]byte, error) { + stream.mu.RLock("Snapshot") + defer stream.mu.RUnlock() + + return json.Marshal((*streamAlias)(stream)) +} + // Snapshot serialises the backend state to JSON. // It implements persistence.Persistable. // Note: shard sequence number counters are now serialised via the NextSeq field. diff --git a/services/kinesis/persistence_race_test.go b/services/kinesis/persistence_race_test.go new file mode 100644 index 0000000000..2016d88e17 --- /dev/null +++ b/services/kinesis/persistence_race_test.go @@ -0,0 +1,83 @@ +package kinesis_test + +import ( + "fmt" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/arn" + "github.com/blackbirdworks/gopherstack/pkgs/config" + "github.com/blackbirdworks/gopherstack/services/kinesis" +) + +// TestSnapshot_RacesWithConsumerWrites reproduces gopherstack-fwd0g: Snapshot marshals streams under b.mu.RLock alone. +// Register/DeregisterStreamConsumer mutate Stream.Consumers under stream.mu after releasing b.mu; run with -race. +func TestSnapshot_RacesWithConsumerWrites(t *testing.T) { + t.Parallel() + + tests := []struct { + mutate func(t *testing.T, b *kinesis.InMemoryBackend, streamARN string) + name string + }{ + { + name: "concurrent_snapshot_and_register_deregister_consumer", + mutate: func(t *testing.T, b *kinesis.InMemoryBackend, streamARN string) { + t.Helper() + + for i := range 100 { + name := fmt.Sprintf("consumer-%03d", i) + + _, err := b.RegisterStreamConsumer(t.Context(), &kinesis.RegisterStreamConsumerInput{ + StreamARN: streamARN, + ConsumerName: name, + }) + require.NoError(t, err) + + err = b.DeregisterStreamConsumer(t.Context(), &kinesis.DeregisterStreamConsumerInput{ + StreamARN: streamARN, + ConsumerName: name, + }) + require.NoError(t, err) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := kinesis.NewInMemoryBackend() + const streamName = "race-snapshot-stream" + + require.NoError(t, b.CreateStream(t.Context(), &kinesis.CreateStreamInput{ + StreamName: streamName, + ShardCount: 1, + })) + + streamARN := arn.Build("kinesis", config.DefaultRegion, config.DefaultAccountID, "stream/"+streamName) + + var wg sync.WaitGroup + + stop := make(chan struct{}) + + wg.Go(func() { + for { + select { + case <-stop: + return + default: + } + + _ = b.Snapshot(t.Context()) + } + }) + + tt.mutate(t, b, streamARN) + close(stop) + wg.Wait() + }) + } +} diff --git a/services/kinesis/persistence_roundtrip_test.go b/services/kinesis/persistence_roundtrip_test.go index 34c406fb9b..42ba51fd65 100644 --- a/services/kinesis/persistence_roundtrip_test.go +++ b/services/kinesis/persistence_roundtrip_test.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "testing" + "testing/synctest" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -20,6 +22,14 @@ import ( func TestInMemoryBackend_FullStateSnapshotRestoreRoundTrip(t *testing.T) { t.Parallel() + synctest.Test(t, func(t *testing.T) { + testInMemoryBackendFullStateSnapshotRestoreRoundTrip(t) + }) +} + +func testInMemoryBackendFullStateSnapshotRestoreRoundTrip(t *testing.T) { + t.Helper() + ctx := context.Background() ctxEast := ctxRegion("us-east-1") ctxWest := ctxRegion("us-west-2") @@ -34,6 +44,7 @@ func TestInMemoryBackend_FullStateSnapshotRestoreRoundTrip(t *testing.T) { StreamName: "beta", ShardCount: 1, })) + time.Sleep(streamSettleWaitInternal) // Inline shard records (hot path, stays inline per Stream -- not decomposed). _, err := original.PutRecord(ctxEast, &PutRecordInput{ diff --git a/services/kinesis/persistence_test.go b/services/kinesis/persistence_test.go index a56334a3f9..138f736761 100644 --- a/services/kinesis/persistence_test.go +++ b/services/kinesis/persistence_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -150,9 +152,18 @@ func TestSnapshot_EmptyShardRecords_NoNull(t *testing.T) { func TestSnapshot_RestoreClearsOldPointers(t *testing.T) { t.Parallel() + synctest.Test(t, func(t *testing.T) { + testSnapshotRestoreClearsOldPointers(t) + }) +} + +func testSnapshotRestoreClearsOldPointers(t *testing.T) { + t.Helper() + // Create a backend with records in it. bk := kinesis.NewInMemoryBackendWithConfig("000000000000", "us-east-1") require.NoError(t, bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "ptr-stream"})) + time.Sleep(streamSettleWait) for range 5 { _, err := bk.PutRecord(context.Background(), &kinesis.PutRecordInput{ diff --git a/services/kinesis/realclient_stream_encryption_test.go b/services/kinesis/realclient_stream_encryption_test.go index a1ef185c09..9cae27ba0e 100644 --- a/services/kinesis/realclient_stream_encryption_test.go +++ b/services/kinesis/realclient_stream_encryption_test.go @@ -2,6 +2,7 @@ package kinesis_test import ( "testing" + "time" "github.com/aws/aws-sdk-go-v2/aws" kinesissdk "github.com/aws/aws-sdk-go-v2/service/kinesis" @@ -17,7 +18,8 @@ import ( func TestRealClient_StopStreamEncryption(t *testing.T) { t.Parallel() - h := kinesis.NewHandler(kinesis.NewInMemoryBackend()) + clock := newFakeClock(time.Now()) + h := kinesis.NewHandler(kinesis.NewInMemoryBackend().WithClock(clock.Now)) client := newTestKinesisClient(t, h) streamName := "s11-encryption-stream" @@ -26,6 +28,7 @@ func TestRealClient_StopStreamEncryption(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + clock.Advance(streamSettleWait) _, err = client.StartStreamEncryption(t.Context(), &kinesissdk.StartStreamEncryptionInput{ StreamName: aws.String(streamName), @@ -33,6 +36,7 @@ func TestRealClient_StopStreamEncryption(t *testing.T) { KeyId: aws.String("alias/aws/kinesis"), }) require.NoError(t, err) + clock.Advance(streamSettleWait) _, err = client.StopStreamEncryption(t.Context(), &kinesissdk.StopStreamEncryptionInput{ StreamName: aws.String(streamName), diff --git a/services/kinesis/records.go b/services/kinesis/records.go index 5e0b9004bd..08b989c007 100644 --- a/services/kinesis/records.go +++ b/services/kinesis/records.go @@ -50,8 +50,16 @@ func (b *InMemoryBackend) putRecordLocked( return nil, "", ErrProvisionedThroughputExceeded } - // Reject writes if the stream is not active (e.g. CREATING/DELETING). - if stream.Status != streamStatusActive { + // Real AWS rejects PutRecord while CREATING (stream not ready yet) but + // documents UPDATING as accepting reads/writes ("Updating or applying + // encryption normally takes a few seconds ... You can continue to read + // and write data to your stream while its status is UPDATING" -- + // api_op_StartStreamEncryption.go); DELETING is treated conservatively + // as rejecting too. Uses the lazily-resolved effective status (not the + // possibly-stale stored field) since PutRecord does not itself hold + // b.mu for writing -- see effectiveStreamStatus. + switch effectiveStreamStatus(stream, b.nowFunc()) { + case streamStatusCreating, streamStatusDeleting: return nil, "", ErrInvalidArgument } @@ -231,6 +239,10 @@ func (b *InMemoryBackend) GetRecords(ctx context.Context, input *GetRecordsInput b.mu.RUnlock() defer stream.mu.RUnlock() + if streamEffectivelyGone(stream, b.nowFunc()) { + return nil, ErrStreamNotFound + } + if b.isThroughputFaultActive(region, it.StreamName) { return nil, ErrProvisionedThroughputExceeded } diff --git a/services/kinesis/records_get_test.go b/services/kinesis/records_get_test.go index 579bba985f..596fb9a82c 100644 --- a/services/kinesis/records_get_test.go +++ b/services/kinesis/records_get_test.go @@ -8,6 +8,7 @@ import ( "net/http" "strings" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -35,6 +36,7 @@ func TestGetRecords_SizeCap_ExcludesPartitionKey(t *testing.T) { "ShardCount": 1, }) require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": streamName}) require.Equal(t, http.StatusOK, rec.Code) @@ -120,6 +122,7 @@ func TestGetRecords_SizeCap_ExcludesPartitionKey(t *testing.T) { "ShardCount": 1, }) require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": streamName}) require.Equal(t, http.StatusOK, rec.Code) @@ -186,7 +189,7 @@ func TestGetRecords_SizeCap_ExcludesPartitionKey(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { t.Parallel() - tc.run(t) + synctest.Test(t, tc.run) }) } } @@ -194,58 +197,75 @@ func TestGetRecords_SizeCap_ExcludesPartitionKey(t *testing.T) { func TestKinesisBackend_GetRecordsDeletedStream(t *testing.T) { t.Parallel() - bk := kinesis.NewInMemoryBackend() - require.NoError(t, bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "deleted-stream"})) + synctest.Test(t, func(t *testing.T) { + bk := kinesis.NewInMemoryBackend() + require.NoError( + t, + bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "deleted-stream"}), + ) + time.Sleep(streamSettleWait) - desc, err := bk.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: "deleted-stream"}) - require.NoError(t, err) - shardID := desc.Shards[0].ShardID + desc, err := bk.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: "deleted-stream"}) + require.NoError(t, err) + shardID := desc.Shards[0].ShardID - iterOut, err := bk.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ - StreamName: "deleted-stream", - ShardID: shardID, - ShardIteratorType: "TRIM_HORIZON", - }) - require.NoError(t, err) + iterOut, err := bk.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + StreamName: "deleted-stream", + ShardID: shardID, + ShardIteratorType: "TRIM_HORIZON", + }) + require.NoError(t, err) - // Delete stream - require.NoError(t, bk.DeleteStream(context.Background(), &kinesis.DeleteStreamInput{StreamName: "deleted-stream"})) + // Delete stream + require.NoError( + t, + bk.DeleteStream(context.Background(), &kinesis.DeleteStreamInput{StreamName: "deleted-stream"}), + ) + time.Sleep(streamSettleWait) - // GetRecords should return stream not found - _, err = bk.GetRecords(context.Background(), &kinesis.GetRecordsInput{ShardIterator: iterOut.ShardIterator}) - assert.ErrorIs(t, err, kinesis.ErrStreamNotFound) + // GetRecords should return stream not found + _, err = bk.GetRecords(context.Background(), &kinesis.GetRecordsInput{ShardIterator: iterOut.ShardIterator}) + assert.ErrorIs(t, err, kinesis.ErrStreamNotFound) + }) } func TestKinesisBackend_GetRecordsInvalidShard(t *testing.T) { t.Parallel() - bk := kinesis.NewInMemoryBackend() - require.NoError( - t, - bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "shard-gone-stream"}), - ) + synctest.Test(t, func(t *testing.T) { + bk := kinesis.NewInMemoryBackend() + require.NoError( + t, + bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "shard-gone-stream"}), + ) + time.Sleep(streamSettleWait) + + desc, err := bk.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: "shard-gone-stream"}, + ) + require.NoError(t, err) + shardID := desc.Shards[0].ShardID - desc, err := bk.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: "shard-gone-stream"}) - require.NoError(t, err) - shardID := desc.Shards[0].ShardID + iterOut, err := bk.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + StreamName: "shard-gone-stream", + ShardID: shardID, + ShardIteratorType: "TRIM_HORIZON", + }) + require.NoError(t, err) - iterOut, err := bk.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ - StreamName: "shard-gone-stream", - ShardID: shardID, - ShardIteratorType: "TRIM_HORIZON", + // Delete and recreate the stream (new shards will have the same IDs so this won't test the gap, + // but we can test invalid shard via ListShards with wrong stream name) + require.NoError( + t, + bk.DeleteStream(context.Background(), &kinesis.DeleteStreamInput{StreamName: "shard-gone-stream"}), + ) + time.Sleep(streamSettleWait) + + // Recreate stream (iterator now points to deleted stream) + _, err = bk.GetRecords(context.Background(), &kinesis.GetRecordsInput{ShardIterator: iterOut.ShardIterator}) + assert.Error(t, err) }) - require.NoError(t, err) - - // Delete and recreate the stream (new shards will have the same IDs so this won't test the gap, - // but we can test invalid shard via ListShards with wrong stream name) - require.NoError( - t, - bk.DeleteStream(context.Background(), &kinesis.DeleteStreamInput{StreamName: "shard-gone-stream"}), - ) - - // Recreate stream (iterator now points to deleted stream) - _, err = bk.GetRecords(context.Background(), &kinesis.GetRecordsInput{ShardIterator: iterOut.ShardIterator}) - assert.Error(t, err) } func TestGetRecords_MillisBehindLatest(t *testing.T) { @@ -285,649 +305,688 @@ func TestGetRecords_MillisBehindLatest(t *testing.T) { t.Run("zero when fully caught up", func(t *testing.T) { t.Parallel() - b := newParityBackend(t) - ctx := context.Background() + synctest.Test(t, func(t *testing.T) { + testMillisBehindLatestZeroWhenCaughtUp(t) + }) + }) +} - createParityStream(t, b, "mbl-zero", 1) +func testMillisBehindLatestZeroWhenCaughtUp(t *testing.T) { + t.Helper() - _, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ - StreamName: "mbl-zero", - PartitionKey: "pk", - Data: []byte("x"), - }) - require.NoError(t, err) + b := newParityBackend(t) + ctx := context.Background() - itOut, err := b.GetShardIterator(ctx, &kinesis.GetShardIteratorInput{ - StreamName: "mbl-zero", - ShardID: "shardId-000000000000", - ShardIteratorType: "TRIM_HORIZON", - }) - require.NoError(t, err) + createParityStream(t, b, "mbl-zero", 1) + time.Sleep(streamSettleWait) - rOut, err := b.GetRecords(ctx, &kinesis.GetRecordsInput{ShardIterator: itOut.ShardIterator}) - require.NoError(t, err) - assert.Len(t, rOut.Records, 1) - assert.Equal(t, int64(0), rOut.MillisBehindLatest) + _, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ + StreamName: "mbl-zero", + PartitionKey: "pk", + Data: []byte("x"), + }) + require.NoError(t, err) + + itOut, err := b.GetShardIterator(ctx, &kinesis.GetShardIteratorInput{ + StreamName: "mbl-zero", + ShardID: "shardId-000000000000", + ShardIteratorType: "TRIM_HORIZON", }) + require.NoError(t, err) + + rOut, err := b.GetRecords(ctx, &kinesis.GetRecordsInput{ShardIterator: itOut.ShardIterator}) + require.NoError(t, err) + assert.Len(t, rOut.Records, 1) + assert.Equal(t, int64(0), rOut.MillisBehindLatest) } func TestPutAndGetRecords(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - // Create stream with 1 shard - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "records-stream", - "ShardCount": 1, - }) - require.Equal(t, http.StatusOK, rec.Code) + // Create stream with 1 shard + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "records-stream", + "ShardCount": 1, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) - // Describe to find shard ID - rec = doRequest(t, h, "DescribeStream", map[string]any{ - "StreamName": "records-stream", - }) - require.Equal(t, http.StatusOK, rec.Code) - - var descResp struct { - StreamDescription struct { - Shards []struct { - ShardID string `json:"ShardId"` - } `json:"Shards"` - } `json:"StreamDescription"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - require.NotEmpty(t, descResp.StreamDescription.Shards) - shardID := descResp.StreamDescription.Shards[0].ShardID - - // PutRecord - rec = doRequest(t, h, "PutRecord", map[string]any{ - "StreamName": "records-stream", - "PartitionKey": "pk-1", - "Data": []byte("hello world"), - }) - require.Equal(t, http.StatusOK, rec.Code) + // Describe to find shard ID + rec = doRequest(t, h, "DescribeStream", map[string]any{ + "StreamName": "records-stream", + }) + require.Equal(t, http.StatusOK, rec.Code) - var putResp struct { - ShardID string `json:"ShardId"` - SequenceNumber string `json:"SequenceNumber"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &putResp)) - assert.NotEmpty(t, putResp.ShardID) - assert.NotEmpty(t, putResp.SequenceNumber) - firstSeq := putResp.SequenceNumber - - // PutRecords (batch) - rec = doRequest(t, h, "PutRecords", map[string]any{ - "StreamName": "records-stream", - "Records": []map[string]any{ - {"PartitionKey": "pk-2", "Data": []byte("record 2")}, - {"PartitionKey": "pk-3", "Data": []byte("record 3")}, - }, - }) - require.Equal(t, http.StatusOK, rec.Code) + var descResp struct { + StreamDescription struct { + Shards []struct { + ShardID string `json:"ShardId"` + } `json:"Shards"` + } `json:"StreamDescription"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + require.NotEmpty(t, descResp.StreamDescription.Shards) + shardID := descResp.StreamDescription.Shards[0].ShardID - var batchResp struct { - Records []struct { + // PutRecord + rec = doRequest(t, h, "PutRecord", map[string]any{ + "StreamName": "records-stream", + "PartitionKey": "pk-1", + "Data": []byte("hello world"), + }) + require.Equal(t, http.StatusOK, rec.Code) + + var putResp struct { ShardID string `json:"ShardId"` SequenceNumber string `json:"SequenceNumber"` - } `json:"Records"` - FailedRecordCount int `json:"FailedRecordCount"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &batchResp)) - assert.Equal(t, 0, batchResp.FailedRecordCount) - assert.Len(t, batchResp.Records, 2) - - // GetShardIterator - TRIM_HORIZON (reads from beginning) - rec = doRequest(t, h, "GetShardIterator", map[string]any{ - "StreamName": "records-stream", - "ShardId": shardID, - "ShardIteratorType": "TRIM_HORIZON", - }) - require.Equal(t, http.StatusOK, rec.Code) + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &putResp)) + assert.NotEmpty(t, putResp.ShardID) + assert.NotEmpty(t, putResp.SequenceNumber) + firstSeq := putResp.SequenceNumber + + // PutRecords (batch) + rec = doRequest(t, h, "PutRecords", map[string]any{ + "StreamName": "records-stream", + "Records": []map[string]any{ + {"PartitionKey": "pk-2", "Data": []byte("record 2")}, + {"PartitionKey": "pk-3", "Data": []byte("record 3")}, + }, + }) + require.Equal(t, http.StatusOK, rec.Code) - var iterResp struct { - ShardIterator string `json:"ShardIterator"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &iterResp)) - assert.NotEmpty(t, iterResp.ShardIterator) + var batchResp struct { + Records []struct { + ShardID string `json:"ShardId"` + SequenceNumber string `json:"SequenceNumber"` + } `json:"Records"` + FailedRecordCount int `json:"FailedRecordCount"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &batchResp)) + assert.Equal(t, 0, batchResp.FailedRecordCount) + assert.Len(t, batchResp.Records, 2) + + // GetShardIterator - TRIM_HORIZON (reads from beginning) + rec = doRequest(t, h, "GetShardIterator", map[string]any{ + "StreamName": "records-stream", + "ShardId": shardID, + "ShardIteratorType": "TRIM_HORIZON", + }) + require.Equal(t, http.StatusOK, rec.Code) - // GetRecords - rec = doRequest(t, h, "GetRecords", map[string]any{ - "ShardIterator": iterResp.ShardIterator, - "Limit": 10, - }) - require.Equal(t, http.StatusOK, rec.Code) + var iterResp struct { + ShardIterator string `json:"ShardIterator"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &iterResp)) + assert.NotEmpty(t, iterResp.ShardIterator) - var getResp struct { - NextShardIterator string `json:"NextShardIterator"` - Records []struct { - PartitionKey string `json:"PartitionKey"` - SequenceNumber string `json:"SequenceNumber"` - Data []byte `json:"Data"` - } `json:"Records"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &getResp)) - assert.Len(t, getResp.Records, 3) // 1 + 2 batch - assert.NotEmpty(t, getResp.NextShardIterator) - - // GetShardIterator - AT_SEQUENCE_NUMBER - rec = doRequest(t, h, "GetShardIterator", map[string]any{ - "StreamName": "records-stream", - "ShardId": shardID, - "ShardIteratorType": "AT_SEQUENCE_NUMBER", - "StartingSequenceNumber": firstSeq, - }) - require.Equal(t, http.StatusOK, rec.Code) + // GetRecords + rec = doRequest(t, h, "GetRecords", map[string]any{ + "ShardIterator": iterResp.ShardIterator, + "Limit": 10, + }) + require.Equal(t, http.StatusOK, rec.Code) - var atSeqIterResp struct { - ShardIterator string `json:"ShardIterator"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &atSeqIterResp)) + var getResp struct { + NextShardIterator string `json:"NextShardIterator"` + Records []struct { + PartitionKey string `json:"PartitionKey"` + SequenceNumber string `json:"SequenceNumber"` + Data []byte `json:"Data"` + } `json:"Records"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &getResp)) + assert.Len(t, getResp.Records, 3) // 1 + 2 batch + assert.NotEmpty(t, getResp.NextShardIterator) + + // GetShardIterator - AT_SEQUENCE_NUMBER + rec = doRequest(t, h, "GetShardIterator", map[string]any{ + "StreamName": "records-stream", + "ShardId": shardID, + "ShardIteratorType": "AT_SEQUENCE_NUMBER", + "StartingSequenceNumber": firstSeq, + }) + require.Equal(t, http.StatusOK, rec.Code) - rec = doRequest(t, h, "GetRecords", map[string]any{ - "ShardIterator": atSeqIterResp.ShardIterator, - "Limit": 10, - }) - require.Equal(t, http.StatusOK, rec.Code) + var atSeqIterResp struct { + ShardIterator string `json:"ShardIterator"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &atSeqIterResp)) - var atSeqResp struct { - Records []struct { - SequenceNumber string `json:"SequenceNumber"` - } `json:"Records"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &atSeqResp)) - // AT_SEQUENCE_NUMBER starts at the given record (inclusive) - require.NotEmpty(t, atSeqResp.Records) - assert.Equal(t, firstSeq, atSeqResp.Records[0].SequenceNumber) - - // GetShardIterator - AFTER_SEQUENCE_NUMBER - rec = doRequest(t, h, "GetShardIterator", map[string]any{ - "StreamName": "records-stream", - "ShardId": shardID, - "ShardIteratorType": "AFTER_SEQUENCE_NUMBER", - "StartingSequenceNumber": firstSeq, - }) - require.Equal(t, http.StatusOK, rec.Code) + rec = doRequest(t, h, "GetRecords", map[string]any{ + "ShardIterator": atSeqIterResp.ShardIterator, + "Limit": 10, + }) + require.Equal(t, http.StatusOK, rec.Code) - var afterIterResp struct { - ShardIterator string `json:"ShardIterator"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &afterIterResp)) + var atSeqResp struct { + Records []struct { + SequenceNumber string `json:"SequenceNumber"` + } `json:"Records"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &atSeqResp)) + // AT_SEQUENCE_NUMBER starts at the given record (inclusive) + require.NotEmpty(t, atSeqResp.Records) + assert.Equal(t, firstSeq, atSeqResp.Records[0].SequenceNumber) + + // GetShardIterator - AFTER_SEQUENCE_NUMBER + rec = doRequest(t, h, "GetShardIterator", map[string]any{ + "StreamName": "records-stream", + "ShardId": shardID, + "ShardIteratorType": "AFTER_SEQUENCE_NUMBER", + "StartingSequenceNumber": firstSeq, + }) + require.Equal(t, http.StatusOK, rec.Code) - rec = doRequest(t, h, "GetRecords", map[string]any{ - "ShardIterator": afterIterResp.ShardIterator, - "Limit": 10, - }) - require.Equal(t, http.StatusOK, rec.Code) + var afterIterResp struct { + ShardIterator string `json:"ShardIterator"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &afterIterResp)) - var afterSeqResp struct { - Records []struct { - SequenceNumber string `json:"SequenceNumber"` - } `json:"Records"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &afterSeqResp)) - // AFTER_SEQUENCE_NUMBER skips the given record - assert.Len(t, afterSeqResp.Records, 2) - - // GetShardIterator - LATEST (no new records) - rec = doRequest(t, h, "GetShardIterator", map[string]any{ - "StreamName": "records-stream", - "ShardId": shardID, - "ShardIteratorType": "LATEST", - }) - require.Equal(t, http.StatusOK, rec.Code) + rec = doRequest(t, h, "GetRecords", map[string]any{ + "ShardIterator": afterIterResp.ShardIterator, + "Limit": 10, + }) + require.Equal(t, http.StatusOK, rec.Code) - var latestIterResp struct { - ShardIterator string `json:"ShardIterator"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &latestIterResp)) + var afterSeqResp struct { + Records []struct { + SequenceNumber string `json:"SequenceNumber"` + } `json:"Records"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &afterSeqResp)) + // AFTER_SEQUENCE_NUMBER skips the given record + assert.Len(t, afterSeqResp.Records, 2) + + // GetShardIterator - LATEST (no new records) + rec = doRequest(t, h, "GetShardIterator", map[string]any{ + "StreamName": "records-stream", + "ShardId": shardID, + "ShardIteratorType": "LATEST", + }) + require.Equal(t, http.StatusOK, rec.Code) - rec = doRequest(t, h, "GetRecords", map[string]any{ - "ShardIterator": latestIterResp.ShardIterator, - }) - require.Equal(t, http.StatusOK, rec.Code) + var latestIterResp struct { + ShardIterator string `json:"ShardIterator"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &latestIterResp)) - var latestResp struct { - Records []any `json:"Records"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &latestResp)) - assert.Empty(t, latestResp.Records) // No new records since iterator was created + rec = doRequest(t, h, "GetRecords", map[string]any{ + "ShardIterator": latestIterResp.ShardIterator, + }) + require.Equal(t, http.StatusOK, rec.Code) + + var latestResp struct { + Records []any `json:"Records"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &latestResp)) + assert.Empty(t, latestResp.Records) // No new records since iterator was created + }) } func TestSequenceNumberOrdering(t *testing.T) { t.Parallel() - h := newTestHandler(t) - - // Create stream - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "order-stream", - "ShardCount": 1, - }) - require.Equal(t, http.StatusOK, rec.Code) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - // Get shard ID - rec = doRequest(t, h, "DescribeStream", map[string]any{ - "StreamName": "order-stream", - }) - require.Equal(t, http.StatusOK, rec.Code) - - var descResp struct { - StreamDescription struct { - Shards []struct { - ShardID string `json:"ShardId"` - } `json:"Shards"` - } `json:"StreamDescription"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - shardID := descResp.StreamDescription.Shards[0].ShardID + // Create stream + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "order-stream", + "ShardCount": 1, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) - // Put 5 records - seqNums := make([]string, 5) - for i := range 5 { - rec = doRequest(t, h, "PutRecord", map[string]any{ - "StreamName": "order-stream", - "PartitionKey": "pk", - "Data": []byte("data"), + // Get shard ID + rec = doRequest(t, h, "DescribeStream", map[string]any{ + "StreamName": "order-stream", }) require.Equal(t, http.StatusOK, rec.Code) - var putResp struct { - SequenceNumber string `json:"SequenceNumber"` + var descResp struct { + StreamDescription struct { + Shards []struct { + ShardID string `json:"ShardId"` + } `json:"Shards"` + } `json:"StreamDescription"` } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &putResp)) - seqNums[i] = putResp.SequenceNumber - } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + shardID := descResp.StreamDescription.Shards[0].ShardID + + // Put 5 records + seqNums := make([]string, 5) + for i := range 5 { + rec = doRequest(t, h, "PutRecord", map[string]any{ + "StreamName": "order-stream", + "PartitionKey": "pk", + "Data": []byte("data"), + }) + require.Equal(t, http.StatusOK, rec.Code) - // Verify ordering - for i := 1; i < len(seqNums); i++ { - assert.Greater(t, seqNums[i], seqNums[i-1], - "sequence numbers should be strictly increasing: %s <= %s", seqNums[i], seqNums[i-1]) - } + var putResp struct { + SequenceNumber string `json:"SequenceNumber"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &putResp)) + seqNums[i] = putResp.SequenceNumber + } - // Read back and verify order - rec = doRequest(t, h, "GetShardIterator", map[string]any{ - "StreamName": "order-stream", - "ShardId": shardID, - "ShardIteratorType": "TRIM_HORIZON", - }) - require.Equal(t, http.StatusOK, rec.Code) + // Verify ordering + for i := 1; i < len(seqNums); i++ { + assert.Greater(t, seqNums[i], seqNums[i-1], + "sequence numbers should be strictly increasing: %s <= %s", seqNums[i], seqNums[i-1]) + } - var iterResp struct { - ShardIterator string `json:"ShardIterator"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &iterResp)) + // Read back and verify order + rec = doRequest(t, h, "GetShardIterator", map[string]any{ + "StreamName": "order-stream", + "ShardId": shardID, + "ShardIteratorType": "TRIM_HORIZON", + }) + require.Equal(t, http.StatusOK, rec.Code) - rec = doRequest(t, h, "GetRecords", map[string]any{ - "ShardIterator": iterResp.ShardIterator, - "Limit": 10, - }) - require.Equal(t, http.StatusOK, rec.Code) + var iterResp struct { + ShardIterator string `json:"ShardIterator"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &iterResp)) - var getResp struct { - Records []struct { - SequenceNumber string `json:"SequenceNumber"` - } `json:"Records"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &getResp)) - require.Len(t, getResp.Records, 5) + rec = doRequest(t, h, "GetRecords", map[string]any{ + "ShardIterator": iterResp.ShardIterator, + "Limit": 10, + }) + require.Equal(t, http.StatusOK, rec.Code) - for i, r := range getResp.Records { - assert.Equal(t, seqNums[i], r.SequenceNumber) - } + var getResp struct { + Records []struct { + SequenceNumber string `json:"SequenceNumber"` + } `json:"Records"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &getResp)) + require.Len(t, getResp.Records, 5) + + for i, r := range getResp.Records { + assert.Equal(t, seqNums[i], r.SequenceNumber) + } + }) } func TestGetRecords_10MBCap_StopsAtLimit(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "big-records-stream", - ShardCount: 1, - })) + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "big-records-stream", + ShardCount: 1, + })) + time.Sleep(streamSettleWait) - // Each record is ~1 MiB of data. - oneMiB := make([]byte, 1_048_576) + // Each record is ~1 MiB of data. + oneMiB := make([]byte, 1_048_576) - // Put 12 records (12 MiB total, well above the 10 MiB cap). - for i := range 12 { - _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "big-records-stream", - PartitionKey: fmt.Sprintf("pk%d", i), - Data: oneMiB, + // Put 12 records (12 MiB total, well above the 10 MiB cap). + for i := range 12 { + _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "big-records-stream", + PartitionKey: fmt.Sprintf("pk%d", i), + Data: oneMiB, + }) + require.NoError(t, err) + } + + out, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + StreamName: "big-records-stream", + ShardID: "shardId-000000000000", + ShardIteratorType: "TRIM_HORIZON", }) require.NoError(t, err) - } - out, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ - StreamName: "big-records-stream", - ShardID: "shardId-000000000000", - ShardIteratorType: "TRIM_HORIZON", - }) - require.NoError(t, err) + rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ + ShardIterator: out.ShardIterator, + Limit: 10000, + }) + require.NoError(t, err) - rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ - ShardIterator: out.ShardIterator, - Limit: 10000, + // Must have received fewer than 12 records due to 10 MiB cap. + assert.Less(t, len(rec.Records), 12, "10 MiB cap should limit response to fewer than 12 records") + assert.NotEmpty(t, rec.NextShardIterator, "should still have a next iterator") }) - require.NoError(t, err) - - // Must have received fewer than 12 records due to 10 MiB cap. - assert.Less(t, len(rec.Records), 12, "10 MiB cap should limit response to fewer than 12 records") - assert.NotEmpty(t, rec.NextShardIterator, "should still have a next iterator") } func TestGetRecords_10MBCap_SingleLargeRecordAllowed(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "single-big-record", - ShardCount: 1, - })) + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "single-big-record", + ShardCount: 1, + })) + time.Sleep(streamSettleWait) - // Increase the record size limit to 10 MiB first. - require.NoError(t, b.UpdateMaxRecordSize(context.Background(), &kinesis.UpdateMaxRecordSizeInput{ - StreamARN: mustStreamARN(t, b, "single-big-record"), - MaxRecordSizeInKiB: 10_485_760 / 1024, - })) + // Increase the record size limit to 10 MiB first. + require.NoError(t, b.UpdateMaxRecordSize(context.Background(), &kinesis.UpdateMaxRecordSizeInput{ + StreamARN: mustStreamARN(t, b, "single-big-record"), + MaxRecordSizeInKiB: 10_485_760 / 1024, + })) - tenMiB := make([]byte, 10_485_760) - _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "single-big-record", - PartitionKey: "pk", - Data: tenMiB, - }) - require.NoError(t, err) + tenMiB := make([]byte, 10_485_760) + _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "single-big-record", + PartitionKey: "pk", + Data: tenMiB, + }) + require.NoError(t, err) - // Put a second record so we can verify MillisBehindLatest. - _, err = b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "single-big-record", - PartitionKey: "pk2", - Data: []byte("small"), - }) - require.NoError(t, err) + // Put a second record so we can verify MillisBehindLatest. + _, err = b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "single-big-record", + PartitionKey: "pk2", + Data: []byte("small"), + }) + require.NoError(t, err) - out, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ - StreamName: "single-big-record", - ShardID: "shardId-000000000000", - ShardIteratorType: "TRIM_HORIZON", - }) - require.NoError(t, err) + out, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + StreamName: "single-big-record", + ShardID: "shardId-000000000000", + ShardIteratorType: "TRIM_HORIZON", + }) + require.NoError(t, err) - rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ - ShardIterator: out.ShardIterator, - Limit: 10000, - }) - require.NoError(t, err) + rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ + ShardIterator: out.ShardIterator, + Limit: 10000, + }) + require.NoError(t, err) - // A single record that exceeds the cap is still returned (cap is applied - // as "stop adding AFTER limit is hit if at least 1 record consumed"). - assert.GreaterOrEqual(t, len(rec.Records), 1, "at least one record should be returned") + // A single record that exceeds the cap is still returned (cap is applied + // as "stop adding AFTER limit is hit if at least 1 record consumed"). + assert.GreaterOrEqual(t, len(rec.Records), 1, "at least one record should be returned") + }) } func TestGetRecords_10MBCap_IteratorAdvancesCorrectly(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) - - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "cap-advance-stream", - ShardCount: 1, - })) - - // Use UpdateMaxRecordSize to allow 6 MiB records (> default 1 MiB limit). - require.NoError(t, b.UpdateMaxRecordSize(context.Background(), &kinesis.UpdateMaxRecordSizeInput{ - StreamARN: mustStreamARN(t, b, "cap-advance-stream"), - MaxRecordSizeInKiB: 10_485_760 / 1024, - })) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) + + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "cap-advance-stream", + ShardCount: 1, + })) + time.Sleep(streamSettleWait) + + // Use UpdateMaxRecordSize to allow 6 MiB records (> default 1 MiB limit). + require.NoError(t, b.UpdateMaxRecordSize(context.Background(), &kinesis.UpdateMaxRecordSizeInput{ + StreamARN: mustStreamARN(t, b, "cap-advance-stream"), + MaxRecordSizeInKiB: 10_485_760 / 1024, + })) + + // 4 MiB records × 3 = 12 MiB total: first call gets 2 (8MB), second call gets 1. + fourMiB := make([]byte, 4_194_304) + for i := range 3 { + _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "cap-advance-stream", + PartitionKey: fmt.Sprintf("pk%d", i), + Data: fourMiB, + }) + require.NoError(t, err) + } - // 4 MiB records × 3 = 12 MiB total: first call gets 2 (8MB), second call gets 1. - fourMiB := make([]byte, 4_194_304) - for i := range 3 { - _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "cap-advance-stream", - PartitionKey: fmt.Sprintf("pk%d", i), - Data: fourMiB, + iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + StreamName: "cap-advance-stream", + ShardID: "shardId-000000000000", + ShardIteratorType: "TRIM_HORIZON", }) require.NoError(t, err) - } - - iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ - StreamName: "cap-advance-stream", - ShardID: "shardId-000000000000", - ShardIteratorType: "TRIM_HORIZON", - }) - require.NoError(t, err) - first, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ - ShardIterator: iterOut.ShardIterator, - Limit: 10000, - }) - require.NoError(t, err) - require.Less(t, len(first.Records), 3, "should not return all 3 records due to 10 MiB cap") - require.NotEmpty(t, first.NextShardIterator) + first, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ + ShardIterator: iterOut.ShardIterator, + Limit: 10000, + }) + require.NoError(t, err) + require.Less(t, len(first.Records), 3, "should not return all 3 records due to 10 MiB cap") + require.NotEmpty(t, first.NextShardIterator) - // Second call should return the remaining records. - second, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ - ShardIterator: first.NextShardIterator, - Limit: 10000, + // Second call should return the remaining records. + second, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ + ShardIterator: first.NextShardIterator, + Limit: 10000, + }) + require.NoError(t, err) + total := len(first.Records) + len(second.Records) + assert.Equal(t, 3, total, "all records should be reachable via pagination") }) - require.NoError(t, err) - total := len(first.Records) + len(second.Records) - assert.Equal(t, 3, total, "all records should be reachable via pagination") } func TestGetRecords_MillisBehindLatest_UsesLastRecord(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "millis-behind-stream", - ShardCount: 1, - })) + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "millis-behind-stream", + ShardCount: 1, + })) + time.Sleep(streamSettleWait) - // Put 3 records and introduce a small delay so their timestamps are in the past. - for i := range 3 { - _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "millis-behind-stream", - PartitionKey: fmt.Sprintf("pk%d", i), - Data: []byte("d"), + // Put 3 records and introduce a small delay so their timestamps are in the past. + for i := range 3 { + _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "millis-behind-stream", + PartitionKey: fmt.Sprintf("pk%d", i), + Data: []byte("d"), + }) + require.NoError(t, err) + } + + // Wait briefly so the records have a measurable age. + time.Sleep(5 * time.Millisecond) + + iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + StreamName: "millis-behind-stream", + ShardID: "shardId-000000000000", + ShardIteratorType: "TRIM_HORIZON", }) require.NoError(t, err) - } - - // Wait briefly so the records have a measurable age. - time.Sleep(5 * time.Millisecond) - iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ - StreamName: "millis-behind-stream", - ShardID: "shardId-000000000000", - ShardIteratorType: "TRIM_HORIZON", - }) - require.NoError(t, err) + // Get only 1 record (leaving 2 unread). + rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ + ShardIterator: iterOut.ShardIterator, + Limit: 1, + }) + require.NoError(t, err) + require.Len(t, rec.Records, 1) - // Get only 1 record (leaving 2 unread). - rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ - ShardIterator: iterOut.ShardIterator, - Limit: 1, + // MillisBehindLatest should be the lag from the LAST record (record 3), not the next unread. + assert.Positive(t, rec.MillisBehindLatest) }) - require.NoError(t, err) - require.Len(t, rec.Records, 1) - - // MillisBehindLatest should be the lag from the LAST record (record 3), not the next unread. - assert.Positive(t, rec.MillisBehindLatest) } func TestGetRecords_MillisBehindLatest_ZeroWhenCaughtUp(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "millis-caught-up", - ShardCount: 1, - })) + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "millis-caught-up", + ShardCount: 1, + })) + time.Sleep(streamSettleWait) - _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "millis-caught-up", - PartitionKey: "pk", - Data: []byte("d"), - }) - require.NoError(t, err) + _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "millis-caught-up", + PartitionKey: "pk", + Data: []byte("d"), + }) + require.NoError(t, err) - iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ - StreamName: "millis-caught-up", - ShardID: "shardId-000000000000", - ShardIteratorType: "TRIM_HORIZON", - }) - require.NoError(t, err) + iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + StreamName: "millis-caught-up", + ShardID: "shardId-000000000000", + ShardIteratorType: "TRIM_HORIZON", + }) + require.NoError(t, err) - // Consume all records. - rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ - ShardIterator: iterOut.ShardIterator, - Limit: 10000, - }) - require.NoError(t, err) - require.Len(t, rec.Records, 1) + // Consume all records. + rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ + ShardIterator: iterOut.ShardIterator, + Limit: 10000, + }) + require.NoError(t, err) + require.Len(t, rec.Records, 1) - // Consumer is now at the tip → MillisBehindLatest should be 0. - assert.Equal(t, int64(0), rec.MillisBehindLatest) + // Consumer is now at the tip → MillisBehindLatest should be 0. + assert.Equal(t, int64(0), rec.MillisBehindLatest) + }) } func TestGetRecords_SmallRecords_NoCap(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "small-records-stream", - ShardCount: 1, - })) + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "small-records-stream", + ShardCount: 1, + })) + time.Sleep(streamSettleWait) - // Put 100 small records (well under 10 MiB). - for i := range 100 { - _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "small-records-stream", - PartitionKey: fmt.Sprintf("pk%d", i), - Data: []byte("hello"), + // Put 100 small records (well under 10 MiB). + for i := range 100 { + _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "small-records-stream", + PartitionKey: fmt.Sprintf("pk%d", i), + Data: []byte("hello"), + }) + require.NoError(t, err) + } + + iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + StreamName: "small-records-stream", + ShardID: "shardId-000000000000", + ShardIteratorType: "TRIM_HORIZON", }) require.NoError(t, err) - } - iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ - StreamName: "small-records-stream", - ShardID: "shardId-000000000000", - ShardIteratorType: "TRIM_HORIZON", - }) - require.NoError(t, err) - - rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ - ShardIterator: iterOut.ShardIterator, - Limit: 10000, + rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ + ShardIterator: iterOut.ShardIterator, + Limit: 10000, + }) + require.NoError(t, err) + // All 100 small records should be returned in one call. + assert.Len(t, rec.Records, 100) }) - require.NoError(t, err) - // All 100 small records should be returned in one call. - assert.Len(t, rec.Records, 100) } func TestGetRecords_10MBCap_ExactlyAtLimit(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) - - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "exact-cap-stream", - ShardCount: 1, - })) - - require.NoError(t, b.UpdateMaxRecordSize(context.Background(), &kinesis.UpdateMaxRecordSizeInput{ - StreamARN: mustStreamARN(t, b, "exact-cap-stream"), - MaxRecordSizeInKiB: 10_485_760 / 1024, - })) - - // Two 5 MiB records = exactly 10 MiB; both should fit in one response. - fiveMiB := make([]byte, 5_242_880) - for i := range 2 { + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) + + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "exact-cap-stream", + ShardCount: 1, + })) + time.Sleep(streamSettleWait) + + require.NoError(t, b.UpdateMaxRecordSize(context.Background(), &kinesis.UpdateMaxRecordSizeInput{ + StreamARN: mustStreamARN(t, b, "exact-cap-stream"), + MaxRecordSizeInKiB: 10_485_760 / 1024, + })) + + // Two 5 MiB records = exactly 10 MiB; both should fit in one response. + fiveMiB := make([]byte, 5_242_880) + for i := range 2 { + _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "exact-cap-stream", + PartitionKey: fmt.Sprintf("pk%d", i), + Data: fiveMiB, + }) + require.NoError(t, err) + } + // Third 1-byte record (so we can check lag). _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ StreamName: "exact-cap-stream", - PartitionKey: fmt.Sprintf("pk%d", i), - Data: fiveMiB, + PartitionKey: "extra", + Data: []byte("x"), }) require.NoError(t, err) - } - // Third 1-byte record (so we can check lag). - _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "exact-cap-stream", - PartitionKey: "extra", - Data: []byte("x"), - }) - require.NoError(t, err) - iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ - StreamName: "exact-cap-stream", - ShardID: "shardId-000000000000", - ShardIteratorType: "TRIM_HORIZON", - }) - require.NoError(t, err) + iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + StreamName: "exact-cap-stream", + ShardID: "shardId-000000000000", + ShardIteratorType: "TRIM_HORIZON", + }) + require.NoError(t, err) - rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ - ShardIterator: iterOut.ShardIterator, - Limit: 10000, - }) - require.NoError(t, err) + rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ + ShardIterator: iterOut.ShardIterator, + Limit: 10000, + }) + require.NoError(t, err) - // Both 5 MiB records (10 MiB total) should be returned; third should remain. - assert.Len(t, rec.Records, 2) - assert.NotEmpty(t, rec.NextShardIterator) + // Both 5 MiB records (10 MiB total) should be returned; third should remain. + assert.Len(t, rec.Records, 2) + assert.NotEmpty(t, rec.NextShardIterator) + }) } func TestGetRecords_ZeroLimitUsesDefault(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "default-limit-stream", - ShardCount: 1, - })) + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "default-limit-stream", + ShardCount: 1, + })) + time.Sleep(streamSettleWait) - // Put more than defaultGetRecordsLimit records. - for i := range 5 { - _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "default-limit-stream", - PartitionKey: fmt.Sprintf("pk%d", i), - Data: []byte("d"), + // Put more than defaultGetRecordsLimit records. + for i := range 5 { + _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "default-limit-stream", + PartitionKey: fmt.Sprintf("pk%d", i), + Data: []byte("d"), + }) + require.NoError(t, err) + } + + iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + StreamName: "default-limit-stream", + ShardID: "shardId-000000000000", + ShardIteratorType: "TRIM_HORIZON", }) require.NoError(t, err) - } - iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ - StreamName: "default-limit-stream", - ShardID: "shardId-000000000000", - ShardIteratorType: "TRIM_HORIZON", - }) - require.NoError(t, err) - - // Limit=0 uses the default (10000). - rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ - ShardIterator: iterOut.ShardIterator, - Limit: 0, + // Limit=0 uses the default (10000). + rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ + ShardIterator: iterOut.ShardIterator, + Limit: 0, + }) + require.NoError(t, err) + assert.Len(t, rec.Records, 5, "all 5 records should be returned with default limit") }) - require.NoError(t, err) - assert.Len(t, rec.Records, 5, "all 5 records should be returned with default limit") } // TestGetRecords_ZeroLimitDefaultsTo10000 verifies that omitting Limit falls @@ -937,45 +996,48 @@ func TestGetRecords_ZeroLimitUsesDefault(t *testing.T) { func TestGetRecords_ZeroLimitDefaultsTo10000(t *testing.T) { t.Parallel() - b := kinesis.NewInMemoryBackend() - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "default-10000-stream", - ShardCount: 1, - })) - - const ( - totalRecords = 10500 - putRecordsBatchLimit = 500 - ) - - for start := 0; start < totalRecords; start += putRecordsBatchLimit { - batch := make([]kinesis.PutRecordsEntry, 0, putRecordsBatchLimit) - for i := start; i < start+putRecordsBatchLimit && i < totalRecords; i++ { - batch = append(batch, kinesis.PutRecordsEntry{ - PartitionKey: fmt.Sprintf("pk%d", i), - Data: []byte("d"), + synctest.Test(t, func(t *testing.T) { + b := kinesis.NewInMemoryBackend() + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "default-10000-stream", + ShardCount: 1, + })) + time.Sleep(streamSettleWait) + + const ( + totalRecords = 10500 + putRecordsBatchLimit = 500 + ) + + for start := 0; start < totalRecords; start += putRecordsBatchLimit { + batch := make([]kinesis.PutRecordsEntry, 0, putRecordsBatchLimit) + for i := start; i < start+putRecordsBatchLimit && i < totalRecords; i++ { + batch = append(batch, kinesis.PutRecordsEntry{ + PartitionKey: fmt.Sprintf("pk%d", i), + Data: []byte("d"), + }) + } + out, err := b.PutRecords(context.Background(), &kinesis.PutRecordsInput{ + StreamName: "default-10000-stream", + Records: batch, }) + require.NoError(t, err) + require.Zero(t, out.FailedRecordCount) } - out, err := b.PutRecords(context.Background(), &kinesis.PutRecordsInput{ - StreamName: "default-10000-stream", - Records: batch, + + iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + StreamName: "default-10000-stream", + ShardID: "shardId-000000000000", + ShardIteratorType: "TRIM_HORIZON", }) require.NoError(t, err) - require.Zero(t, out.FailedRecordCount) - } - iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ - StreamName: "default-10000-stream", - ShardID: "shardId-000000000000", - ShardIteratorType: "TRIM_HORIZON", - }) - require.NoError(t, err) - - rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ - ShardIterator: iterOut.ShardIterator, + rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ + ShardIterator: iterOut.ShardIterator, + }) + require.NoError(t, err) + assert.Len(t, rec.Records, 10000, "default page size must be AWS's documented 10000, not fewer") }) - require.NoError(t, err) - assert.Len(t, rec.Records, 10000, "default page size must be AWS's documented 10000, not fewer") } func TestGetRecords_EmptyShard_MillisBehindZero(t *testing.T) { @@ -1006,101 +1068,107 @@ func TestGetRecords_EmptyShard_MillisBehindZero(t *testing.T) { func TestGetRecords_10MBCap_RecordsBeforeCapNotDropped(t *testing.T) { t.Parallel() - b := kinesis.NewInMemoryBackend() - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "precap-records", - ShardCount: 1, - })) - - require.NoError(t, b.UpdateMaxRecordSize(context.Background(), &kinesis.UpdateMaxRecordSizeInput{ - StreamARN: mustStreamARN(t, b, "precap-records"), - MaxRecordSizeInKiB: 10_485_760 / 1024, - })) + synctest.Test(t, func(t *testing.T) { + b := kinesis.NewInMemoryBackend() + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "precap-records", + ShardCount: 1, + })) + time.Sleep(streamSettleWait) + + require.NoError(t, b.UpdateMaxRecordSize(context.Background(), &kinesis.UpdateMaxRecordSizeInput{ + StreamARN: mustStreamARN(t, b, "precap-records"), + MaxRecordSizeInKiB: 10_485_760 / 1024, + })) + + // Put 3 small + 1 huge record (order matters for iteration). + for i := range 3 { + _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "precap-records", + PartitionKey: fmt.Sprintf("small%d", i), + Data: []byte("tiny"), + }) + require.NoError(t, err) + } - // Put 3 small + 1 huge record (order matters for iteration). - for i := range 3 { + bigData := make([]byte, 9_000_000) _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ StreamName: "precap-records", - PartitionKey: fmt.Sprintf("small%d", i), - Data: []byte("tiny"), + PartitionKey: "big", + Data: bigData, }) require.NoError(t, err) - } - bigData := make([]byte, 9_000_000) - _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "precap-records", - PartitionKey: "big", - Data: bigData, - }) - require.NoError(t, err) + iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + StreamName: "precap-records", + ShardID: "shardId-000000000000", + ShardIteratorType: "TRIM_HORIZON", + }) + require.NoError(t, err) - iterOut, err := b.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ - StreamName: "precap-records", - ShardID: "shardId-000000000000", - ShardIteratorType: "TRIM_HORIZON", - }) - require.NoError(t, err) + rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ + ShardIterator: iterOut.ShardIterator, + Limit: 10000, + }) + require.NoError(t, err) - rec, err := b.GetRecords(context.Background(), &kinesis.GetRecordsInput{ - ShardIterator: iterOut.ShardIterator, - Limit: 10000, + // All 3 small records + the 9MB record fit within 10MB. + assert.Len(t, rec.Records, 4) }) - require.NoError(t, err) - - // All 3 small records + the 9MB record fit within 10MB. - assert.Len(t, rec.Records, 4) } func TestGetRecords_MillisBehindLatest_ViaHandler(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "millis-handler-stream", - "ShardCount": 1, - }) - require.Equal(t, http.StatusOK, rec.Code) - - // Put 3 records. - for i := range 3 { - doRequest(t, h, "PutRecord", map[string]any{ - "StreamName": "millis-handler-stream", - "PartitionKey": fmt.Sprintf("pk%d", i), - "Data": []byte("x"), + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "millis-handler-stream", + "ShardCount": 1, }) - } + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) + + // Put 3 records. + for i := range 3 { + doRequest(t, h, "PutRecord", map[string]any{ + "StreamName": "millis-handler-stream", + "PartitionKey": fmt.Sprintf("pk%d", i), + "Data": []byte("x"), + }) + } - // Sleep briefly to ensure records have a measurable age. - time.Sleep(2 * time.Millisecond) + // Sleep briefly to ensure records have a measurable age. + time.Sleep(2 * time.Millisecond) - // Get shard iterator at trim horizon. - iterRec := doRequest(t, h, "GetShardIterator", map[string]any{ - "StreamName": "millis-handler-stream", - "ShardId": "shardId-000000000000", - "ShardIteratorType": "TRIM_HORIZON", - }) - require.Equal(t, http.StatusOK, iterRec.Code) + // Get shard iterator at trim horizon. + iterRec := doRequest(t, h, "GetShardIterator", map[string]any{ + "StreamName": "millis-handler-stream", + "ShardId": "shardId-000000000000", + "ShardIteratorType": "TRIM_HORIZON", + }) + require.Equal(t, http.StatusOK, iterRec.Code) - var iterResp struct { - ShardIterator string `json:"ShardIterator"` - } - require.NoError(t, json.Unmarshal(iterRec.Body.Bytes(), &iterResp)) + var iterResp struct { + ShardIterator string `json:"ShardIterator"` + } + require.NoError(t, json.Unmarshal(iterRec.Body.Bytes(), &iterResp)) - // Fetch 1 record (leaving 2 behind). - rec = doRequest(t, h, "GetRecords", map[string]any{ - "ShardIterator": iterResp.ShardIterator, - "Limit": 1, - }) - require.Equal(t, http.StatusOK, rec.Code) + // Fetch 1 record (leaving 2 behind). + rec = doRequest(t, h, "GetRecords", map[string]any{ + "ShardIterator": iterResp.ShardIterator, + "Limit": 1, + }) + require.Equal(t, http.StatusOK, rec.Code) - var getResp struct { - Records []any `json:"Records"` - MillisBehindLatest int64 `json:"MillisBehindLatest"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &getResp)) - assert.Len(t, getResp.Records, 1) - // Should be behind the last record, not just the next one. - assert.GreaterOrEqual(t, getResp.MillisBehindLatest, int64(0)) + var getResp struct { + Records []any `json:"Records"` + MillisBehindLatest int64 `json:"MillisBehindLatest"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &getResp)) + assert.Len(t, getResp.Records, 1) + // Should be behind the last record, not just the next one. + assert.GreaterOrEqual(t, getResp.MillisBehindLatest, int64(0)) + }) } diff --git a/services/kinesis/records_test.go b/services/kinesis/records_test.go index bd0d0f64f4..62092defba 100644 --- a/services/kinesis/records_test.go +++ b/services/kinesis/records_test.go @@ -7,6 +7,8 @@ import ( "math/big" "net/http" "testing" + "testing/synctest" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -17,15 +19,26 @@ import ( func TestPutRecord_ByARN(t *testing.T) { t.Parallel() - h := newTestHandler(t) - doRequest(t, h, "CreateStream", map[string]any{"StreamName": "put-record-arn-stream", "ShardCount": 1}) - - b := h.Backend.(*kinesis.InMemoryBackend) - desc, err := b.DescribeStream( - context.Background(), - &kinesis.DescribeStreamInput{StreamName: "put-record-arn-stream"}, - ) - require.NoError(t, err) + var h *kinesis.Handler + var desc *kinesis.DescribeStreamOutput + + // Setup runs inside a synctest bubble so CreateStream's CREATING window + // lazily elapses without a real time.Sleep; the subtests below only call + // PutRecord, which needs ACTIVE only at call time (already satisfied by + // then) and does not itself gate on stream status transitions. + synctest.Test(t, func(t *testing.T) { + h = newTestHandler(t) + doRequest(t, h, "CreateStream", map[string]any{"StreamName": "put-record-arn-stream", "ShardCount": 1}) + time.Sleep(streamSettleWait) + + b := h.Backend.(*kinesis.InMemoryBackend) + d, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: "put-record-arn-stream"}, + ) + require.NoError(t, err) + desc = d + }) tests := []struct { body map[string]any @@ -70,15 +83,22 @@ func TestPutRecord_ByARN(t *testing.T) { func TestPutRecords_ByARN(t *testing.T) { t.Parallel() - h := newTestHandler(t) - doRequest(t, h, "CreateStream", map[string]any{"StreamName": "put-records-arn-stream", "ShardCount": 1}) - - b := h.Backend.(*kinesis.InMemoryBackend) - desc, err := b.DescribeStream( - context.Background(), - &kinesis.DescribeStreamInput{StreamName: "put-records-arn-stream"}, - ) - require.NoError(t, err) + var h *kinesis.Handler + var desc *kinesis.DescribeStreamOutput + + synctest.Test(t, func(t *testing.T) { + h = newTestHandler(t) + doRequest(t, h, "CreateStream", map[string]any{"StreamName": "put-records-arn-stream", "ShardCount": 1}) + time.Sleep(streamSettleWait) + + b := h.Backend.(*kinesis.InMemoryBackend) + d, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: "put-records-arn-stream"}, + ) + require.NoError(t, err) + desc = d + }) records := []map[string]any{ {"PartitionKey": "pk1", "Data": []byte("r1")}, @@ -146,31 +166,34 @@ func TestPutRecords_ThroughputErrorCode(t *testing.T) { func TestExplicitHashKey_OverridesPartitionKey(t *testing.T) { t.Parallel() - b := newParityBackend(t) - ctx := context.Background() + synctest.Test(t, func(t *testing.T) { + b := newParityBackend(t) + ctx := context.Background() - createParityStream(t, b, "explicit-key", 2) + createParityStream(t, b, "explicit-key", 2) + time.Sleep(streamSettleWait) - out0, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ - StreamName: "explicit-key", - PartitionKey: "anything", - ExplicitHashKey: "0", - Data: []byte("to-shard-0"), - }) - require.NoError(t, err) - assert.Equal(t, "shardId-000000000000", out0.ShardID) + out0, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ + StreamName: "explicit-key", + PartitionKey: "anything", + ExplicitHashKey: "0", + Data: []byte("to-shard-0"), + }) + require.NoError(t, err) + assert.Equal(t, "shardId-000000000000", out0.ShardID) - // shard 1 start = 2^127. - shard1Start := new(big.Int).Lsh(big.NewInt(1), 127) + // shard 1 start = 2^127. + shard1Start := new(big.Int).Lsh(big.NewInt(1), 127) - out1, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ - StreamName: "explicit-key", - PartitionKey: "anything", - ExplicitHashKey: shard1Start.String(), - Data: []byte("to-shard-1"), + out1, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ + StreamName: "explicit-key", + PartitionKey: "anything", + ExplicitHashKey: shard1Start.String(), + Data: []byte("to-shard-1"), + }) + require.NoError(t, err) + assert.Equal(t, "shardId-000000000001", out1.ShardID) }) - require.NoError(t, err) - assert.Equal(t, "shardId-000000000001", out1.ShardID) } func TestPutRecord_ExplicitHashKey(t *testing.T) { @@ -194,22 +217,25 @@ func TestPutRecord_ExplicitHashKey(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) - - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "ehk-stream-" + tt.name, - ShardCount: tt.shardCount, - })) - - out, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "ehk-stream-" + tt.name, - PartitionKey: "some-key", - ExplicitHashKey: tt.explicitHashKey, - Data: []byte("data"), + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) + + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "ehk-stream-" + tt.name, + ShardCount: tt.shardCount, + })) + time.Sleep(streamSettleWait) + + out, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "ehk-stream-" + tt.name, + PartitionKey: "some-key", + ExplicitHashKey: tt.explicitHashKey, + Data: []byte("data"), + }) + require.NoError(t, err) + assert.Equal(t, tt.wantShard, out.ShardID) }) - require.NoError(t, err) - assert.Equal(t, tt.wantShard, out.ShardID) }) } } @@ -238,34 +264,37 @@ func TestPutRecordsNotFound(t *testing.T) { func TestMultipleShardRouting(t *testing.T) { t.Parallel() - h := newTestHandler(t) - - // Create stream with 4 shards - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "multi-shard-stream", - "ShardCount": 4, - }) - require.Equal(t, http.StatusOK, rec.Code) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - // Put records with different partition keys - shardIDs := make(map[string]bool) - for i := range 10 { - rec = doRequest(t, h, "PutRecord", map[string]any{ - "StreamName": "multi-shard-stream", - "PartitionKey": fmt.Sprintf("pk-%d", i), - "Data": []byte("data"), + // Create stream with 4 shards + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "multi-shard-stream", + "ShardCount": 4, }) require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) + + // Put records with different partition keys + shardIDs := make(map[string]bool) + for i := range 10 { + rec = doRequest(t, h, "PutRecord", map[string]any{ + "StreamName": "multi-shard-stream", + "PartitionKey": fmt.Sprintf("pk-%d", i), + "Data": []byte("data"), + }) + require.Equal(t, http.StatusOK, rec.Code) - var putResp struct { - ShardID string `json:"ShardId"` + var putResp struct { + ShardID string `json:"ShardId"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &putResp)) + shardIDs[putResp.ShardID] = true } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &putResp)) - shardIDs[putResp.ShardID] = true - } - // With 10 records and 4 shards, we should get records on more than 1 shard - assert.GreaterOrEqual(t, len(shardIDs), 1) + // With 10 records and 4 shards, we should get records on more than 1 shard + assert.GreaterOrEqual(t, len(shardIDs), 1) + }) } func TestPutRecordMaxRecords(t *testing.T) { @@ -283,47 +312,50 @@ func TestPutRecordMaxRecords(t *testing.T) { func TestPutRecords_OversizeRecordReturnsValidationException(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "putrecords-err-stream", - "ShardCount": 1, - }) - require.Equal(t, http.StatusOK, rec.Code) + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "putrecords-err-stream", + "ShardCount": 1, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) - // Build a batch: first record is valid, second is oversize. - smallData := make([]byte, 100) - oversizeData := make([]byte, 1_048_577) // 1 MiB + 1 byte + // Build a batch: first record is valid, second is oversize. + smallData := make([]byte, 100) + oversizeData := make([]byte, 1_048_577) // 1 MiB + 1 byte - rec = doRequest(t, h, "PutRecords", map[string]any{ - "StreamName": "putrecords-err-stream", - "Records": []map[string]any{ - {"PartitionKey": "pk1", "Data": smallData}, - {"PartitionKey": "pk2", "Data": oversizeData}, - }, - }) - require.Equal(t, http.StatusOK, rec.Code) + rec = doRequest(t, h, "PutRecords", map[string]any{ + "StreamName": "putrecords-err-stream", + "Records": []map[string]any{ + {"PartitionKey": "pk1", "Data": smallData}, + {"PartitionKey": "pk2", "Data": oversizeData}, + }, + }) + require.Equal(t, http.StatusOK, rec.Code) - var resp struct { - Records []struct { - ErrorCode string `json:"ErrorCode"` - ErrorMessage string `json:"ErrorMessage"` - ShardID string `json:"ShardId"` - } `json:"Records"` - FailedRecordCount int `json:"FailedRecordCount"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + var resp struct { + Records []struct { + ErrorCode string `json:"ErrorCode"` + ErrorMessage string `json:"ErrorMessage"` + ShardID string `json:"ShardId"` + } `json:"Records"` + FailedRecordCount int `json:"FailedRecordCount"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) - assert.Equal(t, 1, resp.FailedRecordCount) - require.Len(t, resp.Records, 2) + assert.Equal(t, 1, resp.FailedRecordCount) + require.Len(t, resp.Records, 2) - // First record succeeded. - assert.Empty(t, resp.Records[0].ErrorCode) - assert.NotEmpty(t, resp.Records[0].ShardID) + // First record succeeded. + assert.Empty(t, resp.Records[0].ErrorCode) + assert.NotEmpty(t, resp.Records[0].ShardID) - // Second record failed with ValidationException (not InternalFailure). - assert.Equal(t, "ValidationException", resp.Records[1].ErrorCode) - assert.NotEmpty(t, resp.Records[1].ErrorMessage) + // Second record failed with ValidationException (not InternalFailure). + assert.Equal(t, "ValidationException", resp.Records[1].ErrorCode) + assert.NotEmpty(t, resp.Records[1].ErrorMessage) + }) } func TestPutRecords_ThrottledRecordReturnsProvisionedThroughputException(t *testing.T) { @@ -363,37 +395,40 @@ func TestPutRecords_ThrottledRecordReturnsProvisionedThroughputException(t *test func TestPutRecords_AllValidRecordsSucceed(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "putrecords-all-ok", - "ShardCount": 1, - }) - require.Equal(t, http.StatusOK, rec.Code) + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "putrecords-all-ok", + "ShardCount": 1, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) + + rec = doRequest(t, h, "PutRecords", map[string]any{ + "StreamName": "putrecords-all-ok", + "Records": []map[string]any{ + {"PartitionKey": "pk1", "Data": []byte("a")}, + {"PartitionKey": "pk2", "Data": []byte("b")}, + {"PartitionKey": "pk3", "Data": []byte("c")}, + }, + }) + require.Equal(t, http.StatusOK, rec.Code) - rec = doRequest(t, h, "PutRecords", map[string]any{ - "StreamName": "putrecords-all-ok", - "Records": []map[string]any{ - {"PartitionKey": "pk1", "Data": []byte("a")}, - {"PartitionKey": "pk2", "Data": []byte("b")}, - {"PartitionKey": "pk3", "Data": []byte("c")}, - }, + var resp struct { + Records []struct { + ErrorCode string `json:"ErrorCode"` + ShardID string `json:"ShardId"` + } `json:"Records"` + FailedRecordCount int `json:"FailedRecordCount"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + assert.Equal(t, 0, resp.FailedRecordCount) + for _, r := range resp.Records { + assert.Empty(t, r.ErrorCode) + assert.NotEmpty(t, r.ShardID) + } }) - require.Equal(t, http.StatusOK, rec.Code) - - var resp struct { - Records []struct { - ErrorCode string `json:"ErrorCode"` - ShardID string `json:"ShardId"` - } `json:"Records"` - FailedRecordCount int `json:"FailedRecordCount"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) - assert.Equal(t, 0, resp.FailedRecordCount) - for _, r := range resp.Records { - assert.Empty(t, r.ErrorCode) - assert.NotEmpty(t, r.ShardID) - } } func TestPutRecord_ExplicitHashKey_AboveMaxRejected(t *testing.T) { @@ -441,43 +476,49 @@ func TestPutRecord_ExplicitHashKey_NegativeRejected(t *testing.T) { func TestPutRecord_ExplicitHashKey_ZeroAccepted(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) - - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "hashkey-zero-stream", - ShardCount: 1, - })) - - _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "hashkey-zero-stream", - PartitionKey: "pk", - ExplicitHashKey: "0", - Data: []byte("d"), + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) + + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "hashkey-zero-stream", + ShardCount: 1, + })) + time.Sleep(streamSettleWait) + + _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "hashkey-zero-stream", + PartitionKey: "pk", + ExplicitHashKey: "0", + Data: []byte("d"), + }) + require.NoError(t, err) }) - require.NoError(t, err) } func TestPutRecord_ExplicitHashKey_MaxAccepted(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) - - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "hashkey-maxval-stream", - ShardCount: 1, - })) - - // 2^128-1 is the maximum valid hash key. - maxKey := "340282366920938463463374607431768211455" - _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "hashkey-maxval-stream", - PartitionKey: "pk", - ExplicitHashKey: maxKey, - Data: []byte("d"), + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) + + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "hashkey-maxval-stream", + ShardCount: 1, + })) + time.Sleep(streamSettleWait) + + // 2^128-1 is the maximum valid hash key. + maxKey := "340282366920938463463374607431768211455" + _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "hashkey-maxval-stream", + PartitionKey: "pk", + ExplicitHashKey: maxKey, + Data: []byte("d"), + }) + require.NoError(t, err) }) - require.NoError(t, err) } func TestPutRecord_ExplicitHashKey_ViaHandler_AboveMaxRejected(t *testing.T) { @@ -504,24 +545,27 @@ func TestPutRecord_ExplicitHashKey_ViaHandler_AboveMaxRejected(t *testing.T) { func TestExplicitHashKey_PartitionKeyOverride(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) - - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "explicit-hash-override", - ShardCount: 2, - })) - - // Use a hash key in the upper half to target the second shard. - upperHalfKey := "255211775190703847597592248818726428672" - out, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "explicit-hash-override", - PartitionKey: "ignored-partition-key", - ExplicitHashKey: upperHalfKey, - Data: []byte("d"), + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) + + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "explicit-hash-override", + ShardCount: 2, + })) + time.Sleep(streamSettleWait) + + // Use a hash key in the upper half to target the second shard. + upperHalfKey := "255211775190703847597592248818726428672" + out, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "explicit-hash-override", + PartitionKey: "ignored-partition-key", + ExplicitHashKey: upperHalfKey, + Data: []byte("d"), + }) + require.NoError(t, err) + assert.NotEmpty(t, out.ShardID) }) - require.NoError(t, err) - assert.NotEmpty(t, out.ShardID) } func TestPutRecord_ExplicitHashKey_OneAboveMax(t *testing.T) { @@ -547,48 +591,54 @@ func TestPutRecord_ExplicitHashKey_OneAboveMax(t *testing.T) { func TestPutRecords_MixedOversizeAndValid(t *testing.T) { t.Parallel() - b := kinesis.NewInMemoryBackend() - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "putrecords-mixed", - ShardCount: 1, - })) - - // 3 records: valid, oversize, valid. - oversize := make([]byte, 1_048_577) // 1 MiB + 1 byte - out, err := b.PutRecords(context.Background(), &kinesis.PutRecordsInput{ - StreamName: "putrecords-mixed", - Records: []kinesis.PutRecordsEntry{ - {PartitionKey: "pk1", Data: []byte("ok1")}, - {PartitionKey: "pk2", Data: oversize}, - {PartitionKey: "pk3", Data: []byte("ok3")}, - }, + synctest.Test(t, func(t *testing.T) { + b := kinesis.NewInMemoryBackend() + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "putrecords-mixed", + ShardCount: 1, + })) + time.Sleep(streamSettleWait) + + // 3 records: valid, oversize, valid. + oversize := make([]byte, 1_048_577) // 1 MiB + 1 byte + out, err := b.PutRecords(context.Background(), &kinesis.PutRecordsInput{ + StreamName: "putrecords-mixed", + Records: []kinesis.PutRecordsEntry{ + {PartitionKey: "pk1", Data: []byte("ok1")}, + {PartitionKey: "pk2", Data: oversize}, + {PartitionKey: "pk3", Data: []byte("ok3")}, + }, + }) + require.NoError(t, err) + require.Len(t, out.Records, 3) + assert.Equal(t, 1, out.FailedRecordCount) + assert.Empty(t, out.Records[0].ErrorCode) + assert.Equal(t, "ValidationException", out.Records[1].ErrorCode) + assert.Empty(t, out.Records[2].ErrorCode) }) - require.NoError(t, err) - require.Len(t, out.Records, 3) - assert.Equal(t, 1, out.FailedRecordCount) - assert.Empty(t, out.Records[0].ErrorCode) - assert.Equal(t, "ValidationException", out.Records[1].ErrorCode) - assert.Empty(t, out.Records[2].ErrorCode) } func TestExplicitHashKey_ValidMidRange(t *testing.T) { t.Parallel() - b := kinesis.NewInMemoryBackend() - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "midrange-hash", - ShardCount: 2, - })) - - // Hash key exactly at the midpoint of 2^128 space. - midpoint := "170141183460469231731687303715884105728" - _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "midrange-hash", - PartitionKey: "pk", - ExplicitHashKey: midpoint, - Data: []byte("d"), + synctest.Test(t, func(t *testing.T) { + b := kinesis.NewInMemoryBackend() + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "midrange-hash", + ShardCount: 2, + })) + time.Sleep(streamSettleWait) + + // Hash key exactly at the midpoint of 2^128 space. + midpoint := "170141183460469231731687303715884105728" + _, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "midrange-hash", + PartitionKey: "pk", + ExplicitHashKey: midpoint, + Data: []byte("d"), + }) + require.NoError(t, err) }) - require.NoError(t, err) } func TestPutRecords_EmptyBatch(t *testing.T) { diff --git a/services/kinesis/resharding.go b/services/kinesis/resharding.go index ffc8b19a36..bc5a287fe4 100644 --- a/services/kinesis/resharding.go +++ b/services/kinesis/resharding.go @@ -47,13 +47,17 @@ func (b *InMemoryBackend) UpdateShardCount( b.mu.Lock("UpdateShardCount") defer b.mu.Unlock() - stream, ok := b.streams.Get(streamKey(region, input.StreamName)) - if !ok { - return nil, ErrStreamNotFound + stream, err := b.resolveStreamTransitionLocked(region, input.StreamName) + if err != nil { + return nil, err } stream.mu.Lock("UpdateShardCount.stream") defer stream.mu.Unlock() + if stream.Status != streamStatusActive { + return nil, ErrStreamNotActive + } + if stream.StreamMode == streamModeOnDemand { return nil, ErrInvalidArgument } @@ -83,6 +87,8 @@ func (b *InMemoryBackend) UpdateShardCount( } reshardTo(stream, targetCount) + stream.Status = streamStatusUpdating + stream.ReadyAt = b.nowFunc().Add(streamTransitionDelay) return &UpdateShardCountOutput{ StreamName: input.StreamName, @@ -189,6 +195,38 @@ func nextShardID(shards []*Shard) string { return fmt.Sprintf("shardId-%012d", nextShardIDIndex(shards)) } +// mergedHashRange returns the combined [start, end] hash key range of two +// adjacent shards, or ok=false if they are not actually adjacent (a gap +// between their ranges). +func mergedHashRange(shard1, shard2 *Shard) (*big.Int, *big.Int, bool) { + s1Start := new(big.Int) + s1Start.SetString(shard1.HashKeyRangeStart, hashKeyDecimalBase) + s2Start := new(big.Int) + s2Start.SetString(shard2.HashKeyRangeStart, hashKeyDecimalBase) + s1End := new(big.Int) + s1End.SetString(shard1.HashKeyRangeEnd, hashKeyDecimalBase) + s2End := new(big.Int) + s2End.SetString(shard2.HashKeyRangeEnd, hashKeyDecimalBase) + + s1EndPlusOne := new(big.Int).Add(s1End, big.NewInt(1)) + s2EndPlusOne := new(big.Int).Add(s2End, big.NewInt(1)) + if s1EndPlusOne.Cmp(s2Start) != 0 && s2EndPlusOne.Cmp(s1Start) != 0 { + return nil, nil, false + } + + start := s1Start + if s2Start.Cmp(s1Start) < 0 { + start = s2Start + } + + end := s1End + if s2End.Cmp(s1End) > 0 { + end = s2End + } + + return start, end, true +} + // MergeShards merges two adjacent shards into one. // The merged shard spans the combined hash key range of both parent shards. func (b *InMemoryBackend) MergeShards(ctx context.Context, input *MergeShardsInput) error { @@ -202,13 +240,17 @@ func (b *InMemoryBackend) MergeShards(ctx context.Context, input *MergeShardsInp streamName = streamNameFromARN(input.StreamARN) } - stream, ok := b.streams.Get(streamKey(region, streamName)) - if !ok { - return ErrStreamNotFound + stream, err := b.resolveStreamTransitionLocked(region, streamName) + if err != nil { + return err } stream.mu.Lock("MergeShards.stream") defer stream.mu.Unlock() + if stream.Status != streamStatusActive { + return ErrStreamNotActive + } + if stream.StreamMode == streamModeOnDemand { return ErrInvalidArgument } @@ -223,32 +265,11 @@ func (b *InMemoryBackend) MergeShards(ctx context.Context, input *MergeShardsInp return ErrInvalidArgument } - // Determine the merged range: min start, max end. - s1Start := new(big.Int) - s1Start.SetString(shard1.HashKeyRangeStart, hashKeyDecimalBase) - s2Start := new(big.Int) - s2Start.SetString(shard2.HashKeyRangeStart, hashKeyDecimalBase) - s1End := new(big.Int) - s1End.SetString(shard1.HashKeyRangeEnd, hashKeyDecimalBase) - s2End := new(big.Int) - s2End.SetString(shard2.HashKeyRangeEnd, hashKeyDecimalBase) - - s1EndPlusOne := new(big.Int).Add(s1End, big.NewInt(1)) - s2EndPlusOne := new(big.Int).Add(s2End, big.NewInt(1)) - if s1EndPlusOne.Cmp(s2Start) != 0 && s2EndPlusOne.Cmp(s1Start) != 0 { + startKey, endKey, ok := mergedHashRange(shard1, shard2) + if !ok { return ErrInvalidArgument } - startKey := s1Start - if s2Start.Cmp(s1Start) < 0 { - startKey = s2Start - } - - endKey := s1End - if s2End.Cmp(s1End) > 0 { - endKey = s2End - } - mergedID := nextShardID(stream.Shards) merged := &Shard{ ID: mergedID, @@ -268,6 +289,8 @@ func (b *InMemoryBackend) MergeShards(ctx context.Context, input *MergeShardsInp newShards = append(newShards, stream.Shards...) newShards = append(newShards, merged) stream.Shards = newShards + stream.Status = streamStatusUpdating + stream.ReadyAt = b.nowFunc().Add(streamTransitionDelay) return nil } @@ -284,13 +307,17 @@ func (b *InMemoryBackend) SplitShard(ctx context.Context, input *SplitShardInput streamName = streamNameFromARN(input.StreamARN) } - stream, ok := b.streams.Get(streamKey(region, streamName)) - if !ok { - return ErrStreamNotFound + stream, err := b.resolveStreamTransitionLocked(region, streamName) + if err != nil { + return err } stream.mu.Lock("SplitShard.stream") defer stream.mu.Unlock() + if stream.Status != streamStatusActive { + return ErrStreamNotActive + } + if stream.StreamMode == streamModeOnDemand { return ErrInvalidArgument } @@ -320,7 +347,7 @@ func (b *InMemoryBackend) SplitShard(ctx context.Context, input *SplitShardInput shard1ID := nextShardID(stream.Shards) var shard1Idx int - if _, err := fmt.Sscanf(shard1ID, "shardId-%012d", &shard1Idx); err != nil { + if _, scanErr := fmt.Sscanf(shard1ID, "shardId-%012d", &shard1Idx); scanErr != nil { // nextShardID guarantees the format; this path is unreachable in practice. shard1Idx = len(stream.Shards) } @@ -352,6 +379,8 @@ func (b *InMemoryBackend) SplitShard(ctx context.Context, input *SplitShardInput newShards = append(newShards, stream.Shards...) newShards = append(newShards, shard1, shard2) stream.Shards = newShards + stream.Status = streamStatusUpdating + stream.ReadyAt = b.nowFunc().Add(streamTransitionDelay) return nil } diff --git a/services/kinesis/resharding_shard_count_test.go b/services/kinesis/resharding_shard_count_test.go index 3eade7cbc2..90c205befb 100644 --- a/services/kinesis/resharding_shard_count_test.go +++ b/services/kinesis/resharding_shard_count_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -45,41 +47,44 @@ func TestUpdateShardCount(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newTestHandler(t) - streamName := "reshard-stream-" + tt.name - - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": streamName, - "ShardCount": tt.initialShards, + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + streamName := "reshard-stream-" + tt.name + + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": streamName, + "ShardCount": tt.initialShards, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) + + rec = doRequest(t, h, "UpdateShardCount", map[string]any{ + "StreamName": streamName, + "TargetShardCount": tt.targetShards, + "ScalingType": "UNIFORM_SCALING", + }) + require.Equal(t, tt.wantCode, rec.Code) + + var resp struct { + StreamName string `json:"StreamName"` + CurrentShardCount int `json:"CurrentShardCount"` + TargetShardCount int `json:"TargetShardCount"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + assert.Equal(t, streamName, resp.StreamName) + assert.Equal(t, tt.wantCurrentCount, resp.CurrentShardCount) + assert.Equal(t, tt.wantTargetCount, resp.TargetShardCount) + + // Verify new shard count via ListShards. + rec = doRequest(t, h, "ListShards", map[string]any{"StreamName": streamName}) + require.Equal(t, http.StatusOK, rec.Code) + + var shardsResp struct { + Shards []any `json:"Shards"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &shardsResp)) + assert.Len(t, shardsResp.Shards, tt.targetShards) }) - require.Equal(t, http.StatusOK, rec.Code) - - rec = doRequest(t, h, "UpdateShardCount", map[string]any{ - "StreamName": streamName, - "TargetShardCount": tt.targetShards, - "ScalingType": "UNIFORM_SCALING", - }) - require.Equal(t, tt.wantCode, rec.Code) - - var resp struct { - StreamName string `json:"StreamName"` - CurrentShardCount int `json:"CurrentShardCount"` - TargetShardCount int `json:"TargetShardCount"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) - assert.Equal(t, streamName, resp.StreamName) - assert.Equal(t, tt.wantCurrentCount, resp.CurrentShardCount) - assert.Equal(t, tt.wantTargetCount, resp.TargetShardCount) - - // Verify new shard count via ListShards. - rec = doRequest(t, h, "ListShards", map[string]any{"StreamName": streamName}) - require.Equal(t, http.StatusOK, rec.Code) - - var shardsResp struct { - Shards []any `json:"Shards"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &shardsResp)) - assert.Len(t, shardsResp.Shards, tt.targetShards) }) } } @@ -123,222 +128,245 @@ func TestUpdateShardCountErrors(t *testing.T) { func TestUpdateShardCount_OldShardsMarkedClosed(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) - - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "update-shardcount-closed", - ShardCount: 2, - })) - - out, err := b.DescribeStream( - context.Background(), - &kinesis.DescribeStreamInput{StreamName: "update-shardcount-closed"}, - ) - require.NoError(t, err) - require.Len(t, out.Shards, 2) - - // Scale up to 4. - _, err = b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ - StreamName: "update-shardcount-closed", - TargetShardCount: 4, - ScalingType: "UNIFORM_SCALING", - }) - require.NoError(t, err) - - // DescribeStream must include old closed shards + new open ones. - out2, err := b.DescribeStream( - context.Background(), - &kinesis.DescribeStreamInput{StreamName: "update-shardcount-closed"}, - ) - require.NoError(t, err) - - openCount := 0 - closedCount := 0 - for _, s := range out2.Shards { - if s.Closed { - closedCount++ - } else { - openCount++ + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) + + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "update-shardcount-closed", + ShardCount: 2, + })) + time.Sleep(streamSettleWait) + + out, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: "update-shardcount-closed"}, + ) + require.NoError(t, err) + require.Len(t, out.Shards, 2) + + // Scale up to 4. + _, err = b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ + StreamName: "update-shardcount-closed", + TargetShardCount: 4, + ScalingType: "UNIFORM_SCALING", + }) + require.NoError(t, err) + + // DescribeStream must include old closed shards + new open ones. + out2, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: "update-shardcount-closed"}, + ) + require.NoError(t, err) + + openCount := 0 + closedCount := 0 + for _, s := range out2.Shards { + if s.Closed { + closedCount++ + } else { + openCount++ + } } - } - assert.Equal(t, 4, openCount, "should have 4 new open shards") - assert.Equal(t, 2, closedCount, "old 2 shards should be marked closed") - assert.Len(t, out2.Shards, 6, "total 6 shards (2 closed + 4 open)") + assert.Equal(t, 4, openCount, "should have 4 new open shards") + assert.Equal(t, 2, closedCount, "old 2 shards should be marked closed") + assert.Len(t, out2.Shards, 6, "total 6 shards (2 closed + 4 open)") + }) } func TestUpdateShardCount_ListShardsOnlyReturnsOpenShards(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "update-listshard-stream", - ShardCount: 2, - })) + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "update-listshard-stream", + ShardCount: 2, + })) + time.Sleep(streamSettleWait) - _, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ - StreamName: "update-listshard-stream", - TargetShardCount: 3, - ScalingType: "UNIFORM_SCALING", - }) - require.NoError(t, err) + _, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ + StreamName: "update-listshard-stream", + TargetShardCount: 3, + ScalingType: "UNIFORM_SCALING", + }) + require.NoError(t, err) - // ListShards default = open shards only. - list, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "update-listshard-stream"}) - require.NoError(t, err) - assert.Len(t, list.Shards, 3, "ListShards should return only the 3 new open shards") + // ListShards default = open shards only. + list, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "update-listshard-stream"}) + require.NoError(t, err) + assert.Len(t, list.Shards, 3, "ListShards should return only the 3 new open shards") + }) } func TestUpdateShardCount_CurrentCountIsOpenShards(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "update-currentcount-stream", - ShardCount: 4, - })) + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "update-currentcount-stream", + ShardCount: 4, + })) + time.Sleep(streamSettleWait) - out, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ - StreamName: "update-currentcount-stream", - TargetShardCount: 2, - ScalingType: "UNIFORM_SCALING", - }) - require.NoError(t, err) + out, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ + StreamName: "update-currentcount-stream", + TargetShardCount: 2, + ScalingType: "UNIFORM_SCALING", + }) + require.NoError(t, err) - // CurrentShardCount should reflect the 4 open shards before the operation. - assert.Equal(t, 4, out.CurrentShardCount) - assert.Equal(t, 2, out.TargetShardCount) + // CurrentShardCount should reflect the 4 open shards before the operation. + assert.Equal(t, 4, out.CurrentShardCount) + assert.Equal(t, 2, out.TargetShardCount) + }) } func TestUpdateShardCount_UniqueShardIDs(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "update-uniqueids-stream", - ShardCount: 2, - })) + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "update-uniqueids-stream", + ShardCount: 2, + })) + time.Sleep(streamSettleWait) - _, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ - StreamName: "update-uniqueids-stream", - TargetShardCount: 3, - ScalingType: "UNIFORM_SCALING", - }) - require.NoError(t, err) - - // Scale again (3 -> 2 stays within the AWS 50%-200% per-call window). - _, err = b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ - StreamName: "update-uniqueids-stream", - TargetShardCount: 2, - ScalingType: "UNIFORM_SCALING", + _, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ + StreamName: "update-uniqueids-stream", + TargetShardCount: 3, + ScalingType: "UNIFORM_SCALING", + }) + require.NoError(t, err) + time.Sleep(streamSettleWait) + + // Scale again (3 -> 2 stays within the AWS 50%-200% per-call window). + _, err = b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ + StreamName: "update-uniqueids-stream", + TargetShardCount: 2, + ScalingType: "UNIFORM_SCALING", + }) + require.NoError(t, err) + + out, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: "update-uniqueids-stream"}, + ) + require.NoError(t, err) + + seen := make(map[string]struct{}) + for _, s := range out.Shards { + assert.NotContains(t, seen, s.ShardID, "duplicate shard ID %q", s.ShardID) + seen[s.ShardID] = struct{}{} + } }) - require.NoError(t, err) - - out, err := b.DescribeStream( - context.Background(), - &kinesis.DescribeStreamInput{StreamName: "update-uniqueids-stream"}, - ) - require.NoError(t, err) - - seen := make(map[string]struct{}) - for _, s := range out.Shards { - assert.NotContains(t, seen, s.ShardID, "duplicate shard ID %q", s.ShardID) - seen[s.ShardID] = struct{}{} - } } func TestUpdateShardCount_ViaHandler_OpenShardsOnly(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "handler-update-shard", - "ShardCount": 2, - }) - require.Equal(t, http.StatusOK, rec.Code) + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "handler-update-shard", + "ShardCount": 2, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) - rec = doRequest(t, h, "UpdateShardCount", map[string]any{ - "StreamName": "handler-update-shard", - "TargetShardCount": 4, - "ScalingType": "UNIFORM_SCALING", - }) - require.Equal(t, http.StatusOK, rec.Code) + rec = doRequest(t, h, "UpdateShardCount", map[string]any{ + "StreamName": "handler-update-shard", + "TargetShardCount": 4, + "ScalingType": "UNIFORM_SCALING", + }) + require.Equal(t, http.StatusOK, rec.Code) - var updateResp struct { - CurrentShardCount int `json:"CurrentShardCount"` - TargetShardCount int `json:"TargetShardCount"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &updateResp)) - assert.Equal(t, 2, updateResp.CurrentShardCount) - assert.Equal(t, 4, updateResp.TargetShardCount) + var updateResp struct { + CurrentShardCount int `json:"CurrentShardCount"` + TargetShardCount int `json:"TargetShardCount"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &updateResp)) + assert.Equal(t, 2, updateResp.CurrentShardCount) + assert.Equal(t, 4, updateResp.TargetShardCount) - // ListShards returns only open shards → should see 4 new open shards. - rec = doRequest(t, h, "ListShards", map[string]any{"StreamName": "handler-update-shard"}) - require.Equal(t, http.StatusOK, rec.Code) + // ListShards returns only open shards → should see 4 new open shards. + rec = doRequest(t, h, "ListShards", map[string]any{"StreamName": "handler-update-shard"}) + require.Equal(t, http.StatusOK, rec.Code) - var listResp struct { - Shards []any `json:"Shards"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &listResp)) - assert.Len(t, listResp.Shards, 4) + var listResp struct { + Shards []any `json:"Shards"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &listResp)) + assert.Len(t, listResp.Shards, 4) + }) } func TestUpdateShardCount_SecondScaleStillWorks(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "double-scale-stream", - ShardCount: 2, - })) + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "double-scale-stream", + ShardCount: 2, + })) + time.Sleep(streamSettleWait) - _, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ - StreamName: "double-scale-stream", - TargetShardCount: 4, - ScalingType: "UNIFORM_SCALING", - }) - require.NoError(t, err) + _, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ + StreamName: "double-scale-stream", + TargetShardCount: 4, + ScalingType: "UNIFORM_SCALING", + }) + require.NoError(t, err) + time.Sleep(streamSettleWait) - out2, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ - StreamName: "double-scale-stream", - TargetShardCount: 2, - ScalingType: "UNIFORM_SCALING", + out2, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ + StreamName: "double-scale-stream", + TargetShardCount: 2, + ScalingType: "UNIFORM_SCALING", + }) + require.NoError(t, err) + assert.Equal(t, 4, out2.CurrentShardCount, "current count after first scale is 4 open shards") + assert.Equal(t, 2, out2.TargetShardCount) }) - require.NoError(t, err) - assert.Equal(t, 4, out2.CurrentShardCount, "current count after first scale is 4 open shards") - assert.Equal(t, 2, out2.TargetShardCount) } func TestUpdateShardCount_LargeScale(t *testing.T) { t.Parallel() - b := kinesis.NewInMemoryBackend() - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "large-scale-stream", - ShardCount: 5, - })) - - out, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ - StreamName: "large-scale-stream", - TargetShardCount: 10, - ScalingType: "UNIFORM_SCALING", + synctest.Test(t, func(t *testing.T) { + b := kinesis.NewInMemoryBackend() + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "large-scale-stream", + ShardCount: 5, + })) + time.Sleep(streamSettleWait) + + out, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ + StreamName: "large-scale-stream", + TargetShardCount: 10, + ScalingType: "UNIFORM_SCALING", + }) + require.NoError(t, err) + assert.Equal(t, 5, out.CurrentShardCount) + assert.Equal(t, 10, out.TargetShardCount) + + // Verify 10 open shards via ListShards. + list, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "large-scale-stream"}) + require.NoError(t, err) + assert.Len(t, list.Shards, 10) }) - require.NoError(t, err) - assert.Equal(t, 5, out.CurrentShardCount) - assert.Equal(t, 10, out.TargetShardCount) - - // Verify 10 open shards via ListShards. - list, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "large-scale-stream"}) - require.NoError(t, err) - assert.Len(t, list.Shards, 10) } diff --git a/services/kinesis/resharding_test.go b/services/kinesis/resharding_test.go index 52405b7e78..05e132e016 100644 --- a/services/kinesis/resharding_test.go +++ b/services/kinesis/resharding_test.go @@ -6,6 +6,8 @@ import ( "math/big" "net/http" "testing" + "testing/synctest" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -96,32 +98,35 @@ func TestScalingCap(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := kinesis.NewInMemoryBackend() - name := "cap-" + tt.name - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: name, - ShardCount: tt.initialShards, - })) - - out, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ - StreamName: name, - TargetShardCount: tt.targetShards, - ScalingType: "UNIFORM_SCALING", - }) + synctest.Test(t, func(t *testing.T) { + b := kinesis.NewInMemoryBackend() + name := "cap-" + tt.name + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: name, + ShardCount: tt.initialShards, + })) + time.Sleep(streamSettleWait) - if tt.wantErr { - require.Error(t, err) - require.ErrorIs(t, err, kinesis.ErrShardCountScaling) - // Rejected calls must not mutate the open shard count. - assert.Len(t, openShards(t, b, name), tt.initialShards) + out, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ + StreamName: name, + TargetShardCount: tt.targetShards, + ScalingType: "UNIFORM_SCALING", + }) - return - } + if tt.wantErr { + require.Error(t, err) + require.ErrorIs(t, err, kinesis.ErrShardCountScaling) + // Rejected calls must not mutate the open shard count. + assert.Len(t, openShards(t, b, name), tt.initialShards) - require.NoError(t, err) - assert.Equal(t, tt.initialShards, out.CurrentShardCount) - assert.Equal(t, tt.targetShards, out.TargetShardCount) - assert.Len(t, openShards(t, b, name), tt.targetShards) + return + } + + require.NoError(t, err) + assert.Equal(t, tt.initialShards, out.CurrentShardCount) + assert.Equal(t, tt.targetShards, out.TargetShardCount) + assert.Len(t, openShards(t, b, name), tt.targetShards) + }) }) } } @@ -165,31 +170,34 @@ func TestScalingCap_HandlerValidationException(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newTestHandler(t) - name := "cap-h-" + tt.name + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + name := "cap-h-" + tt.name - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": name, - "ShardCount": tt.initialShard, - }) - require.Equal(t, http.StatusOK, rec.Code) - - rec = doRequest(t, h, "UpdateShardCount", map[string]any{ - "StreamName": name, - "TargetShardCount": tt.targetShard, - "ScalingType": "UNIFORM_SCALING", - }) - assert.Equal(t, tt.wantCode, rec.Code) + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": name, + "ShardCount": tt.initialShard, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) - if tt.wantType != "" { - var resp struct { - Type string `json:"__type"` - Message string `json:"message"` + rec = doRequest(t, h, "UpdateShardCount", map[string]any{ + "StreamName": name, + "TargetShardCount": tt.targetShard, + "ScalingType": "UNIFORM_SCALING", + }) + assert.Equal(t, tt.wantCode, rec.Code) + + if tt.wantType != "" { + var resp struct { + Type string `json:"__type"` + Message string `json:"message"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + assert.Equal(t, tt.wantType, resp.Type) + assert.NotEmpty(t, resp.Message) } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) - assert.Equal(t, tt.wantType, resp.Type) - assert.NotEmpty(t, resp.Message) - } + }) }) } } @@ -214,38 +222,41 @@ func TestChildLineage(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := kinesis.NewInMemoryBackend() - name := "lineage-" + tt.name - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: name, - ShardCount: tt.initialShards, - })) - - _, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ - StreamName: name, - TargetShardCount: tt.targetShards, - ScalingType: "UNIFORM_SCALING", - }) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := kinesis.NewInMemoryBackend() + name := "lineage-" + tt.name + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: name, + ShardCount: tt.initialShards, + })) + time.Sleep(streamSettleWait) - all := allShards(t, b, name) - byID := make(map[string]kinesis.ShardDescription, len(all)) - for _, s := range all { - byID[s.ShardID] = s - } + _, err := b.UpdateShardCount(context.Background(), &kinesis.UpdateShardCountInput{ + StreamName: name, + TargetShardCount: tt.targetShards, + ScalingType: "UNIFORM_SCALING", + }) + require.NoError(t, err) - open := openShards(t, b, name) - require.Len(t, open, tt.targetShards) + all := allShards(t, b, name) + byID := make(map[string]kinesis.ShardDescription, len(all)) + for _, s := range all { + byID[s.ShardID] = s + } - for _, child := range open { - require.NotEmpty(t, child.ParentShardID, - "open child %q must record its parent lineage", child.ShardID) + open := openShards(t, b, name) + require.Len(t, open, tt.targetShards) - parent, ok := byID[child.ParentShardID] - require.True(t, ok, "parent %q of child %q must exist", child.ParentShardID, child.ShardID) - assert.True(t, parent.Closed, - "parent %q of child %q must be CLOSED after resharding", parent.ShardID, child.ShardID) - } + for _, child := range open { + require.NotEmpty(t, child.ParentShardID, + "open child %q must record its parent lineage", child.ShardID) + + parent, ok := byID[child.ParentShardID] + require.True(t, ok, "parent %q of child %q must exist", child.ParentShardID, child.ShardID) + assert.True(t, parent.Closed, + "parent %q of child %q must be CLOSED after resharding", parent.ShardID, child.ShardID) + } + }) }) } } @@ -256,62 +267,65 @@ func TestChildLineage(t *testing.T) { func TestSequenceNumbersShardScopedAndOrdered(t *testing.T) { t.Parallel() - b := kinesis.NewInMemoryBackend() - name := "seq-scope-stream" - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: name, - ShardCount: 2, - })) - - open := openShards(t, b, name) - require.Len(t, open, 2) - - type putResult struct { - shardID string - seq string - } + synctest.Test(t, func(t *testing.T) { + b := kinesis.NewInMemoryBackend() + name := "seq-scope-stream" + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: name, + ShardCount: 2, + })) + time.Sleep(streamSettleWait) + + open := openShards(t, b, name) + require.Len(t, open, 2) + + type putResult struct { + shardID string + seq string + } - // Route two records to each shard using the shard's own starting hash key. - results := make([]putResult, 0, 2*len(open)) - for _, shard := range open { - for range 2 { - out, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: name, - PartitionKey: "pk-" + shard.ShardID, - ExplicitHashKey: shard.HashKeyRangeStart, - Data: []byte("payload"), - }) - require.NoError(t, err) - results = append(results, putResult{shardID: out.ShardID, seq: out.SequenceNumber}) + // Route two records to each shard using the shard's own starting hash key. + results := make([]putResult, 0, 2*len(open)) + for _, shard := range open { + for range 2 { + out, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: name, + PartitionKey: "pk-" + shard.ShardID, + ExplicitHashKey: shard.HashKeyRangeStart, + Data: []byte("payload"), + }) + require.NoError(t, err) + results = append(results, putResult{shardID: out.ShardID, seq: out.SequenceNumber}) + } } - } - require.Len(t, results, 4) - - // Group sequence numbers by the shard they landed on. - bySeq := map[string][]string{} - for _, r := range results { - require.GreaterOrEqual(t, len(r.seq), 40, "AWS-style sequence number is 40+ chars") - assert.Equal(t, "49", r.seq[:2], "AWS sequence numbers begin with the 49 version prefix") - bySeq[r.shardID] = append(bySeq[r.shardID], r.seq) - } - require.Len(t, bySeq, 2, "records must be shard-scoped across the two shards") + require.Len(t, results, 4) + + // Group sequence numbers by the shard they landed on. + bySeq := map[string][]string{} + for _, r := range results { + require.GreaterOrEqual(t, len(r.seq), 40, "AWS-style sequence number is 40+ chars") + assert.Equal(t, "49", r.seq[:2], "AWS sequence numbers begin with the 49 version prefix") + bySeq[r.shardID] = append(bySeq[r.shardID], r.seq) + } + require.Len(t, bySeq, 2, "records must be shard-scoped across the two shards") - // Within each shard, sequence numbers strictly increase. - for shardID, seqs := range bySeq { - require.Len(t, seqs, 2) - assert.Less(t, seqs[0], seqs[1], - "sequence numbers within shard %q must be monotonically ordered", shardID) - } + // Within each shard, sequence numbers strictly increase. + for shardID, seqs := range bySeq { + require.Len(t, seqs, 2) + assert.Less(t, seqs[0], seqs[1], + "sequence numbers within shard %q must be monotonically ordered", shardID) + } - // The encoded shard-index segment differs across the two shards, proving - // the sequence number is shard-scoped rather than a flat global counter. - shardIDs := make([]string, 0, len(bySeq)) - for id := range bySeq { - shardIDs = append(shardIDs, id) - } - segA := bySeq[shardIDs[0]][0][16:20] - segB := bySeq[shardIDs[1]][0][16:20] - assert.NotEqual(t, segA, segB, "sequence numbers must encode a per-shard segment") + // The encoded shard-index segment differs across the two shards, proving + // the sequence number is shard-scoped rather than a flat global counter. + shardIDs := make([]string, 0, len(bySeq)) + for id := range bySeq { + shardIDs = append(shardIDs, id) + } + segA := bySeq[shardIDs[0]][0][16:20] + segB := bySeq[shardIDs[1]][0][16:20] + assert.NotEqual(t, segA, segB, "sequence numbers must encode a per-shard segment") + }) } // TestMergeRequiresOpenShards asserts MergeShards verifies both @@ -331,51 +345,55 @@ func TestMergeRequiresOpenShards(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := kinesis.NewInMemoryBackend() - name := "merge-open-" + tt.name - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: name, - ShardCount: 2, - })) - - open := openShards(t, b, name) - require.Len(t, open, 2) - s0, s1 := open[0].ShardID, open[1].ShardID - - if tt.closeFirst { - // Split s0 to close it, leaving s1 open but s0 CLOSED. - mid := midHashKey(t, open[0]) - require.NoError(t, b.SplitShard(context.Background(), &kinesis.SplitShardInput{ - StreamName: name, - ShardToSplit: s0, - NewStartingHashKey: mid, + synctest.Test(t, func(t *testing.T) { + b := kinesis.NewInMemoryBackend() + name := "merge-open-" + tt.name + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: name, + ShardCount: 2, })) + time.Sleep(streamSettleWait) + + open := openShards(t, b, name) + require.Len(t, open, 2) + s0, s1 := open[0].ShardID, open[1].ShardID + + if tt.closeFirst { + // Split s0 to close it, leaving s1 open but s0 CLOSED. + mid := midHashKey(t, open[0]) + require.NoError(t, b.SplitShard(context.Background(), &kinesis.SplitShardInput{ + StreamName: name, + ShardToSplit: s0, + NewStartingHashKey: mid, + })) + time.Sleep(streamSettleWait) + + err := b.MergeShards(context.Background(), &kinesis.MergeShardsInput{ + StreamName: name, + ShardToMerge: s0, + AdjacentShardToMerge: s1, + }) + require.Error(t, err, "merging a CLOSED parent must be rejected") + require.ErrorIs(t, err, kinesis.ErrInvalidArgument) + + return + } err := b.MergeShards(context.Background(), &kinesis.MergeShardsInput{ StreamName: name, ShardToMerge: s0, AdjacentShardToMerge: s1, }) - require.Error(t, err, "merging a CLOSED parent must be rejected") - require.ErrorIs(t, err, kinesis.ErrInvalidArgument) - - return - } - - err := b.MergeShards(context.Background(), &kinesis.MergeShardsInput{ - StreamName: name, - ShardToMerge: s0, - AdjacentShardToMerge: s1, - }) - require.NoError(t, err) - - // The merged child records both parents in its lineage. - for _, s := range openShards(t, b, name) { - if s.ParentShardID != "" { - assert.Equal(t, s0, s.ParentShardID) - assert.Equal(t, s1, s.AdjacentParentShardID) + require.NoError(t, err) + + // The merged child records both parents in its lineage. + for _, s := range openShards(t, b, name) { + if s.ParentShardID != "" { + assert.Equal(t, s0, s.ParentShardID) + assert.Equal(t, s1, s.AdjacentParentShardID) + } } - } + }) }) } } @@ -409,39 +427,42 @@ func TestSplitStrictInterior(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := kinesis.NewInMemoryBackend() - name := "split-interior-" + tt.name - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: name, - ShardCount: 1, - })) - - open := openShards(t, b, name) - require.Len(t, open, 1) - shard := open[0] - - var hashKey string - switch tt.key { - case splitKeyStart: - hashKey = shard.HashKeyRangeStart - case splitKeyEnd: - hashKey = shard.HashKeyRangeEnd - case splitKeyMid: - hashKey = midHashKey(t, shard) - } + synctest.Test(t, func(t *testing.T) { + b := kinesis.NewInMemoryBackend() + name := "split-interior-" + tt.name + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: name, + ShardCount: 1, + })) + time.Sleep(streamSettleWait) + + open := openShards(t, b, name) + require.Len(t, open, 1) + shard := open[0] + + var hashKey string + switch tt.key { + case splitKeyStart: + hashKey = shard.HashKeyRangeStart + case splitKeyEnd: + hashKey = shard.HashKeyRangeEnd + case splitKeyMid: + hashKey = midHashKey(t, shard) + } - err := b.SplitShard(context.Background(), &kinesis.SplitShardInput{ - StreamName: name, - ShardToSplit: shard.ShardID, - NewStartingHashKey: hashKey, - }) - if tt.wantErr { - require.Error(t, err) - require.ErrorIs(t, err, kinesis.ErrInvalidArgument) + err := b.SplitShard(context.Background(), &kinesis.SplitShardInput{ + StreamName: name, + ShardToSplit: shard.ShardID, + NewStartingHashKey: hashKey, + }) + if tt.wantErr { + require.Error(t, err) + require.ErrorIs(t, err, kinesis.ErrInvalidArgument) - return - } - require.NoError(t, err) + return + } + require.NoError(t, err) + }) }) } } @@ -450,307 +471,329 @@ func TestSplitStrictInterior(t *testing.T) { func TestMergeShards_ARNSupport(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "arn-merge-stream", - "ShardCount": 2, - }) - require.Equal(t, http.StatusOK, rec.Code) - - // Get ARN and shard IDs. - rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "arn-merge-stream"}) - require.Equal(t, http.StatusOK, rec.Code) - - var descResp struct { - StreamDescription struct { - StreamARN string `json:"StreamARN"` - Shards []struct { - ShardID string `json:"ShardId"` - } `json:"Shards"` - } `json:"StreamDescription"` - } + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "arn-merge-stream", + "ShardCount": 2, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) + + // Get ARN and shard IDs. + rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "arn-merge-stream"}) + require.Equal(t, http.StatusOK, rec.Code) + + var descResp struct { + StreamDescription struct { + StreamARN string `json:"StreamARN"` + Shards []struct { + ShardID string `json:"ShardId"` + } `json:"Shards"` + } `json:"StreamDescription"` + } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - require.Len(t, descResp.StreamDescription.Shards, 2) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + require.Len(t, descResp.StreamDescription.Shards, 2) - streamARN := descResp.StreamDescription.StreamARN - shard0 := descResp.StreamDescription.Shards[0].ShardID - shard1 := descResp.StreamDescription.Shards[1].ShardID + streamARN := descResp.StreamDescription.StreamARN + shard0 := descResp.StreamDescription.Shards[0].ShardID + shard1 := descResp.StreamDescription.Shards[1].ShardID - // Merge using ARN (no StreamName). - rec = doRequest(t, h, "MergeShards", map[string]any{ - "StreamARN": streamARN, - "ShardToMerge": shard0, - "AdjacentShardToMerge": shard1, + // Merge using ARN (no StreamName). + rec = doRequest(t, h, "MergeShards", map[string]any{ + "StreamARN": streamARN, + "ShardToMerge": shard0, + "AdjacentShardToMerge": shard1, + }) + assert.Equal(t, http.StatusOK, rec.Code) }) - assert.Equal(t, http.StatusOK, rec.Code) } // TestSplitShard_ARNSupport verifies SplitShard accepts StreamARN. func TestSplitShard_ARNSupport(t *testing.T) { t.Parallel() - const splitKey = "170141183460469231731687303715884105728" + synctest.Test(t, func(t *testing.T) { + const splitKey = "170141183460469231731687303715884105728" - h := newTestHandler(t) + h := newTestHandler(t) - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "arn-split-stream", - "ShardCount": 1, - }) - require.Equal(t, http.StatusOK, rec.Code) - - rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "arn-split-stream"}) - require.Equal(t, http.StatusOK, rec.Code) - - var descResp struct { - StreamDescription struct { - StreamARN string `json:"StreamARN"` - Shards []struct { - ShardID string `json:"ShardId"` - } `json:"Shards"` - } `json:"StreamDescription"` - } + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "arn-split-stream", + "ShardCount": 1, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - require.Len(t, descResp.StreamDescription.Shards, 1) + rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "arn-split-stream"}) + require.Equal(t, http.StatusOK, rec.Code) - rec = doRequest(t, h, "SplitShard", map[string]any{ - "StreamARN": descResp.StreamDescription.StreamARN, - "ShardToSplit": descResp.StreamDescription.Shards[0].ShardID, - "NewStartingHashKey": splitKey, + var descResp struct { + StreamDescription struct { + StreamARN string `json:"StreamARN"` + Shards []struct { + ShardID string `json:"ShardId"` + } `json:"Shards"` + } `json:"StreamDescription"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + require.Len(t, descResp.StreamDescription.Shards, 1) + + rec = doRequest(t, h, "SplitShard", map[string]any{ + "StreamARN": descResp.StreamDescription.StreamARN, + "ShardToSplit": descResp.StreamDescription.Shards[0].ShardID, + "NewStartingHashKey": splitKey, + }) + assert.Equal(t, http.StatusOK, rec.Code) }) - assert.Equal(t, http.StatusOK, rec.Code) } func TestMergeShards_OnDemandRejected(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "on-demand-merge", - "ShardCount": 1, - "StreamModeDetails": map[string]any{"StreamMode": "ON_DEMAND"}, - }) + doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "on-demand-merge", + "ShardCount": 1, + "StreamModeDetails": map[string]any{"StreamMode": "ON_DEMAND"}, + }) + time.Sleep(streamSettleWait) - rec := doRequest(t, h, "MergeShards", map[string]any{ - "StreamName": "on-demand-merge", - "ShardToMerge": "shardId-000000000000", - "AdjacentShardToMerge": "shardId-000000000001", - }) - assert.Equal(t, http.StatusBadRequest, rec.Code) + rec := doRequest(t, h, "MergeShards", map[string]any{ + "StreamName": "on-demand-merge", + "ShardToMerge": "shardId-000000000000", + "AdjacentShardToMerge": "shardId-000000000001", + }) + assert.Equal(t, http.StatusBadRequest, rec.Code) - var resp struct { - Type string `json:"__type"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) - assert.Equal(t, "InvalidArgumentException", resp.Type) + var resp struct { + Type string `json:"__type"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + assert.Equal(t, "InvalidArgumentException", resp.Type) + }) } func TestSplitShard_OnDemandRejected(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "on-demand-split", - "ShardCount": 1, - "StreamModeDetails": map[string]any{"StreamMode": "ON_DEMAND"}, - }) + doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "on-demand-split", + "ShardCount": 1, + "StreamModeDetails": map[string]any{"StreamMode": "ON_DEMAND"}, + }) + time.Sleep(streamSettleWait) - rec := doRequest(t, h, "SplitShard", map[string]any{ - "StreamName": "on-demand-split", - "ShardToSplit": "shardId-000000000000", - "NewStartingHashKey": "170141183460469231731687303715884105728", - }) - assert.Equal(t, http.StatusBadRequest, rec.Code) + rec := doRequest(t, h, "SplitShard", map[string]any{ + "StreamName": "on-demand-split", + "ShardToSplit": "shardId-000000000000", + "NewStartingHashKey": "170141183460469231731687303715884105728", + }) + assert.Equal(t, http.StatusBadRequest, rec.Code) - var resp struct { - Type string `json:"__type"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) - assert.Equal(t, "InvalidArgumentException", resp.Type) + var resp struct { + Type string `json:"__type"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + assert.Equal(t, "InvalidArgumentException", resp.Type) + }) } func TestMergeShards_ProvisionedAllowed(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - doRequest(t, h, "CreateStream", map[string]any{"StreamName": "prov-merge", "ShardCount": 2}) + doRequest(t, h, "CreateStream", map[string]any{"StreamName": "prov-merge", "ShardCount": 2}) + time.Sleep(streamSettleWait) - b := h.Backend.(*kinesis.InMemoryBackend) - out, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "prov-merge"}) - require.NoError(t, err) - require.Len(t, out.Shards, 2) + b := h.Backend.(*kinesis.InMemoryBackend) + out, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "prov-merge"}) + require.NoError(t, err) + require.Len(t, out.Shards, 2) - rec := doRequest(t, h, "MergeShards", map[string]any{ - "StreamName": "prov-merge", - "ShardToMerge": out.Shards[0].ShardID, - "AdjacentShardToMerge": out.Shards[1].ShardID, + rec := doRequest(t, h, "MergeShards", map[string]any{ + "StreamName": "prov-merge", + "ShardToMerge": out.Shards[0].ShardID, + "AdjacentShardToMerge": out.Shards[1].ShardID, + }) + assert.Equal(t, http.StatusOK, rec.Code) }) - assert.Equal(t, http.StatusOK, rec.Code) } // TestMergeShards verifies that two adjacent shards can be merged into one. func TestMergeShards(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - // Create stream with 2 shards. - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "merge-stream", - "ShardCount": 2, - }) - require.Equal(t, http.StatusOK, rec.Code) + // Create stream with 2 shards. + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "merge-stream", + "ShardCount": 2, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) - // Get the shard IDs. - rec = doRequest(t, h, "DescribeStream", map[string]any{ - "StreamName": "merge-stream", - }) - require.Equal(t, http.StatusOK, rec.Code) - - var descResp struct { - StreamDescription struct { - Shards []struct { - ShardID string `json:"ShardId"` - } `json:"Shards"` - } `json:"StreamDescription"` - } + // Get the shard IDs. + rec = doRequest(t, h, "DescribeStream", map[string]any{ + "StreamName": "merge-stream", + }) + require.Equal(t, http.StatusOK, rec.Code) + + var descResp struct { + StreamDescription struct { + Shards []struct { + ShardID string `json:"ShardId"` + } `json:"Shards"` + } `json:"StreamDescription"` + } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - require.Len(t, descResp.StreamDescription.Shards, 2) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + require.Len(t, descResp.StreamDescription.Shards, 2) - shard0 := descResp.StreamDescription.Shards[0].ShardID - shard1 := descResp.StreamDescription.Shards[1].ShardID + shard0 := descResp.StreamDescription.Shards[0].ShardID + shard1 := descResp.StreamDescription.Shards[1].ShardID - // Merge the two shards. - rec = doRequest(t, h, "MergeShards", map[string]any{ - "StreamName": "merge-stream", - "ShardToMerge": shard0, - "AdjacentShardToMerge": shard1, - }) - require.Equal(t, http.StatusOK, rec.Code) + // Merge the two shards. + rec = doRequest(t, h, "MergeShards", map[string]any{ + "StreamName": "merge-stream", + "ShardToMerge": shard0, + "AdjacentShardToMerge": shard1, + }) + require.Equal(t, http.StatusOK, rec.Code) + + // AWS DescribeStream returns ALL shards including closed parent shards. + // After merging 2 → 1, expect 3 total: 2 closed parents + 1 open merged. + rec = doRequest(t, h, "DescribeStream", map[string]any{ + "StreamName": "merge-stream", + }) + require.Equal(t, http.StatusOK, rec.Code) - // AWS DescribeStream returns ALL shards including closed parent shards. - // After merging 2 → 1, expect 3 total: 2 closed parents + 1 open merged. - rec = doRequest(t, h, "DescribeStream", map[string]any{ - "StreamName": "merge-stream", + var descResp2 struct { + StreamDescription struct { + Shards []struct { + ShardID string `json:"ShardId"` + SequenceNumberRange struct { + EndingSequenceNumber string `json:"EndingSequenceNumber"` + } `json:"SequenceNumberRange"` + } `json:"Shards"` + } `json:"StreamDescription"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp2)) + assert.Len(t, descResp2.StreamDescription.Shards, 3) }) - require.Equal(t, http.StatusOK, rec.Code) - - var descResp2 struct { - StreamDescription struct { - Shards []struct { - ShardID string `json:"ShardId"` - SequenceNumberRange struct { - EndingSequenceNumber string `json:"EndingSequenceNumber"` - } `json:"SequenceNumberRange"` - } `json:"Shards"` - } `json:"StreamDescription"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp2)) - assert.Len(t, descResp2.StreamDescription.Shards, 3) } // TestMergeAndSplitShardIDs verifies that shard IDs remain unique after merge+split operations. func TestMergeAndSplitShardIDs(t *testing.T) { t.Parallel() - h := newTestHandler(t) - - // Create stream with 4 shards (IDs 0-3). - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "id-check-stream", - "ShardCount": 4, - }) - require.Equal(t, http.StatusOK, rec.Code) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - type shardEntry struct { - ShardID string `json:"ShardId"` - } - - // getAllShards returns all shards (open + closed) from DescribeStream. - getAllShards := func() []shardEntry { - r := doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "id-check-stream"}) - require.Equal(t, http.StatusOK, r.Code) + // Create stream with 4 shards (IDs 0-3). + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "id-check-stream", + "ShardCount": 4, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) - var d struct { - StreamDescription struct { - Shards []shardEntry `json:"Shards"` - } `json:"StreamDescription"` + type shardEntry struct { + ShardID string `json:"ShardId"` } - require.NoError(t, json.Unmarshal(r.Body.Bytes(), &d)) + // getAllShards returns all shards (open + closed) from DescribeStream. + getAllShards := func() []shardEntry { + r := doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "id-check-stream"}) + require.Equal(t, http.StatusOK, r.Code) - return d.StreamDescription.Shards - } + var d struct { + StreamDescription struct { + Shards []shardEntry `json:"Shards"` + } `json:"StreamDescription"` + } - // getOpenShards returns only open (non-closed) shards via ListShards. - getOpenShards := func() []shardEntry { - r := doRequest(t, h, "ListShards", map[string]any{"StreamName": "id-check-stream"}) - require.Equal(t, http.StatusOK, r.Code) + require.NoError(t, json.Unmarshal(r.Body.Bytes(), &d)) - var d struct { - Shards []shardEntry `json:"Shards"` + return d.StreamDescription.Shards } - require.NoError(t, json.Unmarshal(r.Body.Bytes(), &d)) + // getOpenShards returns only open (non-closed) shards via ListShards. + getOpenShards := func() []shardEntry { + r := doRequest(t, h, "ListShards", map[string]any{"StreamName": "id-check-stream"}) + require.Equal(t, http.StatusOK, r.Code) - return d.Shards - } + var d struct { + Shards []shardEntry `json:"Shards"` + } - all := getAllShards() - require.Len(t, all, 4) + require.NoError(t, json.Unmarshal(r.Body.Bytes(), &d)) - // Merge shards 0 and 1 → should produce shard with a new unique ID (4). - rec = doRequest(t, h, "MergeShards", map[string]any{ - "StreamName": "id-check-stream", - "ShardToMerge": all[0].ShardID, - "AdjacentShardToMerge": all[1].ShardID, - }) - require.Equal(t, http.StatusOK, rec.Code) + return d.Shards + } - // DescribeStream returns all shards (2 closed parents + 3 open = 5 total). - all = getAllShards() - require.Len(t, all, 5) + all := getAllShards() + require.Len(t, all, 4) - // Verify all IDs are unique. - seen := map[string]struct{}{} - for _, s := range all { - assert.NotContains(t, seen, s.ShardID, "duplicate shard ID %q detected", s.ShardID) - seen[s.ShardID] = struct{}{} - } + // Merge shards 0 and 1 → should produce shard with a new unique ID (4). + rec = doRequest(t, h, "MergeShards", map[string]any{ + "StreamName": "id-check-stream", + "ShardToMerge": all[0].ShardID, + "AdjacentShardToMerge": all[1].ShardID, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) + + // DescribeStream returns all shards (2 closed parents + 3 open = 5 total). + all = getAllShards() + require.Len(t, all, 5) + + // Verify all IDs are unique. + seen := map[string]struct{}{} + for _, s := range all { + assert.NotContains(t, seen, s.ShardID, "duplicate shard ID %q detected", s.ShardID) + seen[s.ShardID] = struct{}{} + } - // Split one of the open shards. Use a key strictly inside shard 2's range - // (170141183460469231731687303715884105728 to 255211775190703847598956918694523764991). - const splitKey = "200000000000000000000000000000000000000" - openShards := getOpenShards() - require.NotEmpty(t, openShards) - - // splitKey 200000000000000000000000000000000000000 falls in shard 2's range - // (170141183460469231731687303715884105728..255211775190703847597592248818726428671). - // openShards[0] is shard 2 (first open shard after merge). - rec = doRequest(t, h, "SplitShard", map[string]any{ - "StreamName": "id-check-stream", - "ShardToSplit": openShards[0].ShardID, - "NewStartingHashKey": splitKey, - }) - require.Equal(t, http.StatusOK, rec.Code) + // Split one of the open shards. Use a key strictly inside shard 2's range + // (170141183460469231731687303715884105728 to 255211775190703847598956918694523764991). + const splitKey = "200000000000000000000000000000000000000" + openShards := getOpenShards() + require.NotEmpty(t, openShards) + + // splitKey 200000000000000000000000000000000000000 falls in shard 2's range + // (170141183460469231731687303715884105728..255211775190703847597592248818726428671). + // openShards[0] is shard 2 (first open shard after merge). + rec = doRequest(t, h, "SplitShard", map[string]any{ + "StreamName": "id-check-stream", + "ShardToSplit": openShards[0].ShardID, + "NewStartingHashKey": splitKey, + }) + require.Equal(t, http.StatusOK, rec.Code) - // After split: 5 previous + 1 newly closed (parent of split) + 2 children = 7 total. - all = getAllShards() - require.Len(t, all, 7) + // After split: 5 previous + 1 newly closed (parent of split) + 2 children = 7 total. + all = getAllShards() + require.Len(t, all, 7) - // Verify all IDs are still unique after the split. - seen = map[string]struct{}{} - for _, s := range all { - assert.NotContains(t, seen, s.ShardID, "duplicate shard ID %q detected after split", s.ShardID) - seen[s.ShardID] = struct{}{} - } + // Verify all IDs are still unique after the split. + seen = map[string]struct{}{} + for _, s := range all { + assert.NotContains(t, seen, s.ShardID, "duplicate shard ID %q detected after split", s.ShardID) + seen[s.ShardID] = struct{}{} + } + }) } // TestMergeShards_Errors verifies error cases for MergeShards. @@ -785,13 +828,15 @@ func TestMergeShards_Errors(t *testing.T) { }, } - h := newTestHandler(t) + clock := newFakeClock(time.Now()) + h := newTestHandlerWithBackend(t, kinesis.NewInMemoryBackend().WithClock(clock.Now)) // Create the stream used in shard_not_found test. setup := doRequest(t, h, "CreateStream", map[string]any{ "StreamName": "merge-err-stream", "ShardCount": 1, }) require.Equal(t, http.StatusOK, setup.Code) + clock.Advance(streamSettleWait) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -816,48 +861,51 @@ func TestMergeShards_Errors(t *testing.T) { func TestSplitShard(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - // Create stream with 1 shard. - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "split-stream", - "ShardCount": 1, - }) - require.Equal(t, http.StatusOK, rec.Code) - - // Get shard details. - rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "split-stream"}) - require.Equal(t, http.StatusOK, rec.Code) - - var descResp struct { - StreamDescription struct { - Shards []struct { - ShardID string `json:"ShardId"` - } `json:"Shards"` - } `json:"StreamDescription"` - } + // Create stream with 1 shard. + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "split-stream", + "ShardCount": 1, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - require.Len(t, descResp.StreamDescription.Shards, 1) + // Get shard details. + rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "split-stream"}) + require.Equal(t, http.StatusOK, rec.Code) - shardID := descResp.StreamDescription.Shards[0].ShardID + var descResp struct { + StreamDescription struct { + Shards []struct { + ShardID string `json:"ShardId"` + } `json:"Shards"` + } `json:"StreamDescription"` + } - // Split at midpoint (half of 2^128). - const midKey = "170141183460469231731687303715884105728" + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + require.Len(t, descResp.StreamDescription.Shards, 1) - rec = doRequest(t, h, "SplitShard", map[string]any{ - "StreamName": "split-stream", - "ShardToSplit": shardID, - "NewStartingHashKey": midKey, + shardID := descResp.StreamDescription.Shards[0].ShardID + + // Split at midpoint (half of 2^128). + const midKey = "170141183460469231731687303715884105728" + + rec = doRequest(t, h, "SplitShard", map[string]any{ + "StreamName": "split-stream", + "ShardToSplit": shardID, + "NewStartingHashKey": midKey, + }) + require.Equal(t, http.StatusOK, rec.Code) + + // AWS DescribeStream returns ALL shards including closed parent. + // After splitting 1 → 2, expect 3 total: 1 closed parent + 2 open children. + rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "split-stream"}) + require.Equal(t, http.StatusOK, rec.Code) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + assert.Len(t, descResp.StreamDescription.Shards, 3) }) - require.Equal(t, http.StatusOK, rec.Code) - - // AWS DescribeStream returns ALL shards including closed parent. - // After splitting 1 → 2, expect 3 total: 1 closed parent + 2 open children. - rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "split-stream"}) - require.Equal(t, http.StatusOK, rec.Code) - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - assert.Len(t, descResp.StreamDescription.Shards, 3) } // TestSplitShard_Errors verifies error cases for SplitShard. @@ -892,12 +940,14 @@ func TestSplitShard_Errors(t *testing.T) { }, } - h := newTestHandler(t) + clock := newFakeClock(time.Now()) + h := newTestHandlerWithBackend(t, kinesis.NewInMemoryBackend().WithClock(clock.Now)) setup := doRequest(t, h, "CreateStream", map[string]any{ "StreamName": "split-err-stream", "ShardCount": 1, }) require.Equal(t, http.StatusOK, setup.Code) + clock.Advance(streamSettleWait) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -921,153 +971,168 @@ func TestSplitShard_Errors(t *testing.T) { func TestSplitShard_Basic(t *testing.T) { t.Parallel() - bk := kinesis.NewInMemoryBackend() - require.NoError( - t, - bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "split-stream", ShardCount: 1}), - ) - - // Get the initial shard list. - listOut, err := bk.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "split-stream"}) - require.NoError(t, err) - require.Len(t, listOut.Shards, 1) + synctest.Test(t, func(t *testing.T) { + bk := kinesis.NewInMemoryBackend() + require.NoError( + t, + bk.CreateStream( + context.Background(), + &kinesis.CreateStreamInput{StreamName: "split-stream", ShardCount: 1}, + ), + ) + time.Sleep(streamSettleWait) + + // Get the initial shard list. + listOut, err := bk.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "split-stream"}) + require.NoError(t, err) + require.Len(t, listOut.Shards, 1) - parentID := listOut.Shards[0].ShardID - // Split at a midpoint well inside the shard range. - splitKey := "170141183460469231731687303715884105728" // 2^127 / 1 + parentID := listOut.Shards[0].ShardID + // Split at a midpoint well inside the shard range. + splitKey := "170141183460469231731687303715884105728" // 2^127 / 1 - err = bk.SplitShard(context.Background(), &kinesis.SplitShardInput{ - StreamName: "split-stream", - ShardToSplit: parentID, - NewStartingHashKey: splitKey, - }) - require.NoError(t, err) + err = bk.SplitShard(context.Background(), &kinesis.SplitShardInput{ + StreamName: "split-stream", + ShardToSplit: parentID, + NewStartingHashKey: splitKey, + }) + require.NoError(t, err) - // Default list (open shards only) should now have 2 shards. - listOut, err = bk.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "split-stream"}) - require.NoError(t, err) - assert.Len(t, listOut.Shards, 2, "split should produce 2 open child shards") + // Default list (open shards only) should now have 2 shards. + listOut, err = bk.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "split-stream"}) + require.NoError(t, err) + assert.Len(t, listOut.Shards, 2, "split should produce 2 open child shards") - // Both child shards reference the parent. - for _, s := range listOut.Shards { - assert.Equal(t, parentID, s.ParentShardID) - } + // Both child shards reference the parent. + for _, s := range listOut.Shards { + assert.Equal(t, parentID, s.ParentShardID) + } - // Full list includes the closed parent + 2 children. - fullOut, err := bk.ListShards(context.Background(), &kinesis.ListShardsInput{ - StreamName: "split-stream", - ShardFilter: "FROM_TRIM_HORIZON", + // Full list includes the closed parent + 2 children. + fullOut, err := bk.ListShards(context.Background(), &kinesis.ListShardsInput{ + StreamName: "split-stream", + ShardFilter: "FROM_TRIM_HORIZON", + }) + require.NoError(t, err) + assert.Len(t, fullOut.Shards, 3, "FROM_TRIM_HORIZON should include closed parent") }) - require.NoError(t, err) - assert.Len(t, fullOut.Shards, 3, "FROM_TRIM_HORIZON should include closed parent") } func TestMergeShards_Basic(t *testing.T) { t.Parallel() - bk := kinesis.NewInMemoryBackend() - require.NoError( - t, - bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "merge-stream", ShardCount: 2}), - ) - - listOut, err := bk.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "merge-stream"}) - require.NoError(t, err) - require.Len(t, listOut.Shards, 2) + synctest.Test(t, func(t *testing.T) { + bk := kinesis.NewInMemoryBackend() + require.NoError( + t, + bk.CreateStream( + context.Background(), + &kinesis.CreateStreamInput{StreamName: "merge-stream", ShardCount: 2}, + ), + ) + time.Sleep(streamSettleWait) + + listOut, err := bk.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "merge-stream"}) + require.NoError(t, err) + require.Len(t, listOut.Shards, 2) - shard1 := listOut.Shards[0].ShardID - shard2 := listOut.Shards[1].ShardID + shard1 := listOut.Shards[0].ShardID + shard2 := listOut.Shards[1].ShardID - err = bk.MergeShards(context.Background(), &kinesis.MergeShardsInput{ - StreamName: "merge-stream", - ShardToMerge: shard1, - AdjacentShardToMerge: shard2, - }) - require.NoError(t, err) + err = bk.MergeShards(context.Background(), &kinesis.MergeShardsInput{ + StreamName: "merge-stream", + ShardToMerge: shard1, + AdjacentShardToMerge: shard2, + }) + require.NoError(t, err) - // Only 1 open shard (the merged one). - openOut, err := bk.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "merge-stream"}) - require.NoError(t, err) - assert.Len(t, openOut.Shards, 1) + // Only 1 open shard (the merged one). + openOut, err := bk.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "merge-stream"}) + require.NoError(t, err) + assert.Len(t, openOut.Shards, 1) - merged := openOut.Shards[0] - assert.Equal(t, shard1, merged.ParentShardID) - assert.Equal(t, shard2, merged.AdjacentParentShardID) + merged := openOut.Shards[0] + assert.Equal(t, shard1, merged.ParentShardID) + assert.Equal(t, shard2, merged.AdjacentParentShardID) - // Full list: 2 closed parents + 1 open merged = 3. - fullOut, err := bk.ListShards(context.Background(), &kinesis.ListShardsInput{ - StreamName: "merge-stream", - ShardFilter: "FROM_TRIM_HORIZON", + // Full list: 2 closed parents + 1 open merged = 3. + fullOut, err := bk.ListShards(context.Background(), &kinesis.ListShardsInput{ + StreamName: "merge-stream", + ShardFilter: "FROM_TRIM_HORIZON", + }) + require.NoError(t, err) + assert.Len(t, fullOut.Shards, 3) }) - require.NoError(t, err) - assert.Len(t, fullOut.Shards, 3) } func TestSplitShard_ParentClosedChildrenAcceptRecords(t *testing.T) { t.Parallel() - b := newParityBackend(t) - ctx := context.Background() + synctest.Test(t, func(t *testing.T) { + b := newParityBackend(t) + ctx := context.Background() - createParityStream(t, b, "split-test", 1) + createParityStream(t, b, "split-test", 1) + time.Sleep(streamSettleWait) - _, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ - StreamName: "split-test", - PartitionKey: "pre-split", - Data: []byte("before"), - }) - require.NoError(t, err) + _, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ + StreamName: "split-test", + PartitionKey: "pre-split", + Data: []byte("before"), + }) + require.NoError(t, err) - mid := new(big.Int).Lsh(big.NewInt(1), 127) + mid := new(big.Int).Lsh(big.NewInt(1), 127) - err = b.SplitShard(ctx, &kinesis.SplitShardInput{ - StreamName: "split-test", - ShardToSplit: "shardId-000000000000", - NewStartingHashKey: mid.String(), - }) - require.NoError(t, err) + err = b.SplitShard(ctx, &kinesis.SplitShardInput{ + StreamName: "split-test", + ShardToSplit: "shardId-000000000000", + NewStartingHashKey: mid.String(), + }) + require.NoError(t, err) - desc, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "split-test"}) - require.NoError(t, err) + desc, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "split-test"}) + require.NoError(t, err) - var parent *kinesis.ShardDescription - var children []*kinesis.ShardDescription + var parent *kinesis.ShardDescription + var children []*kinesis.ShardDescription - for i := range desc.Shards { - s := &desc.Shards[i] - if s.ShardID == "shardId-000000000000" { - parent = s - } else { - children = append(children, s) + for i := range desc.Shards { + s := &desc.Shards[i] + if s.ShardID == "shardId-000000000000" { + parent = s + } else { + children = append(children, s) + } } - } - require.NotNil(t, parent) - assert.True(t, parent.Closed, "parent shard must be closed after SplitShard") - assert.Len(t, children, 2, "SplitShard must produce exactly 2 child shards") + require.NotNil(t, parent) + assert.True(t, parent.Closed, "parent shard must be closed after SplitShard") + assert.Len(t, children, 2, "SplitShard must produce exactly 2 child shards") - putOut, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ - StreamName: "split-test", - PartitionKey: "post-split", - Data: []byte("after"), - }) - require.NoError(t, err) - assert.NotEqual(t, "shardId-000000000000", putOut.ShardID, - "new record must land in a child shard, not the closed parent") - - // Parent records still readable. - itOut, err := b.GetShardIterator(ctx, &kinesis.GetShardIteratorInput{ - StreamName: "split-test", - ShardID: "shardId-000000000000", - ShardIteratorType: "TRIM_HORIZON", - }) - require.NoError(t, err) + putOut, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ + StreamName: "split-test", + PartitionKey: "post-split", + Data: []byte("after"), + }) + require.NoError(t, err) + assert.NotEqual(t, "shardId-000000000000", putOut.ShardID, + "new record must land in a child shard, not the closed parent") + + // Parent records still readable. + itOut, err := b.GetShardIterator(ctx, &kinesis.GetShardIteratorInput{ + StreamName: "split-test", + ShardID: "shardId-000000000000", + ShardIteratorType: "TRIM_HORIZON", + }) + require.NoError(t, err) - rOut, err := b.GetRecords(ctx, &kinesis.GetRecordsInput{ShardIterator: itOut.ShardIterator}) - require.NoError(t, err) - assert.Len(t, rOut.Records, 1, "pre-split record must still be readable from the parent shard") - assert.Empty(t, rOut.NextShardIterator, - "closed shard with all records consumed must return empty NextShardIterator") + rOut, err := b.GetRecords(ctx, &kinesis.GetRecordsInput{ShardIterator: itOut.ShardIterator}) + require.NoError(t, err) + assert.Len(t, rOut.Records, 1, "pre-split record must still be readable from the parent shard") + assert.Empty(t, rOut.NextShardIterator, + "closed shard with all records consumed must return empty NextShardIterator") + }) } func TestMergeShards_AdjacencyRequired(t *testing.T) { @@ -1076,86 +1141,95 @@ func TestMergeShards_AdjacencyRequired(t *testing.T) { t.Run("adjacent shards merge successfully", func(t *testing.T) { t.Parallel() - b := newParityBackend(t) - ctx := context.Background() + synctest.Test(t, func(t *testing.T) { + b := newParityBackend(t) + ctx := context.Background() - createParityStream(t, b, "merge-ok", 3) + createParityStream(t, b, "merge-ok", 3) + time.Sleep(streamSettleWait) - err := b.MergeShards(ctx, &kinesis.MergeShardsInput{ - StreamName: "merge-ok", - ShardToMerge: "shardId-000000000000", - AdjacentShardToMerge: "shardId-000000000001", - }) - require.NoError(t, err) + err := b.MergeShards(ctx, &kinesis.MergeShardsInput{ + StreamName: "merge-ok", + ShardToMerge: "shardId-000000000000", + AdjacentShardToMerge: "shardId-000000000001", + }) + require.NoError(t, err) - desc, descErr := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "merge-ok"}) - require.NoError(t, descErr) + desc, descErr := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "merge-ok"}) + require.NoError(t, descErr) - closed, open := 0, 0 - for _, s := range desc.Shards { - if s.Closed { - closed++ - } else { - open++ + closed, open := 0, 0 + for _, s := range desc.Shards { + if s.Closed { + closed++ + } else { + open++ + } } - } - assert.Equal(t, 2, closed) - assert.Equal(t, 2, open) + assert.Equal(t, 2, closed) + assert.Equal(t, 2, open) + }) }) t.Run("non-adjacent shards are rejected", func(t *testing.T) { t.Parallel() - b := newParityBackend(t) - ctx := context.Background() + synctest.Test(t, func(t *testing.T) { + b := newParityBackend(t) + ctx := context.Background() - createParityStream(t, b, "merge-fail", 3) + createParityStream(t, b, "merge-fail", 3) + time.Sleep(streamSettleWait) - err := b.MergeShards(ctx, &kinesis.MergeShardsInput{ - StreamName: "merge-fail", - ShardToMerge: "shardId-000000000000", - AdjacentShardToMerge: "shardId-000000000002", + err := b.MergeShards(ctx, &kinesis.MergeShardsInput{ + StreamName: "merge-fail", + ShardToMerge: "shardId-000000000000", + AdjacentShardToMerge: "shardId-000000000002", + }) + assert.Error(t, err) }) - assert.Error(t, err) }) t.Run("merged shard spans combined hash range", func(t *testing.T) { t.Parallel() - b := newParityBackend(t) - ctx := context.Background() + synctest.Test(t, func(t *testing.T) { + b := newParityBackend(t) + ctx := context.Background() - createParityStream(t, b, "merge-range", 2) + createParityStream(t, b, "merge-range", 2) + time.Sleep(streamSettleWait) - desc0, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "merge-range"}) - require.NoError(t, err) + desc0, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "merge-range"}) + require.NoError(t, err) - s0Start := desc0.Shards[0].HashKeyRangeStart - s1End := desc0.Shards[1].HashKeyRangeEnd + s0Start := desc0.Shards[0].HashKeyRangeStart + s1End := desc0.Shards[1].HashKeyRangeEnd - err = b.MergeShards(ctx, &kinesis.MergeShardsInput{ - StreamName: "merge-range", - ShardToMerge: "shardId-000000000000", - AdjacentShardToMerge: "shardId-000000000001", - }) - require.NoError(t, err) + err = b.MergeShards(ctx, &kinesis.MergeShardsInput{ + StreamName: "merge-range", + ShardToMerge: "shardId-000000000000", + AdjacentShardToMerge: "shardId-000000000001", + }) + require.NoError(t, err) - desc1, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "merge-range"}) - require.NoError(t, err) + desc1, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "merge-range"}) + require.NoError(t, err) - var merged *kinesis.ShardDescription - for i := range desc1.Shards { - if !desc1.Shards[i].Closed { - merged = &desc1.Shards[i] + var merged *kinesis.ShardDescription + for i := range desc1.Shards { + if !desc1.Shards[i].Closed { + merged = &desc1.Shards[i] - break + break + } } - } - require.NotNil(t, merged) - assert.Equal(t, s0Start, merged.HashKeyRangeStart) - assert.Equal(t, s1End, merged.HashKeyRangeEnd) + require.NotNil(t, merged) + assert.Equal(t, s0Start, merged.HashKeyRangeStart) + assert.Equal(t, s1End, merged.HashKeyRangeEnd) + }) }) } @@ -1182,36 +1256,45 @@ func TestMergeShards_KeepsClosedShards(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: tt.streamName, - ShardCount: tt.shardCount, - })) - - out, err := b.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: tt.streamName}) - require.NoError(t, err) - require.Len(t, out.Shards, 2) - - require.NoError(t, b.MergeShards(context.Background(), &kinesis.MergeShardsInput{ - StreamName: tt.streamName, - ShardToMerge: out.Shards[0].ShardID, - AdjacentShardToMerge: out.Shards[1].ShardID, - })) + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: tt.streamName, + ShardCount: tt.shardCount, + })) + time.Sleep(streamSettleWait) + + out, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: tt.streamName}, + ) + require.NoError(t, err) + require.Len(t, out.Shards, 2) + + require.NoError(t, b.MergeShards(context.Background(), &kinesis.MergeShardsInput{ + StreamName: tt.streamName, + ShardToMerge: out.Shards[0].ShardID, + AdjacentShardToMerge: out.Shards[1].ShardID, + })) - out2, err := b.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: tt.streamName}) - require.NoError(t, err) + out2, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: tt.streamName}, + ) + require.NoError(t, err) - assert.Len(t, out2.Shards, tt.wantTotalShards) + assert.Len(t, out2.Shards, tt.wantTotalShards) - openCount := 0 - for _, s := range out2.Shards { - if !s.Closed { - openCount++ + openCount := 0 + for _, s := range out2.Shards { + if !s.Closed { + openCount++ + } } - } - assert.Equal(t, tt.wantOpenShards, openCount) + assert.Equal(t, tt.wantOpenShards, openCount) + }) }) } } @@ -1237,38 +1320,47 @@ func TestSplitShard_KeepsClosedShards(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: tt.streamName, - ShardCount: 1, - })) + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: tt.streamName, + ShardCount: 1, + })) + time.Sleep(streamSettleWait) - out, err := b.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: tt.streamName}) - require.NoError(t, err) - require.Len(t, out.Shards, 1) + out, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: tt.streamName}, + ) + require.NoError(t, err) + require.Len(t, out.Shards, 1) - newHashKey := "170141183460469231731687303715884105727" + newHashKey := "170141183460469231731687303715884105727" - require.NoError(t, b.SplitShard(context.Background(), &kinesis.SplitShardInput{ - StreamName: tt.streamName, - ShardToSplit: out.Shards[0].ShardID, - NewStartingHashKey: newHashKey, - })) + require.NoError(t, b.SplitShard(context.Background(), &kinesis.SplitShardInput{ + StreamName: tt.streamName, + ShardToSplit: out.Shards[0].ShardID, + NewStartingHashKey: newHashKey, + })) - out2, err := b.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: tt.streamName}) - require.NoError(t, err) + out2, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: tt.streamName}, + ) + require.NoError(t, err) - assert.Len(t, out2.Shards, tt.wantTotalShards) + assert.Len(t, out2.Shards, tt.wantTotalShards) - openCount := 0 - for _, s := range out2.Shards { - if !s.Closed { - openCount++ + openCount := 0 + for _, s := range out2.Shards { + if !s.Closed { + openCount++ + } } - } - assert.Equal(t, tt.wantOpenShards, openCount) + assert.Equal(t, tt.wantOpenShards, openCount) + }) }) } } diff --git a/services/kinesis/retention_iterator_test.go b/services/kinesis/retention_iterator_test.go index 3f36189c0a..2c5bb99bdb 100644 --- a/services/kinesis/retention_iterator_test.go +++ b/services/kinesis/retention_iterator_test.go @@ -3,6 +3,7 @@ package kinesis_test import ( "context" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -62,32 +63,38 @@ func TestGetShardIterator_HonoursRetentionWindow(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := kinesis.NewInMemoryBackend() - ctx := context.Background() - streamName := "retention-iter-" + tt.name - - require.NoError(t, b.CreateStream(ctx, &kinesis.CreateStreamInput{StreamName: streamName, ShardCount: 1})) - require.NoError(t, b.SetRetentionPeriodForTest(streamName, tt.retentionHrs)) - require.NoError(t, b.PushOldRecordForTest(streamName, 0, time.Duration(tt.expiredAgeHrs)*time.Hour)) - - _, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ - StreamName: streamName, - PartitionKey: "pk", - Data: []byte(tt.wantData), + synctest.Test(t, func(t *testing.T) { + b := kinesis.NewInMemoryBackend() + ctx := context.Background() + streamName := "retention-iter-" + tt.name + + require.NoError( + t, + b.CreateStream(ctx, &kinesis.CreateStreamInput{StreamName: streamName, ShardCount: 1}), + ) + time.Sleep(streamSettleWait) + require.NoError(t, b.SetRetentionPeriodForTest(streamName, tt.retentionHrs)) + require.NoError(t, b.PushOldRecordForTest(streamName, 0, time.Duration(tt.expiredAgeHrs)*time.Hour)) + + _, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ + StreamName: streamName, + PartitionKey: "pk", + Data: []byte(tt.wantData), + }) + require.NoError(t, err) + + input := tt.iterator() + input.StreamName = streamName + input.ShardID = "shardId-000000000000" + + itOut, err := b.GetShardIterator(ctx, &input) + require.NoError(t, err) + + rOut, err := b.GetRecords(ctx, &kinesis.GetRecordsInput{ShardIterator: itOut.ShardIterator}) + require.NoError(t, err) + require.Len(t, rOut.Records, 1, "only the record within the retention window should be returned") + assert.Equal(t, tt.wantData, string(rOut.Records[0].Data)) }) - require.NoError(t, err) - - input := tt.iterator() - input.StreamName = streamName - input.ShardID = "shardId-000000000000" - - itOut, err := b.GetShardIterator(ctx, &input) - require.NoError(t, err) - - rOut, err := b.GetRecords(ctx, &kinesis.GetRecordsInput{ShardIterator: itOut.ShardIterator}) - require.NoError(t, err) - require.Len(t, rOut.Records, 1, "only the record within the retention window should be returned") - assert.Equal(t, tt.wantData, string(rOut.Records[0].Data)) }) } } @@ -103,11 +110,20 @@ func TestGetShardIterator_HonoursRetentionWindow(t *testing.T) { func TestGetShardIterator_RetentionDecreaseAppliesBeforeJanitorSweep(t *testing.T) { t.Parallel() + synctest.Test(t, func(t *testing.T) { + testGetShardIteratorRetentionDecreaseAppliesBeforeJanitorSweep(t) + }) +} + +func testGetShardIteratorRetentionDecreaseAppliesBeforeJanitorSweep(t *testing.T) { + t.Helper() + b := kinesis.NewInMemoryBackend() ctx := context.Background() streamName := "retention-decrease-before-sweep" require.NoError(t, b.CreateStream(ctx, &kinesis.CreateStreamInput{StreamName: streamName, ShardCount: 1})) + time.Sleep(streamSettleWait) // Retention can only ever decrease to minRetentionHours (24h) at the // lowest, so widen it first: raise to 48h, then a 30h-old record sits @@ -158,11 +174,20 @@ func TestGetShardIterator_RetentionDecreaseAppliesBeforeJanitorSweep(t *testing. func TestSubscribeToShard_HonoursRetentionWindow(t *testing.T) { t.Parallel() + synctest.Test(t, func(t *testing.T) { + testSubscribeToShardHonoursRetentionWindow(t) + }) +} + +func testSubscribeToShardHonoursRetentionWindow(t *testing.T) { + t.Helper() + b := kinesis.NewInMemoryBackend() ctx := context.Background() streamName := "subscribe-retention" require.NoError(t, b.CreateStream(ctx, &kinesis.CreateStreamInput{StreamName: streamName, ShardCount: 1})) + time.Sleep(streamSettleWait) require.NoError(t, b.SetRetentionPeriodForTest(streamName, 1)) require.NoError(t, b.PushOldRecordForTest(streamName, 0, 2*time.Hour)) diff --git a/services/kinesis/ring_buffer_test.go b/services/kinesis/ring_buffer_test.go index afb19dc37e..67d6bd9bb2 100644 --- a/services/kinesis/ring_buffer_test.go +++ b/services/kinesis/ring_buffer_test.go @@ -3,6 +3,8 @@ package kinesis_test import ( "context" "testing" + "testing/synctest" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -13,77 +15,80 @@ import ( func TestKinesisBackend_FindSequencePositionGaps(t *testing.T) { t.Parallel() - bk := kinesis.NewInMemoryBackend() - require.NoError(t, bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "gap-stream"})) + synctest.Test(t, func(t *testing.T) { + bk := kinesis.NewInMemoryBackend() + require.NoError(t, bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "gap-stream"})) + time.Sleep(streamSettleWait) - desc, err := bk.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: "gap-stream"}) - require.NoError(t, err) - shardID := desc.Shards[0].ShardID + desc, err := bk.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: "gap-stream"}) + require.NoError(t, err) + shardID := desc.Shards[0].ShardID - // Put a record - get seq "00000000000000000001" - out1, err := bk.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "gap-stream", - PartitionKey: "pk", - Data: []byte("first"), - }) - require.NoError(t, err) + // Put a record - get seq "00000000000000000001" + out1, err := bk.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "gap-stream", + PartitionKey: "pk", + Data: []byte("first"), + }) + require.NoError(t, err) - // Put another - get seq "00000000000000000002" - out2, err := bk.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: "gap-stream", - PartitionKey: "pk", - Data: []byte("second"), - }) - require.NoError(t, err) + // Put another - get seq "00000000000000000002" + out2, err := bk.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: "gap-stream", + PartitionKey: "pk", + Data: []byte("second"), + }) + require.NoError(t, err) - // AT_SEQUENCE_NUMBER for out1.SequenceNumber should return index 0 (inclusive) - iterOut, err := bk.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ - StreamName: "gap-stream", - ShardID: shardID, - ShardIteratorType: "AT_SEQUENCE_NUMBER", - StartingSequenceNumber: out1.SequenceNumber, - }) - require.NoError(t, err) + // AT_SEQUENCE_NUMBER for out1.SequenceNumber should return index 0 (inclusive) + iterOut, err := bk.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + StreamName: "gap-stream", + ShardID: shardID, + ShardIteratorType: "AT_SEQUENCE_NUMBER", + StartingSequenceNumber: out1.SequenceNumber, + }) + require.NoError(t, err) - records, err := bk.GetRecords(context.Background(), &kinesis.GetRecordsInput{ - ShardIterator: iterOut.ShardIterator, - Limit: 10, - }) - require.NoError(t, err) - require.Len(t, records.Records, 2) - assert.Equal(t, out1.SequenceNumber, records.Records[0].SequenceNumber) + records, err := bk.GetRecords(context.Background(), &kinesis.GetRecordsInput{ + ShardIterator: iterOut.ShardIterator, + Limit: 10, + }) + require.NoError(t, err) + require.Len(t, records.Records, 2) + assert.Equal(t, out1.SequenceNumber, records.Records[0].SequenceNumber) - // AFTER_SEQUENCE_NUMBER for out1 should start at index 1 - iterOut2, err := bk.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ - StreamName: "gap-stream", - ShardID: shardID, - ShardIteratorType: "AFTER_SEQUENCE_NUMBER", - StartingSequenceNumber: out1.SequenceNumber, - }) - require.NoError(t, err) + // AFTER_SEQUENCE_NUMBER for out1 should start at index 1 + iterOut2, err := bk.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + StreamName: "gap-stream", + ShardID: shardID, + ShardIteratorType: "AFTER_SEQUENCE_NUMBER", + StartingSequenceNumber: out1.SequenceNumber, + }) + require.NoError(t, err) - records2, err := bk.GetRecords(context.Background(), &kinesis.GetRecordsInput{ - ShardIterator: iterOut2.ShardIterator, - Limit: 10, - }) - require.NoError(t, err) - require.Len(t, records2.Records, 1) - assert.Equal(t, out2.SequenceNumber, records2.Records[0].SequenceNumber) + records2, err := bk.GetRecords(context.Background(), &kinesis.GetRecordsInput{ + ShardIterator: iterOut2.ShardIterator, + Limit: 10, + }) + require.NoError(t, err) + require.Len(t, records2.Records, 1) + assert.Equal(t, out2.SequenceNumber, records2.Records[0].SequenceNumber) - // AT_SEQUENCE_NUMBER for a sequence number that is lexicographically larger than all records - // should return empty (positions at end) - iterOut3, err := bk.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ - StreamName: "gap-stream", - ShardID: shardID, - ShardIteratorType: "AT_SEQUENCE_NUMBER", - StartingSequenceNumber: "99999999999999999999", - }) - require.NoError(t, err) + // AT_SEQUENCE_NUMBER for a sequence number that is lexicographically larger than all records + // should return empty (positions at end) + iterOut3, err := bk.GetShardIterator(context.Background(), &kinesis.GetShardIteratorInput{ + StreamName: "gap-stream", + ShardID: shardID, + ShardIteratorType: "AT_SEQUENCE_NUMBER", + StartingSequenceNumber: "99999999999999999999", + }) + require.NoError(t, err) - records3, err := bk.GetRecords(context.Background(), &kinesis.GetRecordsInput{ - ShardIterator: iterOut3.ShardIterator, - Limit: 10, + records3, err := bk.GetRecords(context.Background(), &kinesis.GetRecordsInput{ + ShardIterator: iterOut3.ShardIterator, + Limit: 10, + }) + require.NoError(t, err) + assert.Empty(t, records3.Records) }) - require.NoError(t, err) - assert.Empty(t, records3.Records) } diff --git a/services/kinesis/shard_iterators.go b/services/kinesis/shard_iterators.go index 75324b98f2..e196f87cb2 100644 --- a/services/kinesis/shard_iterators.go +++ b/services/kinesis/shard_iterators.go @@ -55,6 +55,10 @@ func (b *InMemoryBackend) GetShardIterator( b.mu.RUnlock() defer stream.mu.RUnlock() + if streamEffectivelyGone(stream, b.nowFunc()) { + return nil, ErrStreamNotFound + } + // Find the shard shard := findShard(stream.Shards, input.ShardID) diff --git a/services/kinesis/shard_iterators_test.go b/services/kinesis/shard_iterators_test.go index 12aabfc253..4c6cc159a2 100644 --- a/services/kinesis/shard_iterators_test.go +++ b/services/kinesis/shard_iterators_test.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -68,16 +69,23 @@ func TestGetShardIterator_ByARN(t *testing.T) { func TestGetShardIterator_AllTypes(t *testing.T) { t.Parallel() - b := newParityBackend(t) + // A real-time-based fakeClock (not synctest): the TRIM_HORIZON subtest + // below computes a retention cutoff from "now" outside this setup, so + // setup and subtests must share one real-time-rooted clock (a synctest + // bubble's fake epoch would make the just-written records look expired + // once read back with a real time.Now()). + clock := newFakeClock(time.Now()) + b := kinesis.NewInMemoryBackend().WithClock(clock.Now) ctx := context.Background() createParityStream(t, b, "iter-types", 1) + clock.Advance(streamSettleWait) seqs := make([]string, 0, 5) timestamps := make([]time.Time, 0, 5) for i := range 5 { - time.Sleep(time.Millisecond) + clock.Advance(time.Millisecond) out, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ StreamName: "iter-types", PartitionKey: "pk", @@ -85,7 +93,7 @@ func TestGetShardIterator_AllTypes(t *testing.T) { }) require.NoError(t, err) seqs = append(seqs, out.SequenceNumber) - timestamps = append(timestamps, time.Now()) + timestamps = append(timestamps, clock.Now()) } shardID := "shardId-000000000000" @@ -228,6 +236,14 @@ func TestGetShardIteratorNonExistentShard(t *testing.T) { func TestGetShardIteratorAtTimestamp(t *testing.T) { t.Parallel() + synctest.Test(t, func(t *testing.T) { + testGetShardIteratorAtTimestamp(t) + }) +} + +func testGetShardIteratorAtTimestamp(t *testing.T) { + t.Helper() + h := newTestHandler(t) rec := doRequest(t, h, "CreateStream", map[string]any{ @@ -235,6 +251,7 @@ func TestGetShardIteratorAtTimestamp(t *testing.T) { "ShardCount": 1, }) require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) // Get shard ID rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "ts-stream"}) @@ -364,8 +381,17 @@ func TestGetShardIterator_AtTimestampNilRejectedAtBackend(t *testing.T) { func TestGetRecords_NextShardIteratorHasExpiry(t *testing.T) { t.Parallel() + synctest.Test(t, func(t *testing.T) { + testGetRecordsNextShardIteratorHasExpiry(t) + }) +} + +func testGetRecordsNextShardIteratorHasExpiry(t *testing.T) { + t.Helper() + h := newTestHandler(t) doRequest(t, h, "CreateStream", map[string]any{"StreamName": "next-iter-ttl-stream", "ShardCount": 1}) + time.Sleep(streamSettleWait) b := h.Backend.(*kinesis.InMemoryBackend) ctx := context.Background() diff --git a/services/kinesis/shards_test.go b/services/kinesis/shards_test.go index 35121f6faf..4c6884847a 100644 --- a/services/kinesis/shards_test.go +++ b/services/kinesis/shards_test.go @@ -6,6 +6,8 @@ import ( "fmt" "net/http" "testing" + "testing/synctest" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -136,7 +138,8 @@ func TestHashRouting_MD5_MatchesExpectedShard(t *testing.T) { t.Run("known partition keys land on MD5-predicted shard", func(t *testing.T) { t.Parallel() - b := newParityBackend(t) + clock := newFakeClock(time.Now()) + b := kinesis.NewInMemoryBackend().WithClock(clock.Now) ctx := context.Background() const ( @@ -145,6 +148,7 @@ func TestHashRouting_MD5_MatchesExpectedShard(t *testing.T) { ) createParityStream(t, b, streamName, shardCount) + clock.Advance(streamSettleWait) partitionKeys := []string{"hello", "world", "foo", "bar", "kinesis", "test-key-99"} @@ -171,10 +175,12 @@ func TestHashRouting_MD5_MatchesExpectedShard(t *testing.T) { t.Run("multi-shard distribution: records spread across shards", func(t *testing.T) { t.Parallel() - b := newParityBackend(t) + clock := newFakeClock(time.Now()) + b := kinesis.NewInMemoryBackend().WithClock(clock.Now) ctx := context.Background() createParityStream(t, b, "md5-spread", 2) + clock.Advance(streamSettleWait) shardCounts := map[string]int{} @@ -197,29 +203,32 @@ func TestHashRouting_MD5_MatchesExpectedShard(t *testing.T) { func TestSequenceNumber_MonotonicWithinShard(t *testing.T) { t.Parallel() - b := newParityBackend(t) - ctx := context.Background() + synctest.Test(t, func(t *testing.T) { + b := newParityBackend(t) + ctx := context.Background() - createParityStream(t, b, "seq-mono", 1) + createParityStream(t, b, "seq-mono", 1) + time.Sleep(streamSettleWait) - const recordCount = 10 - seqs := make([]string, 0, recordCount) + const recordCount = 10 + seqs := make([]string, 0, recordCount) - for i := range recordCount { - out, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ - StreamName: "seq-mono", - PartitionKey: "pk", - Data: fmt.Appendf(nil, "data-%d", i), - }) - require.NoError(t, err) - seqs = append(seqs, out.SequenceNumber) - } + for i := range recordCount { + out, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ + StreamName: "seq-mono", + PartitionKey: "pk", + Data: fmt.Appendf(nil, "data-%d", i), + }) + require.NoError(t, err) + seqs = append(seqs, out.SequenceNumber) + } - for i := 1; i < recordCount; i++ { - assert.Greater(t, seqs[i], seqs[i-1], - "sequence numbers must be strictly increasing: seqs[%d]=%s seqs[%d]=%s", - i, seqs[i], i-1, seqs[i-1]) - } + for i := 1; i < recordCount; i++ { + assert.Greater(t, seqs[i], seqs[i-1], + "sequence numbers must be strictly increasing: seqs[%d]=%s seqs[%d]=%s", + i, seqs[i], i-1, seqs[i-1]) + } + }) } func TestListShards_Pagination_Complete(t *testing.T) { @@ -277,28 +286,31 @@ func TestCountOpenShards_ExcludesClosedShards(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: tt.streamName, - ShardCount: tt.shardCount, - })) - - if tt.doMerge { - out, err := b.DescribeStream( - context.Background(), - &kinesis.DescribeStreamInput{StreamName: tt.streamName}, - ) - require.NoError(t, err) - require.NoError(t, b.MergeShards(context.Background(), &kinesis.MergeShardsInput{ - StreamName: tt.streamName, - ShardToMerge: out.Shards[0].ShardID, - AdjacentShardToMerge: out.Shards[1].ShardID, + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: tt.streamName, + ShardCount: tt.shardCount, })) - } + time.Sleep(streamSettleWait) + + if tt.doMerge { + out, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: tt.streamName}, + ) + require.NoError(t, err) + require.NoError(t, b.MergeShards(context.Background(), &kinesis.MergeShardsInput{ + StreamName: tt.streamName, + ShardToMerge: out.Shards[0].ShardID, + AdjacentShardToMerge: out.Shards[1].ShardID, + })) + } - assert.Equal(t, tt.wantCount, b.CountOpenShards(context.Background())) + assert.Equal(t, tt.wantCount, b.CountOpenShards(context.Background())) + }) }) } } @@ -355,35 +367,41 @@ func TestListShards_IncludesClosedShards(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: tt.streamName, - ShardCount: tt.shardCount, - })) + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: tt.streamName, + ShardCount: tt.shardCount, + })) + time.Sleep(streamSettleWait) - ds, err := b.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: tt.streamName}) - require.NoError(t, err) + ds, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: tt.streamName}, + ) + require.NoError(t, err) - require.NoError(t, b.MergeShards(context.Background(), &kinesis.MergeShardsInput{ - StreamName: tt.streamName, - ShardToMerge: ds.Shards[0].ShardID, - AdjacentShardToMerge: ds.Shards[1].ShardID, - })) + require.NoError(t, b.MergeShards(context.Background(), &kinesis.MergeShardsInput{ + StreamName: tt.streamName, + ShardToMerge: ds.Shards[0].ShardID, + AdjacentShardToMerge: ds.Shards[1].ShardID, + })) - // Use FROM_TRIM_HORIZON filter to retrieve all shards including closed ones. - out, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{ - StreamName: tt.streamName, - ShardFilter: "FROM_TRIM_HORIZON", - }) - require.NoError(t, err) - assert.Len(t, out.Shards, tt.wantTotalShards) + // Use FROM_TRIM_HORIZON filter to retrieve all shards including closed ones. + out, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{ + StreamName: tt.streamName, + ShardFilter: "FROM_TRIM_HORIZON", + }) + require.NoError(t, err) + assert.Len(t, out.Shards, tt.wantTotalShards) - // Without a filter, only open shards are returned (matching AWS default behavior). - openOut, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: tt.streamName}) - require.NoError(t, err) - assert.Len(t, openOut.Shards, 1, "expected only the 1 open (merged) shard without filter") + // Without a filter, only open shards are returned (matching AWS default behavior). + openOut, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: tt.streamName}) + require.NoError(t, err) + assert.Len(t, openOut.Shards, 1, "expected only the 1 open (merged) shard without filter") + }) }) } } @@ -402,40 +420,49 @@ func TestShardDescription_ParentShardId(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: tt.streamName, - ShardCount: 2, - })) + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: tt.streamName, + ShardCount: 2, + })) + time.Sleep(streamSettleWait) - ds, err := b.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: tt.streamName}) - require.NoError(t, err) + ds, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: tt.streamName}, + ) + require.NoError(t, err) - shard0ID := ds.Shards[0].ShardID - shard1ID := ds.Shards[1].ShardID + shard0ID := ds.Shards[0].ShardID + shard1ID := ds.Shards[1].ShardID - require.NoError(t, b.MergeShards(context.Background(), &kinesis.MergeShardsInput{ - StreamName: tt.streamName, - ShardToMerge: shard0ID, - AdjacentShardToMerge: shard1ID, - })) + require.NoError(t, b.MergeShards(context.Background(), &kinesis.MergeShardsInput{ + StreamName: tt.streamName, + ShardToMerge: shard0ID, + AdjacentShardToMerge: shard1ID, + })) - ds2, err := b.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: tt.streamName}) - require.NoError(t, err) + ds2, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: tt.streamName}, + ) + require.NoError(t, err) - var mergedShard *kinesis.ShardDescription - for i := range ds2.Shards { - if !ds2.Shards[i].Closed { - mergedShard = &ds2.Shards[i] + var mergedShard *kinesis.ShardDescription + for i := range ds2.Shards { + if !ds2.Shards[i].Closed { + mergedShard = &ds2.Shards[i] - break + break + } } - } - require.NotNil(t, mergedShard) - assert.Equal(t, shard0ID, mergedShard.ParentShardID) - assert.Equal(t, shard1ID, mergedShard.AdjacentParentShardID) + require.NotNil(t, mergedShard) + assert.Equal(t, shard0ID, mergedShard.ParentShardID) + assert.Equal(t, shard1ID, mergedShard.AdjacentParentShardID) + }) }) } } @@ -454,33 +481,36 @@ func TestNextSeq_Serialized(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := kinesis.NewInMemoryBackend() - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: tt.streamName, - ShardCount: 1, - })) + synctest.Test(t, func(t *testing.T) { + b := kinesis.NewInMemoryBackend() + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: tt.streamName, + ShardCount: 1, + })) + time.Sleep(streamSettleWait) - out, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: tt.streamName, - PartitionKey: "key", - Data: []byte("data"), - }) - require.NoError(t, err) - firstSeq := out.SequenceNumber + out, err := b.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: tt.streamName, + PartitionKey: "key", + Data: []byte("data"), + }) + require.NoError(t, err) + firstSeq := out.SequenceNumber - snapshot := b.Snapshot(t.Context()) - require.NotNil(t, snapshot) + snapshot := b.Snapshot(t.Context()) + require.NotNil(t, snapshot) - b2 := kinesis.NewInMemoryBackend() - require.NoError(t, b2.Restore(t.Context(), snapshot)) + b2 := kinesis.NewInMemoryBackend() + require.NoError(t, b2.Restore(t.Context(), snapshot)) - out2, err := b2.PutRecord(context.Background(), &kinesis.PutRecordInput{ - StreamName: tt.streamName, - PartitionKey: "key2", - Data: []byte("data2"), + out2, err := b2.PutRecord(context.Background(), &kinesis.PutRecordInput{ + StreamName: tt.streamName, + PartitionKey: "key2", + Data: []byte("data2"), + }) + require.NoError(t, err) + assert.NotEqual(t, firstSeq, out2.SequenceNumber) }) - require.NoError(t, err) - assert.NotEqual(t, firstSeq, out2.SequenceNumber) }) } } @@ -652,47 +682,50 @@ func TestListShards_MaxResults_ExactlyFits(t *testing.T) { func TestListShards_WithMaxResults_PlusClosedShards(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + b := h.Backend.(*kinesis.InMemoryBackend) - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "listshards-closed-paged", - ShardCount: 2, - })) + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "listshards-closed-paged", + ShardCount: 2, + })) + time.Sleep(streamSettleWait) - out, err := b.DescribeStream( - context.Background(), - &kinesis.DescribeStreamInput{StreamName: "listshards-closed-paged"}, - ) - require.NoError(t, err) - - // Merge to produce 1 open + 2 closed = 3 total. - require.NoError(t, b.MergeShards(context.Background(), &kinesis.MergeShardsInput{ - StreamName: "listshards-closed-paged", - ShardToMerge: out.Shards[0].ShardID, - AdjacentShardToMerge: out.Shards[1].ShardID, - })) + out, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: "listshards-closed-paged"}, + ) + require.NoError(t, err) - // FROM_TRIM_HORIZON includes all shards; MaxResults=2 → page 1 of 2. - list, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{ - StreamName: "listshards-closed-paged", - ShardFilter: "FROM_TRIM_HORIZON", - MaxResults: 2, - }) - require.NoError(t, err) - assert.Len(t, list.Shards, 2) - assert.NotEmpty(t, list.NextToken) - - // Page 2. - list2, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{ - StreamName: "listshards-closed-paged", - ShardFilter: "FROM_TRIM_HORIZON", - MaxResults: 2, - NextToken: list.NextToken, + // Merge to produce 1 open + 2 closed = 3 total. + require.NoError(t, b.MergeShards(context.Background(), &kinesis.MergeShardsInput{ + StreamName: "listshards-closed-paged", + ShardToMerge: out.Shards[0].ShardID, + AdjacentShardToMerge: out.Shards[1].ShardID, + })) + + // FROM_TRIM_HORIZON includes all shards; MaxResults=2 → page 1 of 2. + list, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{ + StreamName: "listshards-closed-paged", + ShardFilter: "FROM_TRIM_HORIZON", + MaxResults: 2, + }) + require.NoError(t, err) + assert.Len(t, list.Shards, 2) + assert.NotEmpty(t, list.NextToken) + + // Page 2. + list2, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{ + StreamName: "listshards-closed-paged", + ShardFilter: "FROM_TRIM_HORIZON", + MaxResults: 2, + NextToken: list.NextToken, + }) + require.NoError(t, err) + assert.Len(t, list2.Shards, 1) + assert.Empty(t, list2.NextToken) }) - require.NoError(t, err) - assert.Len(t, list2.Shards, 1) - assert.Empty(t, list2.NextToken) } func TestListShards_NextToken_SinglePage(t *testing.T) { @@ -750,36 +783,42 @@ func TestListShards_NextToken_OddPageSize(t *testing.T) { func TestListShards_ClosedShards_IncludedWithFilter(t *testing.T) { t.Parallel() - b := kinesis.NewInMemoryBackend() - require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: "listshards-closed-filter", - ShardCount: 2, - })) - - ds, err := b.DescribeStream( - context.Background(), - &kinesis.DescribeStreamInput{StreamName: "listshards-closed-filter"}, - ) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := kinesis.NewInMemoryBackend() + require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: "listshards-closed-filter", + ShardCount: 2, + })) + time.Sleep(streamSettleWait) + + ds, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: "listshards-closed-filter"}, + ) + require.NoError(t, err) - require.NoError(t, b.MergeShards(context.Background(), &kinesis.MergeShardsInput{ - StreamName: "listshards-closed-filter", - ShardToMerge: ds.Shards[0].ShardID, - AdjacentShardToMerge: ds.Shards[1].ShardID, - })) + require.NoError(t, b.MergeShards(context.Background(), &kinesis.MergeShardsInput{ + StreamName: "listshards-closed-filter", + ShardToMerge: ds.Shards[0].ShardID, + AdjacentShardToMerge: ds.Shards[1].ShardID, + })) - // Default: only open shards. - open, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{StreamName: "listshards-closed-filter"}) - require.NoError(t, err) - assert.Len(t, open.Shards, 1) + // Default: only open shards. + open, err := b.ListShards( + context.Background(), + &kinesis.ListShardsInput{StreamName: "listshards-closed-filter"}, + ) + require.NoError(t, err) + assert.Len(t, open.Shards, 1) - // FROM_TRIM_HORIZON: all shards. - all, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{ - StreamName: "listshards-closed-filter", - ShardFilter: "FROM_TRIM_HORIZON", + // FROM_TRIM_HORIZON: all shards. + all, err := b.ListShards(context.Background(), &kinesis.ListShardsInput{ + StreamName: "listshards-closed-filter", + ShardFilter: "FROM_TRIM_HORIZON", + }) + require.NoError(t, err) + assert.Len(t, all.Shards, 3) }) - require.NoError(t, err) - assert.Len(t, all.Shards, 3) } func TestListShards_ExclusiveStart_WithMaxResults(t *testing.T) { @@ -813,48 +852,51 @@ func TestListShards_ExclusiveStart_WithMaxResults(t *testing.T) { func TestListShards_ShardFilterType_AfterShardID(t *testing.T) { t.Parallel() - b := kinesis.NewInMemoryBackend() - ctx := context.Background() - require.NoError(t, b.CreateStream(ctx, &kinesis.CreateStreamInput{ - StreamName: "after-shard-id-stream", - ShardCount: 4, - })) + synctest.Test(t, func(t *testing.T) { + b := kinesis.NewInMemoryBackend() + ctx := context.Background() + require.NoError(t, b.CreateStream(ctx, &kinesis.CreateStreamInput{ + StreamName: "after-shard-id-stream", + ShardCount: 4, + })) + time.Sleep(streamSettleWait) - all, err := b.ListShards(ctx, &kinesis.ListShardsInput{StreamName: "after-shard-id-stream"}) - require.NoError(t, err) - require.Len(t, all.Shards, 4) + all, err := b.ListShards(ctx, &kinesis.ListShardsInput{StreamName: "after-shard-id-stream"}) + require.NoError(t, err) + require.Len(t, all.Shards, 4) - out, err := b.ListShards(ctx, &kinesis.ListShardsInput{ - StreamName: "after-shard-id-stream", - ShardFilterType: "AFTER_SHARD_ID", - ShardFilterShardID: all.Shards[0].ShardID, - }) - require.NoError(t, err) - require.Len(t, out.Shards, 3, "shards after shard 0") - assert.Equal(t, all.Shards[1].ShardID, out.Shards[0].ShardID) - - // Now close a shard via merge and confirm AFTER_SHARD_ID surfaces it too - // (includeAll), where the AT_LATEST default would not. - require.NoError(t, b.MergeShards(ctx, &kinesis.MergeShardsInput{ - StreamName: "after-shard-id-stream", - ShardToMerge: all.Shards[0].ShardID, - AdjacentShardToMerge: all.Shards[1].ShardID, - })) + out, err := b.ListShards(ctx, &kinesis.ListShardsInput{ + StreamName: "after-shard-id-stream", + ShardFilterType: "AFTER_SHARD_ID", + ShardFilterShardID: all.Shards[0].ShardID, + }) + require.NoError(t, err) + require.Len(t, out.Shards, 3, "shards after shard 0") + assert.Equal(t, all.Shards[1].ShardID, out.Shards[0].ShardID) + + // Now close a shard via merge and confirm AFTER_SHARD_ID surfaces it too + // (includeAll), where the AT_LATEST default would not. + require.NoError(t, b.MergeShards(ctx, &kinesis.MergeShardsInput{ + StreamName: "after-shard-id-stream", + ShardToMerge: all.Shards[0].ShardID, + AdjacentShardToMerge: all.Shards[1].ShardID, + })) + + afterAll, err := b.ListShards(ctx, &kinesis.ListShardsInput{ + StreamName: "after-shard-id-stream", + ShardFilterType: "AFTER_SHARD_ID", + ShardFilterShardID: all.Shards[0].ShardID, + }) + require.NoError(t, err) + // shards[1] (closed), shards[2] (open), shards[3] (open), plus the merged shard. + assert.Len(t, afterAll.Shards, 4) - afterAll, err := b.ListShards(ctx, &kinesis.ListShardsInput{ - StreamName: "after-shard-id-stream", - ShardFilterType: "AFTER_SHARD_ID", - ShardFilterShardID: all.Shards[0].ShardID, + defaultOut, err := b.ListShards(ctx, &kinesis.ListShardsInput{StreamName: "after-shard-id-stream"}) + require.NoError(t, err) + // Default (open-only) excludes the two merge parents, keeping only the + // still-open originals plus the new merged shard. + assert.Len(t, defaultOut.Shards, 3) }) - require.NoError(t, err) - // shards[1] (closed), shards[2] (open), shards[3] (open), plus the merged shard. - assert.Len(t, afterAll.Shards, 4) - - defaultOut, err := b.ListShards(ctx, &kinesis.ListShardsInput{StreamName: "after-shard-id-stream"}) - require.NoError(t, err) - // Default (open-only) excludes the two merge parents, keeping only the - // still-open originals plus the new merged shard. - assert.Len(t, defaultOut.Shards, 3) } // TestListShards_ShardFilterType_TimestampRequired verifies AT_TIMESTAMP and diff --git a/services/kinesis/stream_encryption.go b/services/kinesis/stream_encryption.go index b385b5ddd8..4423676521 100644 --- a/services/kinesis/stream_encryption.go +++ b/services/kinesis/stream_encryption.go @@ -89,13 +89,17 @@ func (b *InMemoryBackend) StartStreamEncryption(ctx context.Context, input *Star streamName = streamNameFromARN(input.StreamARN) } - stream, ok := b.streams.Get(streamKey(region, streamName)) - if !ok { - return ErrStreamNotFound + stream, err := b.resolveStreamTransitionLocked(region, streamName) + if err != nil { + return err } stream.mu.Lock("StartStreamEncryption.stream") defer stream.mu.Unlock() + if stream.Status != streamStatusActive { + return ErrStreamNotActive + } + if input.EncryptionType != encryptionTypeKMS { return ErrInvalidArgument } @@ -105,12 +109,14 @@ func (b *InMemoryBackend) StartStreamEncryption(ctx context.Context, input *Star // checks above, matching the "stream not found" test expectations that // predate KMS validation -- a malformed KeyId against a nonexistent // stream still surfaces ResourceNotFoundException, not InvalidArgumentException. - if err := b.resolveKMSKey(ctx, input.KeyID); err != nil { - return err + if kmsErr := b.resolveKMSKey(ctx, input.KeyID); kmsErr != nil { + return kmsErr } stream.EncryptionType = input.EncryptionType stream.KeyID = input.KeyID + stream.Status = streamStatusUpdating + stream.ReadyAt = b.nowFunc().Add(streamTransitionDelay) return nil } @@ -127,13 +133,17 @@ func (b *InMemoryBackend) StopStreamEncryption(ctx context.Context, input *StopS streamName = streamNameFromARN(input.StreamARN) } - stream, ok := b.streams.Get(streamKey(region, streamName)) - if !ok { - return ErrStreamNotFound + stream, err := b.resolveStreamTransitionLocked(region, streamName) + if err != nil { + return err } stream.mu.Lock("StopStreamEncryption.stream") defer stream.mu.Unlock() + if stream.Status != streamStatusActive { + return ErrStreamNotActive + } + // KeyId is a required field on StopStreamEncryptionInput per the real SDK // model even though stopping encryption never needs to look the key up; // only its presence/shape is validated here (no KMS backend call). @@ -143,6 +153,8 @@ func (b *InMemoryBackend) StopStreamEncryption(ctx context.Context, input *StopS stream.EncryptionType = encryptionTypeNone stream.KeyID = "" + stream.Status = streamStatusUpdating + stream.ReadyAt = b.nowFunc().Add(streamTransitionDelay) return nil } diff --git a/services/kinesis/stream_encryption_test.go b/services/kinesis/stream_encryption_test.go index 3de137dd4d..1b0078cc38 100644 --- a/services/kinesis/stream_encryption_test.go +++ b/services/kinesis/stream_encryption_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -16,6 +18,14 @@ import ( func TestStartEncryption_ARNSupport(t *testing.T) { t.Parallel() + synctest.Test(t, func(t *testing.T) { + testStartEncryptionARNSupport(t) + }) +} + +func testStartEncryptionARNSupport(t *testing.T) { + t.Helper() + h := newTestHandler(t) rec := doRequest(t, h, "CreateStream", map[string]any{ @@ -23,6 +33,7 @@ func TestStartEncryption_ARNSupport(t *testing.T) { "ShardCount": 1, }) require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "arn-enc-stream"}) require.Equal(t, http.StatusOK, rec.Code) @@ -41,6 +52,7 @@ func TestStartEncryption_ARNSupport(t *testing.T) { "KeyId": "alias/arn-key", }) assert.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) rec = doRequest(t, h, "StopStreamEncryption", map[string]any{ "StreamARN": descResp.StreamDescription.StreamARN, @@ -103,29 +115,33 @@ func TestStreamEncryption(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newTestHandler(t) - - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": tt.streamName, - "ShardCount": 1, - }) - require.Equal(t, http.StatusOK, rec.Code) - - // Start encryption. - rec = doRequest(t, h, "StartStreamEncryption", map[string]any{ - "StreamName": tt.streamName, - "EncryptionType": tt.encType, - "KeyId": tt.keyID, - }) - assert.Equal(t, tt.wantStartCode, rec.Code) - - // Stop encryption. - rec = doRequest(t, h, "StopStreamEncryption", map[string]any{ - "StreamName": tt.streamName, - "EncryptionType": tt.encType, - "KeyId": tt.keyID, + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": tt.streamName, + "ShardCount": 1, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) + + // Start encryption. + rec = doRequest(t, h, "StartStreamEncryption", map[string]any{ + "StreamName": tt.streamName, + "EncryptionType": tt.encType, + "KeyId": tt.keyID, + }) + assert.Equal(t, tt.wantStartCode, rec.Code) + time.Sleep(streamSettleWait) + + // Stop encryption. + rec = doRequest(t, h, "StopStreamEncryption", map[string]any{ + "StreamName": tt.streamName, + "EncryptionType": tt.encType, + "KeyId": tt.keyID, + }) + assert.Equal(t, tt.wantStopCode, rec.Code) }) - assert.Equal(t, tt.wantStopCode, rec.Code) }) } } @@ -168,12 +184,17 @@ func TestStreamEncryption_Errors(t *testing.T) { }, } - h := newTestHandler(t) - setup := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "enc-err-stream", - "ShardCount": 1, + var h *kinesis.Handler + + synctest.Test(t, func(t *testing.T) { + h = newTestHandler(t) + setup := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "enc-err-stream", + "ShardCount": 1, + }) + require.Equal(t, http.StatusOK, setup.Code) + time.Sleep(streamSettleWait) }) - require.Equal(t, http.StatusOK, setup.Code) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -197,38 +218,42 @@ func TestStreamEncryption_Errors(t *testing.T) { func TestStreamEncryption_StartStop(t *testing.T) { t.Parallel() - bk := kinesis.NewInMemoryBackend() - require.NoError(t, bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "enc-stream"})) - - // Initially no encryption. - descOut, err := bk.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: "enc-stream"}) - require.NoError(t, err) - assert.Equal(t, "NONE", descOut.EncryptionType) - assert.Empty(t, descOut.KeyID) - - // Start encryption. - require.NoError(t, bk.StartStreamEncryption(context.Background(), &kinesis.StartStreamEncryptionInput{ - StreamName: "enc-stream", - EncryptionType: "KMS", - KeyID: "alias/aws/kinesis", - })) - - descOut, err = bk.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: "enc-stream"}) - require.NoError(t, err) - assert.Equal(t, "KMS", descOut.EncryptionType) - assert.Equal(t, "alias/aws/kinesis", descOut.KeyID) - - // Stop encryption. - require.NoError(t, bk.StopStreamEncryption(context.Background(), &kinesis.StopStreamEncryptionInput{ - StreamName: "enc-stream", - EncryptionType: "KMS", - KeyID: "alias/aws/kinesis", - })) - - descOut, err = bk.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: "enc-stream"}) - require.NoError(t, err) - assert.Equal(t, "NONE", descOut.EncryptionType) - assert.Empty(t, descOut.KeyID) + synctest.Test(t, func(t *testing.T) { + bk := kinesis.NewInMemoryBackend() + require.NoError(t, bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "enc-stream"})) + time.Sleep(streamSettleWait) + + // Initially no encryption. + descOut, err := bk.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: "enc-stream"}) + require.NoError(t, err) + assert.Equal(t, "NONE", descOut.EncryptionType) + assert.Empty(t, descOut.KeyID) + + // Start encryption. + require.NoError(t, bk.StartStreamEncryption(context.Background(), &kinesis.StartStreamEncryptionInput{ + StreamName: "enc-stream", + EncryptionType: "KMS", + KeyID: "alias/aws/kinesis", + })) + time.Sleep(streamSettleWait) + + descOut, err = bk.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: "enc-stream"}) + require.NoError(t, err) + assert.Equal(t, "KMS", descOut.EncryptionType) + assert.Equal(t, "alias/aws/kinesis", descOut.KeyID) + + // Stop encryption. + require.NoError(t, bk.StopStreamEncryption(context.Background(), &kinesis.StopStreamEncryptionInput{ + StreamName: "enc-stream", + EncryptionType: "KMS", + KeyID: "alias/aws/kinesis", + })) + + descOut, err = bk.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: "enc-stream"}) + require.NoError(t, err) + assert.Equal(t, "NONE", descOut.EncryptionType) + assert.Empty(t, descOut.KeyID) + }) } // TestStartStreamEncryption_KeyIDFormat verifies StartStreamEncryption's @@ -260,24 +285,27 @@ func TestStartStreamEncryption_KeyIDFormat(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - bk := kinesis.NewInMemoryBackend() - streamName := "kmsfmt-" + tt.name - require.NoError(t, bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: streamName, - })) - - err := bk.StartStreamEncryption(context.Background(), &kinesis.StartStreamEncryptionInput{ - StreamName: streamName, - EncryptionType: "KMS", - KeyID: tt.keyID, + synctest.Test(t, func(t *testing.T) { + bk := kinesis.NewInMemoryBackend() + streamName := "kmsfmt-" + tt.name + require.NoError(t, bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: streamName, + })) + time.Sleep(streamSettleWait) + + err := bk.StartStreamEncryption(context.Background(), &kinesis.StartStreamEncryptionInput{ + StreamName: streamName, + EncryptionType: "KMS", + KeyID: tt.keyID, + }) + + if tt.wantErr { + require.Error(t, err) + assert.ErrorIs(t, err, kinesis.ErrInvalidArgument) + } else { + require.NoError(t, err) + } }) - - if tt.wantErr { - require.Error(t, err) - assert.ErrorIs(t, err, kinesis.ErrInvalidArgument) - } else { - require.NoError(t, err) - } }) } } @@ -322,26 +350,29 @@ func TestStartStreamEncryption_KMSValidator(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - bk := kinesis.NewInMemoryBackend() - streamName := "kmsval-" + tt.name - require.NoError(t, bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{ - StreamName: streamName, - })) - - bk.WithKMSValidator(&fakeKMSValidator{keyID: "alias/scripted-key", err: tt.validatorErr}) - - err := bk.StartStreamEncryption(context.Background(), &kinesis.StartStreamEncryptionInput{ - StreamName: streamName, - EncryptionType: "KMS", - KeyID: "alias/scripted-key", + synctest.Test(t, func(t *testing.T) { + bk := kinesis.NewInMemoryBackend() + streamName := "kmsval-" + tt.name + require.NoError(t, bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{ + StreamName: streamName, + })) + time.Sleep(streamSettleWait) + + bk.WithKMSValidator(&fakeKMSValidator{keyID: "alias/scripted-key", err: tt.validatorErr}) + + err := bk.StartStreamEncryption(context.Background(), &kinesis.StartStreamEncryptionInput{ + StreamName: streamName, + EncryptionType: "KMS", + KeyID: "alias/scripted-key", + }) + + if tt.wantErr != nil { + require.Error(t, err) + assert.ErrorIs(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } }) - - if tt.wantErr != nil { - require.Error(t, err) - assert.ErrorIs(t, err, tt.wantErr) - } else { - require.NoError(t, err) - } }) } } diff --git a/services/kinesis/stream_modes.go b/services/kinesis/stream_modes.go index 028435a722..1dfcb239ab 100644 --- a/services/kinesis/stream_modes.go +++ b/services/kinesis/stream_modes.go @@ -5,10 +5,10 @@ import "context" // UpdateStreamWarmThroughput configures pre-warmed throughput for a stream // (kinesis@v1.46.4 api_op_UpdateStreamWarmThroughput.go:63-70, required // WarmThroughputMiBps). Real AWS applies this asynchronously (stream goes -// UPDATING then back to ACTIVE); this backend has no transient-state model -// for that (streams are always ACTIVE), so the change is applied +// UPDATING then back to ACTIVE); this backend applies the change // synchronously and Current/Target always match on read -- see -// UpdateStreamWarmThroughputOutput and PARITY.md. +// UpdateStreamWarmThroughputOutput and PARITY.md -- but does now reject a +// non-ACTIVE stream with ResourceInUseException, matching the declared error. func (b *InMemoryBackend) UpdateStreamWarmThroughput( ctx context.Context, input *UpdateStreamWarmThroughputInput, @@ -19,25 +19,27 @@ func (b *InMemoryBackend) UpdateStreamWarmThroughput( region := regionFromARNOrCtx(ctx, input.StreamARN, b.region) - b.mu.RLock("UpdateStreamWarmThroughput") + b.mu.Lock("UpdateStreamWarmThroughput") + defer b.mu.Unlock() streamName := input.StreamName if streamName == "" { streamName = streamNameFromARN(input.StreamARN) } - stream, ok := b.streams.Get(streamKey(region, streamName)) - if !ok { - b.mu.RUnlock() - - return nil, ErrStreamNotFound + stream, err := b.resolveStreamTransitionLocked(region, streamName) + if err != nil { + return nil, err } stream.mu.Lock("UpdateStreamWarmThroughput.stream") - b.mu.RUnlock() + defer stream.mu.Unlock() + + if stream.Status != streamStatusActive { + return nil, ErrStreamNotActive + } stream.WarmThroughputMiBps = input.WarmThroughputMiBps arnOut, nameOut := stream.ARN, stream.Name - stream.mu.Unlock() return &UpdateStreamWarmThroughputOutput{ StreamARN: arnOut, @@ -57,13 +59,17 @@ func (b *InMemoryBackend) UpdateStreamMode(ctx context.Context, input *UpdateStr defer b.mu.Unlock() streamName := streamNameFromARN(input.StreamARN) - stream, ok := b.streams.Get(streamKey(region, streamName)) - if !ok { - return ErrStreamNotFound + stream, err := b.resolveStreamTransitionLocked(region, streamName) + if err != nil { + return err } stream.mu.Lock("UpdateStreamMode.stream") defer stream.mu.Unlock() + if stream.Status != streamStatusActive { + return ErrStreamNotActive + } + newMode := input.StreamModeDetails.StreamMode if newMode != streamModeProvisioned && newMode != streamModeOnDemand { return ErrInvalidArgument @@ -98,5 +104,8 @@ func (b *InMemoryBackend) UpdateStreamMode(ctx context.Context, input *UpdateStr stream.WarmThroughputMiBps = v } + stream.Status = streamStatusUpdating + stream.ReadyAt = b.nowFunc().Add(streamTransitionDelay) + return nil } diff --git a/services/kinesis/stream_modes_test.go b/services/kinesis/stream_modes_test.go index 739369eec9..c2a04c2bac 100644 --- a/services/kinesis/stream_modes_test.go +++ b/services/kinesis/stream_modes_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -28,7 +29,8 @@ import ( func TestUpdateStreamWarmThroughput_RoundTrip(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "warm-throughput-stream" @@ -40,6 +42,7 @@ func TestUpdateStreamWarmThroughput_RoundTrip(t *testing.T) { }, }) require.NoError(t, err) + clock.Advance(streamSettleWait) desc, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{StreamName: aws.String(streamName)}) require.NoError(t, err) @@ -74,44 +77,48 @@ func TestUpdateStreamWarmThroughput_RequiredFieldRejected(t *testing.T) { func TestUpdateStreamMode_ProvisionedToOnDemand(t *testing.T) { t.Parallel() - b := newParityBackend(t) - ctx := context.Background() + synctest.Test(t, func(t *testing.T) { + b := newParityBackend(t) + ctx := context.Background() - createParityStream(t, b, "mode-test", 2) + createParityStream(t, b, "mode-test", 2) + time.Sleep(streamSettleWait) - desc0, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "mode-test"}) - require.NoError(t, err) - assert.Equal(t, "PROVISIONED", desc0.StreamMode) + desc0, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "mode-test"}) + require.NoError(t, err) + assert.Equal(t, "PROVISIONED", desc0.StreamMode) - err = b.UpdateStreamMode(ctx, &kinesis.UpdateStreamModeInput{ - StreamARN: desc0.StreamARN, - StreamModeDetails: kinesis.StreamModeDetails{ - StreamMode: "ON_DEMAND", - }, - }) - require.NoError(t, err) + err = b.UpdateStreamMode(ctx, &kinesis.UpdateStreamModeInput{ + StreamARN: desc0.StreamARN, + StreamModeDetails: kinesis.StreamModeDetails{ + StreamMode: "ON_DEMAND", + }, + }) + require.NoError(t, err) + time.Sleep(streamSettleWait) - desc1, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "mode-test"}) - require.NoError(t, err) - assert.Equal(t, "ON_DEMAND", desc1.StreamMode) + desc1, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "mode-test"}) + require.NoError(t, err) + assert.Equal(t, "ON_DEMAND", desc1.StreamMode) - _, err = b.UpdateShardCount(ctx, &kinesis.UpdateShardCountInput{ - StreamName: "mode-test", - TargetShardCount: 4, - }) - require.Error(t, err, "UpdateShardCount must fail for ON_DEMAND streams") + _, err = b.UpdateShardCount(ctx, &kinesis.UpdateShardCountInput{ + StreamName: "mode-test", + TargetShardCount: 4, + }) + require.Error(t, err, "UpdateShardCount must fail for ON_DEMAND streams") - err = b.UpdateStreamMode(ctx, &kinesis.UpdateStreamModeInput{ - StreamARN: desc1.StreamARN, - StreamModeDetails: kinesis.StreamModeDetails{ - StreamMode: "PROVISIONED", - }, - }) - require.NoError(t, err) + err = b.UpdateStreamMode(ctx, &kinesis.UpdateStreamModeInput{ + StreamARN: desc1.StreamARN, + StreamModeDetails: kinesis.StreamModeDetails{ + StreamMode: "PROVISIONED", + }, + }) + require.NoError(t, err) - desc2, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "mode-test"}) - require.NoError(t, err) - assert.Equal(t, "PROVISIONED", desc2.StreamMode) + desc2, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "mode-test"}) + require.NoError(t, err) + assert.Equal(t, "PROVISIONED", desc2.StreamMode) + }) } func TestUpdateStreamMode_Valid(t *testing.T) { @@ -130,44 +137,47 @@ func TestUpdateStreamMode_Valid(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newTestHandler(t) - streamName := "mode-stream-" + tt.name - - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": streamName, - "ShardCount": 1, - }) - require.Equal(t, http.StatusOK, rec.Code) - - rec2 := doRequest(t, h, "DescribeStream", map[string]any{"StreamName": streamName}) - require.Equal(t, http.StatusOK, rec2.Code) - var descResp struct { - StreamDescription struct { - StreamARN string `json:"StreamARN"` - } `json:"StreamDescription"` - } - require.NoError(t, json.Unmarshal(rec2.Body.Bytes(), &descResp)) - - rec3 := doRequest(t, h, "UpdateStreamMode", map[string]any{ - "StreamARN": descResp.StreamDescription.StreamARN, - "StreamModeDetails": map[string]any{ - "StreamMode": tt.newMode, - }, + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + streamName := "mode-stream-" + tt.name + + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": streamName, + "ShardCount": 1, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) + + rec2 := doRequest(t, h, "DescribeStream", map[string]any{"StreamName": streamName}) + require.Equal(t, http.StatusOK, rec2.Code) + var descResp struct { + StreamDescription struct { + StreamARN string `json:"StreamARN"` + } `json:"StreamDescription"` + } + require.NoError(t, json.Unmarshal(rec2.Body.Bytes(), &descResp)) + + rec3 := doRequest(t, h, "UpdateStreamMode", map[string]any{ + "StreamARN": descResp.StreamDescription.StreamARN, + "StreamModeDetails": map[string]any{ + "StreamMode": tt.newMode, + }, + }) + require.Equal(t, http.StatusOK, rec3.Code) + + rec4 := doRequest(t, h, "DescribeStream", map[string]any{"StreamName": streamName}) + require.Equal(t, http.StatusOK, rec4.Code) + var verifyResp struct { + StreamDescription struct { + StreamModeDetails *struct { + StreamMode string `json:"StreamMode"` + } `json:"StreamModeDetails"` + } `json:"StreamDescription"` + } + require.NoError(t, json.Unmarshal(rec4.Body.Bytes(), &verifyResp)) + require.NotNil(t, verifyResp.StreamDescription.StreamModeDetails) + assert.Equal(t, tt.wantMode, verifyResp.StreamDescription.StreamModeDetails.StreamMode) }) - require.Equal(t, http.StatusOK, rec3.Code) - - rec4 := doRequest(t, h, "DescribeStream", map[string]any{"StreamName": streamName}) - require.Equal(t, http.StatusOK, rec4.Code) - var verifyResp struct { - StreamDescription struct { - StreamModeDetails *struct { - StreamMode string `json:"StreamMode"` - } `json:"StreamModeDetails"` - } `json:"StreamDescription"` - } - require.NoError(t, json.Unmarshal(rec4.Body.Bytes(), &verifyResp)) - require.NotNil(t, verifyResp.StreamDescription.StreamModeDetails) - assert.Equal(t, tt.wantMode, verifyResp.StreamDescription.StreamModeDetails.StreamMode) }) } } @@ -234,33 +244,36 @@ func TestUpdateStreamMode_OnDemandTransitionKeepsShardCount(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := newParityBackend(t) - ctx := context.Background() - streamName := "reshard-" + tt.name - - createParityStream(t, b, streamName, tt.startShards) - - descBefore, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: streamName}) - require.NoError(t, err) - require.Len(t, descBefore.Shards, tt.startShards, "sanity: initial open shard count") + synctest.Test(t, func(t *testing.T) { + b := newParityBackend(t) + ctx := context.Background() + streamName := "reshard-" + tt.name - require.NoError(t, b.UpdateStreamMode(ctx, &kinesis.UpdateStreamModeInput{ - StreamARN: descBefore.StreamARN, - StreamModeDetails: kinesis.StreamModeDetails{ - StreamMode: "ON_DEMAND", - }, - })) + createParityStream(t, b, streamName, tt.startShards) + time.Sleep(streamSettleWait) - // ListShards' default (no ShardFilter) only returns open shards, so - // its length is exactly the new open shard count. - openAfter, err := b.ListShards(ctx, &kinesis.ListShardsInput{StreamName: streamName}) - require.NoError(t, err) - assert.Len( - t, - openAfter.Shards, - tt.startShards, - "PROVISIONED -> ON_DEMAND must retain the pre-transition shard count, not floor to defaultOnDemandShardCount", - ) + descBefore, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: streamName}) + require.NoError(t, err) + require.Len(t, descBefore.Shards, tt.startShards, "sanity: initial open shard count") + + require.NoError(t, b.UpdateStreamMode(ctx, &kinesis.UpdateStreamModeInput{ + StreamARN: descBefore.StreamARN, + StreamModeDetails: kinesis.StreamModeDetails{ + StreamMode: "ON_DEMAND", + }, + })) + + // ListShards' default (no ShardFilter) only returns open shards, so + // its length is exactly the new open shard count. + openAfter, err := b.ListShards(ctx, &kinesis.ListShardsInput{StreamName: streamName}) + require.NoError(t, err) + assert.Len( + t, + openAfter.Shards, + tt.startShards, + "PROVISIONED -> ON_DEMAND must retain the pre-transition shard count, not floor to defaultOnDemandShardCount", + ) + }) }) } } @@ -273,7 +286,8 @@ func TestUpdateStreamMode_OnDemandTransitionKeepsShardCount(t *testing.T) { func TestUpdateStreamMode_ProvisionedToOnDemand_RealClientKeepsShardCount(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "real-client-mode-transition" @@ -282,6 +296,7 @@ func TestUpdateStreamMode_ProvisionedToOnDemand_RealClientKeepsShardCount(t *tes ShardCount: aws.Int32(2), }) require.NoError(t, err) + clock.Advance(streamSettleWait) descBefore, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{ StreamName: aws.String(streamName), @@ -326,6 +341,7 @@ func TestUpdateStreamMode_OnDemandAutoScalesOnSustainedWrite(t *testing.T) { StreamName: streamName, StreamMode: "ON_DEMAND", })) + fakeNow = fakeNow.Add(streamSettleWait) descBefore, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: streamName}) require.NoError(t, err) @@ -384,6 +400,7 @@ func TestUpdateStreamMode_OnDemandAutoScaleIgnoresProvisioned(t *testing.T) { StreamName: streamName, ShardCount: 1, })) + fakeNow = fakeNow.Add(streamSettleWait) payload := make([]byte, 600*1024) _, err := b.PutRecord(ctx, &kinesis.PutRecordInput{ @@ -404,26 +421,29 @@ func TestUpdateStreamMode_OnDemandAutoScaleIgnoresProvisioned(t *testing.T) { func TestUpdateStreamMode_OnDemandToProvisionedKeepsShardCount(t *testing.T) { t.Parallel() - b := newParityBackend(t) - ctx := context.Background() + synctest.Test(t, func(t *testing.T) { + b := newParityBackend(t) + ctx := context.Background() - require.NoError(t, b.CreateStream(ctx, &kinesis.CreateStreamInput{ - StreamName: "ondemand-to-prov", - StreamMode: "ON_DEMAND", - })) + require.NoError(t, b.CreateStream(ctx, &kinesis.CreateStreamInput{ + StreamName: "ondemand-to-prov", + StreamMode: "ON_DEMAND", + })) + time.Sleep(streamSettleWait) - descBefore, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "ondemand-to-prov"}) - require.NoError(t, err) - openBefore := len(descBefore.Shards) + descBefore, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "ondemand-to-prov"}) + require.NoError(t, err) + openBefore := len(descBefore.Shards) - require.NoError(t, b.UpdateStreamMode(ctx, &kinesis.UpdateStreamModeInput{ - StreamARN: descBefore.StreamARN, - StreamModeDetails: kinesis.StreamModeDetails{StreamMode: "PROVISIONED"}, - })) + require.NoError(t, b.UpdateStreamMode(ctx, &kinesis.UpdateStreamModeInput{ + StreamARN: descBefore.StreamARN, + StreamModeDetails: kinesis.StreamModeDetails{StreamMode: "PROVISIONED"}, + })) - descAfter, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "ondemand-to-prov"}) - require.NoError(t, err) - assert.Len(t, descAfter.Shards, openBefore, "shard count must be unchanged by ON_DEMAND -> PROVISIONED") + descAfter, err := b.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: "ondemand-to-prov"}) + require.NoError(t, err) + assert.Len(t, descAfter.Shards, openBefore, "shard count must be unchanged by ON_DEMAND -> PROVISIONED") + }) } func TestUpdateStreamMode_NotFound(t *testing.T) { diff --git a/services/kinesis/streams.go b/services/kinesis/streams.go index 75acbc18e9..706637a695 100644 --- a/services/kinesis/streams.go +++ b/services/kinesis/streams.go @@ -4,7 +4,6 @@ import ( "context" "regexp" "sort" - "time" "github.com/blackbirdworks/gopherstack/pkgs/arn" "github.com/blackbirdworks/gopherstack/pkgs/tags" @@ -76,7 +75,7 @@ func (b *InMemoryBackend) CreateStream(ctx context.Context, input *CreateStreamI return ErrValidation } - if b.streams.Has(streamKey(region, input.StreamName)) { + if _, err := b.resolveStreamTransitionLocked(region, input.StreamName); err == nil { return ErrStreamAlreadyExists } @@ -85,7 +84,7 @@ func (b *InMemoryBackend) CreateStream(ctx context.Context, input *CreateStreamI return err } - now := time.Now() + now := b.nowFunc() shards := buildInitialShards(shardCount, now) accountID := b.accountID @@ -94,7 +93,7 @@ func (b *InMemoryBackend) CreateStream(ctx context.Context, input *CreateStreamI } if streamMode == streamModeOnDemand { - if odErr := checkOnDemandLimit(b.streamsByRegion.Get(region), b.onDemandStreamCountLimit); odErr != nil { + if odErr := checkOnDemandLimit(b.resolveRegionStreamsLocked(region), b.onDemandStreamCountLimit); odErr != nil { return odErr } } @@ -114,7 +113,8 @@ func (b *InMemoryBackend) CreateStream(ctx context.Context, input *CreateStreamI Name: input.StreamName, ARN: streamARN, Region: region, - Status: streamStatusActive, + Status: streamStatusCreating, + ReadyAt: now.Add(streamTransitionDelay), Shards: shards, mu: newStreamLock(input.StreamName), Tags: tags.New("kinesis.stream." + input.StreamName + ".tags"), @@ -147,78 +147,52 @@ func (b *InMemoryBackend) DeleteStream(ctx context.Context, input *DeleteStreamI return nil } -// deleteStreamLocked performs DeleteStream's actual state removal. +// deleteStreamLocked marks a resolved, ACTIVE stream DELETING. Real AWS +// deletes asynchronously (StreamStatus DELETING until removal completes, +// still visible via DescribeStreamSummary) -- physical removal happens +// later, lazily, in finishStreamDeletionLocked via +// resolveStreamTransitionLocked, matching CreateStream's CREATING->ACTIVE +// pattern. func (b *InMemoryBackend) deleteStreamLocked(ctx context.Context, input *DeleteStreamInput) error { region := getRegion(ctx, b.region) - var stream *Stream var found bool - var consumerErr error - - // b.mu and stream.mu are both held while the stream is marked DELETING and - // removed from b.streams; b.mu releases as soon as that work is done while - // stream.mu is handed off to the caller (see stream.mu.Unlock below), matching - // the original release timing. handoffOK guards the handoff: if anything in - // this closure panics before the handoff point, stream.mu is still released - // instead of leaking. + var opErr error + func() { b.mu.Lock("DeleteStream") defer b.mu.Unlock() - s, exists := b.streams.Get(streamKey(region, input.StreamName)) - if !exists { + stream, err := b.resolveStreamTransitionLocked(region, input.StreamName) + if err != nil { return } - stream = s found = true stream.mu.Lock("DeleteStream.stream") - handoffOK := false - defer func() { - if !handoffOK { - stream.mu.Unlock() - } - }() + defer stream.mu.Unlock() - if len(stream.Consumers) > 0 && !input.EnforceConsumerDeletion { - consumerErr = ErrStreamHasConsumers + if stream.Status != streamStatusActive { + opErr = ErrStreamNotActive return } - if stream.Tags != nil { - stream.Tags.Close() + if len(stream.Consumers) > 0 && !input.EnforceConsumerDeletion { + opErr = ErrStreamHasConsumers + + return } - // Mark the stream as deleting before removing it (AWS-realistic status transition). stream.Status = streamStatusDeleting - b.streams.Delete(streamKey(region, input.StreamName)) - delete(b.resourcePolicies[region], stream.ARN) - - handoffOK = true + stream.ReadyAt = b.nowFunc().Add(streamTransitionDelay) }() if !found { return ErrStreamNotFound } - if consumerErr != nil { - return consumerErr - } - defer stream.mu.Unlock() - - b.faultsMu.Lock("DeleteStream.faults") - delete(b.faultsStore(region), input.StreamName) - b.faultsMu.Unlock() - - if b.OnStreamPurged != nil { - b.OnStreamPurged(input.StreamName) - } - - // Release lockmetrics resources for the deleted stream to prevent memory leaks. - stream.mu.Close() - - return nil + return opErr } // DescribeStream returns full stream details including shards. @@ -228,16 +202,16 @@ func (b *InMemoryBackend) DescribeStream( ) (*DescribeStreamOutput, error) { region := getRegion(ctx, b.region) - b.mu.RLock("DescribeStream") + b.mu.Lock("DescribeStream") - stream, exists := b.streams.Get(streamKey(region, input.StreamName)) - if !exists { - b.mu.RUnlock() + stream, err := b.resolveStreamTransitionLocked(region, input.StreamName) + if err != nil { + b.mu.Unlock() - return nil, ErrStreamNotFound + return nil, err } stream.mu.RLock("DescribeStream.stream") - b.mu.RUnlock() + b.mu.Unlock() defer stream.mu.RUnlock() // AWS paginates the Shards list: default page size 100, max 10000, resumed @@ -311,11 +285,12 @@ func (b *InMemoryBackend) DescribeStream( func (b *InMemoryBackend) ListStreams(ctx context.Context, input *ListStreamsInput) (*ListStreamsOutput, error) { region := getRegion(ctx, b.region) - b.mu.RLock("ListStreams") - defer b.mu.RUnlock() + b.mu.Lock("ListStreams") + defer b.mu.Unlock() + + regionStreams := b.resolveRegionStreamsLocked(region) // AWS returns streams in alphabetical order by name. - regionStreams := append([]*Stream{}, b.streamsByRegion.Get(region)...) sort.Slice(regionStreams, func(i, j int) bool { return regionStreams[i].Name < regionStreams[j].Name }) // Apply pagination start point: prefer ExclusiveStartStreamName, then NextToken. diff --git a/services/kinesis/streams_describe_test.go b/services/kinesis/streams_describe_test.go index 95b949db65..f589540cf9 100644 --- a/services/kinesis/streams_describe_test.go +++ b/services/kinesis/streams_describe_test.go @@ -7,6 +7,7 @@ import ( "net/http" "strings" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -110,120 +111,129 @@ func TestDescribeStreamSummary_ByARN(t *testing.T) { func TestDescribeStream_EncryptionFields(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "enc-describe-stream", - "ShardCount": 1, - }) - require.Equal(t, http.StatusOK, rec.Code) + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "enc-describe-stream", + "ShardCount": 1, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) - // Start encryption. - rec = doRequest(t, h, "StartStreamEncryption", map[string]any{ - "StreamName": "enc-describe-stream", - "EncryptionType": "KMS", - "KeyId": "alias/my-key-id", - }) - require.Equal(t, http.StatusOK, rec.Code) + // Start encryption. + rec = doRequest(t, h, "StartStreamEncryption", map[string]any{ + "StreamName": "enc-describe-stream", + "EncryptionType": "KMS", + "KeyId": "alias/my-key-id", + }) + require.Equal(t, http.StatusOK, rec.Code) - // DescribeStream should return encryption info. - rec = doRequest(t, h, "DescribeStream", map[string]any{ - "StreamName": "enc-describe-stream", - }) - require.Equal(t, http.StatusOK, rec.Code) + // DescribeStream should return encryption info. + rec = doRequest(t, h, "DescribeStream", map[string]any{ + "StreamName": "enc-describe-stream", + }) + require.Equal(t, http.StatusOK, rec.Code) - var resp struct { - StreamDescription struct { - EncryptionType string `json:"EncryptionType"` - KeyID string `json:"KeyId"` - } `json:"StreamDescription"` - } + var resp struct { + StreamDescription struct { + EncryptionType string `json:"EncryptionType"` + KeyID string `json:"KeyId"` + } `json:"StreamDescription"` + } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) - assert.Equal(t, "KMS", resp.StreamDescription.EncryptionType) - assert.Equal(t, "alias/my-key-id", resp.StreamDescription.KeyID) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + assert.Equal(t, "KMS", resp.StreamDescription.EncryptionType) + assert.Equal(t, "alias/my-key-id", resp.StreamDescription.KeyID) + }) } // TestDescribeStreamSummary_EncryptionFields verifies encryption in summary. func TestDescribeStreamSummary_EncryptionFields(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "enc-summary-stream", - "ShardCount": 1, - }) - require.Equal(t, http.StatusOK, rec.Code) + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "enc-summary-stream", + "ShardCount": 1, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) - rec = doRequest(t, h, "StartStreamEncryption", map[string]any{ - "StreamName": "enc-summary-stream", - "EncryptionType": "KMS", - "KeyId": "alias/summary-key-id", - }) - require.Equal(t, http.StatusOK, rec.Code) + rec = doRequest(t, h, "StartStreamEncryption", map[string]any{ + "StreamName": "enc-summary-stream", + "EncryptionType": "KMS", + "KeyId": "alias/summary-key-id", + }) + require.Equal(t, http.StatusOK, rec.Code) - rec = doRequest(t, h, "DescribeStreamSummary", map[string]any{ - "StreamName": "enc-summary-stream", - }) - require.Equal(t, http.StatusOK, rec.Code) + rec = doRequest(t, h, "DescribeStreamSummary", map[string]any{ + "StreamName": "enc-summary-stream", + }) + require.Equal(t, http.StatusOK, rec.Code) - var resp struct { - StreamDescriptionSummary struct { - EncryptionType string `json:"EncryptionType"` - KeyID string `json:"KeyId"` - } `json:"StreamDescriptionSummary"` - } + var resp struct { + StreamDescriptionSummary struct { + EncryptionType string `json:"EncryptionType"` + KeyID string `json:"KeyId"` + } `json:"StreamDescriptionSummary"` + } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) - assert.Equal(t, "KMS", resp.StreamDescriptionSummary.EncryptionType) - assert.Equal(t, "alias/summary-key-id", resp.StreamDescriptionSummary.KeyID) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + assert.Equal(t, "KMS", resp.StreamDescriptionSummary.EncryptionType) + assert.Equal(t, "alias/summary-key-id", resp.StreamDescriptionSummary.KeyID) + }) } func TestDescribeStreamSummary_OpenShardCountAndConsumerCount(t *testing.T) { t.Parallel() - h := kinesis.NewHandler(kinesis.NewInMemoryBackend()) + synctest.Test(t, func(t *testing.T) { + h := kinesis.NewHandler(kinesis.NewInMemoryBackend()) - rec := doParityRequest(t, h, "CreateStream", - map[string]any{"StreamName": "summary-test", "ShardCount": 3}) - require.Equal(t, http.StatusOK, rec.Code) + rec := doParityRequest(t, h, "CreateStream", + map[string]any{"StreamName": "summary-test", "ShardCount": 3}) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) - descRec := doParityRequest(t, h, "DescribeStream", - map[string]any{"StreamName": "summary-test"}) - require.Equal(t, http.StatusOK, descRec.Code) + descRec := doParityRequest(t, h, "DescribeStream", + map[string]any{"StreamName": "summary-test"}) + require.Equal(t, http.StatusOK, descRec.Code) - var descResp struct { - StreamDescription struct { - StreamARN string `json:"StreamARN"` - } `json:"StreamDescription"` - } + var descResp struct { + StreamDescription struct { + StreamARN string `json:"StreamARN"` + } `json:"StreamDescription"` + } - require.NoError(t, json.NewDecoder(strings.NewReader(descRec.Body.String())).Decode(&descResp)) + require.NoError(t, json.NewDecoder(strings.NewReader(descRec.Body.String())).Decode(&descResp)) - regRec := doParityRequest(t, h, "RegisterStreamConsumer", map[string]any{ - "StreamARN": descResp.StreamDescription.StreamARN, - "ConsumerName": "c1", - }) - require.Equal(t, http.StatusOK, regRec.Code) - - sumRec := doParityRequest(t, h, "DescribeStreamSummary", - map[string]any{"StreamName": "summary-test"}) - require.Equal(t, http.StatusOK, sumRec.Code) - - var sumResp struct { - StreamDescriptionSummary struct { - StreamStatus string `json:"StreamStatus"` - OpenShardCount int `json:"OpenShardCount"` - ConsumerCount int `json:"ConsumerCount"` - } `json:"StreamDescriptionSummary"` - } + regRec := doParityRequest(t, h, "RegisterStreamConsumer", map[string]any{ + "StreamARN": descResp.StreamDescription.StreamARN, + "ConsumerName": "c1", + }) + require.Equal(t, http.StatusOK, regRec.Code) + + sumRec := doParityRequest(t, h, "DescribeStreamSummary", + map[string]any{"StreamName": "summary-test"}) + require.Equal(t, http.StatusOK, sumRec.Code) + + var sumResp struct { + StreamDescriptionSummary struct { + StreamStatus string `json:"StreamStatus"` + OpenShardCount int `json:"OpenShardCount"` + ConsumerCount int `json:"ConsumerCount"` + } `json:"StreamDescriptionSummary"` + } - require.NoError(t, json.NewDecoder(strings.NewReader(sumRec.Body.String())).Decode(&sumResp)) + require.NoError(t, json.NewDecoder(strings.NewReader(sumRec.Body.String())).Decode(&sumResp)) - assert.Equal(t, 3, sumResp.StreamDescriptionSummary.OpenShardCount) - assert.Equal(t, 1, sumResp.StreamDescriptionSummary.ConsumerCount) - assert.Equal(t, "ACTIVE", sumResp.StreamDescriptionSummary.StreamStatus) + assert.Equal(t, 3, sumResp.StreamDescriptionSummary.OpenShardCount) + assert.Equal(t, 1, sumResp.StreamDescriptionSummary.ConsumerCount) + assert.Equal(t, "ACTIVE", sumResp.StreamDescriptionSummary.StreamStatus) + }) } func TestDescribeStream_StreamCreationTimestamp(t *testing.T) { @@ -336,47 +346,50 @@ func TestDescribeStreamSummary_OpenShardCount_AfterMerge(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newTestHandler(t) - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": tt.streamName, - "ShardCount": tt.initialShards, - }) - require.Equal(t, http.StatusOK, rec.Code) - - if tt.doMerge { - rec2 := doRequest(t, h, "ListShards", map[string]any{ + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + rec := doRequest(t, h, "CreateStream", map[string]any{ "StreamName": tt.streamName, + "ShardCount": tt.initialShards, }) - require.Equal(t, http.StatusOK, rec2.Code) - - var shardsResp struct { - Shards []struct { - ShardID string `json:"ShardId"` - } `json:"Shards"` + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) + + if tt.doMerge { + rec2 := doRequest(t, h, "ListShards", map[string]any{ + "StreamName": tt.streamName, + }) + require.Equal(t, http.StatusOK, rec2.Code) + + var shardsResp struct { + Shards []struct { + ShardID string `json:"ShardId"` + } `json:"Shards"` + } + require.NoError(t, json.Unmarshal(rec2.Body.Bytes(), &shardsResp)) + require.Len(t, shardsResp.Shards, 2) + + rec3 := doRequest(t, h, "MergeShards", map[string]any{ + "StreamName": tt.streamName, + "ShardToMerge": shardsResp.Shards[0].ShardID, + "AdjacentShardToMerge": shardsResp.Shards[1].ShardID, + }) + require.Equal(t, http.StatusOK, rec3.Code) } - require.NoError(t, json.Unmarshal(rec2.Body.Bytes(), &shardsResp)) - require.Len(t, shardsResp.Shards, 2) - rec3 := doRequest(t, h, "MergeShards", map[string]any{ - "StreamName": tt.streamName, - "ShardToMerge": shardsResp.Shards[0].ShardID, - "AdjacentShardToMerge": shardsResp.Shards[1].ShardID, + rec4 := doRequest(t, h, "DescribeStreamSummary", map[string]any{ + "StreamName": tt.streamName, }) - require.Equal(t, http.StatusOK, rec3.Code) - } + require.Equal(t, http.StatusOK, rec4.Code) - rec4 := doRequest(t, h, "DescribeStreamSummary", map[string]any{ - "StreamName": tt.streamName, + var summaryResp struct { + StreamDescriptionSummary struct { + OpenShardCount int `json:"OpenShardCount"` + } `json:"StreamDescriptionSummary"` + } + require.NoError(t, json.Unmarshal(rec4.Body.Bytes(), &summaryResp)) + assert.Equal(t, tt.wantOpenShards, summaryResp.StreamDescriptionSummary.OpenShardCount) }) - require.Equal(t, http.StatusOK, rec4.Code) - - var summaryResp struct { - StreamDescriptionSummary struct { - OpenShardCount int `json:"OpenShardCount"` - } `json:"StreamDescriptionSummary"` - } - require.NoError(t, json.Unmarshal(rec4.Body.Bytes(), &summaryResp)) - assert.Equal(t, tt.wantOpenShards, summaryResp.StreamDescriptionSummary.OpenShardCount) }) } } @@ -422,77 +435,83 @@ func TestDescribeStream_EncryptionTypeDefault(t *testing.T) { func TestDescribeStream_IncludesClosedShardsAfterMerge(t *testing.T) { t.Parallel() - h := newTestHandler(t) - - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "describe-closed-merge", - "ShardCount": 2, - }) - require.Equal(t, http.StatusOK, rec.Code) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - var descResp struct { - StreamDescription struct { - Shards []struct { - ShardID string `json:"ShardId"` - } `json:"Shards"` - } `json:"StreamDescription"` - } - rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "describe-closed-merge"}) - require.Equal(t, http.StatusOK, rec.Code) - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - s0 := descResp.StreamDescription.Shards[0].ShardID - s1 := descResp.StreamDescription.Shards[1].ShardID + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "describe-closed-merge", + "ShardCount": 2, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) + + var descResp struct { + StreamDescription struct { + Shards []struct { + ShardID string `json:"ShardId"` + } `json:"Shards"` + } `json:"StreamDescription"` + } + rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "describe-closed-merge"}) + require.Equal(t, http.StatusOK, rec.Code) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + s0 := descResp.StreamDescription.Shards[0].ShardID + s1 := descResp.StreamDescription.Shards[1].ShardID + + rec = doRequest(t, h, "MergeShards", map[string]any{ + "StreamName": "describe-closed-merge", + "ShardToMerge": s0, + "AdjacentShardToMerge": s1, + }) + require.Equal(t, http.StatusOK, rec.Code) - rec = doRequest(t, h, "MergeShards", map[string]any{ - "StreamName": "describe-closed-merge", - "ShardToMerge": s0, - "AdjacentShardToMerge": s1, + // After merge: 2 closed parents + 1 open merged = 3 total. + rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "describe-closed-merge"}) + require.Equal(t, http.StatusOK, rec.Code) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + assert.Len(t, descResp.StreamDescription.Shards, 3) }) - require.Equal(t, http.StatusOK, rec.Code) - - // After merge: 2 closed parents + 1 open merged = 3 total. - rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "describe-closed-merge"}) - require.Equal(t, http.StatusOK, rec.Code) - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - assert.Len(t, descResp.StreamDescription.Shards, 3) } func TestDescribeStream_IncludesClosedShardsAfterSplit(t *testing.T) { t.Parallel() - h := newTestHandler(t) - - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "describe-closed-split", - "ShardCount": 1, - }) - require.Equal(t, http.StatusOK, rec.Code) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - var descResp struct { - StreamDescription struct { - Shards []struct { - ShardID string `json:"ShardId"` - } `json:"Shards"` - } `json:"StreamDescription"` - } - rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "describe-closed-split"}) - require.Equal(t, http.StatusOK, rec.Code) - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - shardID := descResp.StreamDescription.Shards[0].ShardID + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "describe-closed-split", + "ShardCount": 1, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) + + var descResp struct { + StreamDescription struct { + Shards []struct { + ShardID string `json:"ShardId"` + } `json:"Shards"` + } `json:"StreamDescription"` + } + rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "describe-closed-split"}) + require.Equal(t, http.StatusOK, rec.Code) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + shardID := descResp.StreamDescription.Shards[0].ShardID + + const splitKey = "170141183460469231731687303715884105728" + rec = doRequest(t, h, "SplitShard", map[string]any{ + "StreamName": "describe-closed-split", + "ShardToSplit": shardID, + "NewStartingHashKey": splitKey, + }) + require.Equal(t, http.StatusOK, rec.Code) - const splitKey = "170141183460469231731687303715884105728" - rec = doRequest(t, h, "SplitShard", map[string]any{ - "StreamName": "describe-closed-split", - "ShardToSplit": shardID, - "NewStartingHashKey": splitKey, + // After split: 1 closed parent + 2 open children = 3 total. + rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "describe-closed-split"}) + require.Equal(t, http.StatusOK, rec.Code) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + assert.Len(t, descResp.StreamDescription.Shards, 3) }) - require.Equal(t, http.StatusOK, rec.Code) - - // After split: 1 closed parent + 2 open children = 3 total. - rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "describe-closed-split"}) - require.Equal(t, http.StatusOK, rec.Code) - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - assert.Len(t, descResp.StreamDescription.Shards, 3) } func TestDescribeStream_OpenShardNoEndingSequenceNumber(t *testing.T) { @@ -533,76 +552,82 @@ func TestDescribeStream_OpenShardNoEndingSequenceNumber(t *testing.T) { func TestDescribeStream_OpenShardWithRecordsNoEndingSeq(t *testing.T) { t.Parallel() - h := newTestHandler(t) - - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "open-shard-with-records", - "ShardCount": 1, - }) - require.Equal(t, http.StatusOK, rec.Code) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - // Write several records into the single (still open) shard. - for i := range 3 { - rec = doRequest(t, h, "PutRecord", map[string]any{ - "StreamName": "open-shard-with-records", - "PartitionKey": fmt.Sprintf("pk-%d", i), - "Data": []byte("payload"), + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "open-shard-with-records", + "ShardCount": 1, }) require.Equal(t, http.StatusOK, rec.Code) - } - - var descResp struct { - StreamDescription struct { - Shards []struct { - ShardID string `json:"ShardId"` - SequenceNumberRange struct { - StartingSequenceNumber string `json:"StartingSequenceNumber"` - EndingSequenceNumber string `json:"EndingSequenceNumber"` - } `json:"SequenceNumberRange"` - } `json:"Shards"` - } `json:"StreamDescription"` - } - rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "open-shard-with-records"}) - require.Equal(t, http.StatusOK, rec.Code) - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - require.Len(t, descResp.StreamDescription.Shards, 1) + time.Sleep(streamSettleWait) + + // Write several records into the single (still open) shard. + for i := range 3 { + rec = doRequest(t, h, "PutRecord", map[string]any{ + "StreamName": "open-shard-with-records", + "PartitionKey": fmt.Sprintf("pk-%d", i), + "Data": []byte("payload"), + }) + require.Equal(t, http.StatusOK, rec.Code) + } - shard := descResp.StreamDescription.Shards[0] - // A populated open shard still has a starting sequence number... - assert.NotEmpty(t, shard.SequenceNumberRange.StartingSequenceNumber) - // ...but must NOT report an ending sequence number while it remains open. - assert.Empty(t, shard.SequenceNumberRange.EndingSequenceNumber, - "open shard with records must not report EndingSequenceNumber") + var descResp struct { + StreamDescription struct { + Shards []struct { + ShardID string `json:"ShardId"` + SequenceNumberRange struct { + StartingSequenceNumber string `json:"StartingSequenceNumber"` + EndingSequenceNumber string `json:"EndingSequenceNumber"` + } `json:"SequenceNumberRange"` + } `json:"Shards"` + } `json:"StreamDescription"` + } + rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "open-shard-with-records"}) + require.Equal(t, http.StatusOK, rec.Code) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + require.Len(t, descResp.StreamDescription.Shards, 1) + + shard := descResp.StreamDescription.Shards[0] + // A populated open shard still has a starting sequence number... + assert.NotEmpty(t, shard.SequenceNumberRange.StartingSequenceNumber) + // ...but must NOT report an ending sequence number while it remains open. + assert.Empty(t, shard.SequenceNumberRange.EndingSequenceNumber, + "open shard with records must not report EndingSequenceNumber") + }) } func TestDescribeStream_UpdateShardCount_IncludesOldClosedShards(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "describe-usc-stream", - "ShardCount": 3, - }) - require.Equal(t, http.StatusOK, rec.Code) + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "describe-usc-stream", + "ShardCount": 3, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) - rec = doRequest(t, h, "UpdateShardCount", map[string]any{ - "StreamName": "describe-usc-stream", - "TargetShardCount": 2, - "ScalingType": "UNIFORM_SCALING", - }) - require.Equal(t, http.StatusOK, rec.Code) + rec = doRequest(t, h, "UpdateShardCount", map[string]any{ + "StreamName": "describe-usc-stream", + "TargetShardCount": 2, + "ScalingType": "UNIFORM_SCALING", + }) + require.Equal(t, http.StatusOK, rec.Code) - // DescribeStream should include both closed (3 old) and open (2 new) shards. - var descResp struct { - StreamDescription struct { - Shards []any `json:"Shards"` - } `json:"StreamDescription"` - } - rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "describe-usc-stream"}) - require.Equal(t, http.StatusOK, rec.Code) - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - assert.Len(t, descResp.StreamDescription.Shards, 5, "3 closed + 2 open = 5") + // DescribeStream should include both closed (3 old) and open (2 new) shards. + var descResp struct { + StreamDescription struct { + Shards []any `json:"Shards"` + } `json:"StreamDescription"` + } + rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "describe-usc-stream"}) + require.Equal(t, http.StatusOK, rec.Code) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + assert.Len(t, descResp.StreamDescription.Shards, 5, "3 closed + 2 open = 5") + }) } // TestDescribeStream_ShardPagination verifies that DescribeStream @@ -679,58 +704,61 @@ func TestDescribeStream_ShardPagination(t *testing.T) { func TestDescribeStream_ClosedShardHasEndingSequenceNumber(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - rec := doRequest(t, h, "CreateStream", map[string]any{ - "StreamName": "closing-seqnum-stream", - "ShardCount": 1, - }) - require.Equal(t, http.StatusOK, rec.Code) + rec := doRequest(t, h, "CreateStream", map[string]any{ + "StreamName": "closing-seqnum-stream", + "ShardCount": 1, + }) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(streamSettleWait) - // Put a record so the shard has a non-trivial sequence range. - doRequest(t, h, "PutRecord", map[string]any{ - "StreamName": "closing-seqnum-stream", - "PartitionKey": "pk", - "Data": []byte("data"), - }) + // Put a record so the shard has a non-trivial sequence range. + doRequest(t, h, "PutRecord", map[string]any{ + "StreamName": "closing-seqnum-stream", + "PartitionKey": "pk", + "Data": []byte("data"), + }) - var descResp struct { - StreamDescription struct { - Shards []struct { - ShardID string `json:"ShardId"` - SequenceNumberRange struct { - EndingSequenceNumber string `json:"EndingSequenceNumber"` - } `json:"SequenceNumberRange"` - } `json:"Shards"` - } `json:"StreamDescription"` - } + var descResp struct { + StreamDescription struct { + Shards []struct { + ShardID string `json:"ShardId"` + SequenceNumberRange struct { + EndingSequenceNumber string `json:"EndingSequenceNumber"` + } `json:"SequenceNumberRange"` + } `json:"Shards"` + } `json:"StreamDescription"` + } - rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "closing-seqnum-stream"}) - require.Equal(t, http.StatusOK, rec.Code) - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - require.Len(t, descResp.StreamDescription.Shards, 1) - shardID := descResp.StreamDescription.Shards[0].ShardID - - // Split the shard to close it. - const splitKey = "170141183460469231731687303715884105728" - rec = doRequest(t, h, "SplitShard", map[string]any{ - "StreamName": "closing-seqnum-stream", - "ShardToSplit": shardID, - "NewStartingHashKey": splitKey, - }) - require.Equal(t, http.StatusOK, rec.Code) + rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "closing-seqnum-stream"}) + require.Equal(t, http.StatusOK, rec.Code) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + require.Len(t, descResp.StreamDescription.Shards, 1) + shardID := descResp.StreamDescription.Shards[0].ShardID + + // Split the shard to close it. + const splitKey = "170141183460469231731687303715884105728" + rec = doRequest(t, h, "SplitShard", map[string]any{ + "StreamName": "closing-seqnum-stream", + "ShardToSplit": shardID, + "NewStartingHashKey": splitKey, + }) + require.Equal(t, http.StatusOK, rec.Code) - // Re-describe: closed shard should have a non-empty EndingSequenceNumber. - rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "closing-seqnum-stream"}) - require.Equal(t, http.StatusOK, rec.Code) - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - require.Len(t, descResp.StreamDescription.Shards, 3) + // Re-describe: closed shard should have a non-empty EndingSequenceNumber. + rec = doRequest(t, h, "DescribeStream", map[string]any{"StreamName": "closing-seqnum-stream"}) + require.Equal(t, http.StatusOK, rec.Code) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + require.Len(t, descResp.StreamDescription.Shards, 3) - closedCount := 0 - for _, s := range descResp.StreamDescription.Shards { - if s.SequenceNumberRange.EndingSequenceNumber != "" { - closedCount++ + closedCount := 0 + for _, s := range descResp.StreamDescription.Shards { + if s.SequenceNumberRange.EndingSequenceNumber != "" { + closedCount++ + } } - } - assert.Equal(t, 1, closedCount, "exactly one shard should be closed with a non-empty ending seq") + assert.Equal(t, 1, closedCount, "exactly one shard should be closed with a non-empty ending seq") + }) } diff --git a/services/kinesis/streams_test.go b/services/kinesis/streams_test.go index 8f0e34f0f7..23f7df41ed 100644 --- a/services/kinesis/streams_test.go +++ b/services/kinesis/streams_test.go @@ -6,6 +6,8 @@ import ( "fmt" "net/http" "testing" + "testing/synctest" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -195,27 +197,34 @@ func TestDeleteStream_ByARN(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newTestHandler(t) - streamName := "delete-by-" + tt.name - doRequest(t, h, "CreateStream", map[string]any{"StreamName": streamName, "ShardCount": 1}) - - b := h.Backend.(*kinesis.InMemoryBackend) - desc, err := b.DescribeStream(context.Background(), &kinesis.DescribeStreamInput{StreamName: streamName}) - require.NoError(t, err) - - var deleteBody map[string]any - if tt.useARN { - deleteBody = map[string]any{"StreamARN": desc.StreamARN} - } else { - deleteBody = map[string]any{"StreamName": streamName} - } + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + streamName := "delete-by-" + tt.name + doRequest(t, h, "CreateStream", map[string]any{"StreamName": streamName, "ShardCount": 1}) + time.Sleep(streamSettleWait) + + b := h.Backend.(*kinesis.InMemoryBackend) + desc, err := b.DescribeStream( + context.Background(), + &kinesis.DescribeStreamInput{StreamName: streamName}, + ) + require.NoError(t, err) + + var deleteBody map[string]any + if tt.useARN { + deleteBody = map[string]any{"StreamARN": desc.StreamARN} + } else { + deleteBody = map[string]any{"StreamName": streamName} + } - rec := doRequest(t, h, "DeleteStream", deleteBody) - assert.Equal(t, tt.wantStatus, rec.Code) + rec := doRequest(t, h, "DeleteStream", deleteBody) + assert.Equal(t, tt.wantStatus, rec.Code) + time.Sleep(streamSettleWait) - // Verify stream is gone. - descRec := doRequest(t, h, "DescribeStream", map[string]any{"StreamName": streamName}) - assert.Equal(t, http.StatusBadRequest, descRec.Code) + // Verify stream is gone. + descRec := doRequest(t, h, "DescribeStream", map[string]any{"StreamName": streamName}) + assert.Equal(t, http.StatusBadRequest, descRec.Code) + }) }) } } @@ -366,11 +375,14 @@ func TestListStreams_Pagination(t *testing.T) { func TestDeleteStream_ClosesTags(t *testing.T) { t.Parallel() - bk := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + bk := kinesis.NewInMemoryBackend().WithClock(clock.Now) require.NoError(t, bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "tagged-stream"})) + clock.Advance(streamSettleWait) // Delete should not panic (Close is safe to call). require.NoError(t, bk.DeleteStream(context.Background(), &kinesis.DeleteStreamInput{StreamName: "tagged-stream"})) + clock.Advance(streamSettleWait) // Recreating with the same name should succeed (Tags registry released). require.NoError(t, bk.CreateStream(context.Background(), &kinesis.CreateStreamInput{StreamName: "tagged-stream"})) @@ -542,7 +554,8 @@ func TestCreateStream_WithTags(t *testing.T) { func TestStreamLifecycle(t *testing.T) { t.Parallel() - h := newTestHandler(t) + clock := newFakeClock(time.Now()) + h := newTestHandlerWithBackend(t, kinesis.NewInMemoryBackend().WithClock(clock.Now)) // CreateStream rec := doRequest(t, h, "CreateStream", map[string]any{ @@ -550,6 +563,7 @@ func TestStreamLifecycle(t *testing.T) { "ShardCount": 2, }) assert.Equal(t, http.StatusOK, rec.Code) + clock.Advance(streamSettleWait) // ListStreams rec = doRequest(t, h, "ListStreams", nil) @@ -602,6 +616,7 @@ func TestStreamLifecycle(t *testing.T) { "StreamName": "my-stream", }) assert.Equal(t, http.StatusOK, rec.Code) + clock.Advance(streamSettleWait) // Verify gone rec = doRequest(t, h, "DescribeStream", map[string]any{ @@ -746,8 +761,8 @@ func TestCreateStream_ProvisionedNotAffectedByOnDemandLimit(t *testing.T) { func TestCreateStream_OnDemandLimit_DeleteFreesSlot(t *testing.T) { t.Parallel() - h := newTestHandler(t) - b := h.Backend.(*kinesis.InMemoryBackend) + clock := newFakeClock(time.Now()) + b := kinesis.NewInMemoryBackend().WithClock(clock.Now) b.SetOnDemandStreamCountLimit(1) @@ -764,9 +779,13 @@ func TestCreateStream_OnDemandLimit_DeleteFreesSlot(t *testing.T) { StreamMode: "ON_DEMAND", })) + clock.Advance(streamSettleWait) + // Delete the first stream to free the slot. require.NoError(t, b.DeleteStream(context.Background(), &kinesis.DeleteStreamInput{StreamName: "od-del-stream"})) + clock.Advance(streamSettleWait) + // Now the second stream should succeed. require.NoError(t, b.CreateStream(context.Background(), &kinesis.CreateStreamInput{ StreamName: "od-del-stream-2", diff --git a/services/kinesis/subscribe_idle_close_test.go b/services/kinesis/subscribe_idle_close_test.go index 2ccc8e2905..1a4281981f 100644 --- a/services/kinesis/subscribe_idle_close_test.go +++ b/services/kinesis/subscribe_idle_close_test.go @@ -36,7 +36,8 @@ func TestSubscribeToShard_IdleCloseIsGraceful(t *testing.T) { heartbeatInterval = 40 * time.Millisecond ) - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient( t, kinesis.NewHandler(backend).WithSubscribeToShardTiming(streamDuration, pollInterval, heartbeatInterval), @@ -48,6 +49,7 @@ func TestSubscribeToShard_IdleCloseIsGraceful(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + clock.Advance(streamSettleWait) desc, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{ StreamName: aws.String(streamName), diff --git a/services/kinesis/subscribe_roundtrip_test.go b/services/kinesis/subscribe_roundtrip_test.go index 131a0c24f3..8474fd1ed6 100644 --- a/services/kinesis/subscribe_roundtrip_test.go +++ b/services/kinesis/subscribe_roundtrip_test.go @@ -15,7 +15,8 @@ import ( func TestSubscribeToShard_RoundTrip(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "subscribe-smoke-stream" @@ -24,6 +25,7 @@ func TestSubscribeToShard_RoundTrip(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + clock.Advance(streamSettleWait) desc, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{ StreamName: aws.String(streamName), diff --git a/services/kinesis/transitions.go b/services/kinesis/transitions.go new file mode 100644 index 0000000000..e5b535339f --- /dev/null +++ b/services/kinesis/transitions.go @@ -0,0 +1,116 @@ +package kinesis + +import "time" + +// streamDeadlinePassed reports whether stream's ReadyAt deadline has passed +// as of now. +func streamDeadlinePassed(stream *Stream, now time.Time) bool { + return !stream.ReadyAt.IsZero() && !now.Before(stream.ReadyAt) +} + +// effectiveStreamStatus returns stream's status as of now, resolving a due +// CREATING/UPDATING->ACTIVE deadline without mutating stream. Callers that +// only need to know the current status (not report or advance it, e.g. the +// PutRecord hot path) can call this under whatever lock they already hold. +// DELETING is never resolved here -- removing a stream from the backend +// requires b.mu for writing, which callers of this helper may not hold; use +// resolveStreamTransitionLocked for that. +func effectiveStreamStatus(stream *Stream, now time.Time) string { + if (stream.Status == streamStatusCreating || stream.Status == streamStatusUpdating) && + streamDeadlinePassed(stream, now) { + return streamStatusActive + } + + return stream.Status +} + +// streamEffectivelyGone reports whether stream should be treated as already +// removed: DELETING with its removal deadline passed. Pure/non-mutating, for +// hot-path readers (GetRecords, GetShardIterator) that only hold b.mu for +// reading and so cannot perform the physical removal themselves -- see +// resolveStreamTransitionLocked, which does that lazily elsewhere (any +// DescribeStream/ListStreams/mutation call). +func streamEffectivelyGone(stream *Stream, now time.Time) bool { + return stream.Status == streamStatusDeleting && streamDeadlinePassed(stream, now) +} + +// resolveStreamTransitionLocked returns the current stream for region/name +// after resolving any lazy CREATING/UPDATING->ACTIVE transition whose +// deadline has passed, or reports ErrStreamNotFound if a DELETING deadline +// has passed (physically removing the stream). Callers must hold b.mu for +// writing -- see services/dsql's resolveClusterLocked and services/dax's +// sweepClusterTransitionsLocked (commit b42c0fe60) for the same lazy-deadline +// pattern. +func (b *InMemoryBackend) resolveStreamTransitionLocked(region, name string) (*Stream, error) { + key := streamKey(region, name) + + stream, ok := b.streams.Get(key) + if !ok { + return nil, ErrStreamNotFound + } + + stream.mu.Lock("resolveStreamTransition.stream") + now := b.nowFunc() + + switch { + case stream.Status == streamStatusDeleting && streamDeadlinePassed(stream, now): + stream.mu.Unlock() + b.finishStreamDeletionLocked(region, name, stream) + + return nil, ErrStreamNotFound + case (stream.Status == streamStatusCreating || stream.Status == streamStatusUpdating) && + streamDeadlinePassed(stream, now): + stream.Status = streamStatusActive + stream.ReadyAt = time.Time{} + } + stream.mu.Unlock() + + return stream, nil +} + +// resolveRegionStreamsLocked returns every live stream in region after +// resolving each one's due lazy transition (see +// resolveStreamTransitionLocked), pruning any whose DELETING deadline has +// passed. Callers must hold b.mu for writing. +func (b *InMemoryBackend) resolveRegionStreamsLocked(region string) []*Stream { + names := make([]string, 0, len(b.streamsByRegion.Get(region))) + for _, s := range b.streamsByRegion.Get(region) { + names = append(names, s.Name) + } + + live := make([]*Stream, 0, len(names)) + for _, name := range names { + s, err := b.resolveStreamTransitionLocked(region, name) + if err != nil { + continue + } + live = append(live, s) + } + + return live +} + +// finishStreamDeletionLocked physically removes stream -- already past its +// DELETING deadline -- from the backend. Callers must hold b.mu for writing; +// stream.mu must NOT be held. +func (b *InMemoryBackend) finishStreamDeletionLocked(region, name string, stream *Stream) { + stream.mu.Lock("finishStreamDeletion.stream") + if stream.Tags != nil { + stream.Tags.Close() + } + stream.mu.Unlock() + + b.streams.Delete(streamKey(region, name)) + + b.faultsMu.Lock("finishStreamDeletion.faults") + delete(b.faultsStore(region), name) + b.faultsMu.Unlock() + + delete(b.policiesStore(region), stream.ARN) + + stream.mu.Close() + + if b.OnStreamPurged != nil { + b.OnStreamPurged(name) + } +} diff --git a/services/kinesis/whitebox_test.go b/services/kinesis/whitebox_test.go index e430ca73b7..36b38308fb 100644 --- a/services/kinesis/whitebox_test.go +++ b/services/kinesis/whitebox_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -82,12 +83,21 @@ func TestListShards_DefaultMaxResults(t *testing.T) { func TestListShards_ShardFilterType_AtTimestamp(t *testing.T) { t.Parallel() + synctest.Test(t, func(t *testing.T) { + testListShardsShardFilterTypeAtTimestamp(t) + }) +} + +func testListShardsShardFilterTypeAtTimestamp(t *testing.T) { + t.Helper() + b := NewInMemoryBackend() ctx := context.Background() require.NoError(t, b.CreateStream(ctx, &CreateStreamInput{ StreamName: "at-ts-stream", ShardCount: 1, })) + time.Sleep(streamSettleWaitInternal) now := time.Now() oldStart := now.Add(-3 * time.Hour) @@ -137,12 +147,21 @@ func TestListShards_ShardFilterType_AtTimestamp(t *testing.T) { func TestListShards_ShardFilterType_FromTimestamp(t *testing.T) { t.Parallel() + synctest.Test(t, func(t *testing.T) { + testListShardsShardFilterTypeFromTimestamp(t) + }) +} + +func testListShardsShardFilterTypeFromTimestamp(t *testing.T) { + t.Helper() + b := NewInMemoryBackend() ctx := context.Background() require.NoError(t, b.CreateStream(ctx, &CreateStreamInput{ StreamName: "from-ts-stream", ShardCount: 1, })) + time.Sleep(streamSettleWaitInternal) now := time.Now() oldStart := now.Add(-3 * time.Hour) diff --git a/services/kinesis/wire_field_fixes_test.go b/services/kinesis/wire_field_fixes_test.go index 85016848ab..3e3f552959 100644 --- a/services/kinesis/wire_field_fixes_test.go +++ b/services/kinesis/wire_field_fixes_test.go @@ -158,7 +158,8 @@ func TestStreamIdentifiedByARNOnly(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "arn-only-" + tc.name @@ -167,6 +168,7 @@ func TestStreamIdentifiedByARNOnly(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + clock.Advance(streamSettleWait) desc, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{ StreamName: aws.String(streamName), @@ -262,7 +264,8 @@ func TestRegisterStreamConsumer_TagsRoundTrip(t *testing.T) { func TestDescribeStreamSummary_MaxRecordSizeAndWarmThroughput(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "summary-fields-stream" @@ -272,6 +275,7 @@ func TestDescribeStreamSummary_MaxRecordSizeAndWarmThroughput(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + clock.Advance(streamSettleWait) desc, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{StreamName: aws.String(streamName)}) require.NoError(t, err) @@ -312,7 +316,8 @@ func TestDescribeStreamSummary_MaxRecordSizeAndWarmThroughput(t *testing.T) { func TestListStreams_StreamSummaries(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "list-streams-summaries-stream" @@ -322,6 +327,7 @@ func TestListStreams_StreamSummaries(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + clock.Advance(streamSettleWait) desc, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{StreamName: aws.String(streamName)}) require.NoError(t, err) @@ -345,7 +351,8 @@ func TestListStreams_StreamSummaries(t *testing.T) { func TestUpdateShardCount_StreamARN(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "update-shard-count-arn-stream" @@ -355,6 +362,7 @@ func TestUpdateShardCount_StreamARN(t *testing.T) { ShardCount: aws.Int32(2), }) require.NoError(t, err) + clock.Advance(streamSettleWait) desc, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{StreamName: aws.String(streamName)}) require.NoError(t, err) @@ -411,7 +419,8 @@ func TestCreateStream_MaxRecordSizeAndWarmThroughput(t *testing.T) { func TestUpdateStreamMode_WarmThroughputMiBps(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "update-stream-mode-warm" @@ -421,6 +430,7 @@ func TestUpdateStreamMode_WarmThroughputMiBps(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + clock.Advance(streamSettleWait) desc, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{StreamName: aws.String(streamName)}) require.NoError(t, err) @@ -454,7 +464,8 @@ func TestUpdateStreamMode_WarmThroughputMiBps(t *testing.T) { func TestUpdateStreamMode_WarmThroughputMiBps_PreservesOmitted(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "update-stream-mode-warm-preserve" @@ -464,6 +475,7 @@ func TestUpdateStreamMode_WarmThroughputMiBps_PreservesOmitted(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + clock.Advance(streamSettleWait) desc, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{StreamName: aws.String(streamName)}) require.NoError(t, err) @@ -476,6 +488,7 @@ func TestUpdateStreamMode_WarmThroughputMiBps_PreservesOmitted(t *testing.T) { WarmThroughputMiBps: aws.Int32(9), }) require.NoError(t, err) + clock.Advance(streamSettleWait) // Omits WarmThroughputMiBps -- the stored value must survive. _, err = client.UpdateStreamMode(t.Context(), &kinesissdk.UpdateStreamModeInput{ @@ -485,6 +498,7 @@ func TestUpdateStreamMode_WarmThroughputMiBps_PreservesOmitted(t *testing.T) { }, }) require.NoError(t, err) + clock.Advance(streamSettleWait) preserved, err := client.DescribeStreamSummary(t.Context(), &kinesissdk.DescribeStreamSummaryInput{ StreamName: aws.String(streamName), @@ -523,7 +537,8 @@ func TestUpdateStreamMode_WarmThroughputMiBps_PreservesOmitted(t *testing.T) { func TestGetRecords_EncryptionType(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "encryption-type-stream" @@ -532,6 +547,7 @@ func TestGetRecords_EncryptionType(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + clock.Advance(streamSettleWait) desc, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{ StreamName: aws.String(streamName), @@ -578,7 +594,8 @@ func TestGetRecords_EncryptionType(t *testing.T) { func TestSubscribeToShard_EncryptionType(t *testing.T) { t.Parallel() - backend := kinesis.NewInMemoryBackend() + clock := newFakeClock(time.Now()) + backend := kinesis.NewInMemoryBackend().WithClock(clock.Now) client := newTestKinesisClient(t, kinesis.NewHandler(backend)) streamName := "subscribe-encryption-type-stream" @@ -587,6 +604,7 @@ func TestSubscribeToShard_EncryptionType(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + clock.Advance(streamSettleWait) desc, err := client.DescribeStream(t.Context(), &kinesissdk.DescribeStreamInput{ StreamName: aws.String(streamName), diff --git a/services/kinesisanalytics/applications.go b/services/kinesisanalytics/applications.go index fdfe02aa5e..acc05f67c2 100644 --- a/services/kinesisanalytics/applications.go +++ b/services/kinesisanalytics/applications.go @@ -596,7 +596,9 @@ func (b *InMemoryBackend) ListApplications( regionApps := b.appsByRegion.Get(region) all := make([]*Application, 0, len(regionApps)) - all = append(all, regionApps...) + for _, app := range regionApps { + all = append(all, appCopy(app)) + } sort.Slice(all, func(i, j int) bool { return all[i].ApplicationName < all[j].ApplicationName diff --git a/services/kinesisvideo/PARITY.md b/services/kinesisvideo/PARITY.md new file mode 100644 index 0000000000..f0dae66900 --- /dev/null +++ b/services/kinesisvideo/PARITY.md @@ -0,0 +1,71 @@ +--- +service: kinesisvideo +sdk_module: aws-sdk-go-v2/service/kinesisvideo@v1.41.1 +last_audit_commit: 54869319e +last_audit_date: 2026-09-25 +overall: B # new service, control plane only, unit-tested against the real SDK client +ops: + CreateStream: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeStream: {wire: ok, errors: ok, state: ok, persist: ok} + ListStreams: {wire: ok, errors: ok, state: ok, persist: ok, note: "StreamNameCondition BEGINS_WITH filter; opaque NextToken via pkgs/page"} + UpdateStream: {wire: ok, errors: ok, state: ok, persist: ok, note: "CurrentVersion optimistic lock"} + DeleteStream: {wire: ok, errors: ok, state: ok, persist: ok, note: "CurrentVersion optional per AWS docs"} + UpdateDataRetention: {wire: ok, errors: ok, state: ok, persist: ok} + GetDataEndpoint: {wire: ok, errors: ok, state: ok, persist: ok, note: "returns an AWS-shaped emulator hostname; not backed by a real data plane -- see items_still_open"} + TagStream: {wire: ok, errors: ok, state: ok, persist: ok} + UntagStream: {wire: ok, errors: ok, state: ok, persist: ok} + ListTagsForStream: {wire: ok, errors: ok, state: ok, persist: ok} + TagResource: {wire: ok, errors: ok, state: ok, persist: ok, note: "signaling-channel tags only, per AWS docs"} + UntagResource: {wire: ok, errors: ok, state: ok, persist: ok} + ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok} + CreateSignalingChannel: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeSignalingChannel: {wire: ok, errors: ok, state: ok, persist: ok} + ListSignalingChannels: {wire: ok, errors: ok, state: ok, persist: ok, note: "ChannelNameCondition BEGINS_WITH filter"} + UpdateSignalingChannel: {wire: ok, errors: ok, state: ok, persist: ok, note: "CurrentVersion optimistic lock"} + DeleteSignalingChannel: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeImageGenerationConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateImageGenerationConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + DescribeNotificationConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} + UpdateNotificationConfiguration: {wire: ok, errors: ok, state: ok, persist: ok} +families: + Stream: {status: ok, note: "CreateStream/DescribeStream/ListStreams/UpdateStream/DeleteStream/UpdateDataRetention verified end-to-end against the real aws-sdk-go-v2 client over an httptest server -- wire shapes, epoch CreationTime, ARN format, CurrentVersion optimistic locking, and error deserialization (ResourceNotFoundException/ResourceInUseException/VersionMismatchException) all round-trip cleanly."} + SignalingChannel: {status: ok, note: "Same CRUD + optimistic-lock coverage as Stream. SingleMasterConfiguration.MessageTtlSeconds defaults to 60s per AWS docs."} + Tags: {status: ok, note: "Two disjoint tag families, matching real AWS: TagStream/UntagStream/ListTagsForStream key off the stream; TagResource/UntagResource/ListTagsForResource key off a signaling channel ARN only (confirmed against aws-sdk-go-v2 doc comments -- these three ops predate stream tagging and were never extended to cover streams)."} + ImageGenerationConfiguration: {status: ok, note: "Describe/Update round-trip the full nested shape (DestinationConfig/Format/ImageSelectorType/SamplingInterval/Status/FormatConfig/HeightPixels/WidthPixels)."} + NotificationConfiguration: {status: ok, note: "Describe/Update round-trip DestinationConfig.Uri and Status."} +gaps: [] +items_still_open: + - "Media data plane (PutMedia, GetMedia, GetMediaForFragmentList, GetHLSStreamingSessionURL, + GetDASHStreamingSessionURL, GetClip, GetImages, ListFragments) is not implemented -- structural, + out of scope for this pass. This is the aws-sdk-go-v2/service/kinesisvideomedia and + kinesisvideoarchivedmedia client family, a genuinely separate data-plane service with its own + endpoint (obtained via this service's GetDataEndpoint) and its own SDK module; it is not part + of the kinesisvideo control-plane module this backend implements. GetDataEndpoint returns a + wire-accurate, AWS-shaped hostname so control-plane callers (e.g. Rekognition stream processor + setup, which only needs a stream to exist and its ARN) get a realistic response, but nothing is + listening on that hostname." + - "GetSignalingChannelEndpoint, CreateSignalingChannel's WebRTC ingestion, and the Edge Agent / + MediaStorageConfiguration operation family (DescribeEdgeConfiguration, DeleteEdgeConfiguration, + StartEdgeConfigurationUpdate, ListEdgeAgentConfigurations, DescribeMediaStorageConfiguration, + UpdateMediaStorageConfiguration, DescribeMappedResourceConfiguration, + DescribeStreamStorageConfiguration, UpdateStreamStorageConfiguration) are not implemented -- + structural, out of scope for this pass (not needed by the terraform aws_kinesis_video_stream + resource or by Rekognition stream processors, which only need CreateStream/DescribeStream)." + - "CREATING/UPDATING/DELETING transient stream and channel states are not modeled: CreateStream + and CreateSignalingChannel return ACTIVE immediately and DeleteStream/DeleteSignalingChannel + remove the resource immediately, rather than lingering through a transient state on a lazy + deadline the way e.g. services/mediastore's container lifecycle does. This is an accepted + simplification (explicitly allowed for this service by the parity-sweep task that added it), + not a fidelity gap that changes any client-observable outcome other than timing." +--- + +## Notes + +Initial implementation (2026-09-25): control-plane REST-JSON API modeled after +services/mediastore and services/iotanalytics. Every operation mutates/reads +real in-memory state via pkgs/store.Table + pkgs/lockmetrics.RWMutex, with +JSON snapshot/restore wired into pkgs/persistence. Wire shapes and error +codes were verified against the pinned aws-sdk-go-v2/service/kinesisvideo +v1.41.1 serializers.go/deserializers.go, including the real-AWS quirk that +TagResource/UntagResource/ListTagsForResource use PascalCase URI paths +(/TagResource) while every other operation uses camelCase (/createStream). diff --git a/services/kinesisvideo/README.md b/services/kinesisvideo/README.md new file mode 100644 index 0000000000..5a88ced9ac --- /dev/null +++ b/services/kinesisvideo/README.md @@ -0,0 +1,25 @@ + +# Kinesisvideo + +**Parity grade: B** · SDK `aws-sdk-go-v2/service/kinesisvideo@v1.41.1` · last audited 2026-09-25 (`54869319e`) + +## Coverage + +| Metric | Value | +| --- | --- | +| PARITY entries audited | 22 (22 ok) | +| Feature families | 5 (5 ok) | +| Known gaps | 3 | +| Deferred items | 0 | +| Resource leaks | unknown | + +### Known gaps + +- "Media data plane (PutMedia, GetMedia, GetMediaForFragmentList, GetHLSStreamingSessionURL, GetDASHStreamingSessionURL, GetClip, GetImages, ListFragments) is not implemented -- structural, out of scope for this pass. This is the aws-sdk-go-v2/service/kinesisvideomedia and kinesisvideoarchivedmedia client family, a genuinely separate data-plane service with its own endpoint (obtained via this service's GetDataEndpoint) and its own SDK module; it is not part of the kinesisvideo control-plane module this backend implements. GetDataEndpoint returns a wire-accurate, AWS-shaped hostname so control-plane callers (e.g. Rekognition stream processor setup, which only needs a stream to exist and its ARN) get a realistic response, but nothing is listening on that hostname." +- "GetSignalingChannelEndpoint, CreateSignalingChannel's WebRTC ingestion, and the Edge Agent / MediaStorageConfiguration operation family (DescribeEdgeConfiguration, DeleteEdgeConfiguration, StartEdgeConfigurationUpdate, ListEdgeAgentConfigurations, DescribeMediaStorageConfiguration, UpdateMediaStorageConfiguration, DescribeMappedResourceConfiguration, DescribeStreamStorageConfiguration, UpdateStreamStorageConfiguration) are not implemented -- structural, out of scope for this pass (not needed by the terraform aws_kinesis_video_stream resource or by Rekognition stream processors, which only need CreateStream/DescribeStream)." +- "CREATING/UPDATING/DELETING transient stream and channel states are not modeled: CreateStream and CreateSignalingChannel return ACTIVE immediately and DeleteStream/DeleteSignalingChannel remove the resource immediately, rather than lingering through a transient state on a lazy deadline the way e.g. services/mediastore's container lifecycle does. This is an accepted simplification (explicitly allowed for this service by the parity-sweep task that added it), not a fidelity gap that changes any client-observable outcome other than timing." + +## More + +- [Full parity audit](PARITY.md) +- [All services](../../README.md#services) diff --git a/services/kinesisvideo/configs_test.go b/services/kinesisvideo/configs_test.go new file mode 100644 index 0000000000..b95b8fb670 --- /dev/null +++ b/services/kinesisvideo/configs_test.go @@ -0,0 +1,99 @@ +package kinesisvideo_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + kinesisvideosdk "github.com/aws/aws-sdk-go-v2/service/kinesisvideo" + "github.com/aws/aws-sdk-go-v2/service/kinesisvideo/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestImageGenerationConfiguration(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateStream( + ctx, + &kinesisvideosdk.CreateStreamInput{StreamName: aws.String("image-cfg-stream")}, + ) + require.NoError(t, err) + + empty, err := client.DescribeImageGenerationConfiguration( + ctx, + &kinesisvideosdk.DescribeImageGenerationConfigurationInput{ + StreamARN: created.StreamARN, + }, + ) + require.NoError(t, err) + assert.Nil(t, empty.ImageGenerationConfiguration) + + _, err = client.UpdateImageGenerationConfiguration(ctx, &kinesisvideosdk.UpdateImageGenerationConfigurationInput{ + StreamARN: created.StreamARN, + ImageGenerationConfiguration: &types.ImageGenerationConfiguration{ + DestinationConfig: &types.ImageGenerationDestinationConfig{ + DestinationRegion: aws.String("us-east-1"), + Uri: aws.String("s3://bucket/prefix"), + }, + Format: types.FormatJpeg, + ImageSelectorType: types.ImageSelectorTypeServerTimestamp, + SamplingInterval: aws.Int32(1000), + Status: types.ConfigurationStatusEnabled, + }, + }) + require.NoError(t, err) + + out, err := client.DescribeImageGenerationConfiguration( + ctx, + &kinesisvideosdk.DescribeImageGenerationConfigurationInput{ + StreamARN: created.StreamARN, + }, + ) + require.NoError(t, err) + require.NotNil(t, out.ImageGenerationConfiguration) + assert.Equal(t, types.FormatJpeg, out.ImageGenerationConfiguration.Format) + assert.Equal(t, types.ConfigurationStatusEnabled, out.ImageGenerationConfiguration.Status) + assert.EqualValues(t, 1000, aws.ToInt32(out.ImageGenerationConfiguration.SamplingInterval)) + require.NotNil(t, out.ImageGenerationConfiguration.DestinationConfig) + assert.Equal(t, "s3://bucket/prefix", aws.ToString(out.ImageGenerationConfiguration.DestinationConfig.Uri)) +} + +func TestNotificationConfiguration(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateStream( + ctx, + &kinesisvideosdk.CreateStreamInput{StreamName: aws.String("notif-cfg-stream")}, + ) + require.NoError(t, err) + + empty, err := client.DescribeNotificationConfiguration(ctx, &kinesisvideosdk.DescribeNotificationConfigurationInput{ + StreamARN: created.StreamARN, + }) + require.NoError(t, err) + assert.Nil(t, empty.NotificationConfiguration) + + _, err = client.UpdateNotificationConfiguration(ctx, &kinesisvideosdk.UpdateNotificationConfigurationInput{ + StreamARN: created.StreamARN, + NotificationConfiguration: &types.NotificationConfiguration{ + DestinationConfig: &types.NotificationDestinationConfig{Uri: aws.String("https://example.com/notify")}, + Status: types.ConfigurationStatusEnabled, + }, + }) + require.NoError(t, err) + + out, err := client.DescribeNotificationConfiguration(ctx, &kinesisvideosdk.DescribeNotificationConfigurationInput{ + StreamARN: created.StreamARN, + }) + require.NoError(t, err) + require.NotNil(t, out.NotificationConfiguration) + assert.Equal(t, types.ConfigurationStatusEnabled, out.NotificationConfiguration.Status) + require.NotNil(t, out.NotificationConfiguration.DestinationConfig) + assert.Equal(t, "https://example.com/notify", aws.ToString(out.NotificationConfiguration.DestinationConfig.Uri)) +} diff --git a/services/kinesisvideo/errors.go b/services/kinesisvideo/errors.go new file mode 100644 index 0000000000..1e61f7fc01 --- /dev/null +++ b/services/kinesisvideo/errors.go @@ -0,0 +1,18 @@ +package kinesisvideo + +import "github.com/blackbirdworks/gopherstack/pkgs/awserr" + +var ( + // ErrStreamNotFound is returned when a stream does not exist. + ErrStreamNotFound = awserr.New("stream not found", awserr.ErrNotFound) + // ErrStreamAlreadyExists is returned when a stream name is already in use. + ErrStreamAlreadyExists = awserr.New("stream already exists", awserr.ErrAlreadyExists) + // ErrChannelNotFound is returned when a signaling channel does not exist. + ErrChannelNotFound = awserr.New("signaling channel not found", awserr.ErrNotFound) + // ErrChannelAlreadyExists is returned when a channel name is already in use. + ErrChannelAlreadyExists = awserr.New("signaling channel already exists", awserr.ErrAlreadyExists) + // ErrVersionMismatch is returned when CurrentVersion does not match the resource's version. + ErrVersionMismatch = awserr.New("version mismatch", awserr.ErrConflict) + // ErrValidation is returned when request input fails validation. + ErrValidation = awserr.New("invalid argument", awserr.ErrInvalidParameter) +) diff --git a/services/kinesisvideo/handler.go b/services/kinesisvideo/handler.go new file mode 100644 index 0000000000..9a0afb212c --- /dev/null +++ b/services/kinesisvideo/handler.go @@ -0,0 +1,323 @@ +package kinesisvideo + +import ( + "encoding/json" + "errors" + "maps" + "net/http" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/pkgs/awserr" + "github.com/blackbirdworks/gopherstack/pkgs/httputils" + "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/service" +) + +const ( + kinesisVideoService = "kinesisvideo" + kinesisVideoMatchPriority = service.PriorityPathVersioned +) + +// Operation names, matching the AWS API exactly. +const ( + opCreateStream = "CreateStream" + opDescribeStream = "DescribeStream" + opListStreams = "ListStreams" + opUpdateStream = "UpdateStream" + opDeleteStream = "DeleteStream" + opUpdateDataRetention = "UpdateDataRetention" + opGetDataEndpoint = "GetDataEndpoint" + + opTagStream = "TagStream" + opUntagStream = "UntagStream" + opListTagsForStream = "ListTagsForStream" + + opTagResource = "TagResource" + opUntagResource = "UntagResource" + opListTagsForResource = "ListTagsForResource" + + opCreateSignalingChannel = "CreateSignalingChannel" + opDescribeSignalingChannel = "DescribeSignalingChannel" + opListSignalingChannels = "ListSignalingChannels" + opUpdateSignalingChannel = "UpdateSignalingChannel" + opDeleteSignalingChannel = "DeleteSignalingChannel" + + opDescribeImageGenerationConfiguration = "DescribeImageGenerationConfiguration" + opUpdateImageGenerationConfiguration = "UpdateImageGenerationConfiguration" + opDescribeNotificationConfiguration = "DescribeNotificationConfiguration" + opUpdateNotificationConfiguration = "UpdateNotificationConfiguration" +) + +// URI paths. AWS emits camelCase action paths for every operation except the +// generic-tagging trio, which use PascalCase (confirmed against +// aws-sdk-go-v2/service/kinesisvideo@v1.41.1 serializers.go -- /TagResource, +// /UntagResource, /ListTagsForResource vs. e.g. /createStream). +const ( + pathCreateStream = "/createStream" + pathDescribeStream = "/describeStream" + pathListStreams = "/listStreams" + pathUpdateStream = "/updateStream" + pathDeleteStream = "/deleteStream" + pathUpdateDataRetention = "/updateDataRetention" + pathGetDataEndpoint = "/getDataEndpoint" + + pathTagStream = "/tagStream" + pathUntagStream = "/untagStream" + pathListTagsForStream = "/listTagsForStream" + + pathTagResource = "/TagResource" + pathUntagResource = "/UntagResource" + pathListTagsForResource = "/ListTagsForResource" + + pathCreateSignalingChannel = "/createSignalingChannel" + pathDescribeSignalingChannel = "/describeSignalingChannel" + pathListSignalingChannels = "/listSignalingChannels" + pathUpdateSignalingChannel = "/updateSignalingChannel" + pathDeleteSignalingChannel = "/deleteSignalingChannel" + + pathDescribeImageGenerationConfiguration = "/describeImageGenerationConfiguration" + pathUpdateImageGenerationConfiguration = "/updateImageGenerationConfiguration" + pathDescribeNotificationConfiguration = "/describeNotificationConfiguration" + pathUpdateNotificationConfiguration = "/updateNotificationConfiguration" +) + +// kinesisVideoUniquePaths are claimed unconditionally: none of them are +// reused by any other service in this repo (verified by grep across +// services/*/*.go). +var kinesisVideoUniquePaths = map[string]string{ //nolint:gochecknoglobals // package-level routing table + pathCreateStream: opCreateStream, + pathDescribeStream: opDescribeStream, + pathListStreams: opListStreams, + pathUpdateStream: opUpdateStream, + pathDeleteStream: opDeleteStream, + pathUpdateDataRetention: opUpdateDataRetention, + pathGetDataEndpoint: opGetDataEndpoint, + + pathTagStream: opTagStream, + pathUntagStream: opUntagStream, + pathListTagsForStream: opListTagsForStream, + + pathCreateSignalingChannel: opCreateSignalingChannel, + pathDescribeSignalingChannel: opDescribeSignalingChannel, + pathListSignalingChannels: opListSignalingChannels, + pathUpdateSignalingChannel: opUpdateSignalingChannel, + pathDeleteSignalingChannel: opDeleteSignalingChannel, + + pathDescribeImageGenerationConfiguration: opDescribeImageGenerationConfiguration, + pathUpdateImageGenerationConfiguration: opUpdateImageGenerationConfiguration, + pathDescribeNotificationConfiguration: opDescribeNotificationConfiguration, + pathUpdateNotificationConfiguration: opUpdateNotificationConfiguration, +} + +// kinesisVideoSharedPaths are the generic-tagging paths several restjson1 +// services claim verbatim (see services/rolesanywhere and services/xray). +// They are SigV4-scoped instead of claimed unconditionally, per +// .claude/memories -- route-matcher-prefix-collision. +var kinesisVideoSharedPaths = map[string]string{ //nolint:gochecknoglobals // package-level routing table + pathTagResource: opTagResource, + pathUntagResource: opUntagResource, + pathListTagsForResource: opListTagsForResource, +} + +// handlerFunc is the uniform signature for all dispatch operations. +type handlerFunc func(c *echo.Context, body []byte) error + +// Handler is the HTTP handler for the Kinesis Video Streams control-plane REST API. +type Handler struct { + Backend StorageBackend + ops map[string]handlerFunc + AccountID string + DefaultRegion string +} + +// NewHandler creates a new Kinesis Video Streams handler. +func NewHandler(backend StorageBackend) *Handler { + h := &Handler{Backend: backend} + h.ops = h.buildOps() + + return h +} + +// Reset clears all backend state. +func (h *Handler) Reset() { + h.Backend.Reset() +} + +// Name returns the service name. +func (h *Handler) Name() string { return "KinesisVideo" } + +// GetSupportedOperations returns the list of supported operations. +func (h *Handler) GetSupportedOperations() []string { + ops := make([]string, 0, len(kinesisVideoUniquePaths)+len(kinesisVideoSharedPaths)) + for _, op := range kinesisVideoUniquePaths { + ops = append(ops, op) + } + + for _, op := range kinesisVideoSharedPaths { + ops = append(ops, op) + } + + return ops +} + +// ChaosServiceName returns the lowercase AWS service name for fault rule matching. +func (h *Handler) ChaosServiceName() string { return kinesisVideoService } + +// ChaosOperations returns all operations that can be fault-injected. +func (h *Handler) ChaosOperations() []string { return h.GetSupportedOperations() } + +// ChaosRegions returns all regions this handler handles. +func (h *Handler) ChaosRegions() []string { return []string{h.DefaultRegion} } + +// RouteMatcher returns a function that matches Kinesis Video Streams REST API requests. +func (h *Handler) RouteMatcher() service.Matcher { + return func(c *echo.Context) bool { + path := c.Request().URL.Path + + if _, ok := kinesisVideoUniquePaths[path]; ok { + return true + } + + if _, ok := kinesisVideoSharedPaths[path]; ok { + svc := httputils.ExtractServiceFromRequest(c.Request()) + + return svc == "" || svc == kinesisVideoService + } + + return false + } +} + +// MatchPriority returns the routing priority. +func (h *Handler) MatchPriority() int { return kinesisVideoMatchPriority } + +// ExtractOperation extracts the operation name from the request path. +func (h *Handler) ExtractOperation(c *echo.Context) string { + path := c.Request().URL.Path + if op, ok := kinesisVideoUniquePaths[path]; ok { + return op + } + + if op, ok := kinesisVideoSharedPaths[path]; ok { + return op + } + + return "" +} + +// ExtractResource extracts the stream or channel name/ARN from the request body. +func (h *Handler) ExtractResource(c *echo.Context) string { + body, err := httputils.ReadBody(c.Request()) + if err != nil { + return "" + } + + var data map[string]any + if uerr := json.Unmarshal(body, &data); uerr != nil { + return "" + } + + for _, key := range []string{"StreamName", "StreamARN", "ChannelName", "ChannelARN", "ResourceARN"} { + if v, ok := data[key]; ok { + if s, isStr := v.(string); isStr { + return s + } + } + } + + return "" +} + +// Handler returns the Echo handler function for Kinesis Video Streams requests. +func (h *Handler) Handler() echo.HandlerFunc { + return func(c *echo.Context) error { + ctx := c.Request().Context() + log := logger.Load(ctx) + + path := c.Request().URL.Path + + fn, ok := h.ops[path] + if !ok { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "unknown operation") + } + + body, err := httputils.ReadBody(c.Request()) + if err != nil { + log.ErrorContext(ctx, "kinesisvideo: failed to read request body", "error", err) + + return h.writeError( + c, + http.StatusInternalServerError, + "InternalFailureException", + "failed to read request body", + ) + } + + log.DebugContext(ctx, "kinesisvideo request", "path", path) + + return fn(c, body) + } +} + +// buildOps constructs the operation dispatch map keyed by URI path. +func (h *Handler) buildOps() map[string]handlerFunc { + ops := make(map[string]handlerFunc) + + maps.Copy(ops, h.buildStreamOps()) + + maps.Copy(ops, h.buildTagOps()) + + maps.Copy(ops, h.buildSignalingOps()) + + maps.Copy(ops, h.buildConfigOps()) + + return ops +} + +// listAndConvert runs a paginated backend list call and converts each result +// item to its wire DTO. Shared by handleListStreams and +// handleListSignalingChannels, whose only difference is the resource and DTO +// type parameters. +func listAndConvert[T, D any]( + list func() ([]*T, string, error), + toDTO func(*T) D, +) ([]D, string, error) { + items, next, err := list() + if err != nil { + return nil, "", err + } + + dtos := make([]D, 0, len(items)) + for _, item := range items { + dtos = append(dtos, toDTO(item)) + } + + return dtos, next, nil +} + +// writeJSON writes a 200 JSON response. +func (h *Handler) writeJSON(c *echo.Context, v any) error { + return c.JSON(http.StatusOK, v) +} + +// writeError writes a Kinesis Video Streams JSON error response with the AWS __type field. +func (h *Handler) writeError(c *echo.Context, status int, errType, message string) error { + return c.JSON(status, errorResponse{Type: errType, Message: message}) +} + +// writeBackendError maps a backend error to an HTTP error response with the appropriate AWS error type. +func (h *Handler) writeBackendError(c *echo.Context, err error) error { + switch { + case errors.Is(err, awserr.ErrNotFound): + return h.writeError(c, http.StatusNotFound, "ResourceNotFoundException", err.Error()) + case errors.Is(err, awserr.ErrAlreadyExists): + return h.writeError(c, http.StatusBadRequest, "ResourceInUseException", err.Error()) + case errors.Is(err, awserr.ErrConflict): + return h.writeError(c, http.StatusBadRequest, "VersionMismatchException", err.Error()) + case errors.Is(err, awserr.ErrInvalidParameter): + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", err.Error()) + default: + return h.writeError(c, http.StatusInternalServerError, "InternalFailureException", err.Error()) + } +} diff --git a/services/kinesisvideo/handler_configs.go b/services/kinesisvideo/handler_configs.go new file mode 100644 index 0000000000..85011a8ac4 --- /dev/null +++ b/services/kinesisvideo/handler_configs.go @@ -0,0 +1,79 @@ +package kinesisvideo + +import ( + "encoding/json" + "net/http" + + "github.com/labstack/echo/v5" +) + +func (h *Handler) buildConfigOps() map[string]handlerFunc { + return map[string]handlerFunc{ + pathDescribeImageGenerationConfiguration: h.handleDescribeImageGenerationConfiguration, + pathUpdateImageGenerationConfiguration: h.handleUpdateImageGenerationConfiguration, + pathDescribeNotificationConfiguration: h.handleDescribeNotificationConfiguration, + pathUpdateNotificationConfiguration: h.handleUpdateNotificationConfiguration, + } +} + +func (h *Handler) handleDescribeImageGenerationConfiguration(c *echo.Context, body []byte) error { + var req describeImageGenerationConfigurationRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + cfg, err := h.Backend.DescribeImageGenerationConfiguration(req.StreamName, req.StreamARN) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, describeImageGenerationConfigurationResponse{ + ImageGenerationConfiguration: imageGenerationConfigToDTO(cfg), + }) +} + +func (h *Handler) handleUpdateImageGenerationConfiguration(c *echo.Context, body []byte) error { + var req updateImageGenerationConfigurationRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + cfg := imageGenerationConfigFromDTO(req.ImageGenerationConfiguration) + + if err := h.Backend.UpdateImageGenerationConfiguration(req.StreamName, req.StreamARN, cfg); err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, struct{}{}) +} + +func (h *Handler) handleDescribeNotificationConfiguration(c *echo.Context, body []byte) error { + var req describeNotificationConfigurationRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + cfg, err := h.Backend.DescribeNotificationConfiguration(req.StreamName, req.StreamARN) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, describeNotificationConfigurationResponse{ + NotificationConfiguration: notificationConfigToDTO(cfg), + }) +} + +func (h *Handler) handleUpdateNotificationConfiguration(c *echo.Context, body []byte) error { + var req updateNotificationConfigurationRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + cfg := notificationConfigFromDTO(req.NotificationConfiguration) + + if err := h.Backend.UpdateNotificationConfiguration(req.StreamName, req.StreamARN, cfg); err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, struct{}{}) +} diff --git a/services/kinesisvideo/handler_signaling.go b/services/kinesisvideo/handler_signaling.go new file mode 100644 index 0000000000..07b7344086 --- /dev/null +++ b/services/kinesisvideo/handler_signaling.go @@ -0,0 +1,133 @@ +package kinesisvideo + +import ( + "encoding/json" + "net/http" + + "github.com/labstack/echo/v5" +) + +func (h *Handler) buildSignalingOps() map[string]handlerFunc { + return map[string]handlerFunc{ + pathCreateSignalingChannel: h.handleCreateSignalingChannel, + pathDescribeSignalingChannel: h.handleDescribeSignalingChannel, + pathListSignalingChannels: h.handleListSignalingChannels, + pathUpdateSignalingChannel: h.handleUpdateSignalingChannel, + pathDeleteSignalingChannel: h.handleDeleteSignalingChannel, + } +} + +func (h *Handler) handleCreateSignalingChannel(c *echo.Context, body []byte) error { + var req createSignalingChannelRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + if req.ChannelName == "" { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "ChannelName is required") + } + + var ttl int32 + if req.SingleMasterConfiguration != nil { + ttl = req.SingleMasterConfiguration.MessageTTLSeconds + } + + tags := make(map[string]string, len(req.Tags)) + for _, t := range req.Tags { + tags[t.Key] = t.Value + } + + ch, err := h.Backend.CreateSignalingChannel( + h.AccountID, regionFromRequest(c, h.DefaultRegion), req.ChannelName, req.ChannelType, ttl, tags) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, createSignalingChannelResponse{ChannelARN: ch.ARN}) +} + +func (h *Handler) handleDescribeSignalingChannel(c *echo.Context, body []byte) error { + var req describeSignalingChannelRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + ch, err := h.Backend.DescribeSignalingChannel(req.ChannelName, req.ChannelARN) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, describeSignalingChannelResponse{ChannelInfo: channelInfoFromChannel(ch)}) +} + +func (h *Handler) handleListSignalingChannels(c *echo.Context, body []byte) error { + var req listSignalingChannelsRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + var cond *ChannelNameCondition + if req.ChannelNameCondition != nil { + cond = &ChannelNameCondition{ + ComparisonOperator: req.ChannelNameCondition.ComparisonOperator, + ComparisonValue: req.ChannelNameCondition.ComparisonValue, + } + } + + infos, next, err := listAndConvert( + func() ([]*Channel, string, error) { + return h.Backend.ListSignalingChannels(req.NextToken, int(req.MaxResults), cond) + }, + channelInfoFromChannel, + ) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, listSignalingChannelsResponse{ChannelInfoList: infos, NextToken: next}) +} + +func (h *Handler) handleUpdateSignalingChannel(c *echo.Context, body []byte) error { + var req updateSignalingChannelRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + if req.ChannelARN == "" || req.CurrentVersion == "" { + return h.writeError( + c, + http.StatusBadRequest, + "InvalidArgumentException", + "ChannelARN and CurrentVersion are required", + ) + } + + var ttl *int32 + if req.SingleMasterConfiguration != nil { + v := req.SingleMasterConfiguration.MessageTTLSeconds + ttl = &v + } + + if err := h.Backend.UpdateSignalingChannel(req.ChannelARN, req.CurrentVersion, ttl); err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, struct{}{}) +} + +func (h *Handler) handleDeleteSignalingChannel(c *echo.Context, body []byte) error { + var req deleteSignalingChannelRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + if req.ChannelARN == "" { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "ChannelARN is required") + } + + if err := h.Backend.DeleteSignalingChannel(req.ChannelARN, req.CurrentVersion); err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, struct{}{}) +} diff --git a/services/kinesisvideo/handler_streams.go b/services/kinesisvideo/handler_streams.go new file mode 100644 index 0000000000..1840e0f34c --- /dev/null +++ b/services/kinesisvideo/handler_streams.go @@ -0,0 +1,176 @@ +package kinesisvideo + +import ( + "encoding/json" + "net/http" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/pkgs/httputils" +) + +func (h *Handler) buildStreamOps() map[string]handlerFunc { + return map[string]handlerFunc{ + pathCreateStream: h.handleCreateStream, + pathDescribeStream: h.handleDescribeStream, + pathListStreams: h.handleListStreams, + pathUpdateStream: h.handleUpdateStream, + pathDeleteStream: h.handleDeleteStream, + pathUpdateDataRetention: h.handleUpdateDataRetention, + pathGetDataEndpoint: h.handleGetDataEndpoint, + } +} + +func (h *Handler) handleCreateStream(c *echo.Context, body []byte) error { + var req createStreamRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + if req.StreamName == "" { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "StreamName is required") + } + + defaultStorageTier := "" + if req.StreamStorageConfiguration != nil { + defaultStorageTier = req.StreamStorageConfiguration.DefaultStorageTier + } + + s, err := h.Backend.CreateStream( + h.AccountID, regionFromRequest(c, h.DefaultRegion), req.StreamName, req.DeviceName, req.MediaType, + req.KmsKeyID, defaultStorageTier, req.DataRetentionInHours, req.Tags, + ) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, createStreamResponse{StreamARN: s.ARN}) +} + +func (h *Handler) handleDescribeStream(c *echo.Context, body []byte) error { + var req describeStreamRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + s, err := h.Backend.DescribeStream(req.StreamName, req.StreamARN) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, describeStreamResponse{StreamInfo: streamInfoFromStream(s)}) +} + +func (h *Handler) handleListStreams(c *echo.Context, body []byte) error { + var req listStreamsRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + var cond *StreamNameCondition + if req.StreamNameCondition != nil { + cond = &StreamNameCondition{ + ComparisonOperator: req.StreamNameCondition.ComparisonOperator, + ComparisonValue: req.StreamNameCondition.ComparisonValue, + } + } + + infos, next, err := listAndConvert( + func() ([]*Stream, string, error) { + return h.Backend.ListStreams(req.NextToken, int(req.MaxResults), cond) + }, + streamInfoFromStream, + ) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, listStreamsResponse{StreamInfoList: infos, NextToken: next}) +} + +func (h *Handler) handleUpdateStream(c *echo.Context, body []byte) error { + var req updateStreamRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + if req.CurrentVersion == "" { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "CurrentVersion is required") + } + + if err := h.Backend.UpdateStream( + req.StreamName, + req.StreamARN, + req.CurrentVersion, + req.DeviceName, + req.MediaType, + ); err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, struct{}{}) +} + +func (h *Handler) handleDeleteStream(c *echo.Context, body []byte) error { + var req deleteStreamRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + if req.StreamARN == "" { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "StreamARN is required") + } + + if err := h.Backend.DeleteStream(req.StreamARN, req.CurrentVersion); err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, struct{}{}) +} + +func (h *Handler) handleUpdateDataRetention(c *echo.Context, body []byte) error { + var req updateDataRetentionRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + if req.CurrentVersion == "" || req.Operation == "" { + return h.writeError( + c, + http.StatusBadRequest, + "InvalidArgumentException", + "CurrentVersion and Operation are required", + ) + } + + err := h.Backend.UpdateDataRetention( + req.StreamName, req.StreamARN, req.CurrentVersion, req.Operation, req.DataRetentionChangeInHours) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, struct{}{}) +} + +func (h *Handler) handleGetDataEndpoint(c *echo.Context, body []byte) error { + var req getDataEndpointRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + if req.APIName == "" { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "APIName is required") + } + + endpoint, err := h.Backend.GetDataEndpoint( + req.StreamName, req.StreamARN, req.APIName, regionFromRequest(c, h.DefaultRegion)) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, getDataEndpointResponse{DataEndpoint: endpoint}) +} + +func regionFromRequest(c *echo.Context, defaultRegion string) string { + return httputils.ExtractRegionFromRequest(c.Request(), defaultRegion) +} diff --git a/services/kinesisvideo/handler_tags.go b/services/kinesisvideo/handler_tags.go new file mode 100644 index 0000000000..7ca8bf6d15 --- /dev/null +++ b/services/kinesisvideo/handler_tags.go @@ -0,0 +1,116 @@ +package kinesisvideo + +import ( + "encoding/json" + "net/http" + + "github.com/labstack/echo/v5" +) + +func (h *Handler) buildTagOps() map[string]handlerFunc { + return map[string]handlerFunc{ + pathTagStream: h.handleTagStream, + pathUntagStream: h.handleUntagStream, + pathListTagsForStream: h.handleListTagsForStream, + pathTagResource: h.handleTagResource, + pathUntagResource: h.handleUntagResource, + pathListTagsForResource: h.handleListTagsForResource, + } +} + +func (h *Handler) handleTagStream(c *echo.Context, body []byte) error { + var req tagStreamRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + if err := h.Backend.TagStream(req.StreamName, req.StreamARN, req.Tags); err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, struct{}{}) +} + +func (h *Handler) handleUntagStream(c *echo.Context, body []byte) error { + var req untagStreamRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + if err := h.Backend.UntagStream(req.StreamName, req.StreamARN, req.TagKeyList); err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, struct{}{}) +} + +func (h *Handler) handleListTagsForStream(c *echo.Context, body []byte) error { + var req listTagsForStreamRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + tags, err := h.Backend.ListTagsForStream(req.StreamName, req.StreamARN) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, listTagsForStreamResponse{Tags: tags}) +} + +func (h *Handler) handleTagResource(c *echo.Context, body []byte) error { + var req tagResourceRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + if req.ResourceARN == "" { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "ResourceARN is required") + } + + tags := make(map[string]string, len(req.Tags)) + for _, t := range req.Tags { + tags[t.Key] = t.Value + } + + if err := h.Backend.TagResource(req.ResourceARN, tags); err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, struct{}{}) +} + +func (h *Handler) handleUntagResource(c *echo.Context, body []byte) error { + var req untagResourceRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + if req.ResourceARN == "" { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "ResourceARN is required") + } + + if err := h.Backend.UntagResource(req.ResourceARN, req.TagKeyList); err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, struct{}{}) +} + +func (h *Handler) handleListTagsForResource(c *echo.Context, body []byte) error { + var req listTagsForResourceRequest + if err := json.Unmarshal(body, &req); err != nil { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "invalid request body") + } + + if req.ResourceARN == "" { + return h.writeError(c, http.StatusBadRequest, "InvalidArgumentException", "ResourceARN is required") + } + + tags, err := h.Backend.ListTagsForResource(req.ResourceARN) + if err != nil { + return h.writeBackendError(c, err) + } + + return h.writeJSON(c, listTagsForResourceResponse{Tags: tags}) +} diff --git a/services/kinesisvideo/handler_test.go b/services/kinesisvideo/handler_test.go new file mode 100644 index 0000000000..556f250844 --- /dev/null +++ b/services/kinesisvideo/handler_test.go @@ -0,0 +1,55 @@ +package kinesisvideo_test + +import ( + "net/http/httptest" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awscfg "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + kinesisvideosdk "github.com/aws/aws-sdk-go-v2/service/kinesisvideo" + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/kinesisvideo" +) + +const testRegion = "us-east-1" + +// newTestClient stands up the real aws-sdk-go-v2 kinesisvideo client against +// an httptest server running this package's Handler, wired through the same +// pkgs/service registry/router used in production. +func newTestClient(t *testing.T, h *kinesisvideo.Handler) *kinesisvideosdk.Client { + t.Helper() + + e := echo.New() + registry := service.NewRegistry() + require.NoError(t, registry.Register(h)) + e.Use(service.NewServiceRouter(registry).RouteHandler()) + + srv := httptest.NewServer(e) + t.Cleanup(srv.Close) + + cfg, err := awscfg.LoadDefaultConfig( + t.Context(), + awscfg.WithRegion(testRegion), + awscfg.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider("test", "test", ""), + ), + ) + require.NoError(t, err) + + return kinesisvideosdk.NewFromConfig(cfg, func(o *kinesisvideosdk.Options) { + o.BaseEndpoint = aws.String(srv.URL) + }) +} + +func newTestHandler() *kinesisvideo.Handler { + backend := kinesisvideo.NewInMemoryBackend() + h := kinesisvideo.NewHandler(backend) + h.AccountID = "123456789012" + h.DefaultRegion = testRegion + + return h +} diff --git a/services/kinesisvideo/interfaces.go b/services/kinesisvideo/interfaces.go new file mode 100644 index 0000000000..a3d074e572 --- /dev/null +++ b/services/kinesisvideo/interfaces.go @@ -0,0 +1,40 @@ +package kinesisvideo + +// StorageBackend is the interface for the Kinesis Video Streams backend. +type StorageBackend interface { + CreateStream(accountID, region, name, deviceName, mediaType, kmsKeyID, defaultStorageTier string, + dataRetentionInHours int32, tags map[string]string) (*Stream, error) + DescribeStream(name, streamARN string) (*Stream, error) + ListStreams(nextToken string, maxResults int, condition *StreamNameCondition) ([]*Stream, string, error) + UpdateStream(name, streamARN, currentVersion, deviceName, mediaType string) error + DeleteStream(streamARN, currentVersion string) error + UpdateDataRetention(name, streamARN, currentVersion, operation string, changeInHours int32) error + + TagStream(name, streamARN string, tags map[string]string) error + UntagStream(name, streamARN string, tagKeys []string) error + ListTagsForStream(name, streamARN string) (map[string]string, error) + + TagResource(resourceARN string, tags map[string]string) error + UntagResource(resourceARN string, tagKeys []string) error + ListTagsForResource(resourceARN string) (map[string]string, error) + + GetDataEndpoint(name, streamARN, apiName, region string) (string, error) + + DescribeImageGenerationConfiguration(name, streamARN string) (*ImageGenerationConfig, error) + UpdateImageGenerationConfiguration(name, streamARN string, cfg *ImageGenerationConfig) error + DescribeNotificationConfiguration(name, streamARN string) (*NotificationConfig, error) + UpdateNotificationConfiguration(name, streamARN string, cfg *NotificationConfig) error + + CreateSignalingChannel( + accountID, region, name, channelType string, messageTTLSeconds int32, tags map[string]string, + ) (*Channel, error) + DescribeSignalingChannel(name, channelARN string) (*Channel, error) + ListSignalingChannels(nextToken string, maxResults int, condition *ChannelNameCondition) ([]*Channel, string, error) + UpdateSignalingChannel(channelARN, currentVersion string, messageTTLSeconds *int32) error + DeleteSignalingChannel(channelARN, currentVersion string) error + + Reset() +} + +// Compile-time assertion that InMemoryBackend implements StorageBackend. +var _ StorageBackend = (*InMemoryBackend)(nil) diff --git a/services/kinesisvideo/models.go b/services/kinesisvideo/models.go new file mode 100644 index 0000000000..2569adafa2 --- /dev/null +++ b/services/kinesisvideo/models.go @@ -0,0 +1,107 @@ +package kinesisvideo + +import ( + "maps" + "time" +) + +// Stream status values (shared with Channel -- AWS models both under one "Status" enum). +const ( + statusActive = "ACTIVE" +) + +// Stream is the persisted representation of a Kinesis video stream. +type Stream struct { + CreationTime time.Time + ImageGeneration *ImageGenerationConfig + Notification *NotificationConfig + Tags map[string]string + Name string + ARN string + Status string + Version string + DeviceName string + KmsKeyID string + MediaType string + DefaultStorageTier string + DataRetentionInHours int32 +} + +func (s *Stream) clone() *Stream { + if s == nil { + return nil + } + + cp := *s + cp.Tags = make(map[string]string, len(s.Tags)) + maps.Copy(cp.Tags, s.Tags) + + if s.ImageGeneration != nil { + ig := *s.ImageGeneration + ig.FormatConfig = make(map[string]string, len(s.ImageGeneration.FormatConfig)) + maps.Copy(ig.FormatConfig, s.ImageGeneration.FormatConfig) + cp.ImageGeneration = &ig + } + + if s.Notification != nil { + n := *s.Notification + cp.Notification = &n + } + + return &cp +} + +// ImageGenerationConfig mirrors types.ImageGenerationConfiguration. +type ImageGenerationConfig struct { + FormatConfig map[string]string + DestinationRegion string + URI string + Format string + ImageSelectorType string + Status string + SamplingInterval int32 + HeightPixels int32 + WidthPixels int32 +} + +// NotificationConfig mirrors types.NotificationConfiguration. +type NotificationConfig struct { + DestinationURI string + Status string +} + +// Channel is the persisted representation of a signaling channel. +type Channel struct { + CreationTime time.Time + Tags map[string]string + Name string + ARN string + Type string + Status string + Version string + MessageTTLSeconds int32 +} + +func (c *Channel) clone() *Channel { + if c == nil { + return nil + } + + cp := *c + cp.Tags = make(map[string]string, len(c.Tags)) + maps.Copy(cp.Tags, c.Tags) + + return &cp +} + +// StreamNameCondition mirrors types.StreamNameCondition. +type StreamNameCondition struct { + ComparisonOperator string + ComparisonValue string +} + +// ChannelNameCondition mirrors types.ChannelNameCondition. +type ChannelNameCondition struct { + ComparisonOperator string + ComparisonValue string +} diff --git a/services/kinesisvideo/persistence.go b/services/kinesisvideo/persistence.go new file mode 100644 index 0000000000..98240b09a8 --- /dev/null +++ b/services/kinesisvideo/persistence.go @@ -0,0 +1,102 @@ +package kinesisvideo + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/persistence" +) + +// ErrNoSnapshot is returned when a backend does not support snapshot/restore. +var ErrNoSnapshot = errors.New("backend does not support restore") + +// Snapshottable is an optional interface a StorageBackend may implement to +// support snapshot/restore for persistence or test isolation. +type Snapshottable interface { + Snapshot(ctx context.Context) []byte + Restore(context.Context, []byte) error +} + +// kinesisvideoSnapshotVersion identifies the shape of [backendSnapshot]. Bump +// it whenever a change would make an older snapshot unsafe to decode as the +// current shape; Restore discards (rather than partially decodes) any mismatch. +const kinesisvideoSnapshotVersion = 1 + +// backendSnapshot is the top-level on-disk shape for the backend. Tables holds +// one JSON-encoded array per registered table name (streams, channels -- see +// store_setup.go), produced by b.registry.SnapshotAll(). +type backendSnapshot struct { + Tables map[string]json.RawMessage `json:"tables"` + Version int `json:"version"` +} + +// Snapshot serializes backend state to JSON. +func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { + b.mu.RLock("Snapshot") + defer b.mu.RUnlock() + + tables, err := b.registry.SnapshotAll() + if err != nil { + logger.Load(ctx).WarnContext(ctx, "kinesisvideo: snapshot table marshal failed", "error", err) + + return nil + } + + snap := backendSnapshot{ + Version: kinesisvideoSnapshotVersion, + Tables: tables, + } + + return persistence.MarshalSnapshot(ctx, "kinesisvideo", &snap) +} + +// Restore deserializes backend state from a JSON snapshot. +func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { + var snap backendSnapshot + + if err := persistence.UnmarshalSnapshot(ctx, "kinesisvideo", data, &snap); err != nil { + return err + } + + b.mu.Lock("Restore") + defer b.mu.Unlock() + + if snap.Version != kinesisvideoSnapshotVersion { + logger.Load(ctx).WarnContext(ctx, + "kinesisvideo: discarding incompatible snapshot version, starting empty", + "gotVersion", snap.Version, "wantVersion", kinesisvideoSnapshotVersion) + + b.registry.ResetAll() + + return nil + } + + if err := b.registry.RestoreAll(snap.Tables); err != nil { + return fmt.Errorf("kinesisvideo: restore snapshot tables: %w", err) + } + + return nil +} + +// Snapshot implements persistence by delegating to the backend if it supports it. +func (h *Handler) Snapshot(ctx context.Context) []byte { + s, ok := h.Backend.(Snapshottable) + if !ok { + return nil + } + + return s.Snapshot(ctx) +} + +// Restore implements persistence by delegating to the backend if it supports it. +func (h *Handler) Restore(ctx context.Context, data []byte) error { + s, ok := h.Backend.(Snapshottable) + if !ok { + return ErrNoSnapshot + } + + return s.Restore(ctx, data) +} diff --git a/services/kinesisvideo/provider.go b/services/kinesisvideo/provider.go new file mode 100644 index 0000000000..887528ff68 --- /dev/null +++ b/services/kinesisvideo/provider.go @@ -0,0 +1,23 @@ +package kinesisvideo + +import "github.com/blackbirdworks/gopherstack/pkgs/service" + +// Provider implements service.Provider for the Kinesis Video Streams service. +type Provider struct{} + +// Name returns the provider name. +func (p *Provider) Name() string { return "KinesisVideo" } + +// Init initializes the Kinesis Video Streams service backend and handler. +// +//nolint:ireturn,nolintlint // architecturally required to return interface +func (p *Provider) Init(ctx *service.AppContext) (service.Registerable, error) { + accountID, region := service.AccountRegionOrDefault(ctx) + + backend := NewInMemoryBackend() + handler := NewHandler(backend) + handler.AccountID = accountID + handler.DefaultRegion = region + + return handler, nil +} diff --git a/services/kinesisvideo/signaling.go b/services/kinesisvideo/signaling.go new file mode 100644 index 0000000000..2d97fe7fc0 --- /dev/null +++ b/services/kinesisvideo/signaling.go @@ -0,0 +1,147 @@ +package kinesisvideo + +import ( + "maps" + "sort" + "strings" + "time" + + "github.com/blackbirdworks/gopherstack/pkgs/page" +) + +// CreateSignalingChannel creates a new signaling channel. New channels become +// ACTIVE immediately -- see PARITY.md. +func (b *InMemoryBackend) CreateSignalingChannel( + accountID, region, name, channelType string, + messageTTLSeconds int32, + tags map[string]string, +) (*Channel, error) { + if err := validateResourceName(name, maxChannelNameLen); err != nil { + return nil, err + } + + if err := validateTags(tags); err != nil { + return nil, err + } + + if channelType == "" { + channelType = channelTypeSingleMaster + } + + if messageTTLSeconds == 0 { + messageTTLSeconds = defaultMessageTTLSecs + } + + b.mu.Lock("CreateSignalingChannel") + defer b.mu.Unlock() + + if b.channels.Has(name) { + return nil, ErrChannelAlreadyExists + } + + now := time.Now().UTC() + + t := make(map[string]string, len(tags)) + maps.Copy(t, tags) + + c := &Channel{ + Name: name, + ARN: channelARN(region, accountID, name, now.UnixMilli()), + Type: channelType, + Status: statusActive, + Version: newVersion(), + CreationTime: now, + MessageTTLSeconds: messageTTLSeconds, + Tags: t, + } + + b.channels.Put(c) + + return c.clone(), nil +} + +// DescribeSignalingChannel returns the most current information about a channel. +func (b *InMemoryBackend) DescribeSignalingChannel(name, channelARN string) (*Channel, error) { + b.mu.RLock("DescribeSignalingChannel") + defer b.mu.RUnlock() + + c, err := b.resolveChannelLocked(name, channelARN) + if err != nil { + return nil, err + } + + return c.clone(), nil +} + +// ListSignalingChannels returns channels matching condition, paginated by nextToken/maxResults. +func (b *InMemoryBackend) ListSignalingChannels( + nextToken string, + maxResults int, + condition *ChannelNameCondition, +) ([]*Channel, string, error) { + b.mu.RLock("ListSignalingChannels") + defer b.mu.RUnlock() + + all := b.channels.All() + + matched := make([]*Channel, 0, len(all)) + + for _, c := range all { + if condition != nil && condition.ComparisonOperator == comparisonOperatorBeginsWith { + if !strings.HasPrefix(c.Name, condition.ComparisonValue) { + continue + } + } + + matched = append(matched, c.clone()) + } + + sort.Slice(matched, func(i, j int) bool { return matched[i].Name < matched[j].Name }) + + p := page.New(matched, nextToken, maxResults, defaultListLimit) + + return p.Data, p.Next, nil +} + +// UpdateSignalingChannel updates a channel's SingleMasterConfiguration under +// optimistic-lock (CurrentVersion). +func (b *InMemoryBackend) UpdateSignalingChannel(channelARN, currentVersion string, messageTTLSeconds *int32) error { + b.mu.Lock("UpdateSignalingChannel") + defer b.mu.Unlock() + + c, err := b.resolveChannelLocked("", channelARN) + if err != nil { + return err + } + + if c.Version != currentVersion { + return ErrVersionMismatch + } + + if messageTTLSeconds != nil { + c.MessageTTLSeconds = *messageTTLSeconds + } + + c.Version = newVersion() + + return nil +} + +// DeleteSignalingChannel deletes a channel under optimistic-lock (CurrentVersion, when supplied). +func (b *InMemoryBackend) DeleteSignalingChannel(channelARN, currentVersion string) error { + b.mu.Lock("DeleteSignalingChannel") + defer b.mu.Unlock() + + c, err := b.resolveChannelLocked("", channelARN) + if err != nil { + return err + } + + if currentVersion != "" && c.Version != currentVersion { + return ErrVersionMismatch + } + + b.channels.Delete(c.Name) + + return nil +} diff --git a/services/kinesisvideo/signaling_test.go b/services/kinesisvideo/signaling_test.go new file mode 100644 index 0000000000..c3bb16ac82 --- /dev/null +++ b/services/kinesisvideo/signaling_test.go @@ -0,0 +1,193 @@ +package kinesisvideo_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + kinesisvideosdk "github.com/aws/aws-sdk-go-v2/service/kinesisvideo" + "github.com/aws/aws-sdk-go-v2/service/kinesisvideo/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateSignalingChannel(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + out, err := client.CreateSignalingChannel(ctx, &kinesisvideosdk.CreateSignalingChannelInput{ + ChannelName: aws.String("channel-one"), + ChannelType: types.ChannelTypeSingleMaster, + SingleMasterConfiguration: &types.SingleMasterConfiguration{ + MessageTtlSeconds: aws.Int32(120), + }, + }) + require.NoError(t, err) + assert.Contains(t, aws.ToString(out.ChannelARN), "channel/channel-one/") +} + +func TestCreateSignalingChannel_DuplicateNameReturnsResourceInUse(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateSignalingChannel(ctx, &kinesisvideosdk.CreateSignalingChannelInput{ + ChannelName: aws.String("dup-channel"), + }) + require.NoError(t, err) + + _, err = client.CreateSignalingChannel(ctx, &kinesisvideosdk.CreateSignalingChannelInput{ + ChannelName: aws.String("dup-channel"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ResourceInUseException", apiErr.ErrorCode()) +} + +func TestDescribeSignalingChannel(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateSignalingChannel(ctx, &kinesisvideosdk.CreateSignalingChannelInput{ + ChannelName: aws.String("describe-channel"), + }) + require.NoError(t, err) + + out, err := client.DescribeSignalingChannel(ctx, &kinesisvideosdk.DescribeSignalingChannelInput{ + ChannelARN: created.ChannelARN, + }) + require.NoError(t, err) + require.NotNil(t, out.ChannelInfo) + assert.Equal(t, "describe-channel", aws.ToString(out.ChannelInfo.ChannelName)) + assert.Equal(t, types.ChannelTypeSingleMaster, out.ChannelInfo.ChannelType) + assert.Equal(t, types.StatusActive, out.ChannelInfo.ChannelStatus) + require.NotNil(t, out.ChannelInfo.SingleMasterConfiguration) + assert.EqualValues(t, 60, aws.ToInt32(out.ChannelInfo.SingleMasterConfiguration.MessageTtlSeconds)) +} + +func TestDescribeSignalingChannel_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.DescribeSignalingChannel(t.Context(), &kinesisvideosdk.DescribeSignalingChannelInput{ + ChannelName: aws.String("missing"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ResourceNotFoundException", apiErr.ErrorCode()) +} + +func TestListSignalingChannels(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + for _, name := range []string{"cam-1", "cam-2", "door-1"} { + _, err := client.CreateSignalingChannel( + ctx, + &kinesisvideosdk.CreateSignalingChannelInput{ChannelName: aws.String(name)}, + ) + require.NoError(t, err) + } + + out, err := client.ListSignalingChannels(ctx, &kinesisvideosdk.ListSignalingChannelsInput{ + ChannelNameCondition: &types.ChannelNameCondition{ + ComparisonOperator: types.ComparisonOperatorBeginsWith, + ComparisonValue: aws.String("cam-"), + }, + }) + require.NoError(t, err) + require.Len(t, out.ChannelInfoList, 2) +} + +func TestUpdateSignalingChannel(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateSignalingChannel(ctx, &kinesisvideosdk.CreateSignalingChannelInput{ + ChannelName: aws.String("update-channel"), + }) + require.NoError(t, err) + + described, err := client.DescribeSignalingChannel(ctx, &kinesisvideosdk.DescribeSignalingChannelInput{ + ChannelARN: created.ChannelARN, + }) + require.NoError(t, err) + + _, err = client.UpdateSignalingChannel(ctx, &kinesisvideosdk.UpdateSignalingChannelInput{ + ChannelARN: created.ChannelARN, + CurrentVersion: described.ChannelInfo.Version, + SingleMasterConfiguration: &types.SingleMasterConfiguration{ + MessageTtlSeconds: aws.Int32(30), + }, + }) + require.NoError(t, err) + + after, err := client.DescribeSignalingChannel(ctx, &kinesisvideosdk.DescribeSignalingChannelInput{ + ChannelARN: created.ChannelARN, + }) + require.NoError(t, err) + assert.EqualValues(t, 30, aws.ToInt32(after.ChannelInfo.SingleMasterConfiguration.MessageTtlSeconds)) + assert.NotEqual(t, aws.ToString(described.ChannelInfo.Version), aws.ToString(after.ChannelInfo.Version)) +} + +func TestUpdateSignalingChannel_VersionMismatch(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateSignalingChannel(ctx, &kinesisvideosdk.CreateSignalingChannelInput{ + ChannelName: aws.String("stale-channel"), + }) + require.NoError(t, err) + + _, err = client.UpdateSignalingChannel(ctx, &kinesisvideosdk.UpdateSignalingChannelInput{ + ChannelARN: created.ChannelARN, + CurrentVersion: aws.String("not-the-real-version"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "VersionMismatchException", apiErr.ErrorCode()) +} + +func TestDeleteSignalingChannel(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateSignalingChannel(ctx, &kinesisvideosdk.CreateSignalingChannelInput{ + ChannelName: aws.String("delete-channel"), + }) + require.NoError(t, err) + + _, err = client.DeleteSignalingChannel(ctx, &kinesisvideosdk.DeleteSignalingChannelInput{ + ChannelARN: created.ChannelARN, + }) + require.NoError(t, err) + + _, err = client.DescribeSignalingChannel(ctx, &kinesisvideosdk.DescribeSignalingChannelInput{ + ChannelARN: created.ChannelARN, + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ResourceNotFoundException", apiErr.ErrorCode()) +} diff --git a/services/kinesisvideo/store.go b/services/kinesisvideo/store.go new file mode 100644 index 0000000000..c3c360b073 --- /dev/null +++ b/services/kinesisvideo/store.go @@ -0,0 +1,185 @@ +package kinesisvideo + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "regexp" + "strings" + + "github.com/google/uuid" + + "github.com/blackbirdworks/gopherstack/pkgs/arn" + "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" + "github.com/blackbirdworks/gopherstack/pkgs/store" +) + +const ( + maxStreamNameLen = 256 + maxChannelNameLen = 256 + maxTagsPerStream = 50 + minDataRetention = 0 + maxDataRetention = 87600 + defaultListLimit = 500 + + channelTypeSingleMaster = "SINGLE_MASTER" + defaultMessageTTLSecs = int32(60) + + comparisonOperatorBeginsWith = "BEGINS_WITH" + + operationIncreaseDataRetention = "INCREASE_DATA_RETENTION" + operationDecreaseDataRetention = "DECREASE_DATA_RETENTION" +) + +var resourceNameRE = regexp.MustCompile(`^[a-zA-Z0-9_.-]+$`) + +// InMemoryBackend is the in-memory implementation of StorageBackend. +type InMemoryBackend struct { + streams *store.Table[Stream] + channels *store.Table[Channel] + registry *store.Registry + mu *lockmetrics.RWMutex +} + +// NewInMemoryBackend creates a new in-memory Kinesis Video Streams backend. +func NewInMemoryBackend() *InMemoryBackend { + b := &InMemoryBackend{ + registry: store.NewRegistry(), + mu: lockmetrics.New("kinesisvideo"), + } + + registerAllTables(b) + + return b +} + +// Reset clears all backend state. +func (b *InMemoryBackend) Reset() { + b.mu.Lock("Reset") + defer b.mu.Unlock() + + b.registry.ResetAll() +} + +func validateResourceName(name string, maxLen int) error { + if len(name) == 0 || len(name) > maxLen || !resourceNameRE.MatchString(name) { + return ErrValidation + } + + return nil +} + +func validateTags(tags map[string]string) error { + if len(tags) > maxTagsPerStream { + return ErrValidation + } + + for k := range tags { + if k == "" { + return ErrValidation + } + } + + return nil +} + +func newVersion() string { + return strings.ReplaceAll(uuid.NewString(), "-", "")[:16] +} + +// shortHash returns a short, deterministic hex digest of s, used to derive +// stable-looking pseudo-random endpoint hostnames. +func shortHash(s string) string { + sum := sha256.Sum256([]byte(s)) + + return hex.EncodeToString(sum[:])[:8] +} + +func streamARN(region, accountID, name string, creationTime int64) string { + return arn.Build("kinesisvideo", region, accountID, fmt.Sprintf("stream/%s/%d", name, creationTime)) +} + +func channelARN(region, accountID, name string, creationTime int64) string { + return arn.Build("kinesisvideo", region, accountID, fmt.Sprintf("channel/%s/%d", name, creationTime)) +} + +// streamNameFromARN extracts the stream name from a well-formed KVS stream ARN +// (arn:{partition}:kinesisvideo:{region}:{account}:stream/{name}/{creationEpochMillis}). +func streamNameFromARN(streamARNStr string) (string, bool) { + return resourceNameFromARN(streamARNStr, "stream/") +} + +// channelNameFromARN extracts the channel name from a well-formed KVS channel ARN +// (arn:{partition}:kinesisvideo:{region}:{account}:channel/{name}/{creationEpochMillis}). +func channelNameFromARN(channelARNStr string) (string, bool) { + return resourceNameFromARN(channelARNStr, "channel/") +} + +func resourceNameFromARN(arnStr, prefix string) (string, bool) { + parts := strings.SplitN(arnStr, ":", 6) //nolint:mnd // arn:partition:service:region:account:resource + if len(parts) != 6 || !strings.HasPrefix(parts[5], prefix) { + return "", false + } + + rest := strings.TrimPrefix(parts[5], prefix) + + name, _, _ := strings.Cut(rest, "/") + if name == "" { + return "", false + } + + return name, true +} + +// resolveStreamLocked returns the stream identified by name or ARN (name takes +// precedence when both are set, matching AWS's documented behavior). Callers +// must hold b.mu. +func (b *InMemoryBackend) resolveStreamLocked(name, streamARNStr string) (*Stream, error) { + if name != "" { + s, ok := b.streams.Get(name) + if !ok { + return nil, ErrStreamNotFound + } + + return s, nil + } + + if streamARNStr != "" { + resolved, ok := streamNameFromARN(streamARNStr) + if ok { + if s, exists := b.streams.Get(resolved); exists { + return s, nil + } + } + + return nil, ErrStreamNotFound + } + + return nil, ErrValidation +} + +// resolveChannelLocked returns the channel identified by name or ARN. Callers +// must hold b.mu. +func (b *InMemoryBackend) resolveChannelLocked(name, channelARNStr string) (*Channel, error) { + if name != "" { + c, ok := b.channels.Get(name) + if !ok { + return nil, ErrChannelNotFound + } + + return c, nil + } + + if channelARNStr != "" { + resolved, ok := channelNameFromARN(channelARNStr) + if ok { + if c, exists := b.channels.Get(resolved); exists { + return c, nil + } + } + + return nil, ErrChannelNotFound + } + + return nil, ErrValidation +} diff --git a/services/kinesisvideo/store_setup.go b/services/kinesisvideo/store_setup.go new file mode 100644 index 0000000000..0fc815f0bc --- /dev/null +++ b/services/kinesisvideo/store_setup.go @@ -0,0 +1,15 @@ +package kinesisvideo + +import "github.com/blackbirdworks/gopherstack/pkgs/store" + +func streamKeyFn(v *Stream) string { return v.Name } + +func channelKeyFn(v *Channel) string { return v.Name } + +// registerAllTables registers every backend resource table exactly once. +// Must be called during construction only -- store.Register panics on a +// duplicate name. +func registerAllTables(b *InMemoryBackend) { + b.streams = store.Register(b.registry, "streams", store.New(streamKeyFn)) + b.channels = store.Register(b.registry, "channels", store.New(channelKeyFn)) +} diff --git a/services/kinesisvideo/streams.go b/services/kinesisvideo/streams.go new file mode 100644 index 0000000000..d0aa82161f --- /dev/null +++ b/services/kinesisvideo/streams.go @@ -0,0 +1,261 @@ +package kinesisvideo + +import ( + "fmt" + "maps" + "sort" + "strings" + "time" + + "github.com/blackbirdworks/gopherstack/pkgs/page" +) + +// CreateStream creates a new Kinesis video stream. New streams become ACTIVE +// immediately -- see PARITY.md for the CREATING/UPDATING/DELETING transient +// states this backend deliberately does not model. +func (b *InMemoryBackend) CreateStream( + accountID, region, name, deviceName, mediaType, kmsKeyID, defaultStorageTier string, + dataRetentionInHours int32, + tags map[string]string, +) (*Stream, error) { + if err := validateResourceName(name, maxStreamNameLen); err != nil { + return nil, err + } + + if dataRetentionInHours < minDataRetention || dataRetentionInHours > maxDataRetention { + return nil, ErrValidation + } + + if err := validateTags(tags); err != nil { + return nil, err + } + + b.mu.Lock("CreateStream") + defer b.mu.Unlock() + + if b.streams.Has(name) { + return nil, ErrStreamAlreadyExists + } + + now := time.Now().UTC() + + t := make(map[string]string, len(tags)) + maps.Copy(t, tags) + + s := &Stream{ + Name: name, + ARN: streamARN(region, accountID, name, now.UnixMilli()), + Status: statusActive, + Version: newVersion(), + CreationTime: now, + DeviceName: deviceName, + KmsKeyID: kmsKeyID, + MediaType: mediaType, + DefaultStorageTier: defaultStorageTier, + DataRetentionInHours: dataRetentionInHours, + Tags: t, + } + + b.streams.Put(s) + + return s.clone(), nil +} + +// DescribeStream returns the most current information about a stream. +func (b *InMemoryBackend) DescribeStream(name, streamARN string) (*Stream, error) { + b.mu.RLock("DescribeStream") + defer b.mu.RUnlock() + + s, err := b.resolveStreamLocked(name, streamARN) + if err != nil { + return nil, err + } + + return s.clone(), nil +} + +// ListStreams returns streams matching condition, paginated by nextToken/maxResults. +func (b *InMemoryBackend) ListStreams( + nextToken string, + maxResults int, + condition *StreamNameCondition, +) ([]*Stream, string, error) { + b.mu.RLock("ListStreams") + defer b.mu.RUnlock() + + all := b.streams.All() + + matched := make([]*Stream, 0, len(all)) + + for _, s := range all { + if condition != nil && condition.ComparisonOperator == comparisonOperatorBeginsWith { + if !strings.HasPrefix(s.Name, condition.ComparisonValue) { + continue + } + } + + matched = append(matched, s.clone()) + } + + sort.Slice(matched, func(i, j int) bool { return matched[i].Name < matched[j].Name }) + + p := page.New(matched, nextToken, maxResults, defaultListLimit) + + return p.Data, p.Next, nil +} + +// UpdateStream updates a stream's metadata under optimistic-lock (CurrentVersion). +func (b *InMemoryBackend) UpdateStream(name, streamARN, currentVersion, deviceName, mediaType string) error { + b.mu.Lock("UpdateStream") + defer b.mu.Unlock() + + s, err := b.resolveStreamLocked(name, streamARN) + if err != nil { + return err + } + + if s.Version != currentVersion { + return ErrVersionMismatch + } + + if deviceName != "" { + s.DeviceName = deviceName + } + + if mediaType != "" { + s.MediaType = mediaType + } + + s.Version = newVersion() + + return nil +} + +// DeleteStream deletes a stream under optimistic-lock (CurrentVersion, when supplied). +func (b *InMemoryBackend) DeleteStream(streamARN, currentVersion string) error { + b.mu.Lock("DeleteStream") + defer b.mu.Unlock() + + s, err := b.resolveStreamLocked("", streamARN) + if err != nil { + return err + } + + if currentVersion != "" && s.Version != currentVersion { + return ErrVersionMismatch + } + + b.streams.Delete(s.Name) + + return nil +} + +// UpdateDataRetention increases or decreases a stream's data retention period. +func (b *InMemoryBackend) UpdateDataRetention( + name, streamARN, currentVersion, operation string, + changeInHours int32, +) error { + b.mu.Lock("UpdateDataRetention") + defer b.mu.Unlock() + + s, err := b.resolveStreamLocked(name, streamARN) + if err != nil { + return err + } + + if s.Version != currentVersion { + return ErrVersionMismatch + } + + next := s.DataRetentionInHours + + switch operation { + case operationIncreaseDataRetention: + next += changeInHours + case operationDecreaseDataRetention: + next -= changeInHours + default: + return ErrValidation + } + + if next < minDataRetention || next > maxDataRetention { + return ErrValidation + } + + s.DataRetentionInHours = next + s.Version = newVersion() + + return nil +} + +// GetDataEndpoint returns an emulator-hosted, AWS-shaped data-plane endpoint +// for the stream. The KVS data plane (PutMedia/GetMedia/...) is out of scope +// for this backend -- see PARITY.md -- so the endpoint is wire-accurate but +// not backed by a functioning media data plane. +func (b *InMemoryBackend) GetDataEndpoint(name, streamARN, apiName, region string) (string, error) { + b.mu.RLock("GetDataEndpoint") + defer b.mu.RUnlock() + + s, err := b.resolveStreamLocked(name, streamARN) + if err != nil { + return "", err + } + + return fmt.Sprintf("https://s-%s.kinesisvideo.%s.amazonaws.com", shortHash(s.ARN+apiName), region), nil +} + +// DescribeImageGenerationConfiguration returns a stream's image generation config. +func (b *InMemoryBackend) DescribeImageGenerationConfiguration(name, streamARN string) (*ImageGenerationConfig, error) { + b.mu.RLock("DescribeImageGenerationConfiguration") + defer b.mu.RUnlock() + + s, err := b.resolveStreamLocked(name, streamARN) + if err != nil { + return nil, err + } + + return s.clone().ImageGeneration, nil +} + +// UpdateImageGenerationConfiguration sets or clears a stream's image generation config. +func (b *InMemoryBackend) UpdateImageGenerationConfiguration(name, streamARN string, cfg *ImageGenerationConfig) error { + b.mu.Lock("UpdateImageGenerationConfiguration") + defer b.mu.Unlock() + + s, err := b.resolveStreamLocked(name, streamARN) + if err != nil { + return err + } + + s.ImageGeneration = cfg + + return nil +} + +// DescribeNotificationConfiguration returns a stream's notification config. +func (b *InMemoryBackend) DescribeNotificationConfiguration(name, streamARN string) (*NotificationConfig, error) { + b.mu.RLock("DescribeNotificationConfiguration") + defer b.mu.RUnlock() + + s, err := b.resolveStreamLocked(name, streamARN) + if err != nil { + return nil, err + } + + return s.clone().Notification, nil +} + +// UpdateNotificationConfiguration sets or clears a stream's notification config. +func (b *InMemoryBackend) UpdateNotificationConfiguration(name, streamARN string, cfg *NotificationConfig) error { + b.mu.Lock("UpdateNotificationConfiguration") + defer b.mu.Unlock() + + s, err := b.resolveStreamLocked(name, streamARN) + if err != nil { + return err + } + + s.Notification = cfg + + return nil +} diff --git a/services/kinesisvideo/streams_test.go b/services/kinesisvideo/streams_test.go new file mode 100644 index 0000000000..5b5d99a0b4 --- /dev/null +++ b/services/kinesisvideo/streams_test.go @@ -0,0 +1,311 @@ +package kinesisvideo_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + kinesisvideosdk "github.com/aws/aws-sdk-go-v2/service/kinesisvideo" + "github.com/aws/aws-sdk-go-v2/service/kinesisvideo/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateStream(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input kinesisvideosdk.CreateStreamInput + }{ + { + name: "minimal", + input: kinesisvideosdk.CreateStreamInput{StreamName: aws.String("stream-one")}, + }, + { + name: "with retention and media type", + input: kinesisvideosdk.CreateStreamInput{ + StreamName: aws.String("stream-two"), + DataRetentionInHours: aws.Int32(24), + MediaType: aws.String("video/h264"), + DeviceName: aws.String("my-camera"), + Tags: map[string]string{"env": "test"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + out, err := client.CreateStream(t.Context(), &tt.input) + require.NoError(t, err) + assert.Contains(t, aws.ToString(out.StreamARN), "stream/"+aws.ToString(tt.input.StreamName)+"/") + }) + } +} + +func TestCreateStream_DuplicateNameReturnsResourceInUse(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + _, err := client.CreateStream(ctx, &kinesisvideosdk.CreateStreamInput{StreamName: aws.String("dup-stream")}) + require.NoError(t, err) + + _, err = client.CreateStream(ctx, &kinesisvideosdk.CreateStreamInput{StreamName: aws.String("dup-stream")}) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ResourceInUseException", apiErr.ErrorCode()) +} + +func TestDescribeStream(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, createErr := client.CreateStream(ctx, &kinesisvideosdk.CreateStreamInput{ + StreamName: aws.String("describe-me"), + DataRetentionInHours: aws.Int32(5), + }) + require.NoError(t, createErr) + + tests := []struct { + name string + input kinesisvideosdk.DescribeStreamInput + }{ + {name: "by name", input: kinesisvideosdk.DescribeStreamInput{StreamName: aws.String("describe-me")}}, + {name: "by arn", input: kinesisvideosdk.DescribeStreamInput{StreamARN: created.StreamARN}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + out, err := client.DescribeStream(ctx, &tt.input) + require.NoError(t, err) + require.NotNil(t, out.StreamInfo) + assert.Equal(t, "describe-me", aws.ToString(out.StreamInfo.StreamName)) + assert.Equal(t, types.StatusActive, out.StreamInfo.Status) + assert.EqualValues(t, 5, aws.ToInt32(out.StreamInfo.DataRetentionInHours)) + assert.NotZero(t, aws.ToTime(out.StreamInfo.CreationTime)) + assert.NotEmpty(t, aws.ToString(out.StreamInfo.Version)) + }) + } +} + +func TestDescribeStream_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.DescribeStream(t.Context(), &kinesisvideosdk.DescribeStreamInput{ + StreamName: aws.String("does-not-exist"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ResourceNotFoundException", apiErr.ErrorCode()) +} + +func TestListStreams(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + for _, name := range []string{"alpha-1", "alpha-2", "beta-1"} { + _, err := client.CreateStream(ctx, &kinesisvideosdk.CreateStreamInput{StreamName: aws.String(name)}) + require.NoError(t, err) + } + + out, err := client.ListStreams(ctx, &kinesisvideosdk.ListStreamsInput{ + StreamNameCondition: &types.StreamNameCondition{ + ComparisonOperator: types.ComparisonOperatorBeginsWith, + ComparisonValue: aws.String("alpha-"), + }, + }) + require.NoError(t, err) + require.Len(t, out.StreamInfoList, 2) + + names := []string{aws.ToString(out.StreamInfoList[0].StreamName), aws.ToString(out.StreamInfoList[1].StreamName)} + assert.ElementsMatch(t, []string{"alpha-1", "alpha-2"}, names) +} + +func TestUpdateStream(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateStream(ctx, &kinesisvideosdk.CreateStreamInput{StreamName: aws.String("update-me")}) + require.NoError(t, err) + + described, err := client.DescribeStream(ctx, &kinesisvideosdk.DescribeStreamInput{StreamARN: created.StreamARN}) + require.NoError(t, err) + + _, err = client.UpdateStream(ctx, &kinesisvideosdk.UpdateStreamInput{ + StreamARN: created.StreamARN, + CurrentVersion: described.StreamInfo.Version, + MediaType: aws.String("video/h264"), + }) + require.NoError(t, err) + + after, err := client.DescribeStream(ctx, &kinesisvideosdk.DescribeStreamInput{StreamARN: created.StreamARN}) + require.NoError(t, err) + assert.Equal(t, "video/h264", aws.ToString(after.StreamInfo.MediaType)) + assert.NotEqual(t, aws.ToString(described.StreamInfo.Version), aws.ToString(after.StreamInfo.Version)) +} + +func TestUpdateStream_VersionMismatch(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateStream( + ctx, + &kinesisvideosdk.CreateStreamInput{StreamName: aws.String("stale-version")}, + ) + require.NoError(t, err) + + _, err = client.UpdateStream(ctx, &kinesisvideosdk.UpdateStreamInput{ + StreamARN: created.StreamARN, + CurrentVersion: aws.String("not-the-real-version"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "VersionMismatchException", apiErr.ErrorCode()) +} + +func TestDeleteStream(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateStream(ctx, &kinesisvideosdk.CreateStreamInput{StreamName: aws.String("delete-me")}) + require.NoError(t, err) + + _, err = client.DeleteStream(ctx, &kinesisvideosdk.DeleteStreamInput{StreamARN: created.StreamARN}) + require.NoError(t, err) + + _, err = client.DescribeStream(ctx, &kinesisvideosdk.DescribeStreamInput{StreamARN: created.StreamARN}) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ResourceNotFoundException", apiErr.ErrorCode()) +} + +func TestDeleteStream_VersionMismatch(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateStream( + ctx, + &kinesisvideosdk.CreateStreamInput{StreamName: aws.String("delete-mismatch")}, + ) + require.NoError(t, err) + + _, err = client.DeleteStream(ctx, &kinesisvideosdk.DeleteStreamInput{ + StreamARN: created.StreamARN, + CurrentVersion: aws.String("wrong-version"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "VersionMismatchException", apiErr.ErrorCode()) +} + +func TestUpdateDataRetention(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + operation types.UpdateDataRetentionOperation + startHours int32 + changeHours int32 + wantHours int32 + }{ + { + name: "increase", + operation: types.UpdateDataRetentionOperationIncreaseDataRetention, + startHours: 10, + changeHours: 5, + wantHours: 15, + }, + { + name: "decrease", + operation: types.UpdateDataRetentionOperationDecreaseDataRetention, + startHours: 10, + changeHours: 5, + wantHours: 5, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateStream(ctx, &kinesisvideosdk.CreateStreamInput{ + StreamName: aws.String("retention-" + tt.name), + DataRetentionInHours: aws.Int32(tt.startHours), + }) + require.NoError(t, err) + + described, err := client.DescribeStream( + ctx, + &kinesisvideosdk.DescribeStreamInput{StreamARN: created.StreamARN}, + ) + require.NoError(t, err) + + _, err = client.UpdateDataRetention(ctx, &kinesisvideosdk.UpdateDataRetentionInput{ + StreamARN: created.StreamARN, + CurrentVersion: described.StreamInfo.Version, + Operation: tt.operation, + DataRetentionChangeInHours: aws.Int32(tt.changeHours), + }) + require.NoError(t, err) + + after, err := client.DescribeStream(ctx, &kinesisvideosdk.DescribeStreamInput{StreamARN: created.StreamARN}) + require.NoError(t, err) + assert.Equal(t, tt.wantHours, aws.ToInt32(after.StreamInfo.DataRetentionInHours)) + }) + } +} + +func TestGetDataEndpoint(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateStream( + ctx, + &kinesisvideosdk.CreateStreamInput{StreamName: aws.String("endpoint-stream")}, + ) + require.NoError(t, err) + + out, err := client.GetDataEndpoint(ctx, &kinesisvideosdk.GetDataEndpointInput{ + StreamARN: created.StreamARN, + APIName: types.APINamePutMedia, + }) + require.NoError(t, err) + assert.Contains(t, aws.ToString(out.DataEndpoint), ".kinesisvideo."+testRegion+".amazonaws.com") +} diff --git a/services/kinesisvideo/tags.go b/services/kinesisvideo/tags.go new file mode 100644 index 0000000000..ace43e3bb5 --- /dev/null +++ b/services/kinesisvideo/tags.go @@ -0,0 +1,115 @@ +package kinesisvideo + +import "maps" + +// TagStream adds or replaces tags on a stream. +func (b *InMemoryBackend) TagStream(name, streamARN string, tags map[string]string) error { + if err := validateTags(tags); err != nil { + return err + } + + b.mu.Lock("TagStream") + defer b.mu.Unlock() + + s, err := b.resolveStreamLocked(name, streamARN) + if err != nil { + return err + } + + if len(s.Tags)+len(tags) > maxTagsPerStream { + return ErrValidation + } + + maps.Copy(s.Tags, tags) + + return nil +} + +// UntagStream removes tags from a stream by key. +func (b *InMemoryBackend) UntagStream(name, streamARN string, tagKeys []string) error { + b.mu.Lock("UntagStream") + defer b.mu.Unlock() + + s, err := b.resolveStreamLocked(name, streamARN) + if err != nil { + return err + } + + for _, k := range tagKeys { + delete(s.Tags, k) + } + + return nil +} + +// ListTagsForStream returns all tags on a stream. +func (b *InMemoryBackend) ListTagsForStream(name, streamARN string) (map[string]string, error) { + b.mu.RLock("ListTagsForStream") + defer b.mu.RUnlock() + + s, err := b.resolveStreamLocked(name, streamARN) + if err != nil { + return nil, err + } + + out := make(map[string]string, len(s.Tags)) + maps.Copy(out, s.Tags) + + return out, nil +} + +// TagResource adds or replaces tags on a signaling channel. +func (b *InMemoryBackend) TagResource(resourceARN string, tags map[string]string) error { + if err := validateTags(tags); err != nil { + return err + } + + b.mu.Lock("TagResource") + defer b.mu.Unlock() + + c, err := b.resolveChannelLocked("", resourceARN) + if err != nil { + return err + } + + if len(c.Tags)+len(tags) > maxTagsPerStream { + return ErrValidation + } + + maps.Copy(c.Tags, tags) + + return nil +} + +// UntagResource removes tags from a signaling channel by key. +func (b *InMemoryBackend) UntagResource(resourceARN string, tagKeys []string) error { + b.mu.Lock("UntagResource") + defer b.mu.Unlock() + + c, err := b.resolveChannelLocked("", resourceARN) + if err != nil { + return err + } + + for _, k := range tagKeys { + delete(c.Tags, k) + } + + return nil +} + +// ListTagsForResource returns all tags on a signaling channel. +func (b *InMemoryBackend) ListTagsForResource(resourceARN string) (map[string]string, error) { + b.mu.RLock("ListTagsForResource") + defer b.mu.RUnlock() + + c, err := b.resolveChannelLocked("", resourceARN) + if err != nil { + return nil, err + } + + out := make(map[string]string, len(c.Tags)) + maps.Copy(out, c.Tags) + + return out, nil +} diff --git a/services/kinesisvideo/tags_test.go b/services/kinesisvideo/tags_test.go new file mode 100644 index 0000000000..3cfc7c24ad --- /dev/null +++ b/services/kinesisvideo/tags_test.go @@ -0,0 +1,112 @@ +package kinesisvideo_test + +import ( + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + kinesisvideosdk "github.com/aws/aws-sdk-go-v2/service/kinesisvideo" + "github.com/aws/aws-sdk-go-v2/service/kinesisvideo/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStreamTagLifecycle(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateStream(ctx, &kinesisvideosdk.CreateStreamInput{ + StreamName: aws.String("tagged-stream"), + Tags: map[string]string{"team": "video"}, + }) + require.NoError(t, err) + + _, err = client.TagStream(ctx, &kinesisvideosdk.TagStreamInput{ + StreamARN: created.StreamARN, + Tags: map[string]string{"env": "prod"}, + }) + require.NoError(t, err) + + listed, err := client.ListTagsForStream(ctx, &kinesisvideosdk.ListTagsForStreamInput{StreamARN: created.StreamARN}) + require.NoError(t, err) + assert.Equal(t, map[string]string{"team": "video", "env": "prod"}, listed.Tags) + + _, err = client.UntagStream(ctx, &kinesisvideosdk.UntagStreamInput{ + StreamARN: created.StreamARN, + TagKeyList: []string{"team"}, + }) + require.NoError(t, err) + + after, err := client.ListTagsForStream(ctx, &kinesisvideosdk.ListTagsForStreamInput{StreamARN: created.StreamARN}) + require.NoError(t, err) + assert.Equal(t, map[string]string{"env": "prod"}, after.Tags) +} + +func TestListTagsForStream_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.ListTagsForStream(t.Context(), &kinesisvideosdk.ListTagsForStreamInput{ + StreamName: aws.String("missing-stream"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ResourceNotFoundException", apiErr.ErrorCode()) +} + +func TestChannelTagLifecycle(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + ctx := t.Context() + + created, err := client.CreateSignalingChannel(ctx, &kinesisvideosdk.CreateSignalingChannelInput{ + ChannelName: aws.String("tagged-channel"), + Tags: []types.Tag{{Key: aws.String("team"), Value: aws.String("video")}}, + }) + require.NoError(t, err) + + _, err = client.TagResource(ctx, &kinesisvideosdk.TagResourceInput{ + ResourceARN: created.ChannelARN, + Tags: []types.Tag{{Key: aws.String("env"), Value: aws.String("prod")}}, + }) + require.NoError(t, err) + + listed, err := client.ListTagsForResource(ctx, &kinesisvideosdk.ListTagsForResourceInput{ + ResourceARN: created.ChannelARN, + }) + require.NoError(t, err) + assert.Equal(t, map[string]string{"team": "video", "env": "prod"}, listed.Tags) + + _, err = client.UntagResource(ctx, &kinesisvideosdk.UntagResourceInput{ + ResourceARN: created.ChannelARN, + TagKeyList: []string{"team"}, + }) + require.NoError(t, err) + + after, err := client.ListTagsForResource(ctx, &kinesisvideosdk.ListTagsForResourceInput{ + ResourceARN: created.ChannelARN, + }) + require.NoError(t, err) + assert.Equal(t, map[string]string{"env": "prod"}, after.Tags) +} + +func TestListTagsForResource_NotFound(t *testing.T) { + t.Parallel() + + client := newTestClient(t, newTestHandler()) + + _, err := client.ListTagsForResource(t.Context(), &kinesisvideosdk.ListTagsForResourceInput{ + ResourceARN: aws.String("arn:aws:kinesisvideo:us-east-1:123456789012:channel/missing/1"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ResourceNotFoundException", apiErr.ErrorCode()) +} diff --git a/services/kinesisvideo/wire.go b/services/kinesisvideo/wire.go new file mode 100644 index 0000000000..2fa85a4b9c --- /dev/null +++ b/services/kinesisvideo/wire.go @@ -0,0 +1,358 @@ +package kinesisvideo + +import "github.com/blackbirdworks/gopherstack/pkgs/awstime" + +// Wire DTOs for the Kinesis Video Streams REST-JSON control plane. Field +// names match the AWS smithy model exactly (aws-sdk-go-v2/service/ +// kinesisvideo@v1.41.1 serializers.go/deserializers.go emit JSON keys equal +// to the Go struct field names verbatim, no @jsonName overrides). + +type tagDTO struct { + Key string `json:"Key"` + Value string `json:"Value"` +} + +type streamStorageConfigurationDTO struct { + DefaultStorageTier string `json:"DefaultStorageTier,omitempty"` +} + +type streamNameConditionDTO struct { + ComparisonOperator string `json:"ComparisonOperator,omitempty"` + ComparisonValue string `json:"ComparisonValue,omitempty"` +} + +type channelNameConditionDTO struct { + ComparisonOperator string `json:"ComparisonOperator,omitempty"` + ComparisonValue string `json:"ComparisonValue,omitempty"` +} + +type singleMasterConfigurationDTO struct { + MessageTTLSeconds int32 `json:"MessageTtlSeconds,omitempty"` +} + +type streamInfoDTO struct { + DeviceName string `json:"DeviceName,omitempty"` + KmsKeyID string `json:"KmsKeyId,omitempty"` + MediaType string `json:"MediaType,omitempty"` + Status string `json:"Status,omitempty"` + StreamARN string `json:"StreamARN,omitempty"` + StreamName string `json:"StreamName,omitempty"` + Version string `json:"Version,omitempty"` + CreationTime float64 `json:"CreationTime"` + DataRetentionInHours int32 `json:"DataRetentionInHours"` +} + +func streamInfoFromStream(s *Stream) streamInfoDTO { + return streamInfoDTO{ + CreationTime: awstime.Epoch(s.CreationTime), + DataRetentionInHours: s.DataRetentionInHours, + DeviceName: s.DeviceName, + KmsKeyID: s.KmsKeyID, + MediaType: s.MediaType, + Status: s.Status, + StreamARN: s.ARN, + StreamName: s.Name, + Version: s.Version, + } +} + +type channelInfoDTO struct { + SingleMasterConfiguration *singleMasterConfigurationDTO `json:"SingleMasterConfiguration,omitempty"` + ChannelARN string `json:"ChannelARN,omitempty"` + ChannelName string `json:"ChannelName,omitempty"` + ChannelStatus string `json:"ChannelStatus,omitempty"` + ChannelType string `json:"ChannelType,omitempty"` + Version string `json:"Version,omitempty"` + CreationTime float64 `json:"CreationTime"` +} + +func channelInfoFromChannel(c *Channel) channelInfoDTO { + return channelInfoDTO{ + ChannelARN: c.ARN, + ChannelName: c.Name, + ChannelStatus: c.Status, + ChannelType: c.Type, + CreationTime: awstime.Epoch(c.CreationTime), + SingleMasterConfiguration: &singleMasterConfigurationDTO{ + MessageTTLSeconds: c.MessageTTLSeconds, + }, + Version: c.Version, + } +} + +type createStreamRequest struct { + StreamStorageConfiguration *streamStorageConfigurationDTO `json:"StreamStorageConfiguration,omitempty"` + Tags map[string]string `json:"Tags,omitempty"` + StreamName string `json:"StreamName"` + DeviceName string `json:"DeviceName,omitempty"` + KmsKeyID string `json:"KmsKeyId,omitempty"` + MediaType string `json:"MediaType,omitempty"` + DataRetentionInHours int32 `json:"DataRetentionInHours,omitempty"` +} + +type createStreamResponse struct { + StreamARN string `json:"StreamARN"` +} + +type describeStreamRequest struct { + StreamARN string `json:"StreamARN,omitempty"` + StreamName string `json:"StreamName,omitempty"` +} + +type describeStreamResponse struct { + StreamInfo streamInfoDTO `json:"StreamInfo"` +} + +type listStreamsRequest struct { + StreamNameCondition *streamNameConditionDTO `json:"StreamNameCondition,omitempty"` + NextToken string `json:"NextToken,omitempty"` + MaxResults int32 `json:"MaxResults,omitempty"` +} + +type listStreamsResponse struct { + NextToken string `json:"NextToken,omitempty"` + StreamInfoList []streamInfoDTO `json:"StreamInfoList"` +} + +type updateStreamRequest struct { + CurrentVersion string `json:"CurrentVersion"` + DeviceName string `json:"DeviceName,omitempty"` + MediaType string `json:"MediaType,omitempty"` + StreamARN string `json:"StreamARN,omitempty"` + StreamName string `json:"StreamName,omitempty"` +} + +type deleteStreamRequest struct { + StreamARN string `json:"StreamARN"` + CurrentVersion string `json:"CurrentVersion,omitempty"` +} + +type updateDataRetentionRequest struct { + CurrentVersion string `json:"CurrentVersion"` + Operation string `json:"Operation"` + StreamARN string `json:"StreamARN,omitempty"` + StreamName string `json:"StreamName,omitempty"` + DataRetentionChangeInHours int32 `json:"DataRetentionChangeInHours"` +} + +type getDataEndpointRequest struct { + APIName string `json:"APIName"` + StreamARN string `json:"StreamARN,omitempty"` + StreamName string `json:"StreamName,omitempty"` +} + +type getDataEndpointResponse struct { + DataEndpoint string `json:"DataEndpoint"` +} + +type tagStreamRequest struct { + Tags map[string]string `json:"Tags"` + StreamARN string `json:"StreamARN,omitempty"` + StreamName string `json:"StreamName,omitempty"` +} + +type untagStreamRequest struct { + StreamARN string `json:"StreamARN,omitempty"` + StreamName string `json:"StreamName,omitempty"` + TagKeyList []string `json:"TagKeyList"` +} + +type listTagsForStreamRequest struct { + NextToken string `json:"NextToken,omitempty"` + StreamARN string `json:"StreamARN,omitempty"` + StreamName string `json:"StreamName,omitempty"` +} + +type listTagsForStreamResponse struct { + Tags map[string]string `json:"Tags"` + NextToken string `json:"NextToken,omitempty"` +} + +type tagResourceRequest struct { + ResourceARN string `json:"ResourceARN"` + Tags []tagDTO `json:"Tags"` +} + +type untagResourceRequest struct { + ResourceARN string `json:"ResourceARN"` + TagKeyList []string `json:"TagKeyList"` +} + +type listTagsForResourceRequest struct { + ResourceARN string `json:"ResourceARN"` + NextToken string `json:"NextToken,omitempty"` +} + +type listTagsForResourceResponse struct { + Tags map[string]string `json:"Tags"` + NextToken string `json:"NextToken,omitempty"` +} + +type createSignalingChannelRequest struct { + SingleMasterConfiguration *singleMasterConfigurationDTO `json:"SingleMasterConfiguration,omitempty"` + ChannelName string `json:"ChannelName"` + ChannelType string `json:"ChannelType,omitempty"` + Tags []tagDTO `json:"Tags,omitempty"` +} + +type createSignalingChannelResponse struct { + ChannelARN string `json:"ChannelARN"` +} + +type describeSignalingChannelRequest struct { + ChannelARN string `json:"ChannelARN,omitempty"` + ChannelName string `json:"ChannelName,omitempty"` +} + +type describeSignalingChannelResponse struct { + ChannelInfo channelInfoDTO `json:"ChannelInfo"` +} + +type listSignalingChannelsRequest struct { + ChannelNameCondition *channelNameConditionDTO `json:"ChannelNameCondition,omitempty"` + NextToken string `json:"NextToken,omitempty"` + MaxResults int32 `json:"MaxResults,omitempty"` +} + +type listSignalingChannelsResponse struct { + NextToken string `json:"NextToken,omitempty"` + ChannelInfoList []channelInfoDTO `json:"ChannelInfoList"` +} + +type updateSignalingChannelRequest struct { + SingleMasterConfiguration *singleMasterConfigurationDTO `json:"SingleMasterConfiguration,omitempty"` + ChannelARN string `json:"ChannelARN"` + CurrentVersion string `json:"CurrentVersion"` +} + +type deleteSignalingChannelRequest struct { + ChannelARN string `json:"ChannelARN"` + CurrentVersion string `json:"CurrentVersion,omitempty"` +} + +type imageGenerationDestinationConfigDTO struct { + DestinationRegion string `json:"DestinationRegion"` + URI string `json:"Uri"` +} + +type imageGenerationConfigurationDTO struct { + DestinationConfig *imageGenerationDestinationConfigDTO `json:"DestinationConfig"` + FormatConfig map[string]string `json:"FormatConfig,omitempty"` + Format string `json:"Format"` + ImageSelectorType string `json:"ImageSelectorType"` + Status string `json:"Status"` + SamplingInterval int32 `json:"SamplingInterval"` + HeightPixels int32 `json:"HeightPixels,omitempty"` + WidthPixels int32 `json:"WidthPixels,omitempty"` +} + +func imageGenerationConfigFromDTO(dto *imageGenerationConfigurationDTO) *ImageGenerationConfig { + if dto == nil { + return nil + } + + cfg := &ImageGenerationConfig{ + Format: dto.Format, + ImageSelectorType: dto.ImageSelectorType, + Status: dto.Status, + SamplingInterval: dto.SamplingInterval, + HeightPixels: dto.HeightPixels, + WidthPixels: dto.WidthPixels, + FormatConfig: dto.FormatConfig, + } + + if dto.DestinationConfig != nil { + cfg.DestinationRegion = dto.DestinationConfig.DestinationRegion + cfg.URI = dto.DestinationConfig.URI + } + + return cfg +} + +func imageGenerationConfigToDTO(cfg *ImageGenerationConfig) *imageGenerationConfigurationDTO { + if cfg == nil { + return nil + } + + return &imageGenerationConfigurationDTO{ + DestinationConfig: &imageGenerationDestinationConfigDTO{ + DestinationRegion: cfg.DestinationRegion, + URI: cfg.URI, + }, + Format: cfg.Format, + ImageSelectorType: cfg.ImageSelectorType, + Status: cfg.Status, + SamplingInterval: cfg.SamplingInterval, + HeightPixels: cfg.HeightPixels, + WidthPixels: cfg.WidthPixels, + FormatConfig: cfg.FormatConfig, + } +} + +type describeImageGenerationConfigurationRequest struct { + StreamARN string `json:"StreamARN,omitempty"` + StreamName string `json:"StreamName,omitempty"` +} + +type describeImageGenerationConfigurationResponse struct { + ImageGenerationConfiguration *imageGenerationConfigurationDTO `json:"ImageGenerationConfiguration,omitempty"` +} + +type updateImageGenerationConfigurationRequest struct { + ImageGenerationConfiguration *imageGenerationConfigurationDTO `json:"ImageGenerationConfiguration,omitempty"` + StreamARN string `json:"StreamARN,omitempty"` + StreamName string `json:"StreamName,omitempty"` +} + +type notificationDestinationConfigDTO struct { + URI string `json:"Uri"` +} + +type notificationConfigurationDTO struct { + DestinationConfig *notificationDestinationConfigDTO `json:"DestinationConfig"` + Status string `json:"Status"` +} + +func notificationConfigFromDTO(dto *notificationConfigurationDTO) *NotificationConfig { + if dto == nil { + return nil + } + + cfg := &NotificationConfig{Status: dto.Status} + if dto.DestinationConfig != nil { + cfg.DestinationURI = dto.DestinationConfig.URI + } + + return cfg +} + +func notificationConfigToDTO(cfg *NotificationConfig) *notificationConfigurationDTO { + if cfg == nil { + return nil + } + + return ¬ificationConfigurationDTO{ + DestinationConfig: ¬ificationDestinationConfigDTO{URI: cfg.DestinationURI}, + Status: cfg.Status, + } +} + +type describeNotificationConfigurationRequest struct { + StreamARN string `json:"StreamARN,omitempty"` + StreamName string `json:"StreamName,omitempty"` +} + +type describeNotificationConfigurationResponse struct { + NotificationConfiguration *notificationConfigurationDTO `json:"NotificationConfiguration,omitempty"` +} + +type updateNotificationConfigurationRequest struct { + NotificationConfiguration *notificationConfigurationDTO `json:"NotificationConfiguration,omitempty"` + StreamARN string `json:"StreamARN,omitempty"` + StreamName string `json:"StreamName,omitempty"` +} + +type errorResponse struct { + Type string `json:"__type"` + Message string `json:"Message"` +} diff --git a/services/lambda/PARITY.md b/services/lambda/PARITY.md index 4df450877e..5a62dd506c 100644 --- a/services/lambda/PARITY.md +++ b/services/lambda/PARITY.md @@ -3,7 +3,7 @@ service: lambda sdk_module: aws-sdk-go-v2/service/lambda@v1.107.0 last_audit_commit: 51ea2ace0 last_audit_date: 2026-09-19 -overall: A # durable_execution wire-shape rewrite closed the last open gap; all gates green +overall: A # 2026-09-26: Invoke's durable-execution wiring closed the last two items_still_open entries; all gates green protocol: REST-JSON families: resource_policy: {status: ok, note: "PROVEN — RemovePermission StatementId from URI path, Qualifier scoping, EventSourceToken/PrincipalOrgID. This sweep closed the AddPermission deferred item: FunctionUrlAuthType/InvokedViaFunctionUrl are now accepted and rendered as IAM Condition entries (StringEquals lambda:FunctionUrlAuthType, Bool lambda:InvokedViaFunctionUrl — verified against real AWS docs/terraform-provider-aws issue #44829), and RevisionId optimistic concurrency is enforced on AddPermission/RemovePermission/GetPolicy (was hardcoded RevisionId:\"1\" — now a real content-hash of the statement-ID set, changing on every mutation, stable otherwise). Same RevisionId + duplicate-StatementId (ResourceConflictException) treatment extended to AddLayerVersionPermission/RemoveLayerVersionPermission/GetLayerVersionPolicy (layers.go), which had the identical hardcoded-\"1\" bug and silently overwrote a duplicate StatementId instead of rejecting it."} @@ -13,46 +13,186 @@ families: persistence: {status: ok, note: "ce30166a added lambdaSnapshotVersion=1 gate (mirrors sqs/ec2 pilot) — an incompatible/absent Version discards to empty rather than partially decoding. Same known systemic trait as sqs/ec2: on a version-mismatch Restore, only b.registry + b.permissions are reset; raw non-Table fields (versions/layers/eventInvokeConfigs/layerPolicies/functionConcurrencies/accountID/region) are left as-is. Not a lambda-specific regression — identical to services/sqs and services/ec2's Restore; Restore only ever runs once against a freshly-constructed backend in practice. Not flagging as a new bug; tracked here for awareness only. Note: PublishVersion's new RevisionId precondition check deliberately reuses fn.RevisionID (already persisted as part of FunctionConfiguration) rather than adding new persisted state, so this is unaffected."} runtime_lifecycle: {status: ok, note: unchanged since c3b5d46a; PROVEN — LRU eviction, async cleanup semaphore, container stop/remove, port release, dir cleanup. Real Docker exec} function_crud_versions_aliases_layers_concurrency_urls_tags: {status: ok, note: "Field-diffed this sweep (was 'skimmed, not exhaustively re-verified'). Real bug found + fixed: FunctionEventInvokeConfig.LastModified was a time.Time (ISO8601-string wire shape) but the real deserializer (PutFunctionEventInvokeConfig/GetFunctionEventInvokeConfig 'LastModified' case in deserializers.go) parses a json.Number — unlike FunctionConfiguration.LastModified, which IS an ISO8601 string. Fixed to float64 via pkgs/awstime.Epoch, matching the exact bug class documented in parity-principles.md. Also found + fixed a latent double-write bug in handleUpdateFunctionCode/handleUpdateFunctionConfiguration: applyFunctionCodeUpdate returned h.writeError(...)'s own return value as its error signal, but c.JSON (and so writeError) returns nil on ANY successful write — including a written error response — so the `!= nil` check could never detect a validation failure and would silently fall through to a second, conflicting 200 write. Converted to the bool-return convention (see checkRevisionID's doc comment in handler.go). RevisionId optimistic concurrency (previously only on AddPermission) extended to UpdateFunctionConfiguration/UpdateFunctionCode (checked against fn.RevisionID before mutating), UpdateAlias (against alias.RevisionID), and PublishVersion (new PublishVersionWithRevision atomic backend method — kept the existing 2-arg PublishVersion signature untouched since it has ~20 call sites across tests + a CFN caller; the revision check and the publish happen under one lock acquisition via a shared internal publishVersion(name, description, revisionID) to avoid a check-then-act race). Other families (function URL configs, tags, reserved/provisioned concurrency, code signing) spot-checked against the SDK's Output shapes/timestamp wire formats — no further gaps found; CreateFunctionUrlConfig/GetFunctionUrlConfig's CreationTime/LastModifiedTime and ProvisionedConcurrencyConfig.LastModified are correctly ISO8601 strings (verified against deserializers.go), not epoch numbers. Re-checked this pass (wrapper-key sweep) against the sfn TagResource map/array bug class: lambda's own TagResourceInput/UntagResourceInput/ListTagsOutput all genuinely take Tags as map[string]string (api_op_TagResource.go:44, serializers.go:6822-6834) -- unlike sfn, a map here is correct and needed no change; confirmed via a real-client round-trip test (tag_resource_sdk_test.go)."} - durable_execution: {status: ok, note: "CLOSED (was gap) — dedicated rewrite of durable_execution.go/handler_durable_execution.go, field-diffed against api_op_GetDurableExecution.go, api_op_GetDurableExecutionHistory.go, api_op_GetDurableExecutionState.go, api_op_ListDurableExecutionsByFunction.go, api_op_StopDurableExecution.go, api_op_CheckpointDurableExecution.go, api_op_SendDurableExecutionCallback{Success,Failure,Heartbeat}.go and their types.go/serializers.go/deserializers.go on the installed aws-sdk-go-v2/service/lambda@v1.101.2 module (unchanged for these ops/types between v1.97.0 and v1.101.2). All 9 ops confirmed present in the SDK (not a gopherstack-invented family). Fixed: (1) GetDurableExecutionOutput splits DurableExecutionArn/DurableExecutionName (was one merged ExecutionArn), uses Unix-epoch StartTimestamp/EndTimestamp (was ISO8601 StartTime/StopTime), and adds the previously-entirely-absent DurableConfig echo, Error, ExecutionDataIncluded (honors ?IncludeExecutionData=, default true), InputPayload, Result, TraceHeader, Version; (2) DurableExecutionStatus gained TIMED_OUT; (3) GetDurableExecutionHistory's Events use real types.Event field names/types (EventId/epoch EventTimestamp/EventType/Id/Name/ParentId/SubType + the 5 Execution*Details subtypes this emulator's checkpoint-driven state machine can produce), honors IncludeExecutionData (redacts payload/result/error sub-fields via fresh copies, never mutating the stored event) and ReverseOrder, paginates via Marker/MaxItems (pkgs/page) — previously emitted one invented 'Checkpoint' EventType (not a real enum value) with no pagination; (4) GetDurableExecutionState returns real types.Operation-shaped Operations (Id/Type/Status/StartTimestamp/EndTimestamp/Name/ParentId/SubType) tracked through a new CheckpointDurableExecution Updates state machine (Action START/SUCCEED/FAIL/CANCEL/RETRY on STEP/WAIT/CALLBACK/CONTEXT/CHAINED_INVOKE operations, each mapped to its real EventType via a verified (Type,Action)->EventType table) — CheckpointDurableExecutionInput/Output were previously dead types (handler read an untyped map and discarded it; GetDurableExecutionState always echoed only raw StateData with no Operations). Also found (via the required field-diff) and fixed two real ROUTING bugs beyond the named field-shape gap: StopDurableExecution was wired as DELETE on the bare execution path returning the full execution object — real wire is POST .../stop returning {StopTimestamp} (epoch), and an unknown-ARN Stop silently 200'd 'idempotent' — now 404 ResourceNotFoundException matching Get/GetState; ListDurableExecutionsByFunction was wired at GET /2025-12-01/durable-executions?FunctionArn= — the real op is GET /2025-12-01/functions/{FunctionName}/durable-executions, a completely different path family, now correctly routed with DurableExecutionName/Statuses/StartedAfter/StartedBefore/ReverseOrder/Marker/MaxItems all wired. Also fixed: SendDurableExecutionCallback{Success,Failure,Heartbeat} were routed under the durable-executions ARN prefix with suffixes /callback/success|failure|heartbeat — the real wire is a wholly separate resource, POST /2025-12-01/durable-execution-callbacks/{CallbackId}/{succeed|fail|heartbeat} (note succeed/fail, NOT success/failure) keyed by CallbackId alone; now correctly routed, resolved via a callbackOwner index populated when a checkpoint Update starts a CALLBACK operation, and 404s on an unknown CallbackId (previously silently 200'd regardless). Locking hardened as part of the rewrite: durableExecutionStore's raw sync.RWMutex replaced with lockmetrics.RWMutex (pkgs-catalog.md's 'one coarse instrumented mutex per invariant' rule — this file was the one remaining raw-mutex holdout in the package), and every read method now builds its complete wire response — deep-copying any *DurableOperation it returns — while still holding the lock, rather than handing the handler a live internal pointer to read unsynchronized (previously a genuine, if not test-triggered, data race between a concurrent Get and Checkpoint/Stop on the same execution). Deliberately unchanged, pre-existing, out-of-gap-scope limitation: gopherstack has no StartDurableExecution entry point (correctly — neither does the real API; AWS starts an execution implicitly on Invoke) and this emulator's Invoke path does not model durable-execution semantics, so it still auto-creates the execution record on its first CheckpointDurableExecution call. FunctionArn/DurableConfig/InputPayload/Version are therefore wire-correct (right name, right type, will round-trip through the real SDK client) but always empty/nil today, since no caller threads them through that never-built entry point — this is an entry-point/architecture gap, not a wire-shape gap, and rewiring Invoke was out of this task's scope. Also intentionally not populated: the ~19 CONTEXT/STEP/WAIT/CALLBACK/CHAINED_INVOKE *Details sub-objects the real types.Event/types.Operation declare (no step-function-style replay engine exists to produce their contents) — the generic Id/Name/ParentId/SubType/EventType/Status fields ARE populated for those operation types via the Updates state machine, only the type-specific Details payloads are omitted."} + durable_execution: {status: ok, note: "CLOSED (was gap) — dedicated rewrite of durable_execution.go/handler_durable_execution.go, field-diffed against api_op_GetDurableExecution.go, api_op_GetDurableExecutionHistory.go, api_op_GetDurableExecutionState.go, api_op_ListDurableExecutionsByFunction.go, api_op_StopDurableExecution.go, api_op_CheckpointDurableExecution.go, api_op_SendDurableExecutionCallback{Success,Failure,Heartbeat}.go and their types.go/serializers.go/deserializers.go on the installed aws-sdk-go-v2/service/lambda@v1.101.2 module (unchanged for these ops/types between v1.97.0 and v1.101.2). All 9 ops confirmed present in the SDK (not a gopherstack-invented family). Fixed: (1) GetDurableExecutionOutput splits DurableExecutionArn/DurableExecutionName (was one merged ExecutionArn), uses Unix-epoch StartTimestamp/EndTimestamp (was ISO8601 StartTime/StopTime), and adds the previously-entirely-absent DurableConfig echo, Error, ExecutionDataIncluded (honors ?IncludeExecutionData=, default true), InputPayload, Result, TraceHeader, Version; (2) DurableExecutionStatus gained TIMED_OUT; (3) GetDurableExecutionHistory's Events use real types.Event field names/types (EventId/epoch EventTimestamp/EventType/Id/Name/ParentId/SubType + the 5 Execution*Details subtypes this emulator's checkpoint-driven state machine can produce), honors IncludeExecutionData (redacts payload/result/error sub-fields via fresh copies, never mutating the stored event) and ReverseOrder, paginates via Marker/MaxItems (pkgs/page) — previously emitted one invented 'Checkpoint' EventType (not a real enum value) with no pagination; (4) GetDurableExecutionState returns real types.Operation-shaped Operations (Id/Type/Status/StartTimestamp/EndTimestamp/Name/ParentId/SubType) tracked through a new CheckpointDurableExecution Updates state machine (Action START/SUCCEED/FAIL/CANCEL/RETRY on STEP/WAIT/CALLBACK/CONTEXT/CHAINED_INVOKE operations, each mapped to its real EventType via a verified (Type,Action)->EventType table) — CheckpointDurableExecutionInput/Output were previously dead types (handler read an untyped map and discarded it; GetDurableExecutionState always echoed only raw StateData with no Operations). Also found (via the required field-diff) and fixed two real ROUTING bugs beyond the named field-shape gap: StopDurableExecution was wired as DELETE on the bare execution path returning the full execution object — real wire is POST .../stop returning {StopTimestamp} (epoch), and an unknown-ARN Stop silently 200'd 'idempotent' — now 404 ResourceNotFoundException matching Get/GetState; ListDurableExecutionsByFunction was wired at GET /2025-12-01/durable-executions?FunctionArn= — the real op is GET /2025-12-01/functions/{FunctionName}/durable-executions, a completely different path family, now correctly routed with DurableExecutionName/Statuses/StartedAfter/StartedBefore/ReverseOrder/Marker/MaxItems all wired. Also fixed: SendDurableExecutionCallback{Success,Failure,Heartbeat} were routed under the durable-executions ARN prefix with suffixes /callback/success|failure|heartbeat — the real wire is a wholly separate resource, POST /2025-12-01/durable-execution-callbacks/{CallbackId}/{succeed|fail|heartbeat} (note succeed/fail, NOT success/failure) keyed by CallbackId alone; now correctly routed, resolved via a callbackOwner index populated when a checkpoint Update starts a CALLBACK operation, and 404s on an unknown CallbackId (previously silently 200'd regardless). Locking hardened as part of the rewrite: durableExecutionStore's raw sync.RWMutex replaced with lockmetrics.RWMutex (pkgs-catalog.md's 'one coarse instrumented mutex per invariant' rule — this file was the one remaining raw-mutex holdout in the package), and every read method now builds its complete wire response — deep-copying any *DurableOperation it returns — while still holding the lock, rather than handing the handler a live internal pointer to read unsynchronized (previously a genuine, if not test-triggered, data race between a concurrent Get and Checkpoint/Stop on the same execution). gopherstack has no StartDurableExecution entry point (correctly — neither does the real API; AWS starts an execution implicitly on Invoke); CheckpointDurableExecution called directly against an unknown ARN still auto-creates a bare execution record with empty FunctionArn/DurableConfig/InputPayload/Version, unchanged (a client-opaque ARN carries no function identity to assign). 2026-09-26 pass CLOSED the items_still_open Invoke gap: handleInvoke (handler_invocation.go) now reads the X-Amz-Durable-Execution-Name request header (serializers.go:4016-4017, awsRestjson1_serializeOpHttpBindingsInvokeInput) and, when the resolved function/version/alias has DurableConfig set, starts or reuses a DurableExecution via the new durableExecutionStore.startOrReuseExecution, assigning real FunctionArn (qualified with the RESOLVED version, e.g. "...:function:f:2") and Version, and returns the new DurableExecutionArn via the X-Amz-Durable-Execution-Arn response header (deserializers.go:9167-9169, awsRestjson1_deserializeOpHttpBindingsInvokeOutput). The synthesized DurableExecutionArn itself is the invoked (as-called, unresolved) qualified function ARN plus "/durable-execution//", matching a real EventBridge "Durable Execution Status Change" event sample's shape exactly (durableExecutionArn "...:function:my-function:$LATEST/durable-execution//" vs its own separate, differently-qualified functionArn field). Implements the full documented idempotency table (docs.aws.amazon.com/lambda/latest/dg/durable-execution-idempotency.html): no DurableExecutionName always starts a fresh execution; a name never seen before starts one under that name; a name whose existing execution has an IDENTICAL payload is reused WITHOUT re-invoking the function (the closed-execution case replays the stored Result/Error directly — proven in durable_invoke_test.go by a reuse succeeding with no Docker runtime configured, which only works if the function body is never actually called again); a name reused with a DIFFERENT payload returns DurableExecutionAlreadyStartedException (HTTP 409, confirmed against api/API_Invoke.html's Errors table) via the new ErrDurableExecutionAlreadyStarted sentinel. A synchronous (RequestResponse) invocation's real success/failure is recorded as the execution's completion (SUCCEEDED/FAILED, with an ExecutionSucceeded/ExecutionFailed history event) — this is the verbatim, already-known outcome of the one invocation this backend actually performed, not a fabricated replay result; DryRun never starts an execution at all, matching "validate only, don't execute". 2026-09-26 (second pass, same day) CLOSED the remaining Event-invocation gap: AWS documents async invocation of durable functions as fully supported (docs.aws.amazon.com/lambda/latest/dg/durable-invoking.html — "For asynchronous invocations, Lambda returns immediately and the execution continues independently. Use the durable execution APIs to track execution status and retrieve final results."), so an Event invocation's completion is now recorded on the durable execution once the async invoke actually finishes in the existing async retry loop (runAsyncInvocationRetryLoop, invocation.go): SUCCEEDED on the first successful attempt, FAILED only once MaximumRetryAttempts is exhausted (a container timeout counts as a function error for this purpose, same as the existing destinations/DLQ path). The durable execution ARN is threaded from handleInvoke into the background retry loop via a context value (durableExecARNKeyType) rather than by changing InvokeFunctionWithQualifier's signature, since that interface has call sites (event source pollers, the legacy InvokeAsync path) that have no durable-execution context of their own. GetDurableExecution/ListDurableExecutionsByFunction already correctly reflected RUNNING while an Event invocation was pending (DurableExecutionStatusRunning is the constructor default; only completion ever changed it) — that half of the gap required no fix, just the completion wiring. FIXED 2026-09-26 (third pass, same day): the real API's documented "durable functions support DLQs but don’t support Lambda destinations" restriction (same page) is now enforced -- async_destinations.go's resolveAsyncTargets skips OnSuccess/OnFailure destination delivery when the target function has DurableConfig set (DLQ delivery is untouched); AWS documents no error shape for configuring a destination on a durable function, so PutFunctionEventInvokeConfig still accepts one and this is enforced at delivery time instead, per this sweep's standing "prefer behavior-level enforcement over inventing an error" instruction. See the dated section below for the full third-pass writeup (also: Invoke now requires an explicit qualifier for durable functions, and DurableConfig.ExecutionTimeout/RetentionPeriodInDays/Runtime are now range- and allowlist-validated). Found and fixed one real bug blocking this: resolveQualifier's versionToFn (versions_aliases.go) dropped DurableConfig entirely when resolving a published version/alias, so invoking a durable function by anything other than $LATEST would never have been recognized as durable — fixed by copying it through, same as every other invocation-hot-path field. Also fixed a second real bug the new slash-bearing ARN shape exposed: extractDurableExecARN (handler_durable_execution.go) extracted {DurableExecutionArn} by splitting on the first "/", which truncated any ARN containing "/" itself (previously never triggered, since every ARN in this store was either client-supplied via CheckpointDurableExecution using colon-delimited test fixtures, or fabricated with no slashes) — now strips one of the four known trailing suffixes (/checkpoint, /stop, /history, /state) instead, so a real, slash-bearing ARN correctly round-trips through GetDurableExecution/History/State/Stop/Checkpoint. ListDurableExecutionsByFunction's Qualifier filter (previously accepted but never wired — see the former items_still_open entry) now resolves the given qualifier to a concrete version via resolveQualifier and filters on DurableExecution.Version; per the API reference (not the aws-sdk-go-v2 Go doc comment, which is wrong), an absent Qualifier means every version, not $LATEST. The FunctionName-based filter itself needed a fix too: DurableExecution.FunctionArn is always qualified (with the resolved version) while the FunctionName-derived filter ARN is bare, so a naive equality check would never match — durableExecutionMatchesFunction now compares the bare function identity, leaving Qualifier as the independent version filter. Also intentionally not populated: the ~19 CONTEXT/STEP/WAIT/CALLBACK/CHAINED_INVOKE *Details sub-objects the real types.Event/types.Operation declare (no step-function-style replay engine exists to produce their contents) — the generic Id/Name/ParentId/SubType/EventType/Status fields ARE populated for those operation types via the Updates state machine, only the type-specific Details payloads are omitted."} capacity_providers: {status: ok, note: "gopherstack-m53b (required-member sweep pass 4). CreateCapacityProvider read a top-level \"Name\" field that does not exist on the wire -- the real required field is CapacityProviderName (api_op_CreateCapacityProvider.go:28-45 vs the old models.go CreateCapacityProviderInput) -- so every real client request 400'd with \"Name is required\" before ever reaching the backend; PermissionsConfig and VpcConfig, both also required, were dropped entirely. Full-shape read (per this sweep's standing instruction) found the drop was worse than the three named fields: CapacityProvider/CreateCapacityProviderInput/UpdateCapacityProviderInput had a wholesale-fabricated shape -- a TargetOnDemandConcurrency field that appears nowhere in the real API (removed), Status/LastModifiedTime field names that are actually State/LastModified on the wire (renamed), an ACTIVE status value where the real CapacityProviderState enum is title-cased Active/Pending/Failed/Deleting (fixed), and CapacityProviderScalingConfig/InstanceRequirements/KmsKeyArn/PropagateTags/TelemetryConfig(partially)/VpcConfig were entirely un-modeled despite being real CapacityProvider members. Rebuilt CreateCapacityProviderInput/UpdateCapacityProviderInput/CapacityProvider field-for-field against types.CapacityProvider (types/types.go:206-249) and its nested types (CapacityProviderPermissionsConfig/VpcConfig/ScalingConfig/TelemetryConfig, InstanceRequirements, PropagateTags, TargetTrackingScalingPolicy); UpdateCapacityProvider (not itself one of the five named bugs, but sharing the same CapacityProvider model and left broken by a narrower fix) was corrected alongside it -- CapacityProviderName is a URI label there, not a body field (serializers.go:7098-7113), matching the existing name-from-path handler wiring. Get/List now correctly echo the real state instead of a fabricated shape. Existing tests (capacity_providers_test.go) encoded the broken \"Name\"/TargetOnDemandConcurrency shape end to end (3 create/update/list tests + 1 telemetry test); corrected to the real field names, and a Test_SDKRoundTrip_CreateCapacityProvider/Test_SDKRoundTrip_UpdateCapacityProvider pair added, driving the real aws-sdk-go-v2 lambda client end to end -- both fail against the unfixed decode (hand-reverted and confirmed). TestHandlerReset_ClearsState (dispatch_test.go) also encoded the old \"Name\" shape and was corrected. gopherstack-r80d (required-OUTPUT-member sweep): DeleteCapacityProvider returned bare 204 No Content, but DeleteCapacityProviderOutput.CapacityProvider is required on the wire (api_op_DeleteCapacityProvider.go:44-46) -- real AWS returns 200 with the deleted provider's state. The real SDK deserializer treats an empty 204 body as JSON-decode-EOF (not an error), so the old code produced a client-side success with CapacityProvider left nil -- exactly the zero-value-on-success-path bug class. Fixed: DeleteCapacityProvider now returns the pre-deletion snapshot, handler responds 200 with {CapacityProvider}. Test_SDKRoundTrip_DeleteCapacityProvider added, driving the real client; fails against the unfixed handler with 'Expected value not to be nil' on CapacityProvider (hand-reverted and confirmed). Full sweep of the other 20 required-output-member ops in this service's SDK surface (CheckpointDurableExecution, Create/Get/List/UpdateCapacityProvider, Create/Get/UpdateCodeSigningConfig, GetDurableExecution/-History/-State, GetFunctionCodeSigningConfig, Create/Get/List/UpdateFunctionUrlConfig, ListFunctionVersionsByCapacityProvider, PutFunctionCodeSigningConfig, PutRuntimeManagementConfig, StopDurableExecution) found all correctly populated on their success paths -- this was the only miss."} route_reachability: {status: ok, note: "gopherstack-l5ir (2026-08-13). All 85 real lambda ops extracted from serializers.go (request.Method + httpbinding.SplitURI in each op's awsRestjson1_serializeOp.HandleSerialize) and diffed against the route table. Found and fixed 12 ops that were unreachable or misrouted at their true path/method, beyond the two routing bugs durable_execution's rewrite already caught (see that family's note): GetLayerVersionByArn was wired to a fictional literal path /2018-10-31/layers-by-arn -- the real op shares ListLayers' bare /2018-10-31/layers path, disambiguated only by a ?find=LayerVersion query flag (the query-parameter-discriminator class this sweep was told to watch for specifically); ListFunctionEventInvokeConfigs checked a fictional plural suffix /event-invoke-configs instead of the real /event-invoke-config/list; GetFunctionRecursionConfig/PutFunctionRecursionConfig used date 2024-08-28 instead of the real 2024-08-31; GetFunctionScalingConfig/PutFunctionScalingConfig used date 2023-10-26 AND path segment scaling-config instead of the real 2025-11-30 and function-scaling-config (both wrong, independently); ListTags/TagResource/UntagResource used date 2015-03-31 instead of the real 2017-03-31 -- all three tagging operations were unreachable; InvokeAsync's suffix predicate required a trailing slash (/invoke-async/) the real client never sends (real path has none); ListLayerVersions/PublishLayerVersion resolved via a separate parallel implementation (extractLayerOperation, used by ExtractOperation and IAMAction, NOT by the real HTTP dispatch table which was already correct) that left its discriminating segment empty for exactly this path shape, so both ops always fell through to empty/Unknown -- a real IAM-action and CloudTrail-naming gap even though the request itself was correctly handled. Also corrected, not a bug: ExtractOperation previously returned the lambdaOpRoutes table's first-matching entry for POST .../invocations, which was the literal string \"InvokeFunction\" -- that is the correct IAM *action* name for this op (a documented AWS naming quirk where the IAM action differs from the API operation name) but the wrong *operation* name; ExtractOperation now special-cases this path to return the real op name \"Invoke\" while IAMAction is untouched and still correctly returns lambda:InvokeFunction. ExtractOperation, previously covering only ~30 of 85 ops (CRUD, layers, durable exec), was extended to mirror dispatchSpecialRoutes/lambdaOpRoutes/layerOpTable op-for-op so TestExtractOperation_SDKRouteTable (handler_paths_sdk_diff_test.go, one subtest per op) exercises the real dispatch tree directly -- 85/85 pass. Existing tests that encoded the old wrong paths/dates/expected-op-names (tags_test.go, handler_tags_iam_test.go, function_settings_test.go, event_invoke_config_test.go, layers_http_test.go, invocation_test.go, handler_routing_test.go) were corrected to the real shapes rather than preserved. VERIFIED 2026-09-11 (gopherstack-9coa re-audit): the IAMAction/ExtractOperation divergence described above was already fixed in this same pass; re-confirmed against lambda@v1.107.0's api_op_Invoke.go:65 (`c.invokeOperation(ctx, \"Invoke\", ...)` — the real SDK op name, which is also CloudTrail's eventName per https://docs.aws.amazon.com/lambda/latest/dg/logging-using-cloudtrail.html). What remained from that issue was cleanup only: lambdaOpRoutes (handler_dispatch.go) still carried the later, unreachable duplicate `{POST, hasSuffixInvocations, opInvoke}` entry the issue named (first-match-wins made it dead for both IAMAction and ExtractOperation's fallback loop) — removed, and a landmine comment added on the surviving \"InvokeFunction\" entry explaining the IAM-action/op-name split. New test TestHandler_InvokeOp_IAMActionVsExtractOperation (handler_tags_iam_test.go) drives both consumers off the same request table to prove the divergence and that other ops are unaffected."} gaps: [] -items_still_open: - - "ListDurableExecutionsByFunction always returns zero DurableExecutions for - any function: DurableExecution.FunctionARN is never assigned anywhere in - the package (durable_execution.go) because CheckpointDurableExecution -- - the only test/client-reachable creation path -- carries no function - identity, and its DurableExecutionArn is intentionally treated as - client-opaque. Same root cause as the durable_execution family note's - documented FunctionArn-always-empty gap (no StartDurableExecution/Invoke - entry point); this is that gap's consequence for the List op - specifically. Fixing needs the same out-of-scope Invoke rewiring that - gap already defers to. See 2026-09-12 dated section." - - "2026-09-12 (reqfielddiff slice 4), same root cause as the item above: - InvokeInput.DurableExecutionName (an httpHeader binding, - X-Amz-Durable-Execution-Name, confirmed against - awsRestjson1_serializeOpHttpBindingsInvokeInput) is read nowhere in - handler_invocation.go, and InvokeOutput.DurableExecutionArn (the real, - optional response field a durable invocation would echo) does not exist - anywhere in this package's Invoke response shape. Invoke has zero - durable-execution awareness today -- the only way to create a - DurableExecution is to call CheckpointDurableExecution directly against - an already-known arn, bypassing Invoke entirely. Wiring this properly - (Invoke resolves/creates a DurableExecution, sets its real FunctionARN, - and returns DurableExecutionArn) is the same Invoke-rewiring this file - already defers ListDurableExecutionsByFunction's FunctionARN gap to, not - a standalone one-field fix -- not fabricated a bare pass-through with no - backing execution semantics. - ListDurableExecutionsByFunctionInput.Qualifier (httpQuery, - matchesListFilter has no version/qualifier comparison) is unobservable - for the identical reason: DurableExecution.Version is declared - (durable_execution.go) but never assigned anywhere, since nothing - resolves which function version/alias a durable execution actually ran - under absent the same Invoke entry point." +items_still_open: [] deferred: [] leaks: {status: ok, note: "gopherstack-9zx (2026-09-03): 2 real leak-class bugs found + fixed, see dated section below -- cleanupTimedOutRuntime silently dropped container/port/tempdir cleanup when b.cleanupSem was saturated (its two sibling call sites already fell back to inline cleanup; this one just returned), and a genuine async-invocation timeout skipped both retry and DLQ/on-failure destination delivery entirely (AWS treats a runtime timeout as a function error for async purposes). Everything else re-verified clean this pass: event-source pollers + janitor + container lifecycle otherwise leak-conscious; go test -race passes (3/3 clean runs). New PublishVersionWithRevision path adds no new goroutines/locks (reuses the existing PublishVersion lock); layerPolicyRevisionID/policyRevisionID are pure functions with no new backend state (derived from already-persisted b.permissions / b.layerPolicies, so no new persistence surface either). durable_execution rewrite: durableExecutionStore starts no goroutines and holds no live resources (pure in-memory map + mutex), so Shutdown has nothing to drain; every Lock/RLock is immediately followed by a deferred Unlock/RUnlock with no intervening early return; b.durableExecs.reset() (lifecycle.go) clears both the executions map and the callbackOwner index together, so no ghost callbackOwner entries survive a Reset."} --- +## Notes (2026-09-26 third pass — durable-function restriction audit) + +Audited every documented durable-function restriction/limit (durable-functions.html, +durable-invoking.html, durable-invoking-esm.html, durable-supported-runtimes.html, +API_DurableConfig.html, API_Invoke.html, gettingstarted-limits.html) against this +service and closed the four real gaps found: + +- **Destinations vs DLQ** (durable-invoking.html: "durable functions support dead-letter + queues (DLQs) for error handling, but don't support Lambda destinations"): the prior + pass's items_still_open note flagged this as unenforced. Fixed at delivery time — + `resolveAsyncTargets` (async_destinations.go) now skips OnSuccess/OnFailure destination + resolution when the target function has `DurableConfig` set; DLQ delivery is untouched. + AWS documents no error shape for configuring a destination on a durable function, so + `PutFunctionEventInvokeConfig` still accepts one (behavior-level enforcement, not an + invented rejection). +- **Qualified-ARN requirement** (durable-invoking.html#durable-invoking-qualified-arns): + a durable function must be invoked with an explicit version, alias, or literal + `$LATEST` — unlike a standard function, which silently defaults to `$LATEST` when no + qualifier is given. `handleInvoke` previously defaulted an absent qualifier to + `$LATEST` for every function, durable or not. New `requireDurableQualifier` + (handler_invocation.go) rejects an unqualified Invoke of a durable function with + `InvalidParameterValueException` (400) — the documented error for "one of the + parameters in the request is not valid" (API_Invoke.html's Errors table has no + durable-specific exception for this case). +- **DurableConfig range validation** (API_DurableConfig.html): `ExecutionTimeout` + (1-31622400) and `RetentionPeriodInDays` (1-90) were accepted-and-echoed with no range + check. Now validated on CreateFunction and UpdateFunctionConfiguration + (`validateDurableConfigInput`, handler_functions.go), matching the existing + MemorySize/Timeout/EphemeralStorage range-check pattern (same `InvalidParameterValueException` + convention). +- **Supported runtimes** (durable-supported-runtimes.html): a Zip-packaged durable + function must use one of nodejs22.x/nodejs24.x/python3.13/python3.14/java17/java21/java25/ + dotnet8/dotnet10 — container images have no such restriction ("additional runtime + version flexibility"). Added `isDurableSupportedRuntime` (containers.go) and wired it + into CreateFunction's Zip-code validation and UpdateFunctionConfiguration's effective + (existing-fn-overlaid-with-input) Runtime/DurableConfig combination. This also + surfaced that `runtimeBaseImages` was missing nodejs24.x/python3.14/java25/dotnet10 + entirely (rejected as unknown runtimes even outside the durable case) — added. + +Checked and found already correct or out of scope, not changed: + +- Event source mappings, function URLs, and InvokeWithResponseStream are NOT documented + as unsupported for durable functions (durable-invoking-esm.html: "Durable functions + work with all Lambda event source mappings"; configuration-response-streaming.html has + no durable-function restriction at all) — no rejection added for any of these paths. +- The ESM execution-duration limit (15 min default / 90 min on Managed Instances) and the + durable-functions-family quotas (3,000 operations/execution, 100 MB storage/execution, + 5-10M running executions/Region) are real per-Region service quotas, not + request-shape validation — no documented exception name ties them to a specific API + call, and enforcing them meaningfully would require modeling cumulative execution time/ + payload bytes across an execution's whole lifetime. Left unenforced as genuinely + quota-shaped, not a wire-shape or validation gap. +- `DurableConfig.KMSKeyArn` has a documented pattern (`(arn:...)|()`) but this emulator + performs no real KMS encryption of durable-execution payloads (pre-existing, unrelated + to this pass) — not worth pattern-validating a field whose only consumer is echo-back. + +Gates: `gofmt`, `go build ./...`, `go vet`, `go test -race`, `golangci-lint`, `go test +./pkgs/persistence/`, `cmd/parityfmtcheck` all clean; `go.mod`/`go.sum` unchanged. + +## Notes (2026-09-26 pass — Invoke durable-execution wiring, closes items_still_open) + +Closed both remaining durable_execution items_still_open entries by giving +Invoke (handler_invocation.go) real durable-execution awareness — see the +durable_execution family note above for the full description. Summary: + +- `X-Amz-Durable-Execution-Name` request header and `X-Amz-Durable-Execution-Arn` + response header wired (verified against lambda@v1.107.0 + serializers.go:4016-4017 / deserializers.go:9167-9169). +- `DurableExecution.FunctionArn`/`Version` now assigned from the resolved + function/version/alias, unblocking `ListDurableExecutionsByFunction`'s + FunctionName and Qualifier filters for any execution started via Invoke. +- Full idempotency table implemented (no name / new name / identical-payload + reuse / conflicting-payload `DurableExecutionAlreadyStartedException`, + HTTP 409) per docs.aws.amazon.com/lambda/latest/dg/ + durable-execution-idempotency.html and api/API_Invoke.html's Errors table. + A payload-identical reuse against a closed execution never re-invokes the + function — proven in `durable_invoke_test.go` by a reuse succeeding with + no Docker runtime configured. +- Two real bugs found and fixed along the way: `versionToFn` + (versions_aliases.go) dropped `DurableConfig` when resolving a published + version/alias, so a durable function invoked by anything but `$LATEST` + was never recognized as durable; `extractDurableExecARN` + (handler_durable_execution.go) split on the first `/`, which truncates + the real, slash-bearing ARN shape this pass introduces (fixed to strip a + known trailing suffix instead). +- `CheckpointDurableExecution`-only-created executions are unaffected and + still have an empty `FunctionArn` (a client-opaque ARN carries no function + identity) — this remains a correct, narrower simplification, not a gap. + +New tests: `durable_invoke_test.go` (`TestRealClient_DurableInvoke`, 5 +table-driven subtests via the real SDK client over httptest, no Docker). +Gates: `gofmt -l`, `go build ./...`, `go vet ./services/lambda/...`, +`go test -race -count=1 ./services/lambda/...`, `golangci-lint run +./services/lambda/...`, `go test ./pkgs/persistence/...`, `go run +./cmd/parityfmtcheck -dir services` all clean; `git diff --stat go.mod +go.sum` empty. No persisted field changed (durable_execution is +intentionally not wired into Snapshot/Restore, unchanged by this pass). + +## Notes (2026-09-26 second pass — async durable Invoke records completion) + +Closed the remaining durable_execution gap noted above: an Event +(`InvocationType=Event`) invocation of a durable function started an +execution but never recorded its completion, leaving `Status` stuck at +`RUNNING` forever even after the backend's real async invoke actually +finished. Verified against AWS docs first +(docs.aws.amazon.com/lambda/latest/dg/durable-invoking.html) — async +invocation of durable functions is documented as fully supported ("Lambda +returns immediately and the execution continues independently. Use the +durable execution APIs to track execution status and retrieve final +results."), not a special-cased or unsupported combination, so the fix is +to make the emulator behave as documented rather than to reject it. + +- `runAsyncInvocationRetryLoop` (invocation.go) — the existing async + retry/destinations choke point (MaximumRetryAttempts, + MaximumEventAgeInSeconds, DLQ/destination delivery) — now calls the new + `completeAsyncDurableExecution` on the same terminal branch that already + builds the `asyncOutcome` for DLQ/destination delivery: `SUCCEEDED` on + the first non-error attempt, `FAILED` only once `attempt == maxRetries` + (retries exhausted), matching this file's documented default of 2 + retries and any `PutFunctionEventInvokeConfig` override. A container + timeout is treated as a function error for this purpose too, the same + as it already is for destinations (see the `leaks` family note above). +- The durable execution ARN reaches that background goroutine via a new + context key (`durableExecARNKeyType`, `withDurableExecARN`/ + `durableExecARNFromContext`) set by `handleInvoke` and read by + `invokeEvent`/propagated through `scheduleAsyncRetry`'s retry + `pendingInvocation`s — not by adding a parameter to + `InvokeFunctionWithQualifier`, which has call sites (event source + pollers, the legacy `InvokeAsync` API) with no durable-execution + context and would have had to plumb an unused arg through all of them. + `ctxWithAsyncDurableExecARN` factors the "only for Event" check out of + `handleInvoke` to stay under cyclop's complexity budget. +- `GetDurableExecution`/`ListDurableExecutionsByFunction` already + correctly reported `RUNNING` for a pending Event invocation before this + fix (`DurableExecutionStatusRunning` is `newDurableExecution`'s + constructor default, only ever changed by completion) — confirmed, not + changed. +- Found but not fixed (pre-existing, unrelated to completion-recording): + the same AWS page documents durable functions as supporting DLQs but + **not** Lambda destinations; gopherstack does not enforce this — + `PutFunctionEventInvokeConfig` still accepts a `DestinationConfig` on a + durable function and `async_destinations.go` delivers to it + unconditionally. Left as a separate, reported gap. + +New test: `durable_async_invoke_test.go` +(`TestAsyncDurableInvoke_RecordsCompletion`), table-driven + `t.Parallel()` +in both the outer test and its subtests, 2 subtests (success, +failure-after-retries-exhausted). Drives the real `aws-sdk-go-v2` client +over `httptest` against a backend with a mock Docker client + a real +loopback runtime-API server (needed because a durable function's Event +invocation must actually execute and complete, unlike the Docker-less +`durable_invoke_test.go` cases which only prove the pre-invoke ARN +bookkeeping); uses `require.Eventually` to poll the runtime queue and the +execution's terminal status instead of a sleep, since this exercises real +loopback I/O across goroutines and can't be put in a synctest bubble +(same constraint as `TestBackend_InvokeFunction_RequestResponse_WithMockDocker` +in `handler_runtime_test.go`). No `export_test.go` additions — the test +drives the real Lambda Runtime API HTTP endpoints +(`/2018-06-01/runtime/invocation/next`, `.../response`, `.../error`) +directly, the same way a real container would. + +Gates: `gofmt -l services/lambda` (no output), `go build ./...`, `go vet +./services/lambda/...`, `go test -race -count=2 ./services/lambda/...`, +`golangci-lint run ./services/lambda/...` (0 issues — the new +`ctxWithAsyncDurableExecARN` extraction was required to keep +`handleInvoke` under cyclop's limit), `go test ./pkgs/persistence/...`, +`go run ./cmd/parityfmtcheck -dir services` all clean; `git diff --stat +go.mod go.sum` empty. No persisted struct changed (`pendingInvocation` +and `asyncOutcome` are both purely in-memory/transient, never +snapshotted). + ## Notes (2026-09-19 pass — terraform lambda-and-apigateway fixture) Added real-provider fixture coverage for alias/code_signing_config/ diff --git a/services/lambda/README.md b/services/lambda/README.md index 9786e8fba5..2a83eac4e2 100644 --- a/services/lambda/README.md +++ b/services/lambda/README.md @@ -8,15 +8,10 @@ | Metric | Value | | --- | --- | | Feature families | 10 (10 ok) | -| Known gaps | 2 | +| Known gaps | none | | Deferred items | 0 | | Resource leaks | ok | -### Known gaps - -- "ListDurableExecutionsByFunction always returns zero DurableExecutions for any function: DurableExecution.FunctionARN is never assigned anywhere in the package (durable_execution.go) because CheckpointDurableExecution -- the only test/client-reachable creation path -- carries no function identity, and its DurableExecutionArn is intentionally treated as client-opaque. Same root cause as the durable_execution family note's documented FunctionArn-always-empty gap (no StartDurableExecution/Invoke entry point); this is that gap's consequence for the List op specifically. Fixing needs the same out-of-scope Invoke rewiring that gap already defers to. See 2026-09-12 dated section." -- "2026-09-12 (reqfielddiff slice 4), same root cause as the item above: InvokeInput.DurableExecutionName (an httpHeader binding, X-Amz-Durable-Execution-Name, confirmed against awsRestjson1_serializeOpHttpBindingsInvokeInput) is read nowhere in handler_invocation.go, and InvokeOutput.DurableExecutionArn (the real, optional response field a durable invocation would echo) does not exist anywhere in this package's Invoke response shape. Invoke has zero durable-execution awareness today -- the only way to create a DurableExecution is to call CheckpointDurableExecution directly against an already-known arn, bypassing Invoke entirely. Wiring this properly (Invoke resolves/creates a DurableExecution, sets its real FunctionARN, and returns DurableExecutionArn) is the same Invoke-rewiring this file already defers ListDurableExecutionsByFunction's FunctionARN gap to, not a standalone one-field fix -- not fabricated a bare pass-through with no backing execution semantics. ListDurableExecutionsByFunctionInput.Qualifier (httpQuery, matchesListFilter has no version/qualifier comparison) is unobservable for the identical reason: DurableExecution.Version is declared (durable_execution.go) but never assigned anywhere, since nothing resolves which function version/alias a durable execution actually ran under absent the same Invoke entry point." - ## More - [Full parity audit](PARITY.md) diff --git a/services/lambda/async_destinations.go b/services/lambda/async_destinations.go index 6b44c343d7..8eb5435754 100644 --- a/services/lambda/async_destinations.go +++ b/services/lambda/async_destinations.go @@ -123,16 +123,22 @@ func (b *InMemoryBackend) resolveAsyncTargets(out asyncOutcome) ( fn, _ := b.functions.Get(out.functionName) eic := b.eventInvokeConfigs[out.functionName] - var functionArn, dlqTarget string + var functionArn, dlqTarget, destTarget string if fn != nil { functionArn = fn.FunctionArn if fn.DeadLetterConfig != nil { dlqTarget = fn.DeadLetterConfig.TargetArn } + + // Durable functions support DLQs but not Lambda destinations; skip delivery. + // docs.aws.amazon.com/lambda/latest/dg/durable-invoking.html + if fn.DurableConfig == nil { + destTarget = resolveDestinationTarget(eic, out.success) + } } - return delivery, functionArn, dlqTarget, resolveDestinationTarget(eic, out.success) + return delivery, functionArn, dlqTarget, destTarget } // resolveDestinationTarget returns the OnSuccess or OnFailure destination ARN for diff --git a/services/lambda/async_invoke_test.go b/services/lambda/async_invoke_test.go index 52f30bbd3e..8ab7fce5d3 100644 --- a/services/lambda/async_invoke_test.go +++ b/services/lambda/async_invoke_test.go @@ -19,6 +19,7 @@ const ( asyncInvokeSlotLifetimeBase = 18203 // 18203–18204 reserved asyncInvokeRetryBase = 18205 // 18205–18208 reserved asyncInvokeTimeoutDestBase = 18209 // 18209 reserved + asyncInvokeDurableDestBase = 18210 // 18210 reserved ) // newAsyncTestBackend returns a backend with no Docker/port-alloc so that @@ -491,3 +492,48 @@ func TestEnqueueAsync_TimeoutDeliversToFailureDestination(t *testing.T) { assert.Contains(t, fake.targets(), failureARN) } + +// TestEnqueueAsync_DurableFunctionSkipsDestinations verifies a durable +// function's DLQ still fires but its OnFailure destination does not. +// docs.aws.amazon.com/lambda/latest/dg/durable-invoking.html. +func TestEnqueueAsync_DurableFunctionSkipsDestinations(t *testing.T) { + t.Parallel() + + const ( + dlqARN = "arn:aws:sqs:us-east-1:000000000000:durable-dlq" + destFailureARN = "arn:aws:sqs:us-east-1:000000000000:durable-on-failure" + ) + + srv := startAsyncTestServer(t, asyncInvokeDurableDestBase) + bk := newAsyncTestBackend(t) + + require.NoError(t, bk.CreateFunction(&lambda.FunctionConfiguration{ + FunctionName: "fn-durable-timeout-dest", + DurableConfig: &lambda.DurableConfig{}, + DeadLetterConfig: &lambda.DeadLetterConfig{TargetArn: dlqARN}, + })) + + _, err := bk.PutFunctionEventInvokeConfig("fn-durable-timeout-dest", &lambda.PutFunctionEventInvokeConfigInput{ + MaximumRetryAttempts: new(0), + DestinationConfig: &lambda.DestinationConfig{ + OnFailure: &lambda.Destination{Destination: destFailureARN}, + }, + }) + require.NoError(t, err) + + fake := &fakeAsyncDelivery{} + bk.SetAsyncDestinationDelivery(fake) + + // Never simulate any /next or /response call: the container is hung and the + // invocation must time out rather than ever completing. + lambda.EnqueueAsync(t.Context(), bk, srv, "fn-durable-timeout-dest", []byte(`{}`), 200*time.Millisecond, false) + + require.Eventually(t, func() bool { + return len(fake.targets()) > 0 + }, 3*time.Second, 10*time.Millisecond, + "a timed-out durable async invocation must still be delivered to its DLQ") + + assert.Contains(t, fake.targets(), dlqARN) + assert.NotContains(t, fake.targets(), destFailureARN, + "durable functions don't support Lambda destinations") +} diff --git a/services/lambda/capacity_providers.go b/services/lambda/capacity_providers.go index 301f1f45d5..e673b94d4a 100644 --- a/services/lambda/capacity_providers.go +++ b/services/lambda/capacity_providers.go @@ -9,6 +9,15 @@ import ( // --- Capacity providers --- +// cloneCapacityProvider copies cp so a caller can't race Update/Seed, which +// mutate the stored provider in place. +func cloneCapacityProvider(cp *CapacityProvider) *CapacityProvider { + out := *cp + out.AssignedFunctionVersions = append([]string(nil), cp.AssignedFunctionVersions...) + + return &out +} + // CreateCapacityProvider creates a new Lambda capacity provider. func (b *InMemoryBackend) CreateCapacityProvider( input *CreateCapacityProviderInput, @@ -37,7 +46,7 @@ func (b *InMemoryBackend) CreateCapacityProvider( b.capacityProviders.Put(cp) - return cp, nil + return cloneCapacityProvider(cp), nil } // GetCapacityProvider retrieves a capacity provider by name. @@ -50,7 +59,7 @@ func (b *InMemoryBackend) GetCapacityProvider(name string) (*CapacityProvider, e return nil, ErrFunctionNotFound } - return cp, nil + return cloneCapacityProvider(cp), nil } // DeleteCapacityProvider removes a capacity provider by name and returns the @@ -68,7 +77,7 @@ func (b *InMemoryBackend) DeleteCapacityProvider(name string) (*CapacityProvider b.capacityProviders.Delete(name) - return cp, nil + return cloneCapacityProvider(cp), nil } // UpdateCapacityProvider updates an existing capacity provider. @@ -99,7 +108,7 @@ func (b *InMemoryBackend) UpdateCapacityProvider( cp.LastModified = time.Now().UTC().Format(time.RFC3339) b.capacityProviders.Put(cp) - return cp, nil + return cloneCapacityProvider(cp), nil } // ListCapacityProviders returns all capacity providers. @@ -109,11 +118,16 @@ func (b *InMemoryBackend) ListCapacityProviders() []*CapacityProvider { cps := b.capacityProviders.All() - sort.Slice(cps, func(i, j int) bool { - return cps[i].Name < cps[j].Name + out := make([]*CapacityProvider, len(cps)) + for i, cp := range cps { + out[i] = cloneCapacityProvider(cp) + } + + sort.Slice(out, func(i, j int) bool { + return out[i].Name < out[j].Name }) - return cps + return out } // SeedCapacityProviderFunctionVersions assigns the given function-version ARNs to diff --git a/services/lambda/code_signing.go b/services/lambda/code_signing.go index 1efbf015b8..fccb91076b 100644 --- a/services/lambda/code_signing.go +++ b/services/lambda/code_signing.go @@ -41,7 +41,15 @@ func (b *InMemoryBackend) CreateCodeSigningConfig( b.codeSigningConfigs.Put(cfg) - return cfg, nil + return cloneCodeSigningConfig(cfg), nil +} + +// cloneCodeSigningConfig stops a caller from racing UpdateCodeSigningConfig, +// which mutates cfg's fields under the lock. +func cloneCodeSigningConfig(cfg *CodeSigningConfig) *CodeSigningConfig { + cp := *cfg + + return &cp } // GetCodeSigningConfig retrieves a code signing config by ARN. @@ -54,7 +62,7 @@ func (b *InMemoryBackend) GetCodeSigningConfig(cscARN string) (*CodeSigningConfi return nil, ErrFunctionNotFound } - return cfg, nil + return cloneCodeSigningConfig(cfg), nil } // DeleteCodeSigningConfig removes a code signing config by ARN. @@ -99,7 +107,7 @@ func (b *InMemoryBackend) UpdateCodeSigningConfig( cfg.LastModified = time.Now().UTC().Format(time.RFC3339) b.codeSigningConfigs.Put(cfg) - return cfg, nil + return cloneCodeSigningConfig(cfg), nil } // ListCodeSigningConfigs returns all code signing configs. @@ -107,7 +115,12 @@ func (b *InMemoryBackend) ListCodeSigningConfigs(marker string, maxItems int) pa b.mu.RLock("ListCodeSigningConfigs") defer b.mu.RUnlock() - cfgs := b.codeSigningConfigs.All() + stored := b.codeSigningConfigs.All() + cfgs := make([]*CodeSigningConfig, len(stored)) + + for i, cfg := range stored { + cfgs[i] = cloneCodeSigningConfig(cfg) + } sort.Slice(cfgs, func(i, j int) bool { return cfgs[i].CodeSigningConfigID < cfgs[j].CodeSigningConfigID diff --git a/services/lambda/config_race_test.go b/services/lambda/config_race_test.go new file mode 100644 index 0000000000..6b3272e6e6 --- /dev/null +++ b/services/lambda/config_race_test.go @@ -0,0 +1,125 @@ +package lambda_test + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/lambda" +) + +// TestConfigReadConcurrentWithUpdate proves Get/Create/List config accessors +// must not hand back a live pointer that the matching Update mutates in place. +func TestConfigReadConcurrentWithUpdate(t *testing.T) { + t.Parallel() + + tests := []struct { + setup func(t *testing.T) (reader func(), mutator func(i int)) + name string + }{ + { + name: "FunctionURLConfig races UpdateFunctionURLConfig", + setup: func(t *testing.T) (func(), func(int)) { + t.Helper() + + h, bk := newInMemoryHandler(t) + fnName := "url-race-fn" + createFunctionForTest(t, h, fnName) + + _, err := bk.CreateFunctionURLConfig(t.Context(), fnName, "NONE", nil, "BUFFERED") + require.NoError(t, err) + + reader := func() { + cfg, getErr := bk.GetFunctionURLConfig(fnName) + if getErr != nil { + return + } + + _ = cfg.AuthType + _ = cfg.InvokeMode + _ = cfg.LastModifiedTime + } + + mutator := func(i int) { + authType := "NONE" + if i%2 == 0 { + authType = "AWS_IAM" + } + + _, _ = bk.UpdateFunctionURLConfig(fnName, authType, nil, "BUFFERED") + } + + return reader, mutator + }, + }, + { + name: "CapacityProvider races UpdateCapacityProvider", + setup: func(t *testing.T) (func(), func(int)) { + t.Helper() + + bk := newCapacityProviderTestBackend(t) + + _, err := bk.CreateCapacityProvider(&lambda.CreateCapacityProviderInput{ + CapacityProviderName: "race-cp", + PermissionsConfig: &lambda.CapacityProviderPermissionsConfig{ + CapacityProviderOperatorRoleArn: "arn:aws:iam::000000000000:role/cp-role", + }, + VpcConfig: &lambda.CapacityProviderVpcConfig{ + SubnetIDs: []string{"subnet-1"}, + }, + }) + require.NoError(t, err) + + reader := func() { + cp, getErr := bk.GetCapacityProvider("race-cp") + if getErr != nil { + return + } + + _ = cp.LastModified + _ = cp.State + } + + mutator := func(int) { + _, _ = bk.UpdateCapacityProvider("race-cp", &lambda.UpdateCapacityProviderInput{ + PropagateTags: &lambda.PropagateTags{Mode: "TASK_DEFINITION"}, + }) + } + + return reader, mutator + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + reader, mutator := tt.setup(t) + + const iterations = 500 + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + for range iterations { + reader() + } + }() + + go func() { + defer wg.Done() + + for i := range iterations { + mutator(i) + } + }() + + wg.Wait() + }) + } +} diff --git a/services/lambda/containers.go b/services/lambda/containers.go index 20fece00a4..6e4e4c2dd1 100644 --- a/services/lambda/containers.go +++ b/services/lambda/containers.go @@ -415,17 +415,21 @@ func (b *InMemoryBackend) handleContainerStartFailure( // //nolint:gochecknoglobals // intentional package-level lookup table var runtimeBaseImages = map[string]string{ + "python3.14": "public.ecr.aws/lambda/python:3.14", "python3.13": "public.ecr.aws/lambda/python:3.13", "python3.12": "public.ecr.aws/lambda/python:3.12", "python3.11": "public.ecr.aws/lambda/python:3.11", "python3.10": "public.ecr.aws/lambda/python:3.10", "python3.9": "public.ecr.aws/lambda/python:3.9", + "nodejs24.x": "public.ecr.aws/lambda/nodejs:24", "nodejs22.x": "public.ecr.aws/lambda/nodejs:22", "nodejs20.x": "public.ecr.aws/lambda/nodejs:20", "nodejs18.x": "public.ecr.aws/lambda/nodejs:18", + "java25": "public.ecr.aws/lambda/java:25", "java21": "public.ecr.aws/lambda/java:21", "java17": "public.ecr.aws/lambda/java:17", "java11": "public.ecr.aws/lambda/java:11", + "dotnet10": "public.ecr.aws/lambda/dotnet:10", "dotnet9": "public.ecr.aws/lambda/dotnet:9", "dotnet8": "public.ecr.aws/lambda/dotnet:8", "ruby3.3": "public.ecr.aws/lambda/ruby:3.3", @@ -494,6 +498,29 @@ func isValidRuntime(runtime string) bool { return ok } +// durableSupportedRuntimes lists managed runtimes allowed for a Zip durable function. +// docs.aws.amazon.com/lambda/latest/dg/durable-supported-runtimes.html +// +//nolint:gochecknoglobals // static allowlist mirroring a fixed AWS doc table +var durableSupportedRuntimes = map[string]struct{}{ + "nodejs22.x": {}, + "nodejs24.x": {}, + "python3.13": {}, + "python3.14": {}, + "java17": {}, + "java21": {}, + "java25": {}, + "dotnet8": {}, + "dotnet10": {}, +} + +// isDurableSupportedRuntime reports whether runtime supports durable functions. +func isDurableSupportedRuntime(runtime string) bool { + _, ok := durableSupportedRuntimes[runtime] + + return ok +} + // extractZip extracts zip bytes into a new temporary directory and returns the directory path. // The caller is responsible for calling [os.RemoveAll] on the returned path when done. func extractZip(zipData []byte) (string, error) { diff --git a/services/lambda/durable_async_invoke_test.go b/services/lambda/durable_async_invoke_test.go new file mode 100644 index 0000000000..1d420f16ed --- /dev/null +++ b/services/lambda/durable_async_invoke_test.go @@ -0,0 +1,174 @@ +package lambda_test + +import ( + "fmt" + "net/http" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + lambdasdk "github.com/aws/aws-sdk-go-v2/service/lambda" + "github.com/aws/aws-sdk-go-v2/service/lambda/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/portalloc" + "github.com/blackbirdworks/gopherstack/services/lambda" +) + +// TestAsyncDurableInvoke_RecordsCompletion checks an Event invocation of a durable +// function leaves RUNNING once it finishes (docs: lambda/latest/dg/durable-invoking.html). +func TestAsyncDurableInvoke_RecordsCompletion(t *testing.T) { + t.Parallel() + + tests := []struct { + respond func(t *testing.T, port int, requestID string) + wantStatus types.ExecutionStatus + wantEvent types.EventType + name string + fnName string + portBase int + maxRetries int32 + }{ + { + name: "success", + fnName: "durasync-ok", + portBase: 21200, + maxRetries: 0, + respond: func(t *testing.T, port int, requestID string) { + t.Helper() + simulateContainerResponse(t, port, requestID, `{"ok":true}`) + }, + wantStatus: types.ExecutionStatusSucceeded, + wantEvent: types.EventTypeExecutionSucceeded, + }, + { + name: "failure_after_retries_exhausted", + fnName: "durasync-fail", + portBase: 21300, + maxRetries: 0, // no retries: the first (and only) failure is terminal + respond: func(t *testing.T, port int, requestID string) { + t.Helper() + simulateContainerError(t, port, requestID, `{"errorMessage":"boom"}`) + }, + wantStatus: types.ExecutionStatusFailed, + wantEvent: types.EventTypeExecutionFailed, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + portRange := [2]int{tt.portBase, tt.portBase + 50} + pa, err := portalloc.New(portRange[0], portRange[1]) + require.NoError(t, err) + + bk := lambda.NewInMemoryBackend( + newMockDockerClient(), pa, lambda.DefaultSettings(), "000000000000", "us-east-1", + ) + closeBackend(t, bk) + + h := lambda.NewHandler(bk) + h.DefaultRegion = "us-east-1" + h.AccountID = "000000000000" + + client := newTestLambdaClient(t, h) + + fnName := tt.fnName + + _, err = client.CreateFunction(t.Context(), &lambdasdk.CreateFunctionInput{ + FunctionName: aws.String(fnName), + PackageType: types.PackageTypeImage, + Code: &types.FunctionCode{ImageUri: aws.String("myimage:latest")}, + Role: aws.String("arn:aws:iam:::role/r"), + DurableConfig: &types.DurableConfig{ExecutionTimeout: aws.Int32(3600)}, + }) + require.NoError(t, err) + + _, err = client.PutFunctionEventInvokeConfig(t.Context(), &lambdasdk.PutFunctionEventInvokeConfigInput{ + FunctionName: aws.String(fnName), + MaximumRetryAttempts: aws.Int32(tt.maxRetries), + }) + require.NoError(t, err) + + invokeErrCh := make(chan error, 1) + var durableARN string + + go func() { + out, invokeErr := client.Invoke(t.Context(), &lambdasdk.InvokeInput{ + FunctionName: aws.String(fnName), + Qualifier: aws.String("$LATEST"), + InvocationType: types.InvocationTypeEvent, + DurableExecutionName: aws.String("exec-" + tt.name), + Payload: []byte(`{}`), + }) + if out != nil { + durableARN = aws.ToString(out.DurableExecutionArn) + } + invokeErrCh <- invokeErr + }() + + require.NoError(t, <-invokeErrCh) + require.NotEmpty(t, durableARN) + + httpClient := newHTTPClient(t, 200*time.Millisecond) + + var ( + runtimePort int + requestID string + ) + + require.Eventually(t, func() bool { + for p := portRange[0]; p < portRange[1]; p++ { + req, reqErr := http.NewRequestWithContext( + t.Context(), http.MethodGet, + fmt.Sprintf("http://127.0.0.1:%d/2018-06-01/runtime/invocation/next", p), nil, + ) + if reqErr != nil { + continue + } + + resp, doErr := httpClient.Do(req) + if doErr != nil || resp == nil { + continue + } + + id := resp.Header.Get("Lambda-Runtime-Aws-Request-Id") + resp.Body.Close() + + if id != "" { + runtimePort, requestID = p, id + + return true + } + } + + return false + }, 4*time.Second, 50*time.Millisecond, "async invocation was never queued to a runtime") + + tt.respond(t, runtimePort, requestID) + + require.Eventually(t, func() bool { + out, getErr := client.GetDurableExecution(t.Context(), &lambdasdk.GetDurableExecutionInput{ + DurableExecutionArn: aws.String(durableARN), + }) + + return getErr == nil && out.Status == tt.wantStatus + }, 4*time.Second, 50*time.Millisecond, "durable execution never left RUNNING") + + histOut, err := client.GetDurableExecutionHistory(t.Context(), &lambdasdk.GetDurableExecutionHistoryInput{ + DurableExecutionArn: aws.String(durableARN), + }) + require.NoError(t, err) + + var sawEvent bool + for _, ev := range histOut.Events { + if ev.EventType == tt.wantEvent { + sawEvent = true + } + } + assert.True(t, sawEvent, "expected a %s history event", tt.wantEvent) + }) + } +} diff --git a/services/lambda/durable_execution.go b/services/lambda/durable_execution.go index 815ddcf6db..4e3308f59a 100644 --- a/services/lambda/durable_execution.go +++ b/services/lambda/durable_execution.go @@ -397,17 +397,28 @@ func epochPtr(t time.Time) *float64 { return &ts } +// newDurableExecutionARN synthesizes a DurableExecutionArn the way real AWS +// does (verified against a real EventBridge "Durable Execution Status +// Change" event sample: durableExecutionArn +// "...:function:my-function:$LATEST/durable-execution//"): +// the function ARN AS INVOKED (with its qualifier) plus a +// "/durable-execution//" suffix. +func newDurableExecutionARN(invokedFunctionARN string) string { + return invokedFunctionARN + "/durable-execution/" + uuid.New().String() + "/" + uuid.New().String() +} + // newDurableExecution creates a fresh execution record, seeding its // history/operations with the implicit ExecutionStarted event and root // Type=EXECUTION operation every durable execution has. -func newDurableExecution(arn string) *DurableExecution { +func newDurableExecution(arn, inputPayload string) *DurableExecution { now := time.Now().UTC() ex := &DurableExecution{ - ARN: arn, - Name: deriveDurableExecutionName(arn), - Status: DurableExecutionStatusRunning, - StartTime: now, - opIndex: make(map[string]int), + ARN: arn, + Name: deriveDurableExecutionName(arn), + Status: DurableExecutionStatusRunning, + StartTime: now, + InputPayload: inputPayload, + opIndex: make(map[string]int), } ex.appendEvent(DurableExecutionEvent{ @@ -628,6 +639,7 @@ type durableExecutionStore struct { mu *lockmetrics.RWMutex executions map[string]*DurableExecution // key: DurableExecutionArn callbackOwner map[string]string // key: CallbackId (== a CALLBACK operation's Id) -> DurableExecutionArn + byName map[string]string // key: DurableExecutionName -> DurableExecutionArn (Invoke-started only) } func newDurableExecutionStore() *durableExecutionStore { @@ -635,6 +647,7 @@ func newDurableExecutionStore() *durableExecutionStore { mu: lockmetrics.New("lambda.durable_executions"), executions: make(map[string]*DurableExecution), callbackOwner: make(map[string]string), + byName: make(map[string]string), } } @@ -712,7 +725,7 @@ func (s *durableExecutionStore) stateOutput(arn, marker string, maxItems int) (* // internally by tests; the wire-facing handler always resolves a concrete // function ARN from the {FunctionName} URI segment first). func (s *durableExecutionStore) listSummaries( - functionARN, nameFilter string, + functionARN, nameFilter, versionFilter string, statuses []DurableExecutionStatus, startedAfter, startedBefore time.Time, reverseOrder bool, @@ -728,7 +741,7 @@ func (s *durableExecutionStore) listSummaries( var matched []*DurableExecution for _, ex := range s.executions { - if !matchesListFilter(ex, functionARN, nameFilter, statusSet, startedAfter, startedBefore) { + if !matchesListFilter(ex, functionARN, nameFilter, versionFilter, statusSet, startedAfter, startedBefore) { continue } @@ -751,13 +764,27 @@ func (s *durableExecutionStore) listSummaries( return out } +// durableExecutionMatchesFunction reports whether ex's FunctionARN (always +// qualified with the resolved version, e.g. "...:function:my-fn:2") belongs +// to filterARN, a bare function ARN (or "" to match any function — used +// internally by tests; the wire-facing handler always resolves a concrete +// filter first). ListDurableExecutionsByFunction filters by FunctionName +// alone; the separate Qualifier filter is versionFilter above. +func durableExecutionMatchesFunction(exFunctionARN, filterARN string) bool { + if filterARN == "" { + return true + } + + return exFunctionARN == filterARN || strings.HasPrefix(exFunctionARN, filterARN+":") +} + func matchesListFilter( ex *DurableExecution, - functionARN, nameFilter string, + functionARN, nameFilter, versionFilter string, statusSet map[DurableExecutionStatus]bool, startedAfter, startedBefore time.Time, ) bool { - if functionARN != "" && ex.FunctionARN != functionARN { + if !durableExecutionMatchesFunction(ex.FunctionARN, functionARN) { return false } @@ -765,6 +792,14 @@ func matchesListFilter( return false } + // Qualifier ("the function version to filter executions by"): absent means + // every version (verified against the ListDurableExecutionsByFunction API + // reference, not the aws-sdk-go-v2 Go doc comment, which incorrectly + // implies a $LATEST default). + if versionFilter != "" && ex.Version != versionFilter { + return false + } + if len(statusSet) > 0 && !statusSet[ex.Status] { return false } @@ -795,7 +830,7 @@ func (s *durableExecutionStore) checkpoint( ex, ok := s.executions[arn] if !ok { - ex = newDurableExecution(arn) + ex = newDurableExecution(arn, "") s.executions[arn] = ex } @@ -918,4 +953,132 @@ func (s *durableExecutionStore) reset() { s.executions = make(map[string]*DurableExecution) s.callbackOwner = make(map[string]string) + s.byName = make(map[string]string) +} + +// findReusableExecution looks up name in the by-name index (a no-op when +// name is "") and returns the SAME execution when its stored payload +// matches the new one, or ErrDurableExecutionAlreadyStarted when it doesn't. +// found is false for no name, an unknown name, or the defensive case of a +// name whose execution the store has since forgotten (byName and executions +// are always written together, so this never happens in practice). Callers +// must hold the store's write lock. +func (s *durableExecutionStore) findReusableExecution(name, payload string) (*DurableExecution, bool, error) { + if name == "" { + return nil, false, nil + } + + existingARN, ok := s.byName[name] + if !ok { + return nil, false, nil + } + + existing, ok := s.executions[existingARN] + if !ok { + return nil, false, nil + } + + if existing.InputPayload != payload { + return nil, false, ErrDurableExecutionAlreadyStarted + } + + return existing, true, nil +} + +// startOrReuseExecution implements Invoke's durable-execution start +// semantics per docs.aws.amazon.com/lambda/latest/dg/ +// durable-execution-idempotency.html's "Idempotency behavior" table: no name +// always starts a fresh execution; a name never seen before starts a fresh +// execution under that name; a name whose existing execution has an +// IDENTICAL InputPayload returns that SAME execution (reused=true — the +// caller must not invoke the function body again, matching "Lambda returns +// the existing execution instead of creating a duplicate"); a name whose +// existing execution has a DIFFERENT payload returns +// ErrDurableExecutionAlreadyStarted. Execution names are scoped per the +// store (this backend's account+region), matching "Execution names must be +// unique within your account and region.". +func (s *durableExecutionStore) startOrReuseExecution( + invokedFunctionARN, functionARN, version, name string, durableConfig *DurableConfig, inputPayload []byte, +) (*DurableExecution, bool, error) { + s.mu.Lock("StartOrReuseExecution") + defer s.mu.Unlock() + + payload := string(inputPayload) + + existing, found, err := s.findReusableExecution(name, payload) + if err != nil { + return nil, false, err + } + + if found { + return existing, true, nil + } + + newARN := newDurableExecutionARN(invokedFunctionARN) + ex := newDurableExecution(newARN, payload) + ex.FunctionARN = functionARN + ex.Version = version + ex.DurableConfig = durableConfig + + if name != "" { + ex.Name = name + s.byName[name] = newARN + } + + s.executions[newARN] = ex + + return ex, false, nil +} + +// completeExecution records a synchronous Invoke's real outcome as the +// execution's completion. This is not a fabricated replay result: it is the +// verbatim outcome of the one invocation this backend actually performed, +// matching the documented "the durable execution completes" behavior when a +// durable function's invocation "returns a final result or throws an +// unhandled error." Only transitions an execution still RUNNING — a no-op +// for an unknown ARN or an already-closed execution (an idempotent-replay +// reuse never invokes the function again, so never reaches this call). +func (s *durableExecutionStore) completeExecution(arn string, succeeded bool, result string) { + s.mu.Lock("CompleteExecution") + defer s.mu.Unlock() + + ex, ok := s.executions[arn] + if !ok || ex.Status != DurableExecutionStatusRunning { + return + } + + now := time.Now().UTC() + ex.EndTime = now + + if succeeded { + ex.Status = DurableExecutionStatusSucceeded + ex.Result = result + ex.appendEvent(DurableExecutionEvent{ + EventType: eventTypeExecutionSucceeded, + ID: ptrconv.NilIfEmpty(durableExecutionRootOperationID), + ExecutionSucceededDetails: &ExecutionSucceededDetails{ + Result: &EventResult{Payload: ptrconv.NilIfEmpty(result)}, + }, + }) + } else { + ex.Error = &ErrorObject{ErrorMessage: ptrconv.NilIfEmpty(result)} + ex.Status = DurableExecutionStatusFailed + ex.appendEvent(DurableExecutionEvent{ + EventType: eventTypeExecutionFailed, + ID: ptrconv.NilIfEmpty(durableExecutionRootOperationID), + ExecutionFailedDetails: &ExecutionFailedDetails{ + Error: &EventError{Payload: ex.Error}, + }, + }) + } + + if idx, found := ex.opIndex[durableExecutionRootOperationID]; found { + st := DurableOperationStatusSucceeded + if !succeeded { + st = DurableOperationStatusFailed + } + + ex.Operations[idx].Status = st + ex.Operations[idx].EndTimestamp = epochPtr(now) + } } diff --git a/services/lambda/durable_invoke_test.go b/services/lambda/durable_invoke_test.go new file mode 100644 index 0000000000..f1410820ac --- /dev/null +++ b/services/lambda/durable_invoke_test.go @@ -0,0 +1,326 @@ +package lambda_test + +import ( + "strings" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + lambdasdk "github.com/aws/aws-sdk-go-v2/service/lambda" + "github.com/aws/aws-sdk-go-v2/service/lambda/types" + smithy "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestRealClient_DurableInvoke drives Invoke's durable-execution wiring +// (PARITY.md durable_execution items_still_open, closed this pass): the +// X-Amz-Durable-Execution-Name request header / X-Amz-Durable-Execution-Arn +// response header, and the DurableExecution.FunctionARN/Version assignment +// that ListDurableExecutionsByFunction's FunctionName/Qualifier filters +// depend on. This backend has no Docker runtime configured +// (newInMemoryHandler), so a real (non-DryRun) invocation always fails with +// ServiceException -- exactly like every other lambda unit test that +// exercises Invoke without a mocked container -- but a durable execution is +// recorded before that failure, since real AWS starts the execution first +// and only then invokes the function body. +func TestRealClient_DurableInvoke(t *testing.T) { + t.Parallel() + + createDurableFn := func(t *testing.T, client *lambdasdk.Client, name string) { + t.Helper() + + _, err := client.CreateFunction(t.Context(), &lambdasdk.CreateFunctionInput{ + FunctionName: aws.String(name), + PackageType: types.PackageTypeImage, + Code: &types.FunctionCode{ImageUri: aws.String("ecr/myapp:latest")}, + Role: aws.String("arn:aws:iam:::role/r"), + DurableConfig: &types.DurableConfig{ExecutionTimeout: aws.Int32(3600)}, + }) + require.NoError(t, err) + } + + cases := []struct { + run func(t *testing.T) + name string + }{ + { + name: "invoke assigns function arn and version", + run: func(t *testing.T) { + t.Helper() + + h, _ := newInMemoryHandler(t) + client := newTestLambdaClient(t, h) + createDurableFn(t, client, "durinv-basic-fn") + + // DurableExecutionName pins retries to the SAME execution: without + // it, each of the real SDK client's automatic retries of the + // underlying ServiceException would start its own execution (real + // AWS behavior too -- "no name" always starts a new execution, per + // the idempotency table), making the assertion below flaky. + _, err := client.Invoke(t.Context(), &lambdasdk.InvokeInput{ + FunctionName: aws.String("durinv-basic-fn"), + Qualifier: aws.String("$LATEST"), + DurableExecutionName: aws.String("basic-exec"), + Payload: []byte(`{"x":1}`), + }) + require.Error(t, err) // no Docker runtime configured + + listOut, err := client.ListDurableExecutionsByFunction( + t.Context(), + &lambdasdk.ListDurableExecutionsByFunctionInput{FunctionName: aws.String("durinv-basic-fn")}, + ) + require.NoError(t, err) + require.Len(t, listOut.DurableExecutions, 1) + + ex := listOut.DurableExecutions[0] + assert.Equal(t, + "arn:aws:lambda:us-east-1:000000000000:function:durinv-basic-fn:$LATEST", + aws.ToString(ex.FunctionArn), + ) + assert.Contains(t, aws.ToString(ex.DurableExecutionArn), "/durable-execution/") + assert.Equal(t, types.ExecutionStatusRunning, ex.Status) + }, + }, + { + name: "idempotent replay with identical payload does not re-invoke", + run: func(t *testing.T) { + t.Helper() + + h, _ := newInMemoryHandler(t) + client := newTestLambdaClient(t, h) + createDurableFn(t, client, "durinv-idem-fn") + + payload := []byte(`{"orderId":"123"}`) + + _, err := client.Invoke(t.Context(), &lambdasdk.InvokeInput{ + FunctionName: aws.String("durinv-idem-fn"), + Qualifier: aws.String("$LATEST"), + DurableExecutionName: aws.String("idem-exec"), + Payload: payload, + }) + require.Error(t, err) + + listOut, err := client.ListDurableExecutionsByFunction( + t.Context(), + &lambdasdk.ListDurableExecutionsByFunctionInput{FunctionName: aws.String("durinv-idem-fn")}, + ) + require.NoError(t, err) + require.Len(t, listOut.DurableExecutions, 1) + arn := aws.ToString(listOut.DurableExecutions[0].DurableExecutionArn) + + // Close the execution out-of-band (this backend has no Docker + // runtime to complete it for real) so the replay below hits a + // CLOSED execution, per the documented idempotency table. + _, err = client.StopDurableExecution(t.Context(), &lambdasdk.StopDurableExecutionInput{ + DurableExecutionArn: aws.String(arn), + }) + require.NoError(t, err) + + // Same name + identical payload against a CLOSED execution: real + // AWS returns the closed execution's result instead of starting a + // duplicate. This succeeds even with no Docker runtime configured, + // proving the function was NOT invoked again. + out, err := client.Invoke(t.Context(), &lambdasdk.InvokeInput{ + FunctionName: aws.String("durinv-idem-fn"), + Qualifier: aws.String("$LATEST"), + DurableExecutionName: aws.String("idem-exec"), + Payload: payload, + }) + require.NoError(t, err) + assert.Equal(t, "Unhandled", aws.ToString(out.FunctionError)) + assert.Equal(t, arn, aws.ToString(out.DurableExecutionArn)) + + listOut2, err := client.ListDurableExecutionsByFunction( + t.Context(), + &lambdasdk.ListDurableExecutionsByFunctionInput{FunctionName: aws.String("durinv-idem-fn")}, + ) + require.NoError(t, err) + assert.Len(t, listOut2.DurableExecutions, 1, "reuse must not create a duplicate execution") + }, + }, + { + name: "differing payload with same name conflicts", + run: func(t *testing.T) { + t.Helper() + + h, _ := newInMemoryHandler(t) + client := newTestLambdaClient(t, h) + createDurableFn(t, client, "durinv-conflict-fn") + + _, err := client.Invoke(t.Context(), &lambdasdk.InvokeInput{ + FunctionName: aws.String("durinv-conflict-fn"), + Qualifier: aws.String("$LATEST"), + DurableExecutionName: aws.String("conflict-exec"), + Payload: []byte(`{"a":1}`), + }) + require.Error(t, err) // no Docker runtime configured + + _, err = client.Invoke(t.Context(), &lambdasdk.InvokeInput{ + FunctionName: aws.String("durinv-conflict-fn"), + Qualifier: aws.String("$LATEST"), + DurableExecutionName: aws.String("conflict-exec"), + Payload: []byte(`{"a":2}`), + }) + require.Error(t, err) + + var apiErr *types.DurableExecutionAlreadyStartedException + require.ErrorAs(t, err, &apiErr, "expected DurableExecutionAlreadyStartedException, got %v", err) + }, + }, + { + name: "dry run does not start an execution", + run: func(t *testing.T) { + t.Helper() + + h, _ := newInMemoryHandler(t) + client := newTestLambdaClient(t, h) + createDurableFn(t, client, "durinv-dryrun-fn") + + _, err := client.Invoke(t.Context(), &lambdasdk.InvokeInput{ + FunctionName: aws.String("durinv-dryrun-fn"), + Qualifier: aws.String("$LATEST"), + InvocationType: types.InvocationTypeDryRun, + Payload: []byte(`{}`), + }) + require.NoError(t, err) + + listOut, err := client.ListDurableExecutionsByFunction( + t.Context(), + &lambdasdk.ListDurableExecutionsByFunctionInput{FunctionName: aws.String("durinv-dryrun-fn")}, + ) + require.NoError(t, err) + assert.Empty(t, listOut.DurableExecutions) + }, + }, + { + name: "qualifier filters by resolved version", + run: func(t *testing.T) { + t.Helper() + + h, _ := newInMemoryHandler(t) + client := newTestLambdaClient(t, h) + createDurableFn(t, client, "durinv-qual-fn") + + // DurableExecutionName pins each real-client retry to the same + // execution (see the "invoke assigns function arn and version" + // case's comment) so the counts asserted below are stable. + _, err := client.Invoke(t.Context(), &lambdasdk.InvokeInput{ + FunctionName: aws.String("durinv-qual-fn"), + Qualifier: aws.String("$LATEST"), + DurableExecutionName: aws.String("qual-latest-exec"), + Payload: []byte(`{}`), + }) + require.Error(t, err) + + pubOut, err := client.PublishVersion(t.Context(), &lambdasdk.PublishVersionInput{ + FunctionName: aws.String("durinv-qual-fn"), + }) + require.NoError(t, err) + + _, err = client.Invoke(t.Context(), &lambdasdk.InvokeInput{ + FunctionName: aws.String("durinv-qual-fn"), + Qualifier: pubOut.Version, + DurableExecutionName: aws.String("qual-v1-exec"), + Payload: []byte(`{}`), + }) + require.Error(t, err) + + allOut, err := client.ListDurableExecutionsByFunction( + t.Context(), + &lambdasdk.ListDurableExecutionsByFunctionInput{FunctionName: aws.String("durinv-qual-fn")}, + ) + require.NoError(t, err) + require.Len(t, allOut.DurableExecutions, 2, "no Qualifier: executions across every version") + + latestOut, err := client.ListDurableExecutionsByFunction( + t.Context(), + &lambdasdk.ListDurableExecutionsByFunctionInput{ + FunctionName: aws.String("durinv-qual-fn"), + Qualifier: aws.String("$LATEST"), + }, + ) + require.NoError(t, err) + require.Len(t, latestOut.DurableExecutions, 1) + assert.True(t, strings.HasSuffix(aws.ToString(latestOut.DurableExecutions[0].FunctionArn), ":$LATEST")) + + versionOut, err := client.ListDurableExecutionsByFunction( + t.Context(), + &lambdasdk.ListDurableExecutionsByFunctionInput{ + FunctionName: aws.String("durinv-qual-fn"), + Qualifier: pubOut.Version, + }, + ) + require.NoError(t, err) + require.Len(t, versionOut.DurableExecutions, 1) + assert.True(t, strings.HasSuffix( + aws.ToString(versionOut.DurableExecutions[0].FunctionArn), ":"+aws.ToString(pubOut.Version), + )) + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + tc.run(t) + }) + } +} + +// TestInvoke_DurableFunctionRequiresQualifier guards the qualified-ARN requirement. +// docs.aws.amazon.com/lambda/latest/dg/durable-invoking.html#durable-invoking-qualified-arns. +func TestInvoke_DurableFunctionRequiresQualifier(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + qualifier string + durable bool + wantRejected bool + }{ + {name: "durable function, no qualifier is rejected", durable: true, wantRejected: true}, + {name: "durable function, explicit $LATEST is accepted", durable: true, qualifier: "$LATEST"}, + {name: "standard function, no qualifier is accepted"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h, _ := newInMemoryHandler(t) + client := newTestLambdaClient(t, h) + + fnName := "qual-req-" + strings.ReplaceAll(tt.name, " ", "-") + + createInput := &lambdasdk.CreateFunctionInput{ + FunctionName: aws.String(fnName), + PackageType: types.PackageTypeImage, + Code: &types.FunctionCode{ImageUri: aws.String("ecr/myapp:latest")}, + Role: aws.String("arn:aws:iam:::role/r"), + } + if tt.durable { + createInput.DurableConfig = &types.DurableConfig{ExecutionTimeout: aws.Int32(3600)} + } + + _, err := client.CreateFunction(t.Context(), createInput) + require.NoError(t, err) + + invokeInput := &lambdasdk.InvokeInput{FunctionName: aws.String(fnName), Payload: []byte(`{}`)} + if tt.qualifier != "" { + invokeInput.Qualifier = aws.String(tt.qualifier) + } + + _, err = client.Invoke(t.Context(), invokeInput) + require.Error(t, err) // no Docker runtime configured either way + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + + if tt.wantRejected { + assert.Equal(t, "InvalidParameterValueException", apiErr.ErrorCode()) + } else { + assert.NotEqual(t, "InvalidParameterValueException", apiErr.ErrorCode()) + } + }) + } +} diff --git a/services/lambda/errors.go b/services/lambda/errors.go index 88fdfb1cc0..3564d0547a 100644 --- a/services/lambda/errors.go +++ b/services/lambda/errors.go @@ -70,6 +70,11 @@ var ErrDurableExecutionNotFound = errors.New("ResourceNotFoundException") // ErrCallbackNotFound is returned when the specified durable execution callback ID does not exist. var ErrCallbackNotFound = errors.New("ResourceNotFoundException") +// ErrDurableExecutionAlreadyStarted is returned when Invoke's +// DurableExecutionName reuses an existing execution's name with a payload +// that doesn't match the original invocation. +var ErrDurableExecutionAlreadyStarted = errors.New("DurableExecutionAlreadyStartedException") + // ErrVersionReferencedByAlias is returned when DeleteFunction's Qualifier // targets a published version that an alias still points to (real AWS: // "You can't delete a version that an alias references."). diff --git a/services/lambda/event_source_mapping.go b/services/lambda/event_source_mapping.go index 42b1fdf346..0f53932079 100644 --- a/services/lambda/event_source_mapping.go +++ b/services/lambda/event_source_mapping.go @@ -319,7 +319,15 @@ func (b *InMemoryBackend) CreateEventSourceMapping( b.kinesisPoller.Notify() } - return m, nil + return cloneESM(m), nil +} + +// cloneESM stops a caller from racing UpdateEventSourceMapping or the +// janitor's sweepESMs, which mutate m's fields under the lock. +func cloneESM(m *EventSourceMapping) *EventSourceMapping { + cp := *m + + return &cp } // GetEventSourceMapping retrieves an event source mapping by UUID. @@ -332,7 +340,7 @@ func (b *InMemoryBackend) GetEventSourceMapping(uuid string) (*EventSourceMappin return nil, ErrESMNotFound } - return m, nil + return cloneESM(m), nil } // ListEventSourceMappings returns a page of event source mappings, optionally filtered by function name. @@ -356,11 +364,16 @@ func (b *InMemoryBackend) ListEventSourceMappings( result = make([]*EventSourceMapping, 0, len(ids)) for id := range ids { if m, ok := b.eventSourceMappings.Get(id); ok { - result = append(result, m) + result = append(result, cloneESM(m)) } } } else { - result = b.eventSourceMappings.All() + stored := b.eventSourceMappings.All() + result = make([]*EventSourceMapping, len(stored)) + + for i, m := range stored { + result[i] = cloneESM(m) + } } // Apply optional EventSourceArn filter. @@ -402,7 +415,7 @@ func (b *InMemoryBackend) DeleteEventSourceMapping(id string) (*EventSourceMappi b.kinesisPoller.RemoveMapping(id) } - return m, nil + return cloneESM(m), nil } // applyESMUpdate patches esm fields from input (non-zero / non-nil values only). @@ -495,7 +508,7 @@ func (b *InMemoryBackend) UpdateEventSourceMapping( input *UpdateEventSourceMappingInput, ) (*EventSourceMapping, error) { var ( - esm *EventSourceMapping + result *EventSourceMapping found bool nowEnabled bool poller *EventSourcePoller @@ -505,9 +518,7 @@ func (b *InMemoryBackend) UpdateEventSourceMapping( b.mu.Lock("UpdateEventSourceMapping") defer b.mu.Unlock() - var ok bool - - esm, ok = b.eventSourceMappings.Get(id) + esm, ok := b.eventSourceMappings.Get(id) if !ok { return } @@ -515,6 +526,7 @@ func (b *InMemoryBackend) UpdateEventSourceMapping( found = true nowEnabled = applyESMUpdate(esm, input) poller = b.kinesisPoller + result = cloneESM(esm) }() if !found { @@ -525,5 +537,5 @@ func (b *InMemoryBackend) UpdateEventSourceMapping( poller.Notify() } - return esm, nil + return result, nil } diff --git a/services/lambda/function_fields_test.go b/services/lambda/function_fields_test.go index 8d29cabb6c..2f769b5e7a 100644 --- a/services/lambda/function_fields_test.go +++ b/services/lambda/function_fields_test.go @@ -389,6 +389,177 @@ func TestDurableConfig_PublishedVersionCarriesConfig(t *testing.T) { assert.Equal(t, int32(1800), *ver.DurableConfig.ExecutionTimeout) } +// DurableConfig — documented range/runtime restrictions. +// docs.aws.amazon.com/lambda/latest/api/API_DurableConfig.html + +func TestCreateFunction_DurableConfigRangeValidation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + durableConfig string + wantStatusCode int + }{ + { + name: "ExecutionTimeout below minimum is rejected", + durableConfig: `{"ExecutionTimeout":0}`, + wantStatusCode: http.StatusBadRequest, + }, + { + name: "ExecutionTimeout above maximum is rejected", + durableConfig: `{"ExecutionTimeout":31622401}`, + wantStatusCode: http.StatusBadRequest, + }, + { + name: "ExecutionTimeout at the documented boundaries is accepted", + durableConfig: `{"ExecutionTimeout":1,"RetentionPeriodInDays":90}`, + wantStatusCode: http.StatusCreated, + }, + { + name: "RetentionPeriodInDays below minimum is rejected", + durableConfig: `{"RetentionPeriodInDays":0}`, + wantStatusCode: http.StatusBadRequest, + }, + { + name: "RetentionPeriodInDays above maximum is rejected", + durableConfig: `{"RetentionPeriodInDays":91}`, + wantStatusCode: http.StatusBadRequest, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h, _ := newInMemoryHandler(t) + body := fmt.Sprintf( + `{"FunctionName":%q,"PackageType":"Image","Code":{"ImageUri":"ecr/x:latest"},`+ + `"Role":"arn:aws:iam:::role/r","DurableConfig":%s}`, + "durcfg-range-fn", tt.durableConfig, + ) + + rec := auditCreateFunction(t, h, body) + assert.Equal(t, tt.wantStatusCode, rec.Code, rec.Body.String()) + + if tt.wantStatusCode == http.StatusBadRequest { + errBody := lambdaParseBody(t, rec) + assert.Equal(t, "InvalidParameterValueException", errBody["__type"]) + } + }) + } +} + +func TestUpdateFunctionConfiguration_DurableConfigRangeValidation(t *testing.T) { + t.Parallel() + + h, _ := newInMemoryHandler(t) + rec := auditCreateFunction(t, h, baseImageFn("durcfg-range-update-fn")) + require.Equal(t, http.StatusCreated, rec.Code) + + rec2 := auditUpdateConfig(t, h, "durcfg-range-update-fn", `{"DurableConfig":{"RetentionPeriodInDays":91}}`) + assert.Equal(t, http.StatusBadRequest, rec2.Code) + + errBody := lambdaParseBody(t, rec2) + assert.Equal(t, "InvalidParameterValueException", errBody["__type"]) +} + +func TestCreateFunction_DurableRuntimeRestriction(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + fnName string + packageType string + runtime string + durableConfig string + wantStatusCode int + }{ + { + name: "Zip runtime unsupported for durable functions is rejected", + fnName: "durrt-unsupported", + packageType: "Zip", + runtime: "python3.9", + durableConfig: `{"ExecutionTimeout":3600}`, + wantStatusCode: http.StatusBadRequest, + }, + { + name: "Zip runtime supported for durable functions is accepted", + fnName: "durrt-supported", + packageType: "Zip", + runtime: "python3.13", + durableConfig: `{"ExecutionTimeout":3600}`, + wantStatusCode: http.StatusCreated, + }, + { + name: "Zip unsupported runtime without DurableConfig is unaffected", + fnName: "durrt-no-durable", + packageType: "Zip", + runtime: "python3.9", + wantStatusCode: http.StatusCreated, + }, + { + name: "Image package type has no runtime restriction", + fnName: "durrt-image", + packageType: "Image", + durableConfig: `{"ExecutionTimeout":3600}`, + wantStatusCode: http.StatusCreated, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h, _ := newInMemoryHandler(t) + + var codeField, durableField string + + if tt.packageType == "Zip" { + codeField = `"Code":{"ZipFile":"UEsDBA=="},"Handler":"index.handler","Runtime":"` + tt.runtime + `",` + } else { + codeField = `"Code":{"ImageUri":"ecr/x:latest"},` + } + + if tt.durableConfig != "" { + durableField = `"DurableConfig":` + tt.durableConfig + `,` + } + + body := fmt.Sprintf( + `{"FunctionName":%q,"PackageType":%q,%s%s"Role":"arn:aws:iam:::role/r"}`, + tt.fnName, tt.packageType, codeField, durableField, + ) + + rec := auditCreateFunction(t, h, body) + assert.Equal(t, tt.wantStatusCode, rec.Code, rec.Body.String()) + + if tt.wantStatusCode == http.StatusBadRequest { + errBody := lambdaParseBody(t, rec) + assert.Equal(t, "InvalidParameterValueException", errBody["__type"]) + } + }) + } +} + +func TestUpdateFunctionConfiguration_DurableRuntimeRestriction(t *testing.T) { + t.Parallel() + + h, _ := newInMemoryHandler(t) + + body := `{"FunctionName":"durrt-update-fn","PackageType":"Zip","Runtime":"python3.13",` + + `"Handler":"index.handler","Code":{"ZipFile":"UEsDBA=="},"Role":"arn:aws:iam:::role/r",` + + `"DurableConfig":{"ExecutionTimeout":3600}}` + rec := auditCreateFunction(t, h, body) + require.Equal(t, http.StatusCreated, rec.Code, rec.Body.String()) + + // Changing Runtime to one that doesn't support durable functions must be + // rejected using the function's EXISTING DurableConfig (not repeated here). + rec2 := auditUpdateConfig(t, h, "durrt-update-fn", `{"Runtime":"python3.9"}`) + assert.Equal(t, http.StatusBadRequest, rec2.Code, rec2.Body.String()) + + errBody := lambdaParseBody(t, rec2) + assert.Equal(t, "InvalidParameterValueException", errBody["__type"]) +} + // ============================================================ // RevisionId optimistic concurrency on UpdateFunctionConfiguration / // UpdateFunctionCode (PARITY.md deferred item, extended from AddPermission's diff --git a/services/lambda/function_urls.go b/services/lambda/function_urls.go index 566e128e07..324fead8f9 100644 --- a/services/lambda/function_urls.go +++ b/services/lambda/function_urls.go @@ -114,7 +114,7 @@ func (b *InMemoryBackend) CreateFunctionURLConfig( b.functionURLConfigs.Put(cfg) - return cfg, nil + return cloneFunctionURLConfig(cfg), nil } // allocateAndStartURLServerUnlocked allocates a port and starts the HTTP listener @@ -176,6 +176,14 @@ func (b *InMemoryBackend) doAllocateAndStart( return "http://" + net.JoinHostPort("127.0.0.1", strconv.Itoa(port)) + "/", srv, nil } +// cloneFunctionURLConfig copies cfg so a caller can't race UpdateFunctionURLConfig, +// which mutates the stored config's fields in place. +func cloneFunctionURLConfig(cfg *FunctionURLConfig) *FunctionURLConfig { + cp := *cfg + + return &cp +} + // GetFunctionURLConfig returns the function URL config for a function. func (b *InMemoryBackend) GetFunctionURLConfig(functionName string) (*FunctionURLConfig, error) { b.mu.RLock("GetFunctionURLConfig") @@ -186,7 +194,7 @@ func (b *InMemoryBackend) GetFunctionURLConfig(functionName string) (*FunctionUR return nil, ErrFunctionURLNotFound } - return cfg, nil + return cloneFunctionURLConfig(cfg), nil } // DeleteFunctionURLConfig removes the function URL config, stops the listener, and deregisters DNS. @@ -652,7 +660,7 @@ func (b *InMemoryBackend) UpdateFunctionURLConfig( cfg.LastModifiedTime = time.Now().UTC().Format(time.RFC3339) b.functionURLConfigs.Put(cfg) - return cfg, nil + return cloneFunctionURLConfig(cfg), nil } // ListFunctionURLConfigs returns all function URL configs. @@ -662,9 +670,14 @@ func (b *InMemoryBackend) ListFunctionURLConfigs() []*FunctionURLConfig { cfgs := b.functionURLConfigs.All() - sort.Slice(cfgs, func(i, j int) bool { - return cfgs[i].FunctionArn < cfgs[j].FunctionArn + out := make([]*FunctionURLConfig, len(cfgs)) + for i, cfg := range cfgs { + out[i] = cloneFunctionURLConfig(cfg) + } + + sort.Slice(out, func(i, j int) bool { + return out[i].FunctionArn < out[j].FunctionArn }) - return cfgs + return out } diff --git a/services/lambda/functions.go b/services/lambda/functions.go index 0202e0f964..ada3c5921e 100644 --- a/services/lambda/functions.go +++ b/services/lambda/functions.go @@ -159,7 +159,15 @@ func (b *InMemoryBackend) GetFunction(name string) (*FunctionConfiguration, erro return nil, ErrFunctionNotFound } - return fn, nil + return cloneFunctionConfig(fn), nil +} + +// cloneFunctionConfig stops a caller from racing scheduleFunctionActive, +// TagResource or UntagResource, which mutate fn's fields under the lock. +func cloneFunctionConfig(fn *FunctionConfiguration) *FunctionConfiguration { + cp := *fn + + return &cp } // GetFunctionByQualifier returns the configuration for a specific qualifier @@ -238,7 +246,12 @@ func (b *InMemoryBackend) ListFunctions( b.mu.RLock("ListFunctions") defer b.mu.RUnlock() - fns := b.functions.All() + stored := b.functions.All() + fns := make([]*FunctionConfiguration, len(stored)) + + for i, fn := range stored { + fns[i] = cloneFunctionConfig(fn) + } sort.Slice(fns, func(i, j int) bool { return fns[i].FunctionName < fns[j].FunctionName @@ -258,7 +271,12 @@ func (b *InMemoryBackend) ListFunctionsAll( defer b.mu.RUnlock() // Include $LATEST for each function. - fns := b.functions.All() + stored := b.functions.All() + fns := make([]*FunctionConfiguration, 0, len(stored)) + + for _, fn := range stored { + fns = append(fns, cloneFunctionConfig(fn)) + } // Include all published versions. for name, vMap := range b.versionIndex { diff --git a/services/lambda/functions_race_test.go b/services/lambda/functions_race_test.go new file mode 100644 index 0000000000..a9b631f9db --- /dev/null +++ b/services/lambda/functions_race_test.go @@ -0,0 +1,318 @@ +package lambda_test + +import ( + "strconv" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/lambda" +) + +// TestGetFunctionConcurrentWithTagResource proves GetFunction/ListFunctions +// must not hand back the live pointer TagResource/UntagResource mutate. +func TestGetFunctionConcurrentWithTagResource(t *testing.T) { + t.Parallel() + + tests := []struct { + reader func(bk *lambda.InMemoryBackend, fnName string) + name string + }{ + { + name: "GetFunction races tag writer", + reader: func(bk *lambda.InMemoryBackend, fnName string) { + fn, err := bk.GetFunction(fnName) + if err != nil { + return + } + + _ = fn.Description + _ = len(fn.Tags) + }, + }, + { + name: "ListFunctions races tag writer", + reader: func(bk *lambda.InMemoryBackend, _ string) { + p := bk.ListFunctions("", 0) + for _, fn := range p.Data { + _ = fn.Description + _ = len(fn.Tags) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h, bk := newInMemoryHandler(t) + fnName := "race-fn" + createFunctionForTest(t, h, fnName) + + const iterations = 300 + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + for range iterations { + tt.reader(bk, fnName) + } + }() + + go func() { + defer wg.Done() + + for i := range iterations { + if i%2 == 0 { + _ = bk.TagResource(fnName, map[string]string{"k": "v"}) + } else { + _ = bk.UntagResource(fnName, []string{"k"}) + } + } + }() + + wg.Wait() + }) + } +} + +// TestEventSourceMappingConcurrentWithUpdate proves Get/ListEventSourceMappings +// must not hand back the live pointer UpdateEventSourceMapping and sweepESMs mutate. +func TestEventSourceMappingConcurrentWithUpdate(t *testing.T) { + t.Parallel() + + tests := []struct { + reader func(bk *lambda.InMemoryBackend, id string) + name string + }{ + { + name: "GetEventSourceMapping races update", + reader: func(bk *lambda.InMemoryBackend, id string) { + m, err := bk.GetEventSourceMapping(id) + if err != nil { + return + } + + _ = m.State + _ = m.BatchSize + _ = m.LastProcessingResult + }, + }, + { + name: "ListEventSourceMappings races update", + reader: func(bk *lambda.InMemoryBackend, _ string) { + p := bk.ListEventSourceMappings("", "", "", 0) + for _, m := range p.Data { + _ = m.State + _ = m.BatchSize + _ = m.LastProcessingResult + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + _, bk := newInMemoryHandler(t) + + require.NoError(t, bk.CreateFunction(&lambda.FunctionConfiguration{FunctionName: "esm-race-fn"})) + + created, err := bk.CreateEventSourceMapping(&lambda.CreateEventSourceMappingInput{ + EventSourceARN: "arn:aws:kinesis:us-east-1:000000000000:stream/race-stream", + FunctionName: "esm-race-fn", + Enabled: true, + }) + require.NoError(t, err) + + const iterations = 300 + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + for range iterations { + tt.reader(bk, created.UUID) + } + }() + + go func() { + defer wg.Done() + + for i := range iterations { + enabled := i%2 == 0 + batchSize := int32(10 + i%50) + + _, _ = bk.UpdateEventSourceMapping(created.UUID, &lambda.UpdateEventSourceMappingInput{ + Enabled: &enabled, + BatchSize: &batchSize, + }) + } + }() + + wg.Wait() + }) + } +} + +// TestCodeSigningConfigConcurrentWithUpdate proves Get/ListCodeSigningConfigs +// must not hand back the live pointer UpdateCodeSigningConfig mutates. +func TestCodeSigningConfigConcurrentWithUpdate(t *testing.T) { + t.Parallel() + + tests := []struct { + reader func(bk *lambda.InMemoryBackend, arn string) + name string + }{ + { + name: "GetCodeSigningConfig races update", + reader: func(bk *lambda.InMemoryBackend, arn string) { + cfg, err := bk.GetCodeSigningConfig(arn) + if err != nil { + return + } + + _ = cfg.Description + }, + }, + { + name: "ListCodeSigningConfigs races update", + reader: func(bk *lambda.InMemoryBackend, _ string) { + p := bk.ListCodeSigningConfigs("", 0) + for _, cfg := range p.Data { + _ = cfg.Description + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + _, bk := newInMemoryHandler(t) + + created, err := bk.CreateCodeSigningConfig(&lambda.CreateCodeSigningConfigInput{}) + require.NoError(t, err) + + const iterations = 300 + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + for range iterations { + tt.reader(bk, created.CodeSigningConfigArn) + } + }() + + go func() { + defer wg.Done() + + for i := range iterations { + desc := "desc-" + strconv.Itoa(i) + + _, _ = bk.UpdateCodeSigningConfig( + created.CodeSigningConfigArn, + &lambda.UpdateCodeSigningConfigInput{ + Description: &desc, + }, + ) + } + }() + + wg.Wait() + }) + } +} + +// TestAliasConcurrentWithUpdate proves GetAlias/ListAliases must not hand +// back the live pointer UpdateAlias mutates. +func TestAliasConcurrentWithUpdate(t *testing.T) { + t.Parallel() + + tests := []struct { + reader func(bk *lambda.InMemoryBackend, fnName, aliasName string) + name string + }{ + { + name: "GetAlias races update", + reader: func(bk *lambda.InMemoryBackend, fnName, aliasName string) { + alias, err := bk.GetAlias(fnName, aliasName) + if err != nil { + return + } + + _ = alias.Description + _ = alias.RevisionID + }, + }, + { + name: "ListAliases races update", + reader: func(bk *lambda.InMemoryBackend, fnName, _ string) { + p, err := bk.ListAliases(fnName, "", "", 0) + if err != nil { + return + } + + for _, alias := range p.Data { + _ = alias.Description + _ = alias.RevisionID + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h, bk := newInMemoryHandler(t) + fnName := "alias-race-fn" + createFunctionForTest(t, h, fnName) + + _, err := bk.CreateAlias(fnName, &lambda.CreateAliasInput{ + Name: "race-alias", + FunctionVersion: "$LATEST", + }) + require.NoError(t, err) + + const iterations = 300 + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + for range iterations { + tt.reader(bk, fnName, "race-alias") + } + }() + + go func() { + defer wg.Done() + + for i := range iterations { + desc := "desc-" + strconv.Itoa(i) + + _, _ = bk.UpdateAlias(fnName, "race-alias", &lambda.UpdateAliasInput{ + Description: &desc, + }) + } + }() + + wg.Wait() + }) + } +} diff --git a/services/lambda/handler_durable_execution.go b/services/lambda/handler_durable_execution.go index 05da5f470f..2c43335034 100644 --- a/services/lambda/handler_durable_execution.go +++ b/services/lambda/handler_durable_execution.go @@ -95,10 +95,37 @@ func extractDurableExecPathID(path, prefix string) string { return decoded } +// durableExecARNPathSuffixes are the known trailing segments appended after +// {DurableExecutionArn} under lambdaDurableExecPathPrefix. +// +//nolint:gochecknoglobals // static route-suffix table, mirrors lambdaOpRoutes +var durableExecARNPathSuffixes = []string{"/checkpoint", "/stop", "/history", "/state"} + // extractDurableExecARN extracts the DurableExecutionArn from a -// /2025-12-01/durable-executions/{encodedARN}[/...] path. +// /2025-12-01/durable-executions/{encodedARN}[/...] path. Unlike +// extractDurableExecPathID (used for CallbackId, a simple opaque token), this +// cannot split on the first "/": a real DurableExecutionArn legitimately +// contains "/" itself (AWS's own shape is +// "...:function:name:$LATEST/durable-execution//" — verified +// against a real EventBridge "Durable Execution Status Change" event +// sample), so only a known trailing suffix may be stripped. func extractDurableExecARN(path string) string { - return extractDurableExecPathID(path, lambdaDurableExecPathPrefix) + rest := strings.TrimPrefix(path, lambdaDurableExecPathPrefix+"/") + + for _, suffix := range durableExecARNPathSuffixes { + if trimmed, ok := strings.CutSuffix(rest, suffix); ok { + rest = trimmed + + break + } + } + + decoded, err := url.PathUnescape(rest) + if err != nil { + return rest + } + + return decoded } // extractDurableExecCallbackID extracts the CallbackId from a @@ -168,6 +195,100 @@ func durableExecFromBackend(h *Handler) *durableExecutionStore { return bk.durableExecs } +// resolveDurableFunction resolves name/qualifier to a durable function's +// config, or ok=false when the qualifier doesn't resolve or the resolved +// function isn't durable. A resolution failure isn't itself surfaced here: +// the real invoke path resolves the identical qualifier right after and +// produces the correct error response for an unknown qualifier on its own. +func resolveDurableFunction(bk *InMemoryBackend, name, qualifier string) (*FunctionConfiguration, bool) { + resolved, err := bk.resolveQualifier(name, qualifier) + if err != nil { + return nil, false + } + + return resolved, resolved.DurableConfig != nil +} + +// startDurableInvokeExecution is Invoke's half of the durable-execution +// family (PARITY.md durable_execution items_still_open): when name resolves +// to a function with DurableConfig set, it starts or reuses a +// DurableExecution per the documented idempotency table and returns its +// ARN. Returns ("", nil, nil) for a non-durable function or a DryRun +// invocation (DryRun never executes, so it never starts an execution). A +// non-nil reused return means the caller must not invoke the function body +// again (an idempotent-replay hit); a non-nil error is +// ErrDurableExecutionAlreadyStarted (name reused with a different payload). +func (h *Handler) startDurableInvokeExecution( + name, qualifier, invType, execName string, body []byte, +) (string, *DurableExecution, error) { + if invType == InvocationTypeDryRun { + return "", nil, nil + } + + bk, ok := h.Backend.(*InMemoryBackend) + if !ok || bk.durableExecs == nil { + return "", nil, nil + } + + resolved, isDurable := resolveDurableFunction(bk, name, qualifier) + if !isDurable { + return "", nil, nil + } + + invokedQualifier := qualifier + if invokedQualifier == "" { + invokedQualifier = versionLatest + } + + invokedARN := buildARN(h.DefaultRegion, h.AccountID, name) + ":" + invokedQualifier + functionARN := buildARN(h.DefaultRegion, h.AccountID, name) + ":" + resolved.Version + + ex, isReuse, startErr := bk.durableExecs.startOrReuseExecution( + invokedARN, functionARN, resolved.Version, execName, resolved.DurableConfig, body, + ) + if startErr != nil { + return "", nil, startErr + } + + if isReuse { + return ex.ARN, ex, nil + } + + return ex.ARN, nil, nil +} + +// completeDurableInvokeExecution records a freshly-completed synchronous +// invocation's real result against the durable execution arn identifies. +func (h *Handler) completeDurableInvokeExecution(arn string, succeeded bool, result []byte) { + if store := durableExecFromBackend(h); store != nil { + store.completeExecution(arn, succeeded, string(result)) + } +} + +// replayClosedDurableExecution builds the Invoke response for an +// idempotent-replay hit against an already-closed DurableExecution (real +// AWS: "the closed execution result is returned" — no re-invocation). +// InvocationType=Event has no response body regardless of the execution's +// outcome, matching a fresh async accept. +func replayClosedDurableExecution(ex *DurableExecution, invType string) ([]byte, string, int) { + if invType == InvocationTypeEvent { + return nil, "", http.StatusAccepted + } + + if ex.Status == DurableExecutionStatusSucceeded { + return []byte(ex.Result), "", http.StatusOK + } + + msg := "" + if ex.Error != nil { + msg = ptrconv.String(ex.Error.ErrorMessage) + } + + payload, _ := json.Marshal(map[string]string{"errorMessage": msg}) + + return payload, "Unhandled", http.StatusOK +} + // handleCheckpointDurableExecution handles POST /2025-12-01/durable-executions/{arn}/checkpoint. func (h *Handler) handleCheckpointDurableExecution(c *echo.Context, path string) error { store := durableExecFromBackend(h) @@ -268,8 +389,20 @@ func (h *Handler) handleListDurableExecutionsByFunction(c *echo.Context, functio } functionARN := buildARN(h.DefaultRegion, h.AccountID, functionName) + + versionFilter := "" + if qualifier := q.Get("Qualifier"); qualifier != "" { + versionFilter = qualifier + + if bk, ok := h.Backend.(*InMemoryBackend); ok { + if resolved, rErr := bk.resolveQualifier(functionName, qualifier); rErr == nil { + versionFilter = resolved.Version + } + } + } + summaries := store.listSummaries( - functionARN, q.Get("DurableExecutionName"), statuses, + functionARN, q.Get("DurableExecutionName"), versionFilter, statuses, startedAfter, startedBefore, q.Get("ReverseOrder") == "true", ) diff --git a/services/lambda/handler_functions.go b/services/lambda/handler_functions.go index 13a1a7bca9..6bf31c34a7 100644 --- a/services/lambda/handler_functions.go +++ b/services/lambda/handler_functions.go @@ -93,7 +93,11 @@ func (h *Handler) validateCreateFunctionInput(c *echo.Context, input *CreateFunc return false } - return h.validateEphemeralStorageInput(c, input.EphemeralStorage) + if !h.validateEphemeralStorageInput(c, input.EphemeralStorage) { + return false + } + + return h.validateDurableConfigInput(c, input.DurableConfig) } // validateSnapStartInput checks the optional SnapStart.ApplyOn value. AWS only @@ -132,6 +136,34 @@ func (h *Handler) validateEphemeralStorageInput(c *echo.Context, es *EphemeralSt return true } +// validateDurableConfigInput checks ExecutionTimeout/RetentionPeriodInDays ranges. +// docs.aws.amazon.com/lambda/latest/api/API_DurableConfig.html. +func (h *Handler) validateDurableConfigInput(c *echo.Context, dc *DurableConfig) bool { + if dc == nil { + return true + } + + if dc.ExecutionTimeout != nil && + (*dc.ExecutionTimeout < minDurableExecutionTimeout || *dc.ExecutionTimeout > maxDurableExecutionTimeout) { + _ = h.writeError(c, http.StatusBadRequest, "InvalidParameterValueException", + fmt.Sprintf("DurableConfig.ExecutionTimeout must be between %d and %d seconds", + minDurableExecutionTimeout, maxDurableExecutionTimeout)) + + return false + } + + if dc.RetentionPeriodInDays != nil && + (*dc.RetentionPeriodInDays < minDurableRetentionDays || *dc.RetentionPeriodInDays > maxDurableRetentionDays) { + _ = h.writeError(c, http.StatusBadRequest, "InvalidParameterValueException", + fmt.Sprintf("DurableConfig.RetentionPeriodInDays must be between %d and %d", + minDurableRetentionDays, maxDurableRetentionDays)) + + return false + } + + return true +} + // validateMemoryAndTimeout validates MemorySize and Timeout values (both 0 means use defaults). func (h *Handler) validateMemoryAndTimeout(c *echo.Context, memorySize, timeout int) bool { if memorySize != 0 && (memorySize < minMemorySize || memorySize > maxMemorySize) { @@ -234,6 +266,13 @@ func (h *Handler) validateCreateFunctionCode(c *echo.Context, input *CreateFunct return false } + if input.DurableConfig != nil && !isDurableSupportedRuntime(input.Runtime) { + _ = h.writeError(c, http.StatusBadRequest, "InvalidParameterValueException", + fmt.Sprintf("Runtime %q does not support durable functions", input.Runtime)) + + return false + } + if input.Code.ZipFile == nil && (input.Code.S3Bucket == "" || input.Code.S3Key == "") { _ = h.writeError(c, http.StatusBadRequest, "InvalidParameterValueException", "Code.ZipFile or Code.S3Bucket+Code.S3Key is required for Zip package type") @@ -675,6 +714,10 @@ func (h *Handler) handleUpdateFunctionConfiguration(c *echo.Context, name string } } + if !h.validateDurableConfigInput(c, input.DurableConfig) { + return nil + } + fn, getFnErr := h.Backend.GetFunction(name) if getFnErr != nil { if errors.Is(getFnErr, ErrFunctionNotFound) { @@ -689,6 +732,10 @@ func (h *Handler) handleUpdateFunctionConfiguration(c *echo.Context, name string return nil } + if !h.validateDurableRuntimeUpdate(c, fn, &input) { + return nil + } + applyFunctionConfigurationUpdate(fn, &input) fn.LastModified = time.Now().UTC().Format(time.RFC3339) @@ -702,6 +749,39 @@ func (h *Handler) handleUpdateFunctionConfiguration(c *echo.Context, name string return c.JSON(http.StatusOK, toWireFunctionConfiguration(fn)) } +// validateDurableRuntimeUpdate rejects an unsupported runtime for a Zip durable function. +// docs.aws.amazon.com/lambda/latest/dg/durable-supported-runtimes.html. +func (h *Handler) validateDurableRuntimeUpdate( + c *echo.Context, fn *FunctionConfiguration, input *UpdateFunctionConfigurationInput, +) bool { + if fn.PackageType != PackageTypeZip { + return true + } + + durableConfig := fn.DurableConfig + if input.DurableConfig != nil { + durableConfig = input.DurableConfig + } + + if durableConfig == nil { + return true + } + + runtime := fn.Runtime + if input.Runtime != "" { + runtime = input.Runtime + } + + if isDurableSupportedRuntime(runtime) { + return true + } + + _ = h.writeError(c, http.StatusBadRequest, "InvalidParameterValueException", + fmt.Sprintf("Runtime %q does not support durable functions", runtime)) + + return false +} + // applySnapStart sets the SnapStart field on fn based on the input. func applySnapStart(fn *FunctionConfiguration, s *SnapStart) { if s == nil { @@ -861,6 +941,19 @@ const minTimeout = 1 // maxTimeout is the maximum allowed Lambda function timeout in seconds. const maxTimeout = 900 +// ExecutionTimeout's documented range. +// docs.aws.amazon.com/lambda/latest/api/API_DurableConfig.html. +const ( + minDurableExecutionTimeout = 1 + maxDurableExecutionTimeout = 31622400 +) + +// RetentionPeriodInDays's documented range (same API_DurableConfig.html page). +const ( + minDurableRetentionDays = 1 + maxDurableRetentionDays = 90 +) + // handleGetFunctionConfiguration handles GET /2015-03-31/functions/{name}/configuration. // Real AWS returns the function configuration without the code location. func (h *Handler) handleGetFunctionConfiguration(c *echo.Context, name string) error { diff --git a/services/lambda/handler_invocation.go b/services/lambda/handler_invocation.go index c1b672e028..ba12fce168 100644 --- a/services/lambda/handler_invocation.go +++ b/services/lambda/handler_invocation.go @@ -57,7 +57,7 @@ func (h *Handler) handleInvoke(c *echo.Context, name string) error { qualifier := c.Request().URL.Query().Get("Qualifier") - if !h.validateQualifier(c, qualifier) { + if !h.validateInvokeQualifier(c, name, qualifier) { return nil } @@ -72,30 +72,25 @@ func (h *Handler) handleInvoke(c *echo.Context, name string) error { executedVersion := h.resolveExecutedVersion(name, qualifier) - var result []byte - var logResult string - var functionError string - var statusCode int - var invokeErr error - - if qi, ok := h.Backend.(QualifierInvoker); ok { - result, logResult, functionError, statusCode, invokeErr = qi.InvokeFunctionWithQualifier( - ctx, - name, - qualifier, - clientContext, - logType, - invType, - body, - ) - } else { - result, statusCode, invokeErr = h.Backend.InvokeFunction(ctx, name, invType, body) + durableARN, reusedExec, ok := h.beginDurableInvoke(c, name, qualifier, invType, body) + if !ok { + return nil } + ctx = ctxWithAsyncDurableExecARN(ctx, durableARN, invType) + + result, logResult, functionError, statusCode, invokeErr := h.dispatchInvoke( + ctx, name, qualifier, clientContext, logType, invType, body, reusedExec, + ) + if invokeErr != nil { return h.writeInvokeError(c, name, invokeErr) } + if durableARN != "" && reusedExec == nil && invType == InvocationTypeRequestResponse { + h.completeDurableInvokeExecution(durableARN, functionError == "", result) + } + // Set X-Amz-Executed-Version on all successful responses (real AWS always sends this). c.Response().Header().Set("X-Amz-Executed-Version", executedVersion) @@ -128,6 +123,100 @@ func (h *Handler) handleInvoke(c *echo.Context, name string) error { return c.NoContent(http.StatusOK) } +// beginDurableInvoke starts or reuses handleInvoke's durable execution (if +// the target is a durable function) and sets the X-Amz-Durable-Execution-Arn +// response header. When the DurableExecutionName conflicts with a +// differently-payloaded execution, it writes the 409 response itself and +// returns ok=false so the caller stops immediately (matching this file's +// existing "already wrote a response, return nil" convention). +func (h *Handler) beginDurableInvoke( + c *echo.Context, name, qualifier, invType string, body []byte, +) (string, *DurableExecution, bool) { + execName := c.Request().Header.Get("X-Amz-Durable-Execution-Name") + + arn, reusedExec, err := h.startDurableInvokeExecution(name, qualifier, invType, execName, body) + if err != nil { + _ = h.writeError(c, http.StatusConflict, "DurableExecutionAlreadyStartedException", err.Error()) + + return "", nil, false + } + + if arn != "" { + c.Response().Header().Set("X-Amz-Durable-Execution-Arn", arn) + } + + return arn, reusedExec, true +} + +// ctxWithAsyncDurableExecARN carries durableARN for Event invocations so the +// async retry loop can record completion. +func ctxWithAsyncDurableExecARN(ctx context.Context, durableARN, invType string) context.Context { + if durableARN == "" || invType != InvocationTypeEvent { + return ctx + } + + return withDurableExecARN(ctx, durableARN) +} + +// dispatchInvoke performs one Invoke's actual work: replaying an +// idempotent-replay hit against an already-closed durable execution +// (reusedExec set and closed — must NOT invoke the function again), or +// otherwise the real invocation via QualifierInvoker/InvokeFunction exactly +// as before this file gained durable-execution awareness. +func (h *Handler) dispatchInvoke( + ctx context.Context, + name, qualifier, clientContext, logType, invType string, + body []byte, + reusedExec *DurableExecution, +) ([]byte, string, string, int, error) { + if reusedExec != nil && reusedExec.Status != DurableExecutionStatusRunning { + result, functionError, statusCode := replayClosedDurableExecution(reusedExec, invType) + + return result, "", functionError, statusCode, nil + } + + if qi, ok := h.Backend.(QualifierInvoker); ok { + return qi.InvokeFunctionWithQualifier(ctx, name, qualifier, clientContext, logType, invType, body) + } + + result, statusCode, invokeErr := h.Backend.InvokeFunction(ctx, name, invType, body) + + return result, "", "", statusCode, invokeErr +} + +// validateInvokeQualifier checks qualifier well-formedness, then the durable-function requirement. +func (h *Handler) validateInvokeQualifier(c *echo.Context, name, qualifier string) bool { + return h.validateQualifier(c, qualifier) && h.requireDurableQualifier(c, name, qualifier) +} + +// requireDurableQualifier rejects an unqualified Invoke of a durable function. +// docs.aws.amazon.com/lambda/latest/dg/durable-invoking.html#durable-invoking-qualified-arns. +func (h *Handler) requireDurableQualifier(c *echo.Context, name, qualifier string) bool { + if qualifier != "" { + return true + } + + bareName, embeddedQualifier := functionNameAndQualifierFromARN(name) + if embeddedQualifier != "" { + return true + } + + bk, ok := h.Backend.(*InMemoryBackend) + if !ok { + return true + } + + fn, err := bk.GetFunction(bareName) + if err != nil || fn.DurableConfig == nil { + return true + } + + _ = h.writeError(c, http.StatusBadRequest, "InvalidParameterValueException", + "Durable functions require a qualified identifier: specify a version, alias, or $LATEST") + + return false +} + // resolveExecutedVersion returns the version string for the X-Amz-Executed-Version header. func (h *Handler) resolveExecutedVersion(name, qualifier string) string { bk, ok := h.Backend.(*InMemoryBackend) diff --git a/services/lambda/invocation.go b/services/lambda/invocation.go index 47fdb2354c..ede99c6c38 100644 --- a/services/lambda/invocation.go +++ b/services/lambda/invocation.go @@ -45,6 +45,23 @@ func invocationChainContains(ctx context.Context, functionName string) bool { return slices.Contains(chain, functionName) } +// durableExecARNKeyType carries an Event invocation's durable execution ARN to +// invokeEvent without widening InvokeFunctionWithQualifier. +type durableExecARNKeyType struct{} + +// withDurableExecARN returns a context carrying arn for a pending Event invocation. +func withDurableExecARN(ctx context.Context, arn string) context.Context { + return context.WithValue(ctx, durableExecARNKeyType{}, arn) +} + +// durableExecARNFromContext returns the durable execution ARN set by +// withDurableExecARN, or "" when none was set (non-durable or non-Event invocation). +func durableExecARNFromContext(ctx context.Context) string { + arn, _ := ctx.Value(durableExecARNKeyType{}).(string) + + return arn +} + // InvokeFunction invokes a Lambda function without a qualifier (equivalent to "$LATEST"). // For qualified invocations (alias or version number), use InvokeFunctionWithQualifier. func (b *InMemoryBackend) InvokeFunction( @@ -233,12 +250,13 @@ func (b *InMemoryBackend) invokeEvent( trackConcurrency bool, ) { inv := &pendingInvocation{ - requestID: uuid.New().String(), - payload: payload, - clientContext: clientContext, - deadline: time.Now().Add(timeout), - createdAt: time.Now(), - result: make(chan invocationResult, 1), + requestID: uuid.New().String(), + payload: payload, + clientContext: clientContext, + deadline: time.Now().Add(timeout), + createdAt: time.Now(), + result: make(chan invocationResult, 1), + durableExecARN: durableExecARNFromContext(ctx), } b.enqueueAsyncInvocation(ctx, srv, fn.FunctionName, inv, timeout, trackConcurrency) @@ -406,6 +424,7 @@ func (b *InMemoryBackend) runAsyncInvocationRetryLoop( "function", functionName, "attempts", attempt+1) } + b.completeAsyncDurableExecution(currentInv.durableExecARN, !isError, result.payload) b.dispatchAsyncOutcome(context.WithoutCancel(b.ctx), outcome) return @@ -420,6 +439,16 @@ func (b *InMemoryBackend) runAsyncInvocationRetryLoop( } } +// completeAsyncDurableExecution records an Event invocation's final outcome; +// no-op when arn is empty. +func (b *InMemoryBackend) completeAsyncDurableExecution(arn string, succeeded bool, result []byte) { + if arn == "" || b.durableExecs == nil { + return + } + + b.durableExecs.completeExecution(arn, succeeded, string(result)) +} + // readAsyncRetryConfig returns the effective maximum retry attempts and the event-age deadline // for an async invocation. If no event invoke configuration exists, the AWS defaults are used // (2 retries, no age limit). @@ -510,11 +539,12 @@ func scheduleAsyncRetry( } newInv := &pendingInvocation{ - requestID: uuid.New().String(), - payload: original.payload, - deadline: time.Now().Add(timeout), - result: make(chan invocationResult, 1), - createdAt: original.createdAt, + requestID: uuid.New().String(), + payload: original.payload, + deadline: time.Now().Add(timeout), + result: make(chan invocationResult, 1), + createdAt: original.createdAt, + durableExecARN: original.durableExecARN, } ctx, cancel := context.WithTimeout(ctx, asyncInvocationEnqueueTimeout) diff --git a/services/lambda/realclient_durable_execution_test.go b/services/lambda/realclient_durable_execution_test.go index e721699daa..2714f8587a 100644 --- a/services/lambda/realclient_durable_execution_test.go +++ b/services/lambda/realclient_durable_execution_test.go @@ -312,19 +312,16 @@ func TestRealClient_DurableExecution(t *testing.T) { }, ) require.NoError(t, err) - // Real-shape round trip proven (decodes cleanly), but this backend - // can never return a match: CheckpointDurableExecution is the only - // creation path and its request carries no function identity at all - // (DurableExecutionArn is client-opaque, "server-never-parses- - // structure-from-it" per deriveDurableExecutionName's own doc - // comment), so DurableExecution.FunctionARN is never assigned - // anywhere in this package -- confirmed by grep, zero write sites. - // Same root cause as PARITY.md's documented FunctionArn-always-empty - // gap (no StartDurableExecution/Invoke entry point), one step - // further: it also makes this entire op permanently return zero - // results for any function. Recorded in items_still_open rather - // than fixed -- fixing it needs the same out-of-scope Invoke - // rewiring that gap already defers. + // Real-shape round trip proven (decodes cleanly), but this specific + // creation path can never return a match: CheckpointDurableExecution + // carries no function identity at all (DurableExecutionArn is + // client-opaque, "server-never-parses-structure-from-it" per + // deriveDurableExecutionName's own doc comment), so a + // checkpoint-only-created execution's FunctionARN stays empty. + // Invoke's DurableExecutionName wiring (durable_invoke_test.go) now + // DOES assign FunctionARN/Version and makes this op return real + // matches for executions started that way -- this case only proves + // the CheckpointDurableExecution-only path is unaffected. assert.Empty(t, listOut.DurableExecutions) }, }, diff --git a/services/lambda/runtime_api.go b/services/lambda/runtime_api.go index a3a5a6eaea..0f6d6090e5 100644 --- a/services/lambda/runtime_api.go +++ b/services/lambda/runtime_api.go @@ -28,8 +28,11 @@ type pendingInvocation struct { createdAt time.Time // when the event was first received (used for MaximumEventAgeInSeconds) requestID string clientContext string - result chan invocationResult - payload []byte + // durableExecARN is set for an async (Event) invocation of a durable function so + // the retry loop can record the execution's completion once retries are exhausted. + durableExecARN string + result chan invocationResult + payload []byte } // invocationResult holds the outcome of a Lambda container invocation. diff --git a/services/lambda/versions_aliases.go b/services/lambda/versions_aliases.go index 82ae576091..f7c95cf4a6 100644 --- a/services/lambda/versions_aliases.go +++ b/services/lambda/versions_aliases.go @@ -171,7 +171,15 @@ func (b *InMemoryBackend) CreateAlias( b.aliases.Put(alias) - return alias, nil + return cloneAlias(alias), nil +} + +// cloneAlias stops a caller from racing UpdateAlias, which mutates alias's +// fields under the lock. +func cloneAlias(alias *FunctionAlias) *FunctionAlias { + cp := *alias + + return &cp } // GetAlias returns a named alias for a function. @@ -188,7 +196,7 @@ func (b *InMemoryBackend) GetAlias(name, aliasName string) (*FunctionAlias, erro return nil, ErrAliasNotFound } - return alias, nil + return cloneAlias(alias), nil } // ListAliases returns a page of aliases for a function sorted by name. @@ -212,7 +220,7 @@ func (b *InMemoryBackend) ListAliases( continue } - result = append(result, a) + result = append(result, cloneAlias(a)) } sort.Slice(result, func(i, j int) bool { @@ -257,7 +265,7 @@ func (b *InMemoryBackend) UpdateAlias( alias.RevisionID = uuid.New().String() - return alias, nil + return cloneAlias(alias), nil } // DeleteAlias removes a named alias from a function. @@ -376,6 +384,8 @@ func versionToFn(v *FunctionVersion) *FunctionConfiguration { State: v.State, SnapStart: v.SnapStart, Version: v.Version, + // Invoke reads this to detect durable invocations of versions/aliases. + DurableConfig: v.DurableConfig, } } diff --git a/services/macie2/allow_lists.go b/services/macie2/allow_lists.go index 79f030c202..bf2d29a81f 100644 --- a/services/macie2/allow_lists.go +++ b/services/macie2/allow_lists.go @@ -152,7 +152,7 @@ func (b *InMemoryBackend) DeleteAllowList(id string, ignoreJobChecks bool) error // ListAllowLists returns summaries of all allow lists. func (b *InMemoryBackend) ListAllowLists(limit int, token string) ([]*AllowListSummary, string, error) { return listPaginated( - b, "ListAllowLists", b.allowLists.All(), + b, "ListAllowLists", b.allowLists.All, func(al *storedAllowList) (*AllowListSummary, bool) { return &AllowListSummary{ Arn: al.Arn, diff --git a/services/macie2/classification_jobs.go b/services/macie2/classification_jobs.go index 6693e32cf8..e79f3fcea2 100644 --- a/services/macie2/classification_jobs.go +++ b/services/macie2/classification_jobs.go @@ -150,7 +150,7 @@ func (b *InMemoryBackend) ListClassificationJobs( filterCriteria map[string]any, sortBy *ListJobsSortCriteria, maxResults int, nextToken string, ) ([]*ClassificationJobSummary, string, error) { return listPaginated( - b, "ListClassificationJobs", b.classificationJobs.All(), + b, "ListClassificationJobs", b.classificationJobs.All, func(job *ClassificationJob) (*ClassificationJobSummary, bool) { if !matchesJobCriteria(job, filterCriteria) { return nil, false diff --git a/services/macie2/findings_filters.go b/services/macie2/findings_filters.go index 194fbfb329..a896bde419 100644 --- a/services/macie2/findings_filters.go +++ b/services/macie2/findings_filters.go @@ -152,7 +152,7 @@ func (b *InMemoryBackend) DeleteFindingsFilter(id string) error { // ListFindingsFilters returns summaries of all findings filters. func (b *InMemoryBackend) ListFindingsFilters(limit int, token string) ([]*FindingsFilterSummary, string, error) { return listPaginated( - b, "ListFindingsFilters", b.findingsFilters.All(), + b, "ListFindingsFilters", b.findingsFilters.All, func(ff *storedFindingsFilter) (*FindingsFilterSummary, bool) { return &FindingsFilterSummary{ Action: ff.Action, diff --git a/services/macie2/handler_enablement_test.go b/services/macie2/handler_enablement_test.go index e3c0e35333..0832390b60 100644 --- a/services/macie2/handler_enablement_test.go +++ b/services/macie2/handler_enablement_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -311,28 +312,30 @@ func TestSessionTimestampsPresent(t *testing.T) { func TestSessionUpdatedAtAdvances(t *testing.T) { t.Parallel() - h := newTestHandler(t) - doRequest(t, h, http.MethodPost, "/macie", nil) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) + doRequest(t, h, http.MethodPost, "/macie", nil) - rec1 := doRequest(t, h, http.MethodGet, "/macie", nil) - require.Equal(t, http.StatusOK, rec1.Code) + rec1 := doRequest(t, h, http.MethodGet, "/macie", nil) + require.Equal(t, http.StatusOK, rec1.Code) - var before map[string]any - require.NoError(t, json.Unmarshal(rec1.Body.Bytes(), &before)) - createdAt := before["createdAt"].(string) + var before map[string]any + require.NoError(t, json.Unmarshal(rec1.Body.Bytes(), &before)) + createdAt := before["createdAt"].(string) - time.Sleep(1001 * time.Millisecond) + time.Sleep(1001 * time.Millisecond) - doRequest(t, h, http.MethodPatch, "/macie", - map[string]string{"findingPublishingFrequency": "SIX_HOURS"}) + doRequest(t, h, http.MethodPatch, "/macie", + map[string]string{"findingPublishingFrequency": "SIX_HOURS"}) - rec2 := doRequest(t, h, http.MethodGet, "/macie", nil) - require.Equal(t, http.StatusOK, rec2.Code) + rec2 := doRequest(t, h, http.MethodGet, "/macie", nil) + require.Equal(t, http.StatusOK, rec2.Code) - var after map[string]any - require.NoError(t, json.Unmarshal(rec2.Body.Bytes(), &after)) + var after map[string]any + require.NoError(t, json.Unmarshal(rec2.Body.Bytes(), &after)) - assert.Equal(t, createdAt, after["createdAt"].(string), "createdAt must not change after update") - assert.NotEqual(t, after["createdAt"], after["updatedAt"], - "updatedAt must differ from createdAt after UpdateMacieSession") + assert.Equal(t, createdAt, after["createdAt"].(string), "createdAt must not change after update") + assert.NotEqual(t, after["createdAt"], after["updatedAt"], + "updatedAt must differ from createdAt after UpdateMacieSession") + }) } diff --git a/services/macie2/members.go b/services/macie2/members.go index 08c582703b..d4cf8beb1d 100644 --- a/services/macie2/members.go +++ b/services/macie2/members.go @@ -67,7 +67,7 @@ func (b *InMemoryBackend) DeleteMember(accountID string) error { // still associated with this administrator, paginated by limit/token. func (b *InMemoryBackend) ListMembers(onlyAssociated bool, limit int, token string) ([]*Member, string, error) { return listPaginated( - b, "ListMembers", b.members.All(), + b, "ListMembers", b.members.All, func(m *Member) (*Member, bool) { if onlyAssociated && m.RelationshipStatus == "Removed" { return nil, false diff --git a/services/macie2/store.go b/services/macie2/store.go index 9989a7a965..6f86d90ef6 100644 --- a/services/macie2/store.go +++ b/services/macie2/store.go @@ -105,10 +105,12 @@ func NewInMemoryBackend(accountID, region string) *InMemoryBackend { // listPaginated locks for reading, projects/sorts items, and paginates, // returning the page plus continuation token. Shared by the List* methods. +// itemsFn runs under the lock; pass the table's .All method value, not its result, +// or the read races concurrent writers. func listPaginated[T any, R any]( b *InMemoryBackend, lockName string, - items []T, + itemsFn func() []T, mapFn func(T) (R, bool), sortFn func([]R), token string, @@ -117,7 +119,7 @@ func listPaginated[T any, R any]( b.mu.RLock(lockName) defer b.mu.RUnlock() - data, next := mapSortPaginate(items, mapFn, sortFn, token, b.paginationSecret, limit) + data, next := mapSortPaginate(itemsFn(), mapFn, sortFn, token, b.paginationSecret, limit) return data, next, nil } diff --git a/services/mediaconvert/janitor_test.go b/services/mediaconvert/janitor_test.go index 7eb06f37e1..18501b941a 100644 --- a/services/mediaconvert/janitor_test.go +++ b/services/mediaconvert/janitor_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -158,17 +159,19 @@ func TestAdvanceJobPhase_PreservesSubmitTime(t *testing.T) { func TestAdvanceJobPhase_FinishTimeAfterStartTime(t *testing.T) { t.Parallel() - b := mediaconvert.NewInMemoryBackend(testAccountID, testRegion) - j := createTestJobDirect(t, b) + synctest.Test(t, func(t *testing.T) { + b := mediaconvert.NewInMemoryBackend(testAccountID, testRegion) + j := createTestJobDirect(t, b) - for range 4 { - b.AdvanceJobPhase() - time.Sleep(1 * time.Millisecond) - } + for range 4 { + b.AdvanceJobPhase() + time.Sleep(1 * time.Millisecond) + } - got, err := b.GetJob(j.ID) - require.NoError(t, err) - assert.GreaterOrEqual(t, got.Timing.FinishTime, got.Timing.StartTime) + got, err := b.GetJob(j.ID) + require.NoError(t, err) + assert.GreaterOrEqual(t, got.Timing.FinishTime, got.Timing.StartTime) + }) } // TestAdvanceJobPhase_JobPercentComplete100OnComplete verifies 100% on COMPLETE. diff --git a/services/mediastore/containers_test.go b/services/mediastore/containers_test.go index d2e4482da1..0f2d998779 100644 --- a/services/mediastore/containers_test.go +++ b/services/mediastore/containers_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -322,42 +323,6 @@ func TestInMemoryBackend_AccessLogging(t *testing.T) { } } -// waitForContainerStatus polls DescribeContainer until want is observed or -// timeout elapses, returning the final observed status (or an empty string -// if DescribeContainer errored, e.g. because the container was actually -// removed). Modeled on services/redshift's reconciler_test.go waitFor -// helper: since advanceContainerStates only runs lazily (no background -// goroutine), the caller must keep calling a read path for a due transition -// to ever apply. -func waitForContainerStatus( - t *testing.T, - b *mediastore.InMemoryBackend, - name, want string, - timeout time.Duration, -) string { - t.Helper() - - deadline := time.Now().Add(timeout) - last := "" - - for time.Now().Before(deadline) { - c, err := b.DescribeContainer(context.Background(), name) - if err != nil { - last = "" - } else { - last = c.Status - } - - if last == want { - return last - } - - time.Sleep(2 * time.Millisecond) - } - - return last -} - // TestInMemoryBackend_ContainerActivationDelay verifies the CREATING/ // DELETING transient-lifecycle simulation gated by SetActivationDelay: with // no delay configured (the default), transitions stay synchronous (matching @@ -387,56 +352,52 @@ func TestInMemoryBackend_ContainerActivationDelay(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := newBackend() - b.SetActivationDelay(tt.delay) + synctest.Test(t, func(t *testing.T) { + b := newBackend() + b.SetActivationDelay(tt.delay) - container := "activation-delay-test" + container := "activation-delay-test" - created, err := b.CreateContainer(context.Background(), testAccountID, container, nil) - require.NoError(t, err) - - if tt.delay == 0 { - assert.Equal(t, "ACTIVE", created.Status) - } else { - assert.Equal(t, "CREATING", created.Status) - - got := waitForContainerStatus(t, b, container, "ACTIVE", time.Second) - assert.Equal(t, "ACTIVE", got, "container never transitioned to ACTIVE") - } + created, err := b.CreateContainer(context.Background(), testAccountID, container, nil) + require.NoError(t, err) - err = b.DeleteContainer(context.Background(), container) - require.NoError(t, err) + if tt.delay == 0 { + assert.Equal(t, "ACTIVE", created.Status) + } else { + assert.Equal(t, "CREATING", created.Status) - if tt.delay == 0 { - _, err = b.DescribeContainer(context.Background(), container) - require.Error(t, err, "container should be gone immediately with no activation delay") + // advanceContainerStates only runs lazily (no background + // goroutine): sleeping past the delay then reading once is enough. + time.Sleep(tt.delay + time.Millisecond) - return - } + var got *mediastore.Container + got, err = b.DescribeContainer(context.Background(), container) + require.NoError(t, err) + assert.Equal(t, "ACTIVE", got.Status, "container never transitioned to ACTIVE") + } - // With a delay configured, the container must still be visible - // (in DELETING) immediately after DeleteContainer returns... - mid, err := b.DescribeContainer(context.Background(), container) - require.NoError(t, err, "container should still be visible mid-deletion") - assert.Equal(t, "DELETING", mid.Status) + err = b.DeleteContainer(context.Background(), container) + require.NoError(t, err) - // ...and actually gone once the delay elapses. - deadline := time.Now().Add(time.Second) + if tt.delay == 0 { + _, err = b.DescribeContainer(context.Background(), container) + require.Error(t, err, "container should be gone immediately with no activation delay") - for { - _, err = b.DescribeContainer(context.Background(), container) - if err != nil { - break + return } - if time.Now().After(deadline) { - t.Fatal("container was never removed after its deletion delay elapsed") - } + // With a delay configured, the container must still be visible + // (in DELETING) immediately after DeleteContainer returns... + mid, err := b.DescribeContainer(context.Background(), container) + require.NoError(t, err, "container should still be visible mid-deletion") + assert.Equal(t, "DELETING", mid.Status) - time.Sleep(2 * time.Millisecond) - } + // ...and actually gone once the delay elapses. + time.Sleep(tt.delay + time.Millisecond) - require.Error(t, err) + _, err = b.DescribeContainer(context.Background(), container) + require.Error(t, err, "container was never removed after its deletion delay elapsed") + }) }) } } diff --git a/services/opensearch/domain_config.go b/services/opensearch/domain_config.go index 9846c9b041..23a5abafd0 100644 --- a/services/opensearch/domain_config.go +++ b/services/opensearch/domain_config.go @@ -3,6 +3,8 @@ package opensearch import ( "fmt" "time" + + "github.com/google/uuid" ) // CancelDomainConfigChange cancels a pending configuration change on a domain. @@ -148,7 +150,9 @@ func (b *InMemoryBackend) UpdateDomainConfig( applyOperationalConfig(d, input) applyAutoTuneConfig(d, input, b.clock()) - changeID := fmt.Sprintf("change-%s-%d", name, time.Now().UnixNano()) + // ChangeId is a bare UUID (confirmed pattern on ChangeProgressStatusDetails, + // docs.aws.amazon.com/opensearch-service). + changeID := uuid.NewString() d.LastChangeID = changeID b.beginProcessing(d, dpsModifying) diff --git a/services/opensearch/domain_status.go b/services/opensearch/domain_status.go index bf4fc88729..e1e9168a07 100644 --- a/services/opensearch/domain_status.go +++ b/services/opensearch/domain_status.go @@ -5,6 +5,8 @@ import ( "strconv" "time" + "github.com/google/uuid" + "github.com/blackbirdworks/gopherstack/pkgs/awstime" ) @@ -131,7 +133,9 @@ func (b *InMemoryBackend) GetDryRunProgress(domainName string) (*DryRunStatus, e if !exists { now := time.Now().UTC().Format(time.RFC3339) dr = &DryRunStatus{ - DryRunID: fmt.Sprintf("dryrun-%s-%d", domainName, time.Now().UnixNano()), + // DryRunId is a bare UUID (confirmed pattern on DryRunProgressStatus, + // docs.aws.amazon.com/opensearch-service). + DryRunID: uuid.NewString(), DryRunStatus: softwareUpdateCompleted, CreationDate: now, UpdateDate: now, diff --git a/services/opensearch/id_generation_test.go b/services/opensearch/id_generation_test.go new file mode 100644 index 0000000000..cc02f09a2f --- /dev/null +++ b/services/opensearch/id_generation_test.go @@ -0,0 +1,159 @@ +package opensearch_test + +import ( + "regexp" + "testing" + "testing/synctest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/opensearch" +) + +// changeUUIDPattern locks in the fix for DryRunId/ChangeId, previously +// derived from time.Now().UnixNano() and colliding under synctest. +var changeUUIDPattern = regexp.MustCompile( + `^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`, +) + +func TestOpenSearchBackend_ChangeIDs_Unique(t *testing.T) { + t.Parallel() + + tests := []struct { + create func(t *testing.T, b *opensearch.InMemoryBackend) string + name string + }{ + { + name: "update_domain_config_change_id", + create: func(t *testing.T, b *opensearch.InMemoryBackend) string { + t.Helper() + + d, err := b.UpdateDomainConfig("dom-a", opensearch.UpdateDomainConfigInput{}) + require.NoError(t, err) + + return d.LastChangeID + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + b := opensearch.NewInMemoryBackend("000000000000", "us-east-1") + _, err := b.CreateDomain(opensearch.CreateDomainInput{Name: "dom-a"}) + require.NoError(t, err) + + id1 := tt.create(t, b) + id2 := tt.create(t, b) + + assert.NotEqual(t, id1, id2, "two changes created back-to-back must get distinct IDs") + assert.Regexp(t, changeUUIDPattern, id1) + assert.Regexp(t, changeUUIDPattern, id2) + }) + }) + } +} + +func TestOpenSearchBackend_DryRunID_Format(t *testing.T) { + t.Parallel() + + b := opensearch.NewInMemoryBackend("000000000000", "us-east-1") + _, err := b.CreateDomain(opensearch.CreateDomainInput{Name: "dom-b"}) + require.NoError(t, err) + + dr, err := b.GetDryRunProgress("dom-b") + require.NoError(t, err) + assert.Regexp(t, changeUUIDPattern, dr.DryRunID) +} + +// TestOpenSearchBackend_ServerlessPolicyVersions_Unique checks same-instant updates +// get distinct PolicyVersion/ConfigVersion tokens. +func TestOpenSearchBackend_ServerlessPolicyVersions_Unique(t *testing.T) { + t.Parallel() + + tests := []struct { + update func(t *testing.T, b *opensearch.InMemoryBackend) string + name string + }{ + { + name: "access_policy_version", + update: func(t *testing.T, b *opensearch.InMemoryBackend) string { + t.Helper() + + ap, err := b.UpdateServerlessAccessPolicy("data", "pol-a", "desc", `{"a":1}`, "") + require.NoError(t, err) + + return ap.PolicyVersion + }, + }, + { + name: "security_config_version", + update: func(t *testing.T, b *opensearch.InMemoryBackend) string { + t.Helper() + + sc, err := b.UpdateServerlessSecurityConfig("saml/000000000000/1", "desc", "", nil) + require.NoError(t, err) + + return sc.ConfigVersion + }, + }, + { + name: "encryption_policy_version", + update: func(t *testing.T, b *opensearch.InMemoryBackend) string { + t.Helper() + + ep, err := b.UpdateServerlessEncryptionPolicy("data", "pol-e", "desc", `{"e":1}`, "") + require.NoError(t, err) + + return ep.PolicyVersion + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + b := opensearch.NewInMemoryBackend("000000000000", "us-east-1") + + _, err := b.CreateServerlessAccessPolicy("data", "pol-a", "desc", `{"a":0}`) + require.NoError(t, err) + _, err = b.CreateServerlessSecurityConfig("saml", "desc", nil) + require.NoError(t, err) + _, err = b.CreateServerlessEncryptionPolicy("data", "pol-e", "desc", `{"e":0}`) + require.NoError(t, err) + + v1 := tt.update(t, b) + v2 := tt.update(t, b) + + assert.NotEqual(t, v1, v2, "two updates in the same instant must get distinct versions") + }) + }) + } +} + +// TestOpenSearchBackend_LifecyclePolicyVersion_Unique chains updates, since each must +// pass the version returned by the previous call. +func TestOpenSearchBackend_LifecyclePolicyVersion_Unique(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + b := opensearch.NewInMemoryBackend("000000000000", "us-east-1") + + lp, err := b.CreateServerlessLifecyclePolicy("retention", "lp-a", "desc", `{"l":0}`) + require.NoError(t, err) + + lp1, err := b.UpdateServerlessLifecyclePolicy("retention", "lp-a", "desc", `{"l":1}`, lp.PolicyVersion) + require.NoError(t, err) + + lp2, err := b.UpdateServerlessLifecyclePolicy("retention", "lp-a", "desc", `{"l":2}`, lp1.PolicyVersion) + require.NoError(t, err) + + assert.NotEqual(t, lp1.PolicyVersion, lp2.PolicyVersion, + "two updates in the same instant must get distinct versions") + }) +} diff --git a/services/opensearch/lifecycle_policies.go b/services/opensearch/lifecycle_policies.go index 0c0d2b54cf..1348d6bfbc 100644 --- a/services/opensearch/lifecycle_policies.go +++ b/services/opensearch/lifecycle_policies.go @@ -6,6 +6,8 @@ import ( "sort" "strings" "time" + + "github.com/google/uuid" ) // slLifecyclePolicyResourceTypeIndex is the only real ResourceType value @@ -160,7 +162,9 @@ func (b *InMemoryBackend) UpdateServerlessLifecyclePolicy( } lp.LastModifiedDate = float64(time.Now().Unix()) - lp.PolicyVersion = fmt.Sprintf("v%d", time.Now().UnixMilli()) + // uuid suffix: UnixMilli alone collides when two updates land in the same + // synctest instant, breaking the PolicyVersion staleness check above. + lp.PolicyVersion = fmt.Sprintf("v%d-%s", time.Now().UnixMilli(), uuid.NewString()[:8]) cp := *lp diff --git a/services/opensearch/serverless.go b/services/opensearch/serverless.go index 2d038af4b5..dcea2ed211 100644 --- a/services/opensearch/serverless.go +++ b/services/opensearch/serverless.go @@ -6,6 +6,7 @@ import ( "time" "github.com/blackbirdworks/gopherstack/pkgs/arn" + "github.com/google/uuid" ) const defaultSamlSessionTimeoutB64 = "MTY4MDAwMDAwMDAwMA==" @@ -215,6 +216,7 @@ func (b *InMemoryBackend) CreateServerlessCollection( b.slCollections.Put(coll) cp := *coll + cp.Tags = maps.Clone(coll.Tags) resolveCollectionStatus(&cp, b.clock()) return &cp, nil @@ -272,6 +274,7 @@ func (b *InMemoryBackend) BatchGetServerlessCollections(ids, names []string) []* if len(idSet) == 0 && len(nameSet) == 0 { cp := *c + cp.Tags = maps.Clone(c.Tags) resolveCollectionStatus(&cp, now) out = append(out, &cp) @@ -280,6 +283,7 @@ func (b *InMemoryBackend) BatchGetServerlessCollections(ids, names []string) []* if idSet[c.ID] || nameSet[c.Name] { cp := *c + cp.Tags = maps.Clone(c.Tags) resolveCollectionStatus(&cp, now) out = append(out, &cp) } @@ -306,6 +310,7 @@ func (b *InMemoryBackend) DeleteServerlessCollection(id string) (*ServerlessColl if b.processingDelay == 0 { cp := *c + cp.Tags = maps.Clone(c.Tags) cp.Status = statusDeleted b.slCollections.Delete(serverlessCollectionKey(c.Name)) @@ -315,6 +320,7 @@ func (b *InMemoryBackend) DeleteServerlessCollection(id string) (*ServerlessColl c.Status = statusDeleting c.StatusUntil = now.Add(b.processingDelay) cp := *c + cp.Tags = maps.Clone(c.Tags) return &cp, nil } @@ -368,6 +374,7 @@ func (b *InMemoryBackend) UpdateServerlessCollection(id, description string) (*S c.LastModifiedDate = float64(time.Now().Unix()) cp := *c + cp.Tags = maps.Clone(c.Tags) resolveCollectionStatus(&cp, b.clock()) return &cp, nil @@ -461,7 +468,8 @@ func (b *InMemoryBackend) UpdateServerlessAccessPolicy( _ = policyVersion ap.LastModifiedDate = float64(time.Now().Unix()) - ap.PolicyVersion = fmt.Sprintf("v%d", time.Now().UnixMilli()) + // uuid suffix: UnixMilli alone collides across same-instant updates under synctest. + ap.PolicyVersion = fmt.Sprintf("v%d-%s", time.Now().UnixMilli(), uuid.NewString()[:8]) cp := *ap @@ -567,7 +575,8 @@ func (b *InMemoryBackend) UpdateServerlessSecurityConfig( _ = configVersion sc.LastModifiedDate = float64(time.Now().Unix()) - sc.ConfigVersion = fmt.Sprintf("v%d", time.Now().UnixMilli()) + // uuid suffix: UnixMilli alone collides across same-instant updates under synctest. + sc.ConfigVersion = fmt.Sprintf("v%d-%s", time.Now().UnixMilli(), uuid.NewString()[:8]) cp := *sc @@ -677,7 +686,8 @@ func (b *InMemoryBackend) UpdateServerlessEncryptionPolicy( _ = policyVersion ep.LastModifiedDate = float64(time.Now().Unix()) - ep.PolicyVersion = fmt.Sprintf("v%d", time.Now().UnixMilli()) + // uuid suffix: UnixMilli alone collides across same-instant updates under synctest. + ep.PolicyVersion = fmt.Sprintf("v%d-%s", time.Now().UnixMilli(), uuid.NewString()[:8]) cp := *ep diff --git a/services/pipes/enrichment_cleanup_test.go b/services/pipes/enrichment_cleanup_test.go index a66f4e72c6..07ae08a452 100644 --- a/services/pipes/enrichment_cleanup_test.go +++ b/services/pipes/enrichment_cleanup_test.go @@ -3,6 +3,7 @@ package pipes_test import ( "context" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -28,49 +29,35 @@ func createEnrichmentTestPipe(t *testing.T, b *pipes.InMemoryBackend, name strin require.NoError(t, err) } -// waitPipeDeleted waits up to 500ms for a pipe to be fully deleted (removed from store). -func waitPipeDeleted(t *testing.T, b *pipes.InMemoryBackend, name string) { - t.Helper() - - deadline := time.Now().Add(500 * time.Millisecond) - for time.Now().Before(deadline) { - _, err := b.GetPipe(context.Background(), name) - if err != nil { - return // pipe is gone - } - - time.Sleep(5 * time.Millisecond) - } - - t.Fatalf("pipe %q was not deleted within 500ms", name) -} - // TestPipesEnrichmentCallCountPrunedOnDelete verifies that when a pipe is deleted // its enrichment call counter is removed from the index, preventing unbounded growth. func TestPipesEnrichmentCallCountPrunedOnDelete(t *testing.T) { t.Parallel() - b := newPipesBackend(t) + synctest.Test(t, func(t *testing.T) { + b := newPipesBackend(t) - createEnrichmentTestPipe(t, b, "my-pipe") + createEnrichmentTestPipe(t, b, "my-pipe") - // Record some enrichment calls. - b.RecordEnrichmentCall(context.Background(), "my-pipe") - b.RecordEnrichmentCall(context.Background(), "my-pipe") - assert.Equal(t, int64(2), b.EnrichmentCallCountForTest("my-pipe")) - assert.Equal(t, 1, b.EnrichmentIndexSizeForTest()) + // Record some enrichment calls. + b.RecordEnrichmentCall(context.Background(), "my-pipe") + b.RecordEnrichmentCall(context.Background(), "my-pipe") + assert.Equal(t, int64(2), b.EnrichmentCallCountForTest("my-pipe")) + assert.Equal(t, 1, b.EnrichmentIndexSizeForTest()) - // Delete the pipe and wait for the async transition to complete. - _, err := b.DeletePipe(context.Background(), "my-pipe") - require.NoError(t, err) + // Delete the pipe and wait for the async transition to complete. + _, err := b.DeletePipe(context.Background(), "my-pipe") + require.NoError(t, err) - waitPipeDeleted(t, b, "my-pipe") + time.Sleep(20 * time.Millisecond) + synctest.Wait() - // The enrichment counter for the deleted pipe must have been pruned. - assert.Equal(t, int64(0), b.EnrichmentCallCountForTest("my-pipe"), - "enrichment count for deleted pipe must be 0") - assert.Equal(t, 0, b.EnrichmentIndexSizeForTest(), - "enrichment index must be empty after pipe deletion") + // The enrichment counter for the deleted pipe must have been pruned. + assert.Equal(t, int64(0), b.EnrichmentCallCountForTest("my-pipe"), + "enrichment count for deleted pipe must be 0") + assert.Equal(t, 0, b.EnrichmentIndexSizeForTest(), + "enrichment index must be empty after pipe deletion") + }) } // TestPipesEnrichmentCountOnlyPrunesDeletedPipe verifies that deleting one pipe @@ -78,24 +65,28 @@ func TestPipesEnrichmentCallCountPrunedOnDelete(t *testing.T) { func TestPipesEnrichmentCountOnlyPrunesDeletedPipe(t *testing.T) { t.Parallel() - b := newPipesBackend(t) + synctest.Test(t, func(t *testing.T) { + b := newPipesBackend(t) - createEnrichmentTestPipe(t, b, "pipe-a") - createEnrichmentTestPipe(t, b, "pipe-b") + createEnrichmentTestPipe(t, b, "pipe-a") + createEnrichmentTestPipe(t, b, "pipe-b") - b.RecordEnrichmentCall(context.Background(), "pipe-a") - b.RecordEnrichmentCall(context.Background(), "pipe-b") - b.RecordEnrichmentCall(context.Background(), "pipe-b") + b.RecordEnrichmentCall(context.Background(), "pipe-a") + b.RecordEnrichmentCall(context.Background(), "pipe-b") + b.RecordEnrichmentCall(context.Background(), "pipe-b") - assert.Equal(t, 2, b.EnrichmentIndexSizeForTest()) + assert.Equal(t, 2, b.EnrichmentIndexSizeForTest()) - _, err := b.DeletePipe(context.Background(), "pipe-a") - require.NoError(t, err) - waitPipeDeleted(t, b, "pipe-a") + _, err := b.DeletePipe(context.Background(), "pipe-a") + require.NoError(t, err) + + time.Sleep(20 * time.Millisecond) + synctest.Wait() - // pipe-b counter must be untouched. - assert.Equal(t, int64(2), b.EnrichmentCallCountForTest("pipe-b")) - assert.Equal(t, 1, b.EnrichmentIndexSizeForTest(), "only pipe-a should be pruned") + // pipe-b counter must be untouched. + assert.Equal(t, int64(2), b.EnrichmentCallCountForTest("pipe-b")) + assert.Equal(t, 1, b.EnrichmentIndexSizeForTest(), "only pipe-a should be pruned") + }) } // BenchmarkPipesEnrichmentCallCount benchmarks RecordEnrichmentCall to confirm diff --git a/services/pipes/pipe_lifecycle_test.go b/services/pipes/pipe_lifecycle_test.go index de7bcbca4b..2d7ee93b1b 100644 --- a/services/pipes/pipe_lifecycle_test.go +++ b/services/pipes/pipe_lifecycle_test.go @@ -11,6 +11,7 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -94,21 +95,24 @@ func TestLifecycle_CreatingToRunning(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := auditNewBackend() - _, err := b.CreatePipe(context.Background(), pipes.CreatePipeInput{ - RoleARN: "arn:aws:iam::123456789012:role/r", - Name: tt.name, - Source: "arn:aws:sqs:us-west-2:123456789012:q", - Target: "arn:aws:lambda:us-west-2:123456789012:function:fn", - DesiredState: tt.desiredState, - }) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := auditNewBackend() + _, err := b.CreatePipe(context.Background(), pipes.CreatePipeInput{ + RoleARN: "arn:aws:iam::123456789012:role/r", + Name: tt.name, + Source: "arn:aws:sqs:us-west-2:123456789012:q", + Target: "arn:aws:lambda:us-west-2:123456789012:function:fn", + DesiredState: tt.desiredState, + }) + require.NoError(t, err) - require.Eventually(t, func() bool { - p, getErr := b.GetPipe(context.Background(), tt.name) + time.Sleep(20 * time.Millisecond) + synctest.Wait() - return getErr == nil && p.CurrentState == tt.wantEventualState - }, 500*time.Millisecond, 5*time.Millisecond) + p, getErr := b.GetPipe(context.Background(), tt.name) + require.NoError(t, getErr) + assert.Equal(t, tt.wantEventualState, p.CurrentState) + }) }) } } @@ -138,36 +142,39 @@ func TestLifecycle_Updating(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := auditNewBackend() - pipeName := tt.name + "-pipe" - desiredState := "RUNNING" - if tt.wantEventualState == "STOPPED" { - desiredState = "STOPPED" - } - _, err := b.CreatePipe(context.Background(), pipes.CreatePipeInput{ - RoleARN: "arn:aws:iam::123456789012:role/r", - Name: pipeName, - Source: "arn:aws:sqs:us-west-2:123456789012:q", - Target: "arn:aws:lambda:us-west-2:123456789012:function:fn", - DesiredState: "RUNNING", - }) - require.NoError(t, err) - pipes.WaitPipeRunning(t, b, pipeName) + synctest.Test(t, func(t *testing.T) { + b := auditNewBackend() + pipeName := tt.name + "-pipe" + desiredState := "RUNNING" + if tt.wantEventualState == "STOPPED" { + desiredState = "STOPPED" + } + _, err := b.CreatePipe(context.Background(), pipes.CreatePipeInput{ + RoleARN: "arn:aws:iam::123456789012:role/r", + Name: pipeName, + Source: "arn:aws:sqs:us-west-2:123456789012:q", + Target: "arn:aws:lambda:us-west-2:123456789012:function:fn", + DesiredState: "RUNNING", + }) + require.NoError(t, err) + pipes.WaitPipeRunning(t, b, pipeName) + + desc := tt.description + updated, err := b.UpdatePipe(context.Background(), pipeName, pipes.UpdatePipeInput{ + RoleARN: "arn:aws:iam::123456789012:role/r", + Description: &desc, + DesiredState: desiredState, + }) + require.NoError(t, err) + assert.Equal(t, "UPDATING", updated.CurrentState, "UpdatePipe should return UPDATING state") - desc := tt.description - updated, err := b.UpdatePipe(context.Background(), pipeName, pipes.UpdatePipeInput{ - RoleARN: "arn:aws:iam::123456789012:role/r", - Description: &desc, - DesiredState: desiredState, - }) - require.NoError(t, err) - assert.Equal(t, "UPDATING", updated.CurrentState, "UpdatePipe should return UPDATING state") + time.Sleep(20 * time.Millisecond) + synctest.Wait() - require.Eventually(t, func() bool { p, e := b.GetPipe(context.Background(), pipeName) - - return e == nil && p.CurrentState == tt.wantEventualState - }, 500*time.Millisecond, 5*time.Millisecond) + require.NoError(t, e) + assert.Equal(t, tt.wantEventualState, p.CurrentState) + }) }) } } @@ -187,26 +194,28 @@ func TestLifecycle_Deleting(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := auditNewBackend() - pipeName := tt.name + "-pipe" - _, err := b.CreatePipe(context.Background(), pipes.CreatePipeInput{ - RoleARN: "arn:aws:iam::123456789012:role/r", - Name: pipeName, - Source: "arn:aws:sqs:us-west-2:123456789012:q", - Target: "arn:aws:lambda:us-west-2:123456789012:function:fn", - DesiredState: "RUNNING", - }) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := auditNewBackend() + pipeName := tt.name + "-pipe" + _, err := b.CreatePipe(context.Background(), pipes.CreatePipeInput{ + RoleARN: "arn:aws:iam::123456789012:role/r", + Name: pipeName, + Source: "arn:aws:sqs:us-west-2:123456789012:q", + Target: "arn:aws:lambda:us-west-2:123456789012:function:fn", + DesiredState: "RUNNING", + }) + require.NoError(t, err) - deleted, err := b.DeletePipe(context.Background(), pipeName) - require.NoError(t, err) - assert.Equal(t, "DELETING", deleted.CurrentState, "DeletePipe should return DELETING state") + deleted, err := b.DeletePipe(context.Background(), pipeName) + require.NoError(t, err) + assert.Equal(t, "DELETING", deleted.CurrentState, "DeletePipe should return DELETING state") - require.Eventually(t, func() bool { - _, e := b.GetPipe(context.Background(), pipeName) + time.Sleep(20 * time.Millisecond) + synctest.Wait() - return e != nil - }, 500*time.Millisecond, 5*time.Millisecond, "pipe should be removed after DELETING transition") + _, e := b.GetPipe(context.Background(), pipeName) + assert.Error(t, e, "pipe should be removed after DELETING transition") + }) }) } } @@ -261,54 +270,56 @@ func TestLifecycle_StartStop(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := b2Backend() - _, err := b.CreatePipe(context.Background(), pipes.CreatePipeInput{ - RoleARN: "arn:aws:iam::123456789012:role/r", - Name: tt.name, - Source: b2SQSSource, - Target: b2ECSTarget, - DesiredState: "RUNNING", - TargetParameters: &pipes.TargetParameters{ - EcsTaskParameters: &pipes.ECSTaskTargetParameters{ - TaskDefinitionArn: "arn:aws:ecs:us-east-1:123456789012:task-definition/td:1", - LaunchType: "FARGATE", - NetworkConfiguration: &pipes.NetworkConfiguration{ - AwsvpcConfiguration: &pipes.AwsVpcConfiguration{ - Subnets: []string{"subnet-aaa"}, + synctest.Test(t, func(t *testing.T) { + b := b2Backend() + _, err := b.CreatePipe(context.Background(), pipes.CreatePipeInput{ + RoleARN: "arn:aws:iam::123456789012:role/r", + Name: tt.name, + Source: b2SQSSource, + Target: b2ECSTarget, + DesiredState: "RUNNING", + TargetParameters: &pipes.TargetParameters{ + EcsTaskParameters: &pipes.ECSTaskTargetParameters{ + TaskDefinitionArn: "arn:aws:ecs:us-east-1:123456789012:task-definition/td:1", + LaunchType: "FARGATE", + NetworkConfiguration: &pipes.NetworkConfiguration{ + AwsvpcConfiguration: &pipes.AwsVpcConfiguration{ + Subnets: []string{"subnet-aaa"}, + }, }, }, }, - }, - }) - require.NoError(t, err) - pipes.WaitPipeRunning(t, b, tt.name) + }) + require.NoError(t, err) + pipes.WaitPipeRunning(t, b, tt.name) - stopped, err := b.StopPipe(context.Background(), tt.name) - require.NoError(t, err) - assert.Equal(t, "STOPPING", stopped.CurrentState) + stopped, err := b.StopPipe(context.Background(), tt.name) + require.NoError(t, err) + assert.Equal(t, "STOPPING", stopped.CurrentState) - require.Eventually(t, func() bool { - p, e := b.GetPipe(context.Background(), tt.name) + time.Sleep(20 * time.Millisecond) + synctest.Wait() - return e == nil && p.CurrentState == "STOPPED" - }, 500*time.Millisecond, 5*time.Millisecond) + p, e := b.GetPipe(context.Background(), tt.name) + require.NoError(t, e) + assert.Equal(t, "STOPPED", p.CurrentState) - started, err := b.StartPipe(context.Background(), tt.name) - require.NoError(t, err) - assert.Equal(t, "STARTING", started.CurrentState) + started, err := b.StartPipe(context.Background(), tt.name) + require.NoError(t, err) + assert.Equal(t, "STARTING", started.CurrentState) - require.Eventually(t, func() bool { - p, e := b.GetPipe(context.Background(), tt.name) + time.Sleep(20 * time.Millisecond) + synctest.Wait() - return e == nil && p.CurrentState == "RUNNING" - }, 500*time.Millisecond, 5*time.Millisecond) + p, err = b.GetPipe(context.Background(), tt.name) + require.NoError(t, err) + assert.Equal(t, "RUNNING", p.CurrentState) - p, err := b.GetPipe(context.Background(), tt.name) - require.NoError(t, err) - ecs := p.TargetParameters.EcsTaskParameters - require.NotNil(t, ecs.NetworkConfiguration) - assert.Equal(t, "subnet-aaa", - ecs.NetworkConfiguration.AwsvpcConfiguration.Subnets[0]) + ecs := p.TargetParameters.EcsTaskParameters + require.NotNil(t, ecs.NetworkConfiguration) + assert.Equal(t, "subnet-aaa", + ecs.NetworkConfiguration.AwsvpcConfiguration.Subnets[0]) + }) }) } } @@ -328,35 +339,37 @@ func TestLifecycle_Delete(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := b2Backend() - _, err := b.CreatePipe(context.Background(), pipes.CreatePipeInput{ - RoleARN: "arn:aws:iam::123456789012:role/r", - Name: tt.name, - Source: b2SQSSource, - Target: "arn:aws:batch:us-east-1:123456789012:job-queue/q", - TargetParameters: &pipes.TargetParameters{ - BatchJobParameters: &pipes.BatchJobTargetParameters{ - JobDefinition: "jd", - JobName: "job", - DependsOn: []pipes.BatchJobDependency{ - {JobID: "parent-job", Type: "SEQUENTIAL"}, + synctest.Test(t, func(t *testing.T) { + b := b2Backend() + _, err := b.CreatePipe(context.Background(), pipes.CreatePipeInput{ + RoleARN: "arn:aws:iam::123456789012:role/r", + Name: tt.name, + Source: b2SQSSource, + Target: "arn:aws:batch:us-east-1:123456789012:job-queue/q", + TargetParameters: &pipes.TargetParameters{ + BatchJobParameters: &pipes.BatchJobTargetParameters{ + JobDefinition: "jd", + JobName: "job", + DependsOn: []pipes.BatchJobDependency{ + {JobID: "parent-job", Type: "SEQUENTIAL"}, + }, }, }, - }, - }) - require.NoError(t, err) + }) + require.NoError(t, err) - deleted, err := b.DeletePipe(context.Background(), tt.name) - require.NoError(t, err) - assert.Equal(t, "DELETING", deleted.CurrentState) - assert.Equal(t, "parent-job", - deleted.TargetParameters.BatchJobParameters.DependsOn[0].JobID) + deleted, err := b.DeletePipe(context.Background(), tt.name) + require.NoError(t, err) + assert.Equal(t, "DELETING", deleted.CurrentState) + assert.Equal(t, "parent-job", + deleted.TargetParameters.BatchJobParameters.DependsOn[0].JobID) - require.Eventually(t, func() bool { - _, e := b.GetPipe(context.Background(), tt.name) + time.Sleep(20 * time.Millisecond) + synctest.Wait() - return e != nil - }, 500*time.Millisecond, 5*time.Millisecond) + _, e := b.GetPipe(context.Background(), tt.name) + assert.Error(t, e) + }) }) } } @@ -392,39 +405,42 @@ func TestPipeStateTransitions(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := newPipeBackend() - pipeName := "transition-" + tt.name + synctest.Test(t, func(t *testing.T) { + b := newPipeBackend() + pipeName := "transition-" + tt.name - _, err := b.CreatePipe(context.Background(), pipes.CreatePipeInput{ - RoleARN: "arn:aws:iam::123456789012:role/r", - Name: pipeName, - Source: "arn:aws:sqs:us-east-1:000000000000:queue", - Target: "arn:aws:lambda:us-east-1:000000000000:function:fn", - DesiredState: tt.initialState, - }) - require.NoError(t, err) + _, err := b.CreatePipe(context.Background(), pipes.CreatePipeInput{ + RoleARN: "arn:aws:iam::123456789012:role/r", + Name: pipeName, + Source: "arn:aws:sqs:us-east-1:000000000000:queue", + Target: "arn:aws:lambda:us-east-1:000000000000:function:fn", + DesiredState: tt.initialState, + }) + require.NoError(t, err) + + // Perform the action. + var result *pipes.Pipe + switch tt.action { + case "stop": + result, err = b.StopPipe(context.Background(), pipeName) + case "start": + result, err = b.StartPipe(context.Background(), pipeName) + } + require.NoError(t, err) - // Perform the action. - var result *pipes.Pipe - switch tt.action { - case "stop": - result, err = b.StopPipe(context.Background(), pipeName) - case "start": - result, err = b.StartPipe(context.Background(), pipeName) - } - require.NoError(t, err) + // Verify intermediate state in the synchronous return value. + assert.Equal(t, tt.wantImmediate, result.CurrentState, + "expected intermediate state %q", tt.wantImmediate) - // Verify intermediate state in the synchronous return value. - assert.Equal(t, tt.wantImmediate, result.CurrentState, - "expected intermediate state %q", tt.wantImmediate) + // Wait for the async transition to complete. + time.Sleep(30 * time.Millisecond) + synctest.Wait() - // Wait for the async transition to complete. - require.Eventually(t, func() bool { p, e := b.GetPipe(context.Background(), pipeName) - - return e == nil && p.CurrentState == tt.wantEventualFinal - }, 2*time.Second, 10*time.Millisecond, - "timed out waiting for pipe to reach %q", tt.wantEventualFinal) + require.NoError(t, e) + assert.Equal(t, tt.wantEventualFinal, p.CurrentState, + "expected pipe to reach %q", tt.wantEventualFinal) + }) }) } } @@ -785,30 +801,37 @@ func TestUpdatePipe_UpdatesLastModifiedTime(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := auditNewBackend() - _, err := b.CreatePipe(context.Background(), pipes.CreatePipeInput{ - RoleARN: "arn:aws:iam::123456789012:role/r", - Name: tt.name + "-pipe", - Source: "arn:aws:sqs:us-west-2:123456789012:q", - Target: "arn:aws:lambda:us-west-2:123456789012:function:fn", - DesiredState: "RUNNING", - }) - require.NoError(t, err) - pipes.WaitPipeRunning(t, b, tt.name+"-pipe") + synctest.Test(t, func(t *testing.T) { + b := auditNewBackend() + _, err := b.CreatePipe(context.Background(), pipes.CreatePipeInput{ + RoleARN: "arn:aws:iam::123456789012:role/r", + Name: tt.name + "-pipe", + Source: "arn:aws:sqs:us-west-2:123456789012:q", + Target: "arn:aws:lambda:us-west-2:123456789012:function:fn", + DesiredState: "RUNNING", + }) + require.NoError(t, err) + pipes.WaitPipeRunning(t, b, tt.name+"-pipe") - before, _ := b.GetPipe(context.Background(), tt.name+"-pipe") - time.Sleep(2 * time.Millisecond) + before, _ := b.GetPipe(context.Background(), tt.name+"-pipe") + time.Sleep(2 * time.Millisecond) - updatedDesc := "updated" - _, err = b.UpdatePipe(context.Background(), tt.name+"-pipe", pipes.UpdatePipeInput{ - RoleARN: "arn:aws:iam::123456789012:role/r", - Description: &updatedDesc, - }) - require.NoError(t, err) + updatedDesc := "updated" + _, err = b.UpdatePipe(context.Background(), tt.name+"-pipe", pipes.UpdatePipeInput{ + RoleARN: "arn:aws:iam::123456789012:role/r", + Description: &updatedDesc, + }) + require.NoError(t, err) - after, _ := b.GetPipe(context.Background(), tt.name+"-pipe") - assert.True(t, after.LastModifiedTime.After(before.LastModifiedTime), - "LastModifiedTime should increase after update") + after, _ := b.GetPipe(context.Background(), tt.name+"-pipe") + assert.True(t, after.LastModifiedTime.After(before.LastModifiedTime), + "LastModifiedTime should increase after update") + + // Drain UpdatePipe's pending UPDATING->RUNNING transition + // goroutine before the bubble closes. + time.Sleep(20 * time.Millisecond) + synctest.Wait() + }) }) } } diff --git a/services/pipes/runner_test.go b/services/pipes/runner_test.go index c2e7cba5a6..52c1161ec9 100644 --- a/services/pipes/runner_test.go +++ b/services/pipes/runner_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "sync" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -485,51 +486,58 @@ func TestPipeSourceFiltering(t *testing.T) { func TestPipesRunner_ShardIteratorSweep(t *testing.T) { t.Parallel() - backend := newTestPipeBackend(t) - kinesisARN := "arn:aws:kinesis:us-east-1:000000000000:stream/sweep-stream" - lambdaARN := "arn:aws:lambda:us-east-1:000000000000:function:my-fn" - createTestPipe(t, backend, "sweep-pipe", kinesisARN, lambdaARN, "RUNNING") - - reader := &fakeKinesisReader{ - shardIDs: []string{"shard-1"}, - pending: map[string][]pipes.KinesisRecord{}, - } - runner := pipes.NewRunner(backend) - runner.SetKinesisReader(reader) - - ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) - defer cancel() - runner.Start(ctx) - - getIterCalls := func() int { - reader.mu.Lock() - defer reader.mu.Unlock() - - return reader.getIterCalls - } - - // The runner's first background tick polls the running pipe and caches - // one shard iterator for it. - require.Eventually(t, func() bool { return getIterCalls() >= 1 }, 3*time.Second, 20*time.Millisecond, - "expected the runner's first tick to request a shard iterator for the running pipe") - - _, err := backend.StopPipe(context.Background(), "sweep-pipe") - require.NoError(t, err) - pipes.WaitPipeStopped(t, backend, "sweep-pipe") - - // Give the background ticker at least one full cycle while the pipe is - // stopped, so the sweep observes it outside the running set and prunes - // its cached shard iterator (the pipe itself is not polled while - // stopped, so this cannot be observed until it runs again below). - time.Sleep(1500 * time.Millisecond) - - _, err = backend.StartPipe(context.Background(), "sweep-pipe") - require.NoError(t, err) - pipes.WaitPipeRunning(t, backend, "sweep-pipe") - - require.Eventually(t, func() bool { return getIterCalls() >= 2 }, 3*time.Second, 20*time.Millisecond, - "expected a fresh GetShardIterator call after the pipe restarted, proving the sweep "+ - "pruned the stale cache entry while the pipe was stopped") + synctest.Test(t, func(t *testing.T) { + backend := newTestPipeBackend(t) + kinesisARN := "arn:aws:kinesis:us-east-1:000000000000:stream/sweep-stream" + lambdaARN := "arn:aws:lambda:us-east-1:000000000000:function:my-fn" + createTestPipe(t, backend, "sweep-pipe", kinesisARN, lambdaARN, "RUNNING") + + reader := &fakeKinesisReader{ + shardIDs: []string{"shard-1"}, + pending: map[string][]pipes.KinesisRecord{}, + } + runner := pipes.NewRunner(backend) + runner.SetKinesisReader(reader) + + ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) + defer cancel() + runner.Start(ctx) + + getIterCalls := func() int { + reader.mu.Lock() + defer reader.mu.Unlock() + + return reader.getIterCalls + } + + // The runner ticks every second; let the first tick poll the running + // pipe and cache one shard iterator for it. + time.Sleep(1100 * time.Millisecond) + synctest.Wait() + require.GreaterOrEqual(t, getIterCalls(), 1, + "expected the runner's first tick to request a shard iterator for the running pipe") + + _, err := backend.StopPipe(context.Background(), "sweep-pipe") + require.NoError(t, err) + pipes.WaitPipeStopped(t, backend, "sweep-pipe") + + // Give the background ticker at least one full cycle while the pipe is + // stopped, so the sweep observes it outside the running set and prunes + // its cached shard iterator (the pipe itself is not polled while + // stopped, so this cannot be observed until it runs again below). + time.Sleep(1500 * time.Millisecond) + synctest.Wait() + + _, err = backend.StartPipe(context.Background(), "sweep-pipe") + require.NoError(t, err) + pipes.WaitPipeRunning(t, backend, "sweep-pipe") + + time.Sleep(1100 * time.Millisecond) + synctest.Wait() + require.GreaterOrEqual(t, getIterCalls(), 2, + "expected a fresh GetShardIterator call after the pipe restarted, proving the sweep "+ + "pruned the stale cache entry while the pipe was stopped") + }) } // TestPipesRunner_InputTemplate tests that TargetParameters.InputTemplate overrides default payload. diff --git a/services/rds/db_clusters.go b/services/rds/db_clusters.go index c1bdc8e52f..d3c7b084c9 100644 --- a/services/rds/db_clusters.go +++ b/services/rds/db_clusters.go @@ -9,6 +9,12 @@ import ( "time" ) +// cloneDBClusterMutableSlices deep-copies DBClusterMembers before a caller sees them. +// FailoverDBCluster writes IsClusterWriter in place, so a shallow "cp := *cluster" would share the backing array. +func cloneDBClusterMutableSlices(c *DBCluster) { + c.DBClusterMembers = slices.Clone(c.DBClusterMembers) +} + // CreateDBCluster creates a new DB cluster. func (b *InMemoryBackend) CreateDBCluster( id, engine, masterUser, dbName, paramGroupName string, @@ -53,6 +59,7 @@ func (b *InMemoryBackend) CreateDBCluster( } cp := *cluster + cloneDBClusterMutableSlices(&cp) return &cp, nil } @@ -161,6 +168,7 @@ func (b *InMemoryBackend) DescribeDBClusters(id string) ([]DBCluster, error) { return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, id) } cp := *cluster + cloneDBClusterMutableSlices(&cp) b.overlayFailoverStatusRLocked(&cp) return []DBCluster{cp}, nil @@ -168,6 +176,7 @@ func (b *InMemoryBackend) DescribeDBClusters(id string) ([]DBCluster, error) { result := make([]DBCluster, 0, b.clusters.Len()) for _, cluster := range b.clusters.All() { cp := *cluster + cloneDBClusterMutableSlices(&cp) b.overlayFailoverStatusRLocked(&cp) result = append(result, cp) } @@ -309,6 +318,7 @@ func (b *InMemoryBackend) DeleteDBClusterWithOptions( } cp := *cluster + cloneDBClusterMutableSlices(&cp) // Clear the cluster association on any member instances so they appear standalone. for _, member := range cluster.DBClusterMembers { if inst, ok := b.instances.Get(normalizeID(member.DBInstanceIdentifier)); ok { @@ -497,6 +507,7 @@ func (b *InMemoryBackend) ModifyDBCluster( b.cascadeInstanceParameterGroupLocked(cluster, opts.DBInstanceParameterGroupName) } cp := *cluster + cloneDBClusterMutableSlices(&cp) return &cp, nil } @@ -527,6 +538,7 @@ func (b *InMemoryBackend) StartDBCluster(id string) (*DBCluster, error) { } cluster.Status = instanceStatusAvailable cp := *cluster + cloneDBClusterMutableSlices(&cp) return &cp, nil } @@ -544,6 +556,7 @@ func (b *InMemoryBackend) StopDBCluster(id string) (*DBCluster, error) { } cluster.Status = "stopped" cp := *cluster + cloneDBClusterMutableSlices(&cp) return &cp, nil } @@ -592,6 +605,7 @@ func (b *InMemoryBackend) RestoreDBClusterFromSnapshot( } b.clusters.Put(cluster) cp := *cluster + cloneDBClusterMutableSlices(&cp) return &cp, nil } @@ -642,6 +656,7 @@ func (b *InMemoryBackend) RestoreDBClusterToPointInTime( } b.clusters.Put(cluster) cp := *cluster + cloneDBClusterMutableSlices(&cp) return &cp, nil } @@ -884,6 +899,7 @@ func (b *InMemoryBackend) FailoverDBCluster( } cluster.Status = instanceStatusAvailable cp := *cluster + cloneDBClusterMutableSlices(&cp) return &cp, nil } @@ -919,6 +935,7 @@ func (b *InMemoryBackend) RebootDBCluster(clusterID string) (*DBCluster, error) b.clusterReadyAt[cluster.DBClusterIdentifier] = time.Now().Add(instanceTransitionDelay) b.scheduleReconcilerLocked() cp := *cluster + cloneDBClusterMutableSlices(&cp) result = &cp }() @@ -968,6 +985,7 @@ func (b *InMemoryBackend) PromoteReadReplicaDBCluster(clusterID string) (*DBClus cluster.ReplicationSourceIdentifier = "" cluster.Status = instanceStatusAvailable cp := *cluster + cloneDBClusterMutableSlices(&cp) return &cp, nil } @@ -1064,6 +1082,7 @@ func (b *InMemoryBackend) ModifyCurrentDBClusterCapacity( } cluster.ServerlessCapacity = capacity cp := *cluster + cloneDBClusterMutableSlices(&cp) return &cp, nil } @@ -1114,6 +1133,7 @@ func (b *InMemoryBackend) RestoreDBClusterFromS3( } b.clusters.Put(cluster) cp := *cluster + cloneDBClusterMutableSlices(&cp) return &cp, nil } diff --git a/services/rds/db_clusters_operations_test.go b/services/rds/db_clusters_operations_test.go index fa40acd88e..f1db786953 100644 --- a/services/rds/db_clusters_operations_test.go +++ b/services/rds/db_clusters_operations_test.go @@ -5,6 +5,7 @@ import ( "net/http" "net/url" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -571,10 +572,11 @@ func TestDeletionProtectionCanBeDisabled(t *testing.T) { // timing assertions. The backend uses 250ms. const transitionDelay = 250 * time.Millisecond -// TestRebootDBClusterDelayedTransition exercises the delayed lifecycle goroutine -// scheduled by RebootDBCluster via runDelayed. It verifies both that the +// TestRebootDBClusterDelayedTransition exercises the delayed lifecycle +// goroutine scheduled by RebootDBCluster. It verifies both that the // transition still fires after the delay and that Close cancels in-flight // transitions promptly without mutating state after shutdown (the leak fix). +// Runs under synctest so the delay is virtual time, not wall clock. func TestRebootDBClusterDelayedTransition(t *testing.T) { t.Parallel() @@ -601,56 +603,54 @@ func TestRebootDBClusterDelayedTransition(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := rds.NewInMemoryBackend("123456789012", "us-east-1") - t.Cleanup(b.Close) - - _, err := b.CreateDBCluster( - "my-cluster", - "aurora-mysql", - "admin", - "", - "", - 0, - nil, - rds.DBClusterOptions{}, - ) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := rds.NewInMemoryBackend("123456789012", "us-east-1") + defer b.Close() - _, err = b.RebootDBCluster("my-cluster") - require.NoError(t, err) + _, err := b.CreateDBCluster( + "my-cluster", + "aurora-mysql", + "admin", + "", + "", + 0, + nil, + rds.DBClusterOptions{}, + ) + require.NoError(t, err) - if tt.closeEarly { - // Close immediately, before the transition delay elapses. The - // delayed goroutine must observe stopCh and return without - // mutating state. Close must block only briefly on b.wg.Wait(). - start := time.Now() - b.Close() - elapsed := time.Since(start) - - if tt.wantFastClose { - require.Less(t, elapsed, transitionDelay, - "Close should not wait out the full transition delay") - } + _, err = b.RebootDBCluster("my-cluster") + require.NoError(t, err) - clusters, derr := b.DescribeDBClusters("my-cluster") - require.NoError(t, derr) - require.Equal(t, tt.wantStatus, clusters[0].Status) + if tt.closeEarly { + // Close immediately, before the transition delay elapses. The + // delayed goroutine must observe stopCh and return without + // mutating state. Close must block only briefly on b.wg.Wait(). + start := time.Now() + b.Close() + elapsed := time.Since(start) - return - } + if tt.wantFastClose { + require.Less(t, elapsed, transitionDelay, + "Close should not wait out the full transition delay") + } - // Wait for the delayed transition to fire, then verify the status - // and a clean Close afterward. - require.Eventually(t, func() bool { - clusters, derr := b.DescribeDBClusters("my-cluster") - if derr != nil || len(clusters) == 0 { - return false + clusters, derr := b.DescribeDBClusters("my-cluster") + require.NoError(t, derr) + require.Equal(t, tt.wantStatus, clusters[0].Status) + + return } - return clusters[0].Status == tt.wantStatus - }, 2*time.Second, 10*time.Millisecond) + // Advance virtual time past the delay plus the reconciler's own + // tick period, since the transition only lands on a tick boundary. + time.Sleep(2 * transitionDelay) - b.Close() + clusters, derr := b.DescribeDBClusters("my-cluster") + require.NoError(t, derr) + require.Len(t, clusters, 1) + require.Equal(t, tt.wantStatus, clusters[0].Status) + }) }) } } diff --git a/services/rds/db_clusters_race_test.go b/services/rds/db_clusters_race_test.go new file mode 100644 index 0000000000..5d11f92166 --- /dev/null +++ b/services/rds/db_clusters_race_test.go @@ -0,0 +1,109 @@ +package rds_test + +import ( + "fmt" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/rds" +) + +// TestDescribeDBClusters_RacesWithClusterMemberWriters guards cloneDBClusterMutableSlices. +// Each case exercises an in-place DBClusterMembers writer that used to share its backing array with a live cluster. +func TestDescribeDBClusters_RacesWithClusterMemberWriters(t *testing.T) { + t.Parallel() + + tests := []struct { + setup func(t *testing.T, b *rds.InMemoryBackend) + mutate func(b *rds.InMemoryBackend, i int) + name string + }{ + { + name: "failover", + setup: func(t *testing.T, b *rds.InMemoryBackend) { + t.Helper() + + _, err := b.CreateDBInstance("race-writer", "aurora-mysql", "db.r5.large", "", "", "", 20, + rds.DBInstanceOptions{DBClusterIdentifier: "race-cluster"}) + require.NoError(t, err) + _, err = b.CreateDBInstance("race-reader", "aurora-mysql", "db.r5.large", "", "", "", 20, + rds.DBInstanceOptions{DBClusterIdentifier: "race-cluster"}) + require.NoError(t, err) + }, + mutate: func(b *rds.InMemoryBackend, i int) { + target := "race-writer" + if i%2 == 0 { + target = "race-reader" + } + + _, _ = b.FailoverDBCluster("race-cluster", target) + }, + }, + { + // DeleteDBInstance compacts DBClusterMembers with slices.DeleteFunc, a second in-place writer. + // Deleting the non-last of two freshly added members forces DeleteFunc to shift-write the survivor. + name: "delete_instance", + mutate: func(b *rds.InMemoryBackend, i int) { + first := fmt.Sprintf("race-tmp-a-%d", i) + second := fmt.Sprintf("race-tmp-b-%d", i) + + _, _ = b.CreateDBInstance(first, "aurora-mysql", "db.r5.large", "", "", "", 20, + rds.DBInstanceOptions{DBClusterIdentifier: "race-cluster"}) + _, _ = b.CreateDBInstance(second, "aurora-mysql", "db.r5.large", "", "", "", 20, + rds.DBInstanceOptions{DBClusterIdentifier: "race-cluster"}) + _, _ = b.DeleteDBInstanceWithOptions(first, true, "", true) + _, _ = b.DeleteDBInstanceWithOptions(second, true, "", true) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + + _, err := b.CreateDBCluster("race-cluster", "aurora-mysql", "admin", "", "", 0, nil, rds.DBClusterOptions{}) + require.NoError(t, err) + + if tt.setup != nil { + tt.setup(t, b) + } + + const iterations = 2000 + + var wg sync.WaitGroup + + wg.Add(2) + + go func() { + defer wg.Done() + + for range iterations { + clusters, describeErr := b.DescribeDBClusters("race-cluster") + if describeErr != nil { + continue + } + + for _, c := range clusters { + for _, m := range c.DBClusterMembers { + _ = m.IsClusterWriter + } + } + } + }() + + go func() { + defer wg.Done() + + for i := range iterations { + tt.mutate(b, i) + } + }() + + wg.Wait() + }) + } +} diff --git a/services/rds/db_instances_operations_test.go b/services/rds/db_instances_operations_test.go index 2ae0a16b6f..89104a1531 100644 --- a/services/rds/db_instances_operations_test.go +++ b/services/rds/db_instances_operations_test.go @@ -5,6 +5,7 @@ import ( "net/http" "net/url" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -51,39 +52,42 @@ func TestRDSBackend_InstanceModifyTransitionAndDeletePublishesEvents(t *testing. t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := rds.NewInMemoryBackend("000000000000", "us-east-1") - t.Cleanup(b.Close) - const instanceID = "transition-db" + synctest.Test(t, func(t *testing.T) { + b := rds.NewInMemoryBackend("000000000000", "us-east-1") + defer b.Close() + const instanceID = "transition-db" - created, err := b.CreateDBInstance(instanceID, "postgres", "", "", "", "", 20, rds.DBInstanceOptions{}) - require.NoError(t, err) - assert.Equal(t, "creating", created.DBInstanceStatus) + created, err := b.CreateDBInstance(instanceID, "postgres", "", "", "", "", 20, rds.DBInstanceOptions{}) + require.NoError(t, err) + assert.Equal(t, "creating", created.DBInstanceStatus) - modified, err := b.ModifyDBInstance(instanceID, "db.r5.large", 100, rds.DBInstanceOptions{}) - require.NoError(t, err) - assert.Equal(t, "modifying", modified.DBInstanceStatus) + modified, err := b.ModifyDBInstance(instanceID, "db.r5.large", 100, rds.DBInstanceOptions{}) + require.NoError(t, err) + assert.Equal(t, "modifying", modified.DBInstanceStatus) - require.Eventually(t, func() bool { - instances, describeErr := b.DescribeDBInstances(instanceID) - if describeErr != nil || len(instances) != 1 { - return false - } + // Sleep past the delay plus the reconciler's own tick period, + // since the transition only lands on a tick boundary. + time.Sleep(2 * transitionDelay) - return instances[0].DBInstanceStatus == "available" && instances[0].DBInstanceClass == "db.r5.large" - }, 3*time.Second, 20*time.Millisecond) + instances, describeErr := b.DescribeDBInstances(instanceID) + require.NoError(t, describeErr) + require.Len(t, instances, 1) + assert.Equal(t, "available", instances[0].DBInstanceStatus) + assert.Equal(t, "db.r5.large", instances[0].DBInstanceClass) - deleted, err := b.DeleteDBInstance(instanceID) - require.NoError(t, err) - assert.Equal(t, "deleting", deleted.DBInstanceStatus) - _, err = b.DescribeDBInstances(instanceID) - require.ErrorIs(t, err, rds.ErrInstanceNotFound) - - messages := rds.EventMessagesForSource(b, instanceID) - assert.Contains(t, messages, "DB instance created") - assert.Contains(t, messages, "DB instance is now available") - assert.Contains(t, messages, "DB instance modification started") - assert.Contains(t, messages, "DB instance deletion started") - assert.Contains(t, messages, "DB instance deleted") + deleted, err := b.DeleteDBInstance(instanceID) + require.NoError(t, err) + assert.Equal(t, "deleting", deleted.DBInstanceStatus) + _, err = b.DescribeDBInstances(instanceID) + require.ErrorIs(t, err, rds.ErrInstanceNotFound) + + messages := rds.EventMessagesForSource(b, instanceID) + assert.Contains(t, messages, "DB instance created") + assert.Contains(t, messages, "DB instance is now available") + assert.Contains(t, messages, "DB instance modification started") + assert.Contains(t, messages, "DB instance deletion started") + assert.Contains(t, messages, "DB instance deleted") + }) }) } } diff --git a/services/rds/export_test.go b/services/rds/export_test.go index d614fcc384..2cd151f435 100644 --- a/services/rds/export_test.go +++ b/services/rds/export_test.go @@ -21,6 +21,21 @@ func FlushInstanceLifecycle(b *InMemoryBackend) { } } +// FlushClusterLifecycle immediately transitions all rebooting clusters to available. +// This is a test helper that bypasses the reconciler delay. +func FlushClusterLifecycle(b *InMemoryBackend) { + b.mu.Lock("FlushClusterLifecycle") + defer b.mu.Unlock() + + for _, c := range b.clusters.All() { + if c.Status == "rebooting" { + c.Status = instanceStatusAvailable + } + + delete(b.clusterReadyAt, c.DBClusterIdentifier) + } +} + // RDSIDFromARNForTest exposes rdsIDFromARN for unit tests. func RDSIDFromARNForTest(arnOrID string) string { return rdsIDFromARN(arnOrID) diff --git a/services/rds/fis_test.go b/services/rds/fis_test.go index 3baddab025..9da702f6c4 100644 --- a/services/rds/fis_test.go +++ b/services/rds/fis_test.go @@ -3,6 +3,7 @@ package rds_test import ( "context" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -166,27 +167,42 @@ func TestRDS_ExecuteFISAction_FailoverDBCluster(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - h := newFISRDSHandler(t) + run := func(t *testing.T) { + t.Helper() - err := h.ExecuteFISAction(t.Context(), service.FISActionExecution{ - ActionID: "aws:rds:failover-db-cluster", - Targets: tt.targets, - Duration: tt.duration, - }) + h := newFISRDSHandler(t) - if tt.wantErr { - require.Error(t, err) - } else { - require.NoError(t, err) - } + err := h.ExecuteFISAction(t.Context(), service.FISActionExecution{ + ActionID: "aws:rds:failover-db-cluster", + Targets: tt.targets, + Duration: tt.duration, + }) - // For clusters with non-zero duration, the fault should eventually clear. - if tt.duration > 0 && len(tt.targets) > 0 { - time.Sleep(tt.duration + 50*time.Millisecond) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } - id := rdsIDFromARNForTest(tt.targets[0]) - assert.False(t, h.Backend.IsClusterFailoverActive(id), - "failover fault should have expired after duration") + // For clusters with non-zero duration, the fault should eventually clear. + if tt.duration > 0 && len(tt.targets) > 0 { + time.Sleep(tt.duration + 50*time.Millisecond) + synctest.Wait() + + id := rdsIDFromARNForTest(tt.targets[0]) + assert.False(t, h.Backend.IsClusterFailoverActive(id), + "failover fault should have expired after duration") + } + } + + // Only the timed-fault case has a real timer to cross; the + // dur==0 case's fault-clearing goroutine blocks on ctx + // cancellation, which t.Context() only does at test cleanup -- + // after a bubble would have to exit -- so it cannot be bubbled. + if tt.duration > 0 { + synctest.Test(t, run) + } else { + run(t) } }) } @@ -231,29 +247,31 @@ func TestRDS_FISActions_FailoverHasDurationParam(t *testing.T) { func TestRDS_ExecuteFISAction_FailoverDBCluster_CtxCancel(t *testing.T) { t.Parallel() - h := newFISRDSHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newFISRDSHandler(t) - ctx, cancel := context.WithCancel(t.Context()) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() - const clusterTarget = "arn:aws:rds:us-east-1:000000000000:cluster/cancel-cluster" + const clusterTarget = "arn:aws:rds:us-east-1:000000000000:cluster/cancel-cluster" - // Activate indefinite fault (dur==0). - err := h.ExecuteFISAction(ctx, service.FISActionExecution{ - ActionID: "aws:rds:failover-db-cluster", - Targets: []string{clusterTarget}, - Duration: 0, - }) - require.NoError(t, err) + // Activate indefinite fault (dur==0). + err := h.ExecuteFISAction(ctx, service.FISActionExecution{ + ActionID: "aws:rds:failover-db-cluster", + Targets: []string{clusterTarget}, + Duration: 0, + }) + require.NoError(t, err) - assert.True(t, h.Backend.IsClusterFailoverActive("cancel-cluster"), "fault should be active") + assert.True(t, h.Backend.IsClusterFailoverActive("cancel-cluster"), "fault should be active") - // Cancel ctx (simulates StopExperiment). - cancel() + // Cancel ctx (simulates StopExperiment) and let the fault-clearing + // goroutine run to completion. + cancel() + synctest.Wait() - // Fault should clear promptly. - require.Eventually(t, func() bool { - return !h.Backend.IsClusterFailoverActive("cancel-cluster") - }, 2*time.Second, 20*time.Millisecond, "fault should clear after ctx cancel") + assert.False(t, h.Backend.IsClusterFailoverActive("cancel-cluster"), "fault should clear after ctx cancel") + }) } func TestRDS_IsClusterFailoverActive_LazyEviction(t *testing.T) { diff --git a/services/rds/id_generation_test.go b/services/rds/id_generation_test.go new file mode 100644 index 0000000000..b5071d7dc2 --- /dev/null +++ b/services/rds/id_generation_test.go @@ -0,0 +1,37 @@ +package rds_test + +import ( + "regexp" + "testing" + "testing/synctest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/rds" +) + +// reservedDBInstanceIDPattern locks in the fix for +// PurchaseReservedDBInstancesOffering's auto-generated ReservedDBInstanceId, which used to collide under synctest. +var reservedDBInstanceIDPattern = regexp.MustCompile( + `^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`, +) + +func TestRDSBackend_ReservedDBInstanceID_Unique(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + b := rds.NewInMemoryBackend("000000000000", "us-east-1") + + ri1, err := b.PurchaseReservedDBInstancesOffering("some-offering-id", "", 1) + require.NoError(t, err) + + ri2, err := b.PurchaseReservedDBInstancesOffering("some-offering-id", "", 1) + require.NoError(t, err) + + assert.NotEqual(t, ri1.ReservedDBInstanceID, ri2.ReservedDBInstanceID, + "two reservations created back-to-back must get distinct IDs") + assert.Regexp(t, reservedDBInstanceIDPattern, ri1.ReservedDBInstanceID) + assert.Regexp(t, reservedDBInstanceIDPattern, ri2.ReservedDBInstanceID) + }) +} diff --git a/services/rds/realclient_instance_cluster_lifecycle_test.go b/services/rds/realclient_instance_cluster_lifecycle_test.go index c859186ac5..aefc40fa01 100644 --- a/services/rds/realclient_instance_cluster_lifecycle_test.go +++ b/services/rds/realclient_instance_cluster_lifecycle_test.go @@ -2,7 +2,6 @@ package rds_test import ( "testing" - "time" "github.com/aws/aws-sdk-go-v2/aws" rdssdk "github.com/aws/aws-sdk-go-v2/service/rds" @@ -13,28 +12,31 @@ import ( "github.com/blackbirdworks/gopherstack/services/rds" ) -// waitForInstanceStatus polls the backend directly (no HTTP round trip) -// until the named instance reaches wantStatus, matching the repo convention -// of require.Eventually over unbubbled sleeps. +// waitForInstanceStatus forces the pending reconciler transition immediately +// instead of polling wall-clock time (gopherstack-jwr13: Eventually flaked +// under CI load because it depended on the background reconciler goroutine's +// own ticker getting scheduled in time). func waitForInstanceStatus(t *testing.T, backend *rds.InMemoryBackend, id, wantStatus string) { t.Helper() - require.Eventually(t, func() bool { - insts, err := backend.DescribeDBInstances(id) + rds.FlushInstanceLifecycle(backend) - return err == nil && len(insts) == 1 && insts[0].DBInstanceStatus == wantStatus - }, time.Second, 5*time.Millisecond) + insts, err := backend.DescribeDBInstances(id) + require.NoError(t, err) + require.Len(t, insts, 1) + require.Equal(t, wantStatus, insts[0].DBInstanceStatus) } // waitForClusterStatus is waitForInstanceStatus's DB cluster counterpart. func waitForClusterStatus(t *testing.T, backend *rds.InMemoryBackend, id, wantStatus string) { t.Helper() - require.Eventually(t, func() bool { - clusters, err := backend.DescribeDBClusters(id) + rds.FlushClusterLifecycle(backend) - return err == nil && len(clusters) == 1 && clusters[0].Status == wantStatus - }, time.Second, 5*time.Millisecond) + clusters, err := backend.DescribeDBClusters(id) + require.NoError(t, err) + require.Len(t, clusters, 1) + require.Equal(t, wantStatus, clusters[0].Status) } // TestRealClient_InstanceClusterLifecycle covers rds's highest-priority typed-client- diff --git a/services/rds/reserved_instances.go b/services/rds/reserved_instances.go index e653c29d8e..5ea7af1e27 100644 --- a/services/rds/reserved_instances.go +++ b/services/rds/reserved_instances.go @@ -3,6 +3,8 @@ package rds import ( "fmt" "time" + + "github.com/google/uuid" ) // PurchaseReservedDBInstancesOffering purchases a reserved DB instance offering. @@ -36,7 +38,9 @@ func (b *InMemoryBackend) PurchaseReservedDBInstancesOffering( } } if reservedDBInstanceID == "" { - reservedDBInstanceID = fmt.Sprintf("ri-%s-%d", offeringID, time.Now().UnixNano()) + // ReservedDBInstanceId has no documented pattern beyond "customer-specified + // identifier"; a bare UUID is a collision-free, real-shaped default. + reservedDBInstanceID = uuid.NewString() } b.mu.Lock("PurchaseReservedDBInstancesOffering") defer b.mu.Unlock() diff --git a/services/redshift/id_generation_test.go b/services/redshift/id_generation_test.go new file mode 100644 index 0000000000..da2a985089 --- /dev/null +++ b/services/redshift/id_generation_test.go @@ -0,0 +1,73 @@ +package redshift_test + +import ( + "regexp" + "testing" + "testing/synctest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/redshift" +) + +// usageLimitIDPattern/reservedNodeIDPattern lock in the fix for IDs that were +// derived from time.Now().UnixNano() and collided under synctest. +var ( + usageLimitIDPattern = regexp.MustCompile(`^ul-[0-9a-f]{16}$`) + reservedNodeIDPattern = regexp.MustCompile(`^rn-[0-9a-f]{16}$`) +) + +func TestRedshiftBackend_IDs_Unique(t *testing.T) { + t.Parallel() + + tests := []struct { + create func(t *testing.T, b *redshift.InMemoryBackend) string + pattern *regexp.Regexp + name string + }{ + { + name: "usage_limit", + pattern: usageLimitIDPattern, + create: func(t *testing.T, b *redshift.InMemoryBackend) string { + t.Helper() + + ul, err := b.CreateUsageLimit("c1", "spectrum", "time", "log", 100, nil) + require.NoError(t, err) + + return ul.UsageLimitID + }, + }, + { + name: "reserved_node", + pattern: reservedNodeIDPattern, + create: func(t *testing.T, b *redshift.InMemoryBackend) string { + t.Helper() + + node, err := b.PurchaseReservedNodeOffering("offering-dc2-large-1yr-allupfront", "", 1) + require.NoError(t, err) + + return node.ReservedNodeID + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + b := redshift.NewInMemoryBackend("000000000000", "us-east-1") + _, err := b.CreateCluster("c1", "dc2.large", "dev", "admin", nil, "", redshift.CreateClusterOptions{}) + require.NoError(t, err) + + id1 := tt.create(t, b) + id2 := tt.create(t, b) + + assert.NotEqual(t, id1, id2, "two resources created back-to-back must get distinct IDs") + assert.Regexp(t, tt.pattern, id1) + assert.Regexp(t, tt.pattern, id2) + }) + }) + } +} diff --git a/services/redshift/reconciler_test.go b/services/redshift/reconciler_test.go index d75a79f025..4636daff16 100644 --- a/services/redshift/reconciler_test.go +++ b/services/redshift/reconciler_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -39,16 +40,7 @@ func assertStopsPromptly(t *testing.T, timeout time.Duration, stop func()) { func waitFor(t *testing.T, timeout time.Duration, cond func() bool) bool { t.Helper() - deadline := time.Now().Add(timeout) - for time.Now().Before(deadline) { - if cond() { - return true - } - - time.Sleep(2 * time.Millisecond) - } - - return cond() + return assert.Eventually(t, cond, timeout, 2*time.Millisecond) } // describeCount returns the number of clusters, driving the lazy read-time state @@ -340,32 +332,34 @@ func TestReconciler_ContextCancelStops(t *testing.T) { func TestClusterLifecycle_CreatingToAvailable(t *testing.T) { t.Parallel() - b := newRedshiftBackend() - redshift.SetClusterActivationDelay(b, 50*time.Millisecond) + synctest.Test(t, func(t *testing.T) { + b := newRedshiftBackend() + redshift.SetClusterActivationDelay(b, 50*time.Millisecond) - _, err := b.CreateCluster( - "lifecycle-cluster", - "dc2.large", - "dev", - "admin", - nil, - "", - redshift.CreateClusterOptions{}, - ) - require.NoError(t, err) + _, err := b.CreateCluster( + "lifecycle-cluster", + "dc2.large", + "dev", + "admin", + nil, + "", + redshift.CreateClusterOptions{}, + ) + require.NoError(t, err) - // Immediately after create, status should be "creating". - clusters, _, err := b.DescribeClusters("lifecycle-cluster", "", 0, nil, nil) - require.NoError(t, err) - require.Len(t, clusters, 1) - assert.Equal(t, "creating", clusters[0].Status, - "cluster should be in creating state immediately after CreateCluster") + // Immediately after create, status should be "creating". + clusters, _, err := b.DescribeClusters("lifecycle-cluster", "", 0, nil, nil) + require.NoError(t, err) + require.Len(t, clusters, 1) + assert.Equal(t, "creating", clusters[0].Status, + "cluster should be in creating state immediately after CreateCluster") - // After the activation delay, status should be "available". - time.Sleep(200 * time.Millisecond) + // After the activation delay, status should be "available". + time.Sleep(200 * time.Millisecond) - clusters2, _, err := b.DescribeClusters("lifecycle-cluster", "", 0, nil, nil) - require.NoError(t, err) - require.Len(t, clusters2, 1) - assert.Equal(t, "available", clusters2[0].Status, "cluster should be available after activation delay") + clusters2, _, err := b.DescribeClusters("lifecycle-cluster", "", 0, nil, nil) + require.NoError(t, err) + require.Len(t, clusters2, 1) + assert.Equal(t, "available", clusters2[0].Status, "cluster should be available after activation delay") + }) } diff --git a/services/redshift/reserved_nodes.go b/services/redshift/reserved_nodes.go index f84e18e9ac..57f0f62f2c 100644 --- a/services/redshift/reserved_nodes.go +++ b/services/redshift/reserved_nodes.go @@ -5,6 +5,10 @@ import ( "time" ) +// reservedNodeIDHexBytes is the byte length of a ReservedNode's random ID +// suffix (16 hex chars), keeping the existing "rn-" prefix convention. +const reservedNodeIDHexBytes = 8 + const ( offeringClassRegular = "Regular" currencyUSD = "USD" @@ -172,7 +176,7 @@ func (b *InMemoryBackend) PurchaseReservedNodeOffering( } if reservedNodeID == "" { - reservedNodeID = fmt.Sprintf("rn-%d", time.Now().UnixNano()) + reservedNodeID = fmt.Sprintf("rn-%s", randomHex(reservedNodeIDHexBytes)) } b.mu.Lock("PurchaseReservedNodeOffering") diff --git a/services/redshift/usage_limits.go b/services/redshift/usage_limits.go index 3d8c19fbc4..9af0a1d847 100644 --- a/services/redshift/usage_limits.go +++ b/services/redshift/usage_limits.go @@ -2,9 +2,12 @@ package redshift import ( "fmt" - "time" ) +// usageLimitIDHexBytes is the byte length of a UsageLimit's random ID suffix +// (16 hex chars), keeping the existing "ul-" prefix convention. +const usageLimitIDHexBytes = 8 + // CreateUsageLimit creates a new usage limit for a cluster feature. func (b *InMemoryBackend) CreateUsageLimit( clusterID, featureType, limitType, breachAction string, @@ -22,7 +25,7 @@ func (b *InMemoryBackend) CreateUsageLimit( return nil, fmt.Errorf("%w: cluster %s not found", ErrClusterNotFound, clusterID) } - id := fmt.Sprintf("ul-%d", time.Now().UnixNano()) + id := fmt.Sprintf("ul-%s", randomHex(usageLimitIDHexBytes)) ul := &UsageLimit{ UsageLimitID: id, diff --git a/services/redshiftdata/handler.go b/services/redshiftdata/handler.go index e0ab3c31a8..eb83fc09bb 100644 --- a/services/redshiftdata/handler.go +++ b/services/redshiftdata/handler.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "strings" + "sync/atomic" "time" "github.com/labstack/echo/v5" @@ -73,6 +74,8 @@ type Handler struct { idempotency *safemap.Map[string, idempotentStatement] AccountID string Region string + // idempotencyInsertsSinceSweep paces maybeEvictExpiredIdempotency. + idempotencyInsertsSinceSweep atomic.Int64 } // regionFromRequest resolves the AWS region for a request from its SigV4 diff --git a/services/redshiftdata/idempotency.go b/services/redshiftdata/idempotency.go index e23a2918b6..56bf25f347 100644 --- a/services/redshiftdata/idempotency.go +++ b/services/redshiftdata/idempotency.go @@ -13,6 +13,13 @@ import "time" // services/scheduler/idempotency.go, whose 5-minute window this reuses. const clientTokenTTL = 5 * time.Minute +// idempotencyEvictThreshold: cache size that arms the expired-entry sweep; +// unreplayed tokens otherwise live until process exit. +const idempotencyEvictThreshold = 256 + +// idempotencyEvictSweepInterval: inserts between sweeps once armed. +const idempotencyEvictSweepInterval = 64 + // idempotentStatement caches a statement Id created by a ClientToken-bearing // ExecuteStatement/BatchExecuteStatement call. type idempotentStatement struct { @@ -61,4 +68,33 @@ func (h *Handler) storeIdempotentStatement(key, id string) { } h.idempotency.Set(key, idempotentStatement{id: id, expiresAt: time.Now().Add(clientTokenTTL)}) + h.maybeEvictExpiredIdempotency() +} + +// maybeEvictExpiredIdempotency drops expired entries once the cache is large. +func (h *Handler) maybeEvictExpiredIdempotency() { + if h.idempotency.Len() < idempotencyEvictThreshold { + return + } + + if h.idempotencyInsertsSinceSweep.Add(1) < idempotencyEvictSweepInterval { + return + } + + h.idempotencyInsertsSinceSweep.Store(0) + + now := time.Now() + + var expired []string + h.idempotency.Range(func(key string, res idempotentStatement) bool { + if now.After(res.expiresAt) { + expired = append(expired, key) + } + + return true + }) + + for _, key := range expired { + h.idempotency.Delete(key) + } } diff --git a/services/redshiftdata/whitebox_test.go b/services/redshiftdata/whitebox_test.go new file mode 100644 index 0000000000..50e234f0a1 --- /dev/null +++ b/services/redshiftdata/whitebox_test.go @@ -0,0 +1,61 @@ +package redshiftdata + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestRedshiftData_IdempotencyEviction proves storeIdempotentStatement +// opportunistically sweeps expired entries once the cache grows past +// idempotencyEvictThreshold, so an ExecuteStatement/BatchExecuteStatement +// call whose ClientToken is never replayed does not sit in the cache forever +// (only lookupIdempotentStatement pruned before this fix, and only for the +// exact key it was asked about). +func TestRedshiftData_IdempotencyEviction(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + seedExpired int + inserts int + wantSwept bool + }{ + {name: "below threshold keeps expired", seedExpired: 1, inserts: 1}, + { + name: "threshold and sweep interval evicts expired", + seedExpired: idempotencyEvictThreshold + 16, + inserts: idempotencyEvictSweepInterval, + wantSwept: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := NewHandler(NewInMemoryBackend("000000000000", "us-east-1")) + past := time.Now().Add(-time.Hour) + + for i := range tt.seedExpired { + h.idempotency.Set( + fmt.Sprintf("expired-%d", i), + idempotentStatement{id: "stmt-expired", expiresAt: past}, + ) + } + + for i := range tt.inserts { + h.storeIdempotentStatement(fmt.Sprintf("live-%d", i), "stmt-live") + } + + _, stillPresent := h.idempotency.Get("expired-0") + if tt.wantSwept { + assert.False(t, stillPresent, "expired entry should have been swept") + } else { + assert.True(t, stillPresent, "expired entry should remain below the eviction threshold") + } + }) + } +} diff --git a/services/s3/PARITY.md b/services/s3/PARITY.md index e2e0543c68..d48ad29324 100644 --- a/services/s3/PARITY.md +++ b/services/s3/PARITY.md @@ -3,7 +3,7 @@ service: s3 sdk_module: aws-sdk-go-v2/service/s3@v1.111.0 # version audited against (go.mod pin) last_audit_commit: 30db30dd8 last_audit_date: 2026-09-24 -overall: A # gopherstack-3dqa: found+fixed 4 real bugs incl. a race-detector-confirmed data race and a real (not disguised) over-replication bug. gopherstack-zi7k (2026-08-14): implemented the 5-op Object Annotations family that gopherstack-3dqa found entirely missing. gopherstack-3dqa follow-up (2026-08-14b): mechanical struct-field diff (the method that closed the dynamodb sibling pass) found 2 real absent-but-tracked wire bugs; a benchmark-verified ListObjectsV2 allocation fix closed the one axis (optimization) the prior four rounds left as "inspected, not profiled". gopherstack-6flj (2026-08-15): full List/Describe/Get wrapper-key sweep (45 ops), 2 more real bugs fixed (ListObjects/V2 Owner, GetBucketVersioning MFADelete), 1 severe wrong-response-shape finding flagged not fixed (GetBucketMetadataConfiguration/GetBucketMetadataTableConfiguration) -- see families/ops/gaps below. +overall: A # gopherstack-3dqa: found+fixed 4 real bugs incl. a race-detector-confirmed data race and a real (not disguised) over-replication bug. gopherstack-zi7k (2026-08-14): implemented the 5-op Object Annotations family that gopherstack-3dqa found entirely missing. gopherstack-3dqa follow-up (2026-08-14b): mechanical struct-field diff (the method that closed the dynamodb sibling pass) found 2 real absent-but-tracked wire bugs; a benchmark-verified ListObjectsV2 allocation fix closed the one axis (optimization) the prior four rounds left as "inspected, not profiled". gopherstack-6flj (2026-08-15): full List/Describe/Get wrapper-key sweep (45 ops), 2 more real bugs fixed (ListObjects/V2 Owner, GetBucketVersioning MFADelete), 1 severe wrong-response-shape finding flagged not fixed (GetBucketMetadataConfiguration/GetBucketMetadataTableConfiguration) -- see families/ops/gaps below. gopherstack-z2w1a (2026-09-26): S3 Express One Zone (directory buckets) implemented for real -- CreateSession now issues real 5-minute-TTL session credentials, verifyHeaderAuth's Credential-scope check (previously hardcoded to "s3"/"s3-object-lambda") now accepts the "s3express" signing name every S3 Express request uses, and ListDirectoryBuckets' discriminator now keys on the real "x-id" query param instead of dead "list-type=directory" -- see the CreateSession/ListDirectoryBuckets ops rows and the dated Notes section below. protocol: REST-XML families: multipart: {status: ok, note: part-order InvalidPartOrder, non-last EntityTooSmall, ETag=MD5(concat part-MD5s)-N, SSE sealing} @@ -40,22 +40,99 @@ ops: GetBucketVersioning/PutBucketVersioning: {wire: fixed, errors: ok, state: ok, persist: ok, note: "FIXED 2026-08-15 (gopherstack-6flj wrapper-key sweep): GetBucketVersioningOutput.MFADelete (deserializers.go's awsRestxml_deserializeOpDocumentGetBucketVersioningOutput, case \"MfaDelete\", sibling to \"Status\") was read from no request, stored nowhere, and echoed by no response -- a real client's PutBucketVersioning({MFADelete: Enabled}) had the value silently dropped, and GetBucketVersioning's MFADelete was always empty regardless. Real request-side type is types.MFADelete; real response-side type is the DIFFERENT types.MFADeleteStatus (same \"Enabled\"/\"Disabled\" strings, two distinct SDK enums) -- stored as a plain string in StoredBucket to avoid coupling to either. Only emitted once ever configured (omitempty), matching the real doc: \"This element is only returned if the bucket has been configured with MFA delete.\""} GetBucketMetadataTableConfiguration: {wire: ok, errors: n/a, state: n/a, persist: ok, note: "FIXED 2026-09-12 (gopherstack-n3zi typed slice 11) -- see bucket_ops_metadata_table.go's getBucketMetadataTableConfigurationResponse. Was previously the Table half of the finding below; confirmed via a real typed GetBucketMetadataTableConfiguration client call."} GetBucketMetadataConfiguration: {wire: gap, errors: n/a, state: n/a, persist: ok, note: "FOUND, NOT FIXED 2026-08-15 (gopherstack-6flj wrapper-key sweep); re-confirmed still open 2026-09-12 (gopherstack-n3zi). Unlike every other Get*Configuration op in this file (CORS/lifecycle/notification/encryption/logging/replication/analytics/inventory/metrics/intelligent-tiering), where the real GET deserializer parses the response ROOT element directly as the same struct the PUT request root already is (confirmed per-op against deserializers.go), this one does NOT: awsRestxml_deserializeOpGetBucketMetadataConfiguration.HandleDeserialize (deserializers.go) parses the response root directly as types.GetBucketMetadataConfigurationResult, which requires a CHILD element named exactly \"MetadataConfigurationResult\" (types.MetadataConfigurationResult{DestinationResult (required, TableBucketArn/TableBucketType/TableNamespace), AnnotationTableConfigurationResult, InventoryTableConfigurationResult, JournalTableConfigurationResult}) -- a server-computed RESULT shape, structurally different from the client's CreateBucketMetadataConfiguration request body (types.MetadataConfiguration{JournalTableConfiguration, AnnotationTableConfiguration, InventoryTableConfiguration}, no ARNs/status at all). gopherstack's getBucketMetadataConfiguration (bucket_ops_metadata_table.go) echoes the raw stored CREATE request body verbatim -- which has no \"MetadataConfigurationResult\" child element anywhere, so a real typed client's GetBucketMetadataConfigurationOutput.GetBucketMetadataConfigurationResult.MetadataConfigurationResult decodes to nil regardless of what was created. The same OpDocument...Output wrapper function with a matching case IS present in generated code but is dead -- HandleDeserialize never calls it, the same trap gopherstack-ob1g already found and fixed once on GetBucketAbac -- so this is not a simple 'wrong root name' rename. NOT FIXED: producing a real DestinationResult requires an S3 Tables table-bucket ARN/namespace/provisioning-status concept this backend has no model for at all (no CreateBucketMetadataConfiguration path allocates a table bucket or generates an ARN); fabricating plausible-looking ARNs/status would be invented data, not a shape fix. Flagged per this campaign's own precedent for genuinely-unmodeled response shapes (matches securityhub's GetRecommendedPolicyV2 finding) rather than attempted."} + CreateSession (S3 Express One Zone): {wire: fixed, errors: ok, state: ok, persist: n/a, note: "FIXED 2026-09-26 (gopherstack-z2w1a): was a disguised stub returning one hardcoded credential set for any bucket, with no effect on subsequent auth. CreateSession now generates a real random AccessKeyID/SecretAccessKey/SessionToken per call, scoped to the bucket, expiring 5 minutes from issuance (matches the real API's documented TTL); tracked in a safemap.Map keyed by AccessKeyID, swept for expired entries on every CreateSession call so the store cannot grow unbounded. Root cause of the reported 403 SignatureDoesNotMatch was NOT the credentials or a signature bug at all: verifyHeaderAuth (sigv4.go) rejected any Authorization header whose Credential scope's service wasn't literally \"s3\" or \"s3-object-lambda\" -- but every S3 Express request (CreateSession itself included) signs with the \"s3express\" signing name (confirmed against a real client via httptest: s3@v1.111.0 internal/customizations/express_signer.go's SetSigV4SigningName(\"s3express\")), so the very first CreateSession call the SDK makes automatically for a directory-bucket-shaped name was rejected before any real signature check ran. \"s3express\" is now accepted alongside \"s3\"/\"s3-object-lambda\". Requests carrying x-amz-s3session-token are matched against the live session store (bucket+secret+token); an unknown/mismatched/expired token gets 403 ExpiredToken. Full signature re-verification against the session's own secret only happens when PresignSecret is configured, consistent with how every other credential is treated (unverified by default) -- but token liveness (the actual point of the 5-minute TTL) is always checked, independent of that opt-in. SessionMode (ReadOnly/ReadWrite) is accepted but not enforced -- see items_still_open. CreateSession deliberately does NOT require bucketName to already exist (confirmed via a real client against a dumping httptest.Server): CreateBucket's own bindEndpointParams never sets DisableS3ExpressSessionAuth, so with a custom BaseEndpoint the SDK's endpoint ruleset routes CreateBucket itself through the session-credential auth scheme for a directory-bucket-shaped name, before the bucket exists -- rejecting on NoSuchBucket here would make aws_s3_directory_bucket permanently uncreatable through any custom endpoint. Bucket existence is still enforced by every real operation (CreateBucket, PutObject, HeadBucket, ...), just not by this bootstrapping step."} + ListDirectoryBuckets: {wire: ok, errors: ok, state: ok, persist: n/a, note: "FIXED 2026-09-26 (gopherstack-z2w1a): isListDirectoryBucketsRequest previously keyed on \"list-type=directory\", a query param the pinned SDK never sends (gopherstack-0bq8) -- every real ListDirectoryBuckets call silently fell through to listBuckets. Replaced with the \"x-id\" query param (\"x-id=ListDirectoryBuckets\" vs \"x-id=ListBuckets\"), confirmed via a real client against httptest to be present on every S3 restXml request regardless of Express status -- a real, always-present signal, not an invented one. Reaching ListDirectoryBuckets against gopherstack's single custom-BaseEndpoint architecture still requires the caller to set Options.DisableS3ExpressSessionAuth = true: without it, the pinned SDK's own ExpressIdentityResolver.GetIdentity requires a bucket name that this bucket-less operation structurally never has, and the request never reaches the wire (client-side error, not a gopherstack bug) -- this is an SDK-side limitation of driving S3Express-classified operations through a custom endpoint, not something a server-side fix can work around. terraform-provider-aws's aws_s3_directory_bucket resource does not call ListDirectoryBuckets, so this limitation does not affect it."} gaps: [] items_still_open: - "GetBucketMetadataConfiguration returns the wrong response shape entirely for any real typed client (gopherstack-6flj, 2026-08-15) -- the real GET deserializer requires a MetadataConfigurationResult child with a server-computed DestinationResult (table-bucket ARN/namespace/status), and this backend echoes the raw CREATE request body instead. Fixing this needs modeling S3 Tables table-bucket provisioning (ARN/namespace/status), which this backend has no concept of anywhere; fabricating plausible ARNs/status would be invented data, not a shape fix. Kept as a genuinely unmodeled subsystem." - "Object Annotations (gopherstack-zi7k) is implemented and persisted, but two things are deliberately not enforced because they're absent from every relevant op's error switch in the pinned SDK (inventing a rejection would violate this sweep's own no-fabrication rule): the documented 1-byte-to-1-MiB payload size window, and DeleteObjectAnnotation/PutObjectAnnotation's ObjectIfMatch conditional header (read into the request struct but never compared)." - - "CreateSession (S3 Express One Zone) is a disguised stub: it returns hardcoded fake credentials for ANY bucket regardless of IsDirectoryBucket or SessionMode, and the returned session token has no effect on sigv4 validation or any subsequent request. Consistent with this emulator not modeling directory buckets/S3-Express as a distinct bucket type anywhere -- a real fix is a full S3-Express feature addition, not scoped for this pass." - - "RenameObject is applied uniformly to any bucket (general-purpose or directory), but real S3 restricts it to directory buckets only -- a permissive superset rather than a wire-shape bug reachable by a real client, since this emulator has no directory-bucket-vs-general-purpose distinction anywhere (see CreateSession entry above). (DestinationIfMatch/DestinationIfNoneMatch/DestinationIfModifiedSince/DestinationIfUnmodifiedSince precondition enforcement, previously logged as a second gap here, was already fixed and is proven by TestRenameObjectDestinationPreconditions -- stale sub-claim removed this sweep.)" + - "RenameObject is applied uniformly to any bucket (general-purpose or directory), but real S3 restricts it to directory buckets only -- a permissive superset rather than a wire-shape bug reachable by a real client. This emulator now DOES distinguish directory buckets (StoredBucket.IsDirectoryBucket, gopherstack-z2w1a) but RenameObject was not scoped to it this pass. (DestinationIfMatch/DestinationIfNoneMatch/DestinationIfModifiedSince/DestinationIfUnmodifiedSince precondition enforcement, previously logged as a second gap here, was already fixed and is proven by TestRenameObjectDestinationPreconditions -- stale sub-claim removed this sweep.)" + - "CreateSession (S3 Express One Zone) does not check IsDirectoryBucket -- a general-purpose bucket can also successfully call CreateSession, a permissive superset never reachable from an unmodified SDK client (which only ever issues CreateSession for a directory-bucket-shaped name). SessionMode (ReadOnly vs ReadWrite) is accepted and stored nowhere -- real S3 restricts a ReadOnly session's Zonal endpoint calls to GetObject/HeadObject/ListObjectsV2/GetObjectAttributes/ListParts/ListMultipartUploads, which this emulator does not enforce." + - "Directory buckets accept operations real S3 rejects for them beyond the two enforced here (ListObjects V1 rejected; ListObjectsV2 requires Delimiter \"/\") -- e.g. ACLs, tagging, versioning, lifecycle, website, and CORS configuration are all still accepted on a directory bucket though real S3 does not support most of them there. Each such rejection needs its own real S3 error code/message to add honestly rather than guessed; not attempted this pass beyond the two operations the task specifically called out as cheap to model." - "SelectObjectContent ScanRange (partial-object byte-range selection) is not implemented -- requests with a ScanRange element are accepted but the range is ignored and the full object is scanned. Real semantics need record-boundary-aware slicing entangled with evaluateCSVQuery/evaluateJSONQuery's own record-splitting logic -- a real feature addition, not a diff-and-fix." - "List*Configurations (analytics/inventory/metrics/intelligent-tiering) do not implement ContinuationToken-based pagination -- IsTruncated is always false and all stored configs are returned in one response. The underlying config maps also iterate in unspecified Go map order, so real pagination needs a deterministic sort as a prerequisite; only matters for buckets with >100 configs of one type, an edge case unlikely to be exercised by any realistic test." - "object_lambda: GetObject only recognizes a Lambda wired in by bucket name (via SetObjectLambdaConfig), not via genuine access-point-ARN routing (Bucket=). Wiring that needs access-point-ARN parsing on every object route plus a live cross-service lookup into s3control's backend -- and regular (non-Lambda) S3 Access Points have zero ARN-as-bucket routing support anywhere in this service either, so this would be building ARN routing on a foundation that doesn't exist yet. Real, larger cross-service feature." - - "ListDirectoryBuckets is structurally unreachable from any real, unmodified aws-sdk-go-v2 client pointed at gopherstack's single local endpoint (gopherstack-0bq8) -- real AWS distinguishes it from ListBuckets purely by literal hostname (s3express-control.* vs s3.*), which this single-endpoint emulator has no way to key on. The router's isListDirectoryBucketsRequest checks a query key no real client ever sends -- dead code, kept (not deleted) since it's the only way any test can reach the op at all." deferred: [] leaks: {status: clean, note: janitor ctx-parented w/ <-ctx.Done() stop; replication goroutines WaitGroup-drained; Shutdown() cancels; object_lambda config now cleared on DeleteBucket (was previously leaking across bucket-name reuse — see 2026-07-24 section)} --- ## Notes +### 2026-09-26 (S3 Express One Zone / directory buckets, gopherstack-z2w1a) + +**Root cause of the reported 403 SignatureDoesNotMatch on `aws_s3_directory_bucket`**: +reproduced first with a real `aws-sdk-go-v2/service/s3` client against a raw +`httptest.Server` dumping request headers, no gopherstack code involved. With +`UsePathStyle: true` and a directory-bucket-shaped name, the SDK's own +`CreateBucket`/`PutObject` calls sign with +`Credential=...//s3express/aws4_request` -- the credential scope's +*service* is `"s3express"`, never `"s3"` (confirmed: `s3@v1.111.0 +internal/customizations/express_signer.go`, every S3Express endpoint rule +branch calls `smithyhttp.SetSigV4SigningName("s3express")` regardless of +`DisableS3ExpressSessionAuth`). `sigv4.go`'s `verifyHeaderAuth` had a hardcoded +`if scope.service != "s3" && scope.service != "s3-object-lambda"` guard that +ran unconditionally (not gated behind `PresignSecret`) and rejected anything +else with `SignatureDoesNotMatch` -- so the very first `CreateSession` call +the SDK issues automatically for a directory bucket was rejected before any +real signature was ever computed. Fix: accept `"s3express"` too. + +**The express flow, as actually observed** (not from memory -- from driving a +real client against a dumping `httptest.Server`, see the reasoning trail in +the PR): `CreateBucket`/`PutObject`/`GetObject`/etc. on a directory bucket +first trigger `GET /?session=` signed with the caller's own +credentials (still `s3express`-scoped); the response's `Credentials` element +(`SessionToken`/`SecretAccessKey`/`AccessKeyId`/`Expiration`) is then used to +sign the actual request, adding the `x-amz-s3session-token` header and +suppressing the normal `X-Amz-Security-Token`. `ListDirectoryBuckets` sends +`GET /?x-id=ListDirectoryBuckets` -- but only when the client sets +`Options.DisableS3ExpressSessionAuth = true`; without it, the pinned SDK's own +`ExpressIdentityResolver.GetIdentity` needs `GetBucket(ctx)` for this +bucket-less op and errors client-side (`"bucket name is missing"`) before any +request is even built. This is an SDK/harness-side constraint of driving an +S3Express-classified operation through a custom `BaseEndpoint`, not a +gopherstack bug -- terraform-provider-aws's `aws_s3_directory_bucket` resource +never calls `ListDirectoryBuckets`, so it is unaffected. + +**What changed**: `verifyHeaderAuth` accepts the `"s3express"` credential +scope; `CreateSession` (`express_session.go`, new file) issues real random +session credentials scoped to the bucket with a 5-minute TTL, tracked in a +`safemap.Map` keyed by `AccessKeyID` and swept for expired entries on every +`CreateSession` call (bounded, no leak -- proven by +`TestS3ExpressSession_TTLBoundsGrowth`); a request carrying +`x-amz-s3session-token` is checked against that store (`ExpiredToken` 403 if +unknown/mismatched/expired) regardless of whether full signature +cryptographic verification (`PresignSecret`) is enabled, matching this +service's existing "everything else is unverified by default" posture while +still enforcing the one thing the whole feature is about: expiry. +`isListDirectoryBucketsRequest` now keys on the real `x-id` query param. +`ListObjectsV2` on a directory bucket now requires `Delimiter` to be `"/"` or +omitted (`ErrDirectoryBucketDelimiter`, InvalidArgument); `ListObjects` (V1) +on a directory bucket now returns NotImplemented (real S3 docs: "This +operation is not supported for directory buckets", `api_op_ListObjects.go:13`). +`CreateBucket` additionally reads `CreateBucketConfiguration.Bucket.Type` +(alongside the pre-existing `--x-s3` suffix detection) so a caller using the +documented `Bucket{Type: Directory, DataRedundancy: SingleAvailabilityZone}` / +`Location{Type: AvailabilityZone, Name}` shape is never silently ignored. + +**Verified via a real typed client** (`services/s3/express_test.go`): +`TestS3Express_FullFlow` drives `CreateBucket` (directory) -> +`PutObject` -> `GetObject` -> `ListObjectsV2` -> `DeleteObject` -> +`DeleteBucket` through the real SDK end to end, letting the SDK's own +session-credential machinery run untouched; `TestS3Express_ListDirectoryBuckets` +and `TestS3Express_DirectoryBucketSemantics` cover the discriminator and the +two enforced restrictions; `TestS3ExpressSession_Expiry` and +`TestS3ExpressSession_TTLBoundsGrowth` use `testing/synctest` to prove the +5-minute TTL and the sweep, without a real 5-minute sleep. + +**Not attempted this pass** (see items_still_open): `SessionMode` +(ReadOnly/ReadWrite) is accepted but doesn't restrict which Zonal-endpoint ops +a session may authorize; `CreateSession` doesn't reject a general-purpose +bucket; directory-bucket restrictions beyond the two enforced here (ACLs, +tagging, versioning, lifecycle, website, CORS are all real-S3-unsupported on +directory buckets but still accepted here); `RenameObject` is still not +scoped to directory buckets only, despite `IsDirectoryBucket` now existing to +check it against. + ### 2026-09-24 (sorted key-index for ListObjects/V2/ListObjectVersions, gopherstack-0mji2) `processListObjects` (`listing.go`) ranged over every key in `bucket.Objects` diff --git a/services/s3/README.md b/services/s3/README.md index 506631312e..c927060849 100644 --- a/services/s3/README.md +++ b/services/s3/README.md @@ -7,7 +7,7 @@ | Metric | Value | | --- | --- | -| PARITY entries audited | 24 (23 ok, 1 gap) | +| PARITY entries audited | 26 (25 ok, 1 gap) | | Feature families | 8 (8 ok) | | Known gaps | 8 | | Deferred items | 0 | @@ -17,12 +17,12 @@ - GetBucketMetadataConfiguration returns the wrong response shape entirely for any real typed client (gopherstack-6flj, 2026-08-15) -- the real GET deserializer requires a MetadataConfigurationResult child with a server-computed DestinationResult (table-bucket ARN/namespace/status), and this backend echoes the raw CREATE request body instead. Fixing this needs modeling S3 Tables table-bucket provisioning (ARN/namespace/status), which this backend has no concept of anywhere; fabricating plausible ARNs/status would be invented data, not a shape fix. Kept as a genuinely unmodeled subsystem. - Object Annotations (gopherstack-zi7k) is implemented and persisted, but two things are deliberately not enforced because they're absent from every relevant op's error switch in the pinned SDK (inventing a rejection would violate this sweep's own no-fabrication rule): the documented 1-byte-to-1-MiB payload size window, and DeleteObjectAnnotation/PutObjectAnnotation's ObjectIfMatch conditional header (read into the request struct but never compared). -- CreateSession (S3 Express One Zone) is a disguised stub: it returns hardcoded fake credentials for ANY bucket regardless of IsDirectoryBucket or SessionMode, and the returned session token has no effect on sigv4 validation or any subsequent request. Consistent with this emulator not modeling directory buckets/S3-Express as a distinct bucket type anywhere -- a real fix is a full S3-Express feature addition, not scoped for this pass. -- RenameObject is applied uniformly to any bucket (general-purpose or directory), but real S3 restricts it to directory buckets only -- a permissive superset rather than a wire-shape bug reachable by a real client, since this emulator has no directory-bucket-vs-general-purpose distinction anywhere (see CreateSession entry above). (DestinationIfMatch/DestinationIfNoneMatch/DestinationIfModifiedSince/DestinationIfUnmodifiedSince precondition enforcement, previously logged as a second gap here, was already fixed and is proven by TestRenameObjectDestinationPreconditions -- stale sub-claim removed this sweep.) +- RenameObject is applied uniformly to any bucket (general-purpose or directory), but real S3 restricts it to directory buckets only -- a permissive superset rather than a wire-shape bug reachable by a real client. This emulator now DOES distinguish directory buckets (StoredBucket.IsDirectoryBucket, gopherstack-z2w1a) but RenameObject was not scoped to it this pass. (DestinationIfMatch/DestinationIfNoneMatch/DestinationIfModifiedSince/DestinationIfUnmodifiedSince precondition enforcement, previously logged as a second gap here, was already fixed and is proven by TestRenameObjectDestinationPreconditions -- stale sub-claim removed this sweep.) +- CreateSession (S3 Express One Zone) does not check IsDirectoryBucket -- a general-purpose bucket can also successfully call CreateSession, a permissive superset never reachable from an unmodified SDK client (which only ever issues CreateSession for a directory-bucket-shaped name). SessionMode (ReadOnly vs ReadWrite) is accepted and stored nowhere -- real S3 restricts a ReadOnly session's Zonal endpoint calls to GetObject/HeadObject/ListObjectsV2/GetObjectAttributes/ListParts/ListMultipartUploads, which this emulator does not enforce. +- Directory buckets accept operations real S3 rejects for them beyond the two enforced here (ListObjects V1 rejected; ListObjectsV2 requires Delimiter "/") -- e.g. ACLs, tagging, versioning, lifecycle, website, and CORS configuration are all still accepted on a directory bucket though real S3 does not support most of them there. Each such rejection needs its own real S3 error code/message to add honestly rather than guessed; not attempted this pass beyond the two operations the task specifically called out as cheap to model. - SelectObjectContent ScanRange (partial-object byte-range selection) is not implemented -- requests with a ScanRange element are accepted but the range is ignored and the full object is scanned. Real semantics need record-boundary-aware slicing entangled with evaluateCSVQuery/evaluateJSONQuery's own record-splitting logic -- a real feature addition, not a diff-and-fix. - List*Configurations (analytics/inventory/metrics/intelligent-tiering) do not implement ContinuationToken-based pagination -- IsTruncated is always false and all stored configs are returned in one response. The underlying config maps also iterate in unspecified Go map order, so real pagination needs a deterministic sort as a prerequisite; only matters for buckets with >100 configs of one type, an edge case unlikely to be exercised by any realistic test. - object_lambda: GetObject only recognizes a Lambda wired in by bucket name (via SetObjectLambdaConfig), not via genuine access-point-ARN routing (Bucket=). Wiring that needs access-point-ARN parsing on every object route plus a live cross-service lookup into s3control's backend -- and regular (non-Lambda) S3 Access Points have zero ARN-as-bucket routing support anywhere in this service either, so this would be building ARN routing on a foundation that doesn't exist yet. Real, larger cross-service feature. -- ListDirectoryBuckets is structurally unreachable from any real, unmodified aws-sdk-go-v2 client pointed at gopherstack's single local endpoint (gopherstack-0bq8) -- real AWS distinguishes it from ListBuckets purely by literal hostname (s3express-control.* vs s3.*), which this single-endpoint emulator has no way to key on. The router's isListDirectoryBucketsRequest checks a query key no real client ever sends -- dead code, kept (not deleted) since it's the only way any test can reach the op at all. ## More diff --git a/services/s3/access_log_test.go b/services/s3/access_log_test.go index b378a59004..4418d411fa 100644 --- a/services/s3/access_log_test.go +++ b/services/s3/access_log_test.go @@ -7,7 +7,7 @@ import ( "net/http/httptest" "strings" "testing" - "time" + "testing/synctest" "github.com/aws/aws-sdk-go-v2/aws" sdk_s3 "github.com/aws/aws-sdk-go-v2/service/s3" @@ -64,71 +64,56 @@ func TestHandler_AccessLogDispatch(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - handler, backend := newTestHandler(t) - tt.setup(t, backend) - - req := httptest.NewRequest(http.MethodGet, "/"+tt.bucket+"/"+tt.key, nil) - rec := httptest.NewRecorder() - serveS3Handler(handler, rec, req) - require.Equal(t, http.StatusOK, rec.Code) - - if tt.wantLog { - logKey := waitForAccessLog(t, backend) - out, err := backend.GetObject(context.Background(), &sdk_s3.GetObjectInput{ - Bucket: aws.String("log-bkt"), - Key: aws.String(logKey), - }) - require.NoError(t, err) - - body, err := io.ReadAll(out.Body) - require.NoError(t, err) - - line := string(body) - require.Contains(t, line, "REST.GET.OBJECT") - require.Contains(t, line, tt.bucket) - require.Contains(t, line, tt.key) - require.True(t, strings.HasSuffix(line, "\n"), "log line must end with newline") - } - - if !tt.wantLog { - require.Never(t, func() bool { - out, err := backend.ListObjectsV2( - context.Background(), - &sdk_s3.ListObjectsV2Input{ - Bucket: aws.String(tt.bucket), - }, - ) - - return err == nil && len(out.Contents) != tt.wantObjects - }, 100*time.Millisecond, 20*time.Millisecond) - - out, err := backend.ListObjectsV2(context.Background(), &sdk_s3.ListObjectsV2Input{ - Bucket: aws.String(tt.bucket), - }) - require.NoError(t, err) - require.Len(t, out.Contents, tt.wantObjects) - } + synctest.Test(t, func(t *testing.T) { + handler, backend := newTestHandler(t) + tt.setup(t, backend) + + req := httptest.NewRequest(http.MethodGet, "/"+tt.bucket+"/"+tt.key, nil) + rec := httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + synctest.Wait() + + if tt.wantLog { + logKey := findAccessLog(t, backend) + out, err := backend.GetObject(context.Background(), &sdk_s3.GetObjectInput{ + Bucket: aws.String("log-bkt"), + Key: aws.String(logKey), + }) + require.NoError(t, err) + + body, err := io.ReadAll(out.Body) + require.NoError(t, err) + + line := string(body) + require.Contains(t, line, "REST.GET.OBJECT") + require.Contains(t, line, tt.bucket) + require.Contains(t, line, tt.key) + require.True(t, strings.HasSuffix(line, "\n"), "log line must end with newline") + } + + if !tt.wantLog { + out, err := backend.ListObjectsV2(context.Background(), &sdk_s3.ListObjectsV2Input{ + Bucket: aws.String(tt.bucket), + }) + require.NoError(t, err) + require.Len(t, out.Contents, tt.wantObjects) + } + }) }) } } -func waitForAccessLog(t *testing.T, backend *s3.InMemoryBackend) string { +func findAccessLog(t *testing.T, backend *s3.InMemoryBackend) string { t.Helper() - var logKey string - require.Eventually(t, func() bool { - out, err := backend.ListObjectsV2(context.Background(), &sdk_s3.ListObjectsV2Input{ - Bucket: aws.String("log-bkt"), - Prefix: aws.String("logs/"), - }) - if err == nil && len(out.Contents) > 0 { - logKey = aws.ToString(out.Contents[0].Key) - - return true - } - - return false - }, time.Second, 20*time.Millisecond, "expected an access-log object under logs/") + out, err := backend.ListObjectsV2(context.Background(), &sdk_s3.ListObjectsV2Input{ + Bucket: aws.String("log-bkt"), + Prefix: aws.String("logs/"), + }) + require.NoError(t, err) + require.NotEmpty(t, out.Contents, "expected an access-log object under logs/") - return logKey + return aws.ToString(out.Contents[0].Key) } diff --git a/services/s3/bench_test.go b/services/s3/bench_test.go index 02a5368f06..0e2449e41a 100644 --- a/services/s3/bench_test.go +++ b/services/s3/bench_test.go @@ -387,3 +387,34 @@ func BenchmarkGetObject_1MiB(b *testing.B) { } } } + +// BenchmarkUploadPart_5MiB measures backend UploadPart at the 5 MiB minimum part +// size, without HTTP or compression overhead. +func BenchmarkUploadPart_5MiB(b *testing.B) { + backend := s3.NewInMemoryBackend(&s3.GzipCompressor{}).WithSkipMultipartSizeCheck() + bucketName := "bench-uploadpart-5m" + _, _ = backend.CreateBucket(b.Context(), &sdk_s3.CreateBucketInput{Bucket: aws.String(bucketName)}) + key := "bench-key" + initOut, err := backend.CreateMultipartUpload(b.Context(), &sdk_s3.CreateMultipartUploadInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + }) + if err != nil { + b.Fatal(err) + } + partData := bytes.Repeat([]byte("a"), 5*1024*1024) + + b.ReportAllocs() + for b.Loop() { + _, err = backend.UploadPart(b.Context(), &sdk_s3.UploadPartInput{ + Bucket: aws.String(bucketName), + Key: aws.String(key), + UploadId: initOut.UploadId, + PartNumber: aws.Int32(1), + Body: bytes.NewReader(partData), + }) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/services/s3/bucket_ops.go b/services/s3/bucket_ops.go index 9affd39fa4..ea09fc095e 100644 --- a/services/s3/bucket_ops.go +++ b/services/s3/bucket_ops.go @@ -608,7 +608,7 @@ func (h *S3Handler) createBucket( } logger.Load(ctx). - DebugContext(ctx, "S3 createBucket output", "bucket", bucketName, "region", region) + DebugContext(ctx, "S3 createBucket output", "bucket", bucketName) // Set Location header from output if output.Location != nil { @@ -703,6 +703,23 @@ func (h *S3Handler) headBucket( w.WriteHeader(http.StatusOK) } +// createSessionResult is the XML response body for CreateSession +// (s3@v1.111.0 deserializers.go: root element name is never checked by the +// real client, only the nested Credentials element -- "CreateSessionResult" +// matches AWS's documented shape). +type createSessionResult struct { + XMLName xml.Name `xml:"CreateSessionResult"` + Xmlns string `xml:"xmlns,attr"` + Credentials createSessionCreds `xml:"Credentials"` +} + +type createSessionCreds struct { + SessionToken string `xml:"SessionToken"` + SecretAccessKey string `xml:"SecretAccessKey"` + AccessKeyID string `xml:"AccessKeyId"` + Expiration string `xml:"Expiration"` +} + func (h *S3Handler) createSession( ctx context.Context, w http.ResponseWriter, @@ -710,13 +727,23 @@ func (h *S3Handler) createSession( bucket string, ) { h.setOperation(ctx, "CreateSession") - sessionXML, err := h.Backend.CreateSession(ctx, bucket) + + mode := types.SessionMode(r.Header.Get("X-Amz-Create-Session-Mode")) + + creds, err := h.Backend.CreateSession(ctx, bucket, mode) if err != nil { WriteError(ctx, w, r, err) return } - w.Header().Set("Content-Type", "application/xml") - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(sessionXML)) + + httputils.WriteXML(ctx, w, http.StatusOK, createSessionResult{ + Xmlns: xmlNamespaceS3, + Credentials: createSessionCreds{ + SessionToken: creds.SessionToken, + SecretAccessKey: creds.SecretAccessKey, + AccessKeyID: creds.AccessKeyID, + Expiration: creds.Expiration.UTC().Format(time.RFC3339), + }, + }) } diff --git a/services/s3/bucket_ops_listing.go b/services/s3/bucket_ops_listing.go index e9d6317cb5..faaa1ff28e 100644 --- a/services/s3/bucket_ops_listing.go +++ b/services/s3/bucket_ops_listing.go @@ -17,6 +17,23 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/logger" ) +// parseListObjectsMaxKeys parses ListObjects (V1)'s max-keys query param. The +// result is provably in [0, defaultMaxKeys]: it starts at the constant +// default and is only reassigned to a parsed value that is non-negative and +// strictly less than defaultMaxKeys. AWS clamps MaxKeys to [0, 1000] rather +// than rejecting an over-limit value, so a value at or above the limit is +// treated as the limit. +func parseListObjectsMaxKeys(r *http.Request) int32 { + n := defaultMaxKeys + if mk := r.URL.Query().Get("max-keys"); mk != "" { + if v, err := strconv.Atoi(mk); err == nil && v >= 0 && v < defaultMaxKeys { + n = v + } + } + + return int32(n) +} + func (h *S3Handler) listObjects( ctx context.Context, w http.ResponseWriter, @@ -31,6 +48,12 @@ func (h *S3Handler) listObjects( return } + if h.Backend.IsDirectoryBucket(bucketName) { + WriteError(ctx, w, r, ErrListObjectsNotSupportedForDirectoryBucket) + + return + } + prefix := r.URL.Query().Get("prefix") delimiter := r.URL.Query().Get("delimiter") marker := r.URL.Query().Get("marker") @@ -42,18 +65,7 @@ func (h *S3Handler) listObjects( "bucket", bucketName, "prefix", prefix, "delimiter", delimiter, "marker", marker, ) - // n is provably in [0, defaultMaxKeys] before the int32 conversion: it - // starts at the constant default and is only reassigned to a parsed value - // that is non-negative and strictly less than defaultMaxKeys. AWS clamps - // MaxKeys to [0, 1000] rather than rejecting an over-limit value, so a - // value at or above the limit is treated as the limit. - n := defaultMaxKeys - if mk := r.URL.Query().Get("max-keys"); mk != "" { - if v, err := strconv.Atoi(mk); err == nil && v >= 0 && v < defaultMaxKeys { - n = v - } - } - maxKeys := int32(n) + maxKeys := parseListObjectsMaxKeys(r) // Pass marker and delimiter to backend so it can seek and group correctly. out, err := h.Backend.ListObjects(ctx, &s3.ListObjectsInput{ @@ -128,7 +140,11 @@ func (h *S3Handler) listObjects( } } - httputils.WriteXML(ctx, w, http.StatusOK, resp) + buf := httputils.GetBuffer() + defer httputils.PutBuffer(buf) + buf.WriteString(xml.Header) + writeListBucketXML(buf, &resp) + writeListXMLResponse(ctx, w, http.StatusOK, buf) } func (h *S3Handler) mapObjectsToXML( @@ -365,23 +381,23 @@ func (h *S3Handler) handleListDirectoryBuckets( s3DirectoryBucketsResult{Xmlns: xmlNamespaceS3, Buckets: entries}) } -// isListDirectoryBucketsRequest returns true when the request targets ListDirectoryBuckets. +// isListDirectoryBucketsRequest returns true when the request targets +// ListDirectoryBuckets. // -// "list-type=directory" is not a real signal: the pinned SDK -// (s3@v1.106.5 api_op_ListDirectoryBuckets.go/serializers.go) never sends -// it -- ListDirectoryBucketsInput.bindEndpointParams sets -// UseS3ExpressControlEndpoint, and real AWS distinguishes the two ops -// purely by literal hostname (s3express-control..amazonaws.com -// vs s3..amazonaws.com), not by any query/path/header on the -// request itself. Against gopherstack's single local endpoint (the only -// way any client can reach it), a real ListDirectoryBuckets() call is -// wire-identical to ListBuckets() -- no query param, path, or header -// differs -- so this check can never be satisfied by an unmodified SDK -// client and every real call silently falls through to listBuckets -// instead (200 success, wrong bucket set). This is structural, not a -// routing bug fixable by correcting a discriminator: there is no real one -// to key on. Left as documented dead code (gopherstack-0bq8) rather than -// deleted, since it is the only way to reach this op at all in tests. +// Real AWS distinguishes ListDirectoryBuckets from ListBuckets purely by +// literal hostname (s3express-control..amazonaws.com vs +// s3..amazonaws.com), which gopherstack's single local endpoint has +// no way to key on. But every S3 restXml operation the pinned SDK sends +// (confirmed against s3@v1.111.0 by driving both ops through a real client +// with a custom BaseEndpoint) carries its own operation name in the "x-id" +// query parameter -- "GET /?x-id=ListBuckets" vs "GET /?x-id=ListDirectoryBuckets" +// -- so that param is a real, always-present signal rather than an invented +// one. Reaching ListDirectoryBuckets against a custom BaseEndpoint also +// requires the client to set Options.DisableS3ExpressSessionAuth = true: +// without it, the pinned SDK's own S3Express identity resolver requires a +// bucket name that this bucket-less operation never has, and the request +// never reaches the wire at all (client-side error "get identity: bucket +// name is missing", not a gopherstack bug -- see PARITY.md). func isListDirectoryBucketsRequest(r *http.Request) bool { - return r.URL.Query().Get("list-type") == "directory" + return r.URL.Query().Get("x-id") == "ListDirectoryBuckets" } diff --git a/services/s3/bucket_replication_test.go b/services/s3/bucket_replication_test.go index 8ade9ff5c8..ec542d79b7 100644 --- a/services/s3/bucket_replication_test.go +++ b/services/s3/bucket_replication_test.go @@ -7,7 +7,6 @@ import ( "net/http/httptest" "strings" "testing" - "time" sdk_s3 "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/stretchr/testify/assert" @@ -178,16 +177,14 @@ func TestS3BucketReplication_PutObjectReplicates(t *testing.T) { serveS3Handler(handler, rec, req) require.Equal(t, http.StatusOK, rec.Code) - // Allow the async goroutine to run. - testKey := "test.txt" - require.Eventually(t, func() bool { - _, err := bk.GetObject(t.Context(), &sdk_s3.GetObjectInput{ - Bucket: &dst, - Key: &testKey, - }) + bk.DrainReplicationGoroutines() - return err == nil - }, 3*time.Second, 50*time.Millisecond, "replicated object should appear in destination bucket") + testKey := "test.txt" + _, err := bk.GetObject(t.Context(), &sdk_s3.GetObjectInput{ + Bucket: &dst, + Key: &testKey, + }) + require.NoError(t, err, "replicated object should appear in destination bucket") } // TestS3BucketReplication_PrefixFilter verifies that only keys matching the @@ -240,16 +237,14 @@ func TestS3BucketReplication_PrefixFilter(t *testing.T) { require.Equal(t, http.StatusOK, rec.Code) } - // Wait for the replicated key to appear, then verify the non-replicated one is absent. - imgKey := "images/photo.jpg" - require.Eventually(t, func() bool { - _, err := bk.GetObject(t.Context(), &sdk_s3.GetObjectInput{Bucket: &dst, Key: &imgKey}) + bk.DrainReplicationGoroutines() - return err == nil - }, 3*time.Second, 50*time.Millisecond, "images/photo.jpg should be replicated to destination") + imgKey := "images/photo.jpg" + _, err := bk.GetObject(t.Context(), &sdk_s3.GetObjectInput{Bucket: &dst, Key: &imgKey}) + require.NoError(t, err, "images/photo.jpg should be replicated to destination") docKey := "documents/report.pdf" - _, err := bk.GetObject(t.Context(), &sdk_s3.GetObjectInput{Bucket: &dst, Key: &docKey}) + _, err = bk.GetObject(t.Context(), &sdk_s3.GetObjectInput{Bucket: &dst, Key: &docKey}) assert.Error(t, err, "documents/report.pdf should NOT be replicated (prefix filter)") } @@ -378,19 +373,14 @@ func TestS3BucketReplication_DeleteMarker(t *testing.T) { serveS3Handler(handler, rec, req) require.Equal(t, http.StatusNoContent, rec.Code) - noteKey := "note.txt" - require.Eventually(t, func() bool { - out, err := bk.GetObject(t.Context(), &sdk_s3.GetObjectInput{ - Bucket: &dst, - Key: ¬eKey, - }) - if err != nil { - return true // key was deleted - } - _ = out.Body.Close() + bk.DrainReplicationGoroutines() - return false - }, 3*time.Second, 50*time.Millisecond, "delete marker should propagate to destination") + noteKey := "note.txt" + _, err := bk.GetObject(t.Context(), &sdk_s3.GetObjectInput{ + Bucket: &dst, + Key: ¬eKey, + }) + require.Error(t, err, "delete marker should propagate to destination") } func TestS3BucketReplicationCRUD(t *testing.T) { diff --git a/services/s3/buckets.go b/services/s3/buckets.go index d19129699d..c50b07c8ad 100644 --- a/services/s3/buckets.go +++ b/services/s3/buckets.go @@ -71,6 +71,17 @@ func (b *InMemoryBackend) CreateBucket( ownershipControls = buildOwnershipControlsXML(string(input.ObjectOwnership)) } + // A directory bucket is identified by its --{azid}--x-s3 name suffix (the + // only signal a real client's own bucket-naming convention leaves us); + // CreateBucketConfiguration.Bucket.Type/Location (types.go:376,3072) are a + // second, corroborating signal from the same request, read here so a + // caller using them is never silently ignored. + isDirectoryBucket := strings.HasSuffix(bucketName, "--x-s3") + if input.CreateBucketConfiguration != nil && input.CreateBucketConfiguration.Bucket != nil { + isDirectoryBucket = isDirectoryBucket || + input.CreateBucketConfiguration.Bucket.Type == types.BucketTypeDirectory + } + b.buckets.Put(&StoredBucket{ Name: bucketName, Region: region, @@ -87,7 +98,7 @@ func (b *InMemoryBackend) CreateBucket( // S3 Express directory buckets use the naming convention {name}--{az-id}--x-s3. // Detect this at creation time so ListBuckets and ListDirectoryBuckets can // correctly partition general-purpose vs. directory buckets. - IsDirectoryBucket: strings.HasSuffix(bucketName, "--x-s3"), + IsDirectoryBucket: isDirectoryBucket, ObjectLockEnabled: aws.ToBool(input.ObjectLockEnabledForBucket), OwnershipControlsConfig: ownershipControls, OwnerAccountID: awsmeta.Account(ctx), @@ -311,36 +322,6 @@ func (b *InMemoryBackend) BucketsByRegion(region string) []types.Bucket { return buckets } -// CreateSession returns a stub session response for a bucket (S3 Express One -// Zone). It is a stub in more ways than the response body suggests: SessionMode -// (the X-Amz-Create-Session-Mode header) is never read, IsDirectoryBucket is -// never checked -- this emulator has no directory-bucket-vs-general-purpose -// distinction at all -- and the returned SessionToken has no downstream effect; -// nothing validates it on subsequent requests, so it authorizes nothing. -func (b *InMemoryBackend) CreateSession(_ context.Context, bucketName string) (string, error) { - var err error - func() { - b.mu.RLock("CreateSession") - defer b.mu.RUnlock() - - _, err = b.getBucket(bucketName) - }() - - if err != nil { - return "", err - } - - const sessionXML = `` + - `` + - `gopherstack-mock-session-token` + - `gopherstack-mock-secret` + - `gopherstack-mock-access-key` + - `2099-01-01T00:00:00Z` + - `` - - return sessionXML, nil -} - func (b *InMemoryBackend) GetBucketMetadata( _ context.Context, bucketName string, diff --git a/services/s3/buckets_test.go b/services/s3/buckets_test.go index aa5483d95f..17f5f0d48c 100644 --- a/services/s3/buckets_test.go +++ b/services/s3/buckets_test.go @@ -260,8 +260,10 @@ func TestListDirectoryBuckets(t *testing.T) { assert.NotContains(t, listBody, name, "ListBuckets must not contain %q", name) } - // Verify ListDirectoryBuckets via HTTP. - dirReq := httptest.NewRequest(http.MethodGet, "/?list-type=directory", nil) + // Verify ListDirectoryBuckets via HTTP. x-id=ListDirectoryBuckets is + // the real discriminator a client sends (see + // isListDirectoryBucketsRequest) -- list-type=directory never was. + dirReq := httptest.NewRequest(http.MethodGet, "/?x-id=ListDirectoryBuckets", nil) dirRec := httptest.NewRecorder() serveS3Handler(handler, dirRec, dirReq) require.Equal(t, http.StatusOK, dirRec.Code, "ListDirectoryBuckets must return 200") diff --git a/services/s3/compression.go b/services/s3/compression.go index 3cef574639..e5cf530f1c 100644 --- a/services/s3/compression.go +++ b/services/s3/compression.go @@ -3,7 +3,9 @@ package s3 import ( "bytes" "compress/gzip" + "encoding/binary" "io" + "math" ) type GzipCompressor struct{} @@ -12,9 +14,9 @@ type GzipCompressor struct{} // choice (GetObject always decompresses back to the exact original bytes), so // trading ratio for speed here is invisible to callers; DefaultCompression's // CPU cost dominated the object-write hot path under profiling. +// The buffer is not pre-sized to len(data): output is usually much smaller. func (c *GzipCompressor) Compress(data []byte) ([]byte, error) { var buf bytes.Buffer - buf.Grow(len(data)) w, err := gzip.NewWriterLevel(&buf, gzip.BestSpeed) if err != nil { return nil, err @@ -29,6 +31,25 @@ func (c *GzipCompressor) Compress(data []byte) ([]byte, error) { return buf.Bytes(), nil } +// gzipTrailerMinLen is the smallest a valid gzip stream can be: a 10-byte +// header plus an 8-byte trailer (CRC32 + ISIZE). +const gzipTrailerMinLen = 18 + +// gzipISizeHint reads the trailer's ISIZE (uncompressed size mod 2^32, RFC 1952 +// §2.3.1) as a pre-size hint; a wrong value only costs extra growth. +func gzipISizeHint(data []byte) int { + if len(data) < gzipTrailerMinLen { + return 0 + } + + isize := binary.LittleEndian.Uint32(data[len(data)-4:]) + if isize > math.MaxInt32 { + return 0 + } + + return int(isize) +} + func (c *GzipCompressor) Decompress(data []byte) ([]byte, error) { r, err := gzip.NewReader(bytes.NewReader(data)) if err != nil { @@ -36,5 +57,16 @@ func (c *GzipCompressor) Decompress(data []byte) ([]byte, error) { } defer r.Close() - return io.ReadAll(r) + var buf bytes.Buffer + if hint := gzipISizeHint(data); hint > 0 { + // ReadFrom reserves MinRead before its final EOF read; without it the + // buffer doubles once at the end. + buf.Grow(hint + bytes.MinRead) + } + //nolint:gosec // G110: decompresses our own previously Compress'd bytes, not attacker-supplied gzip + if _, err = io.Copy(&buf, r); err != nil { + return nil, err + } + + return buf.Bytes(), nil } diff --git a/services/s3/errors.go b/services/s3/errors.go index 9c6ff06619..f09facc353 100644 --- a/services/s3/errors.go +++ b/services/s3/errors.go @@ -78,6 +78,15 @@ var ( // JSON. The error table maps it to HTTP 400 with code "MalformedPolicy", // matching real S3. ErrMalformedPolicy = errors.New("MalformedPolicy") + // ErrListObjectsNotSupportedForDirectoryBucket is returned by ListObjects + // (V1) on a directory bucket: real S3 documents it as "not supported for + // directory buckets" (s3@v1.111.0 api_op_ListObjects.go:13) -- callers + // must use ListObjectsV2 instead. + ErrListObjectsNotSupportedForDirectoryBucket = errors.New( + "ListObjects is not supported for directory buckets") + // ErrDirectoryBucketDelimiter is returned by ListObjectsV2 on a directory + // bucket when Delimiter is set to anything other than "/". + ErrDirectoryBucketDelimiter = errors.New("directory buckets only support delimiter \"/\"") // ErrAnnotationLimitExceeded and the Object Annotations errors below it // carry codes verified against s3@v1.106.5 deserializers.go's per-op error @@ -238,6 +247,16 @@ func coreErrorTableObject() []s3ErrorEntry { "A header you provided implies functionality that is not implemented.", http.StatusNotImplemented, }}, + {ErrListObjectsNotSupportedForDirectoryBucket, s3ErrorInfo{ + "NotImplemented", + "This operation is not supported for directory buckets. Use ListObjectsV2 instead.", + http.StatusNotImplemented, + }}, + {ErrDirectoryBucketDelimiter, s3ErrorInfo{ + errInvalidArgument, + "Delimiter must be \"/\" for directory buckets.", + http.StatusBadRequest, + }}, {ErrObjectLocked, s3ErrorInfo{errAccessDenied, "Access Denied", http.StatusForbidden}}, {ErrInvalidObjectState, s3ErrorInfo{ "InvalidObjectState", diff --git a/services/s3/express_session.go b/services/s3/express_session.go new file mode 100644 index 0000000000..1f980813ef --- /dev/null +++ b/services/s3/express_session.go @@ -0,0 +1,152 @@ +package s3 + +import ( + "context" + "crypto/rand" + "encoding/hex" + "strconv" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +// s3ExpressSessionTTL matches real S3 Express One Zone: CreateSession +// credentials are scoped to the bucket and expire after 5 minutes +// (s3@v1.111.0 api_op_CreateSession.go doc comment). +const s3ExpressSessionTTL = 5 * time.Minute + +// expressSessionKeyBytes/expressSessionSecretBytes/expressSessionTokenBytes +// size the random components of a generated session credential. Lengths are +// cosmetic (no real client parses them) but kept AWS-shaped (16 hex chars is +// short of a real 20-char AKID, this is a mock, not a forgery target). +const ( + expressSessionKeyBytes = 8 + expressSessionSecretBytes = 20 + expressSessionTokenBytes = 32 +) + +// SessionCredentials is the temporary credential set CreateSession issues, +// scoped to one directory bucket. +type SessionCredentials struct { + Expiration time.Time + AccessKeyID string + SecretAccessKey string + SessionToken string +} + +// expressSession is the backend-side record for a live SessionCredentials, +// keyed by AccessKeyID so verifyHeaderAuth can look it up from the +// Authorization header's Credential without also parsing the session token. +type expressSession struct { + expiresAt time.Time + bucket string + secret string + token string +} + +func randomHex(n int) string { + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + return hex.EncodeToString([]byte(strconv.FormatInt(time.Now().UnixNano(), 10))) + } + + return hex.EncodeToString(b) +} + +// CreateSession issues temporary session credentials scoped to bucketName, +// expiring s3ExpressSessionTTL from now. SessionMode is accepted but not +// enforced (this emulator does not model IAM-policy-scoped ReadOnly vs +// ReadWrite sessions). +// +// Deliberately does NOT require bucketName to already exist: the pinned SDK +// (s3@v1.111.0) issues an implicit CreateSession as part of its own identity +// resolution for ANY operation on a directory-bucket-shaped name when a +// custom BaseEndpoint is configured -- including CreateBucket itself, before +// the bucket exists. Confirmed with a real client against a dumping +// httptest.Server: CreateBucket's own bindEndpointParams never sets +// DisableS3ExpressSessionAuth, so with a custom endpoint the SDK's endpoint +// ruleset routes it through the session-credential auth scheme regardless +// (case selection differs only when there is no endpoint override, i.e. +// real, unmodified AWS, where CreateBucket needs no session at all). This is +// a structural client-side quirk of driving S3Express-classified operations +// through a custom endpoint, not a real permission gate this emulator has +// any other way to model -- bucket existence is still enforced by every +// actual operation (CreateBucket, PutObject, etc.), just not by this +// bootstrapping step. +func (b *InMemoryBackend) CreateSession( + _ context.Context, bucketName string, _ types.SessionMode, +) (SessionCredentials, error) { + b.sweepExpiredSessions() + + now := time.Now() + creds := SessionCredentials{ + AccessKeyID: "ASIAEXPRESS" + randomHex(expressSessionKeyBytes), + SecretAccessKey: randomHex(expressSessionSecretBytes), + SessionToken: randomHex(expressSessionTokenBytes), + Expiration: now.Add(s3ExpressSessionTTL), + } + + b.expressSessions.Set(creds.AccessKeyID, expressSession{ + bucket: bucketName, + secret: creds.SecretAccessKey, + token: creds.SessionToken, + expiresAt: creds.Expiration, + }) + + return creds, nil +} + +// sweepExpiredSessions bounds the session store's size: every CreateSession +// call drops any entry that has since expired, so a client that keeps +// requesting new sessions without ever letting them expire in-process cannot +// leak memory beyond one entry per live 5-minute window. +func (b *InMemoryBackend) sweepExpiredSessions() { + now := time.Now() + + var expired []string + b.expressSessions.Range(func(k string, v expressSession) bool { + if now.After(v.expiresAt) { + expired = append(expired, k) + } + + return true + }) + + for _, k := range expired { + b.expressSessions.Delete(k) + } +} + +// ExpressSessionSecret looks up a live (non-expired) S3 Express session by +// its AccessKeyID and session token, returning the bucket it is scoped to +// and its secret key. A missing, mismatched, or expired session is reported +// as not found; an expired entry is also deleted. +func (b *InMemoryBackend) ExpressSessionSecret(accessKeyID, sessionToken string) (string, string, bool) { + sess, found := b.expressSessions.Get(accessKeyID) + if !found || sess.token != sessionToken { + return "", "", false + } + + if time.Now().After(sess.expiresAt) { + b.expressSessions.Delete(accessKeyID) + + return "", "", false + } + + return sess.bucket, sess.secret, true +} + +// IsDirectoryBucket reports whether bucket is an S3 Express directory +// bucket. Returns false for general-purpose buckets and for buckets that +// don't exist (existence is the caller's own concern). +func (b *InMemoryBackend) IsDirectoryBucket(bucketName string) bool { + b.mu.RLock("IsDirectoryBucket") + defer b.mu.RUnlock() + + bucket, err := b.getBucket(bucketName) + if err != nil { + return false + } + + return bucket.IsDirectoryBucket +} diff --git a/services/s3/express_test.go b/services/s3/express_test.go new file mode 100644 index 0000000000..4cdc236d9a --- /dev/null +++ b/services/s3/express_test.go @@ -0,0 +1,284 @@ +package s3_test + +import ( + "io" + "strings" + "testing" + "testing/synctest" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + sdk_s3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/s3" +) + +// newRealS3ExpressClientTest is newRealS3ClientTest plus +// DisableS3ExpressSessionAuth, which every case in this file needs: without +// it the pinned SDK (s3@v1.111.0) can still drive the CreateSession/ +// PutObject/GetObject flow against a directory bucket fine, but its own +// S3Express identity resolver refuses ListDirectoryBuckets (a no-bucket op) +// with a client-side "bucket name is missing" error before any request +// reaches the wire -- see isListDirectoryBucketsRequest's doc comment. +func newRealS3ExpressClientTest(t *testing.T) *sdk_s3.Client { + t.Helper() + + client := newRealS3ClientTest(t) + + return sdk_s3.New(client.Options(), func(o *sdk_s3.Options) { + o.DisableS3ExpressSessionAuth = aws.Bool(true) + }) +} + +// TestS3Express_FullFlow drives the real aws-sdk-go-v2 client through the +// complete S3 Express One Zone flow -- CreateBucket (directory) triggers the +// SDK's automatic CreateSession, then PutObject/GetObject/ListObjectsV2 sign +// with the returned session credentials and the x-amz-s3session-token header +// -- exercising exactly the flow gopherstack-z2w1a reported as a 403 +// SignatureDoesNotMatch. Regression test for the root cause: verifyHeaderAuth +// rejected any Authorization header whose credential scope wasn't literally +// "s3"/"s3-object-lambda", but S3 Express requests are always signed with the +// "s3express" signing name. +func TestS3Express_FullFlow(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + bucketName string + key string + body string + }{ + { + name: "small text object", + bucketName: "expr-flow-a--use1-az4--x-s3", + key: "hello.txt", + body: "hello from s3 express", + }, + { + name: "empty object", + bucketName: "expr-flow-b--use1-az4--x-s3", + key: "empty.txt", + body: "", + }, + { + name: "nested key", + bucketName: "expr-flow-c--use1-az4--x-s3", + key: "a/b/c.txt", + body: "nested", + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + client := newRealS3ExpressClientTest(t) + ctx := t.Context() + + _, err := client.CreateBucket(ctx, &sdk_s3.CreateBucketInput{ + Bucket: aws.String(tt.bucketName), + CreateBucketConfiguration: &types.CreateBucketConfiguration{ + Bucket: &types.BucketInfo{ + Type: types.BucketTypeDirectory, + DataRedundancy: types.DataRedundancySingleAvailabilityZone, + }, + Location: &types.LocationInfo{ + Type: types.LocationTypeAvailabilityZone, + Name: aws.String("use1-az4"), + }, + }, + }) + require.NoError(t, err, "CreateBucket (directory) must succeed") + + _, err = client.HeadBucket(ctx, &sdk_s3.HeadBucketInput{Bucket: aws.String(tt.bucketName)}) + require.NoError(t, err, "HeadBucket on the new directory bucket must succeed") + + _, err = client.PutObject(ctx, &sdk_s3.PutObjectInput{ + Bucket: aws.String(tt.bucketName), + Key: aws.String(tt.key), + Body: strings.NewReader(tt.body), + }) + require.NoError(t, err, "PutObject via the session-credential flow must succeed") + + getOut, err := client.GetObject(ctx, &sdk_s3.GetObjectInput{ + Bucket: aws.String(tt.bucketName), + Key: aws.String(tt.key), + }) + require.NoError(t, err, "GetObject via the session-credential flow must succeed") + gotBody, err := io.ReadAll(getOut.Body) + require.NoError(t, err) + assert.Equal(t, tt.body, string(gotBody)) + + listOut, err := client.ListObjectsV2(ctx, &sdk_s3.ListObjectsV2Input{ + Bucket: aws.String(tt.bucketName), + Delimiter: aws.String("/"), + }) + require.NoError(t, err, "ListObjectsV2 with delimiter \"/\" must succeed on a directory bucket") + var gotKeys []string + for _, obj := range listOut.Contents { + gotKeys = append(gotKeys, aws.ToString(obj.Key)) + } + if !strings.Contains(tt.key, "/") { + assert.Contains(t, gotKeys, tt.key) + } + + _, err = client.DeleteObject(ctx, &sdk_s3.DeleteObjectInput{ + Bucket: aws.String(tt.bucketName), + Key: aws.String(tt.key), + }) + require.NoError(t, err, "DeleteObject via the session-credential flow must succeed") + + _, err = client.DeleteBucket(ctx, &sdk_s3.DeleteBucketInput{Bucket: aws.String(tt.bucketName)}) + require.NoError(t, err, "DeleteBucket on the now-empty directory bucket must succeed") + }) + } +} + +// TestS3Express_ListDirectoryBuckets confirms ListDirectoryBuckets reaches +// gopherstack and returns only directory buckets, excluding general-purpose +// ones, and that ListBuckets excludes directory buckets in turn. +func TestS3Express_ListDirectoryBuckets(t *testing.T) { + t.Parallel() + + client := newRealS3ExpressClientTest(t) + ctx := t.Context() + + const ( + dirBucket = "expr-list-dir--use1-az4--x-s3" + gpBucket = "expr-list-gp" + ) + + _, err := client.CreateBucket(ctx, &sdk_s3.CreateBucketInput{Bucket: aws.String(dirBucket)}) + require.NoError(t, err) + _, err = client.CreateBucket(ctx, &sdk_s3.CreateBucketInput{Bucket: aws.String(gpBucket)}) + require.NoError(t, err) + + dirOut, err := client.ListDirectoryBuckets(ctx, &sdk_s3.ListDirectoryBucketsInput{}) + require.NoError(t, err, "ListDirectoryBuckets must reach the wire and succeed") + + dirNames := make([]string, 0, len(dirOut.Buckets)) + for _, b := range dirOut.Buckets { + dirNames = append(dirNames, aws.ToString(b.Name)) + } + assert.Contains(t, dirNames, dirBucket) + assert.NotContains(t, dirNames, gpBucket) + + listOut, err := client.ListBuckets(ctx, &sdk_s3.ListBucketsInput{}) + require.NoError(t, err) + + gpNames := make([]string, 0, len(listOut.Buckets)) + for _, b := range listOut.Buckets { + gpNames = append(gpNames, aws.ToString(b.Name)) + } + assert.Contains(t, gpNames, gpBucket) + assert.NotContains(t, gpNames, dirBucket) +} + +// TestS3Express_DirectoryBucketSemantics covers the two cheap-to-model +// directory-bucket restrictions: ListObjectsV2 requires Delimiter "/" (or +// none), and ListObjects (V1) is not supported at all (s3@v1.111.0 +// api_op_ListObjects.go:13). +func TestS3Express_DirectoryBucketSemantics(t *testing.T) { + t.Parallel() + + client := newRealS3ExpressClientTest(t) + ctx := t.Context() + + const bucket = "expr-semantics--use1-az4--x-s3" + + { + _, err := client.CreateBucket(ctx, &sdk_s3.CreateBucketInput{Bucket: aws.String(bucket)}) + require.NoError(t, err) + } + + t.Run("ListObjectsV2 rejects a non-slash delimiter", func(t *testing.T) { + t.Parallel() + + _, err := client.ListObjectsV2(ctx, &sdk_s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Delimiter: aws.String(","), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "InvalidArgument", apiErr.ErrorCode()) + }) + + t.Run("ListObjects (V1) is not supported", func(t *testing.T) { + t.Parallel() + + _, err := client.ListObjects(ctx, &sdk_s3.ListObjectsInput{Bucket: aws.String(bucket)}) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "NotImplemented", apiErr.ErrorCode()) + }) +} + +// TestS3ExpressSession_Expiry is a synctest regression test for the 5-minute +// CreateSession TTL: a session must authenticate requests right up to (but +// not past) its expiry. +func TestS3ExpressSession_Expiry(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + backend := s3.NewInMemoryBackend(&s3.GzipCompressor{}) + mustCreateBucket(t, backend, "expiry--use1-az4--x-s3") + + creds, err := backend.CreateSession(t.Context(), "expiry--use1-az4--x-s3", types.SessionModeReadWrite) + require.NoError(t, err) + + bucket, secret, ok := backend.ExpressSessionSecret(creds.AccessKeyID, creds.SessionToken) + require.True(t, ok, "a freshly issued session must resolve") + assert.Equal(t, "expiry--use1-az4--x-s3", bucket) + assert.Equal(t, creds.SecretAccessKey, secret) + + time.Sleep(4 * time.Minute) + + _, _, ok = backend.ExpressSessionSecret(creds.AccessKeyID, creds.SessionToken) + assert.True(t, ok, "a session must still resolve within its 5-minute TTL") + + time.Sleep(2 * time.Minute) // total elapsed: 6 minutes, past the 5-minute TTL + + _, _, ok = backend.ExpressSessionSecret(creds.AccessKeyID, creds.SessionToken) + assert.False(t, ok, "a session must stop resolving once its TTL has passed") + }) +} + +// TestS3ExpressSession_TTLBoundsGrowth locks in that CreateSession's store +// cannot leak: expired sessions are swept the next time CreateSession is +// called, not retained forever. +func TestS3ExpressSession_TTLBoundsGrowth(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + backend := s3.NewInMemoryBackend(&s3.GzipCompressor{}) + mustCreateBucket(t, backend, "sweep--use1-az4--x-s3") + + old := make([]s3.SessionCredentials, 0, 5) + for range 5 { + c, err := backend.CreateSession(t.Context(), "sweep--use1-az4--x-s3", types.SessionModeReadWrite) + require.NoError(t, err) + old = append(old, c) + } + + time.Sleep(6 * time.Minute) + + fresh, err := backend.CreateSession(t.Context(), "sweep--use1-az4--x-s3", types.SessionModeReadWrite) + require.NoError(t, err) + + for _, c := range old { + _, _, ok := backend.ExpressSessionSecret(c.AccessKeyID, c.SessionToken) + assert.False(t, ok, "expired sessions must be swept, not retained") + } + + _, _, ok := backend.ExpressSessionSecret(fresh.AccessKeyID, fresh.SessionToken) + assert.True(t, ok, "the freshly created session must still resolve") + }) +} diff --git a/services/s3/handler_capabilities_test.go b/services/s3/handler_capabilities_test.go index c2e8b6543d..bb197374e7 100644 --- a/services/s3/handler_capabilities_test.go +++ b/services/s3/handler_capabilities_test.go @@ -274,18 +274,23 @@ func TestS3_CreateSession(t *testing.T) { wantStatus int }{ { - name: "CreateSession returns mock credentials", + name: "CreateSession returns generated credentials", bucket: "session-bucket", path: "/session-bucket?session", wantStatus: http.StatusOK, - wantBody: "gopherstack-mock-session-token", + wantBody: "CreateSessionResult", }, { - name: "CreateSession on missing bucket returns 404", + // CreateSession deliberately does not require the bucket to + // already exist: the pinned SDK issues an implicit CreateSession + // for CreateBucket itself on a directory-bucket-shaped name when + // a custom BaseEndpoint is configured, before the bucket exists + // (see express_session.go's CreateSession doc comment). + name: "CreateSession on not-yet-created bucket still succeeds", bucket: "", - path: "/no-such-bucket?session", - wantStatus: http.StatusNotFound, - wantBody: "NoSuchBucket", + path: "/not-yet-created--use1-az4--x-s3?session", + wantStatus: http.StatusOK, + wantBody: "CreateSessionResult", }, } diff --git a/services/s3/handler_list_v2.go b/services/s3/handler_list_v2.go index 666c6534ef..a53f808694 100644 --- a/services/s3/handler_list_v2.go +++ b/services/s3/handler_list_v2.go @@ -2,6 +2,7 @@ package s3 import ( "context" + "encoding/xml" "errors" "net/http" "net/url" @@ -135,5 +136,9 @@ func (h *S3Handler) renderListObjectsV2Response( } resp.KeyCount = len(resp.Contents) + len(resp.CommonPrefixes) - httputils.WriteXML(ctx, w, http.StatusOK, resp) + buf := httputils.GetBuffer() + defer httputils.PutBuffer(buf) + buf.WriteString(xml.Header) + writeListBucketV2XML(buf, &resp) + writeListXMLResponse(ctx, w, http.StatusOK, buf) } diff --git a/services/s3/interfaces.go b/services/s3/interfaces.go index 18ccbcf819..09d9acb51f 100644 --- a/services/s3/interfaces.go +++ b/services/s3/interfaces.go @@ -219,8 +219,10 @@ type StorageBackend interface { DeleteBucketMetricsConfiguration(ctx context.Context, bucket, id string) error ListBucketMetricsConfigurations(ctx context.Context, bucket string) ([]string, error) - // Session - CreateSession(ctx context.Context, bucket string) (string, error) + // Session (S3 Express One Zone) + CreateSession(ctx context.Context, bucket string, sessionMode types.SessionMode) (SessionCredentials, error) + ExpressSessionSecret(accessKeyID, sessionToken string) (bucket, secret string, ok bool) + IsDirectoryBucket(bucket string) bool // Accelerate / RequestPayment configurations PutBucketAccelerateConfiguration(ctx context.Context, bucket, status string) error diff --git a/services/s3/lifecycle_transition_test.go b/services/s3/lifecycle_transition_test.go index 35036477aa..9c21b072ea 100644 --- a/services/s3/lifecycle_transition_test.go +++ b/services/s3/lifecycle_transition_test.go @@ -2,7 +2,6 @@ package s3_test import ( "bytes" - "context" "testing" "time" @@ -56,14 +55,12 @@ func TestS3Lifecycle_StorageClassTransitions(t *testing.T) { wantClass: "GLACIER", verify: func(t *testing.T, b *s3.InMemoryBackend) { t.Helper() - require.Eventually(t, func() bool { - out, err := b.HeadObject(t.Context(), &sdk_s3.HeadObjectInput{ - Bucket: aws.String("tr-days"), - Key: aws.String("old-obj.txt"), - }) - - return err == nil && string(out.StorageClass) == "GLACIER" - }, 500*time.Millisecond, 10*time.Millisecond, "object must be transitioned to GLACIER") + out, err := b.HeadObject(t.Context(), &sdk_s3.HeadObjectInput{ + Bucket: aws.String("tr-days"), + Key: aws.String("old-obj.txt"), + }) + require.NoError(t, err) + require.Equal(t, "GLACIER", string(out.StorageClass)) }, }, { @@ -86,13 +83,6 @@ func TestS3Lifecycle_StorageClassTransitions(t *testing.T) { wantClass: "STANDARD_IA", verify: func(t *testing.T, b *s3.InMemoryBackend) { t.Helper() - // Wait for transition to fire. - require.Eventually(t, func() bool { - history := s3.StorageClassTransitionsForObject(b, "tr-hist", "doc.txt") - - return len(history) >= 1 - }, 500*time.Millisecond, 10*time.Millisecond, "transition history must be recorded") - history := s3.StorageClassTransitionsForObject(b, "tr-hist", "doc.txt") require.Len(t, history, 1) assert.Equal(t, "STANDARD", history[0].FromClass) @@ -120,14 +110,12 @@ func TestS3Lifecycle_StorageClassTransitions(t *testing.T) { wantClass: "DEEP_ARCHIVE", verify: func(t *testing.T, b *s3.InMemoryBackend) { t.Helper() - require.Eventually(t, func() bool { - out, err := b.HeadObject(t.Context(), &sdk_s3.HeadObjectInput{ - Bucket: aws.String("tr-date"), - Key: aws.String("archive.bin"), - }) - - return err == nil && string(out.StorageClass) == "DEEP_ARCHIVE" - }, 500*time.Millisecond, 10*time.Millisecond, "object must be transitioned to DEEP_ARCHIVE") + out, err := b.HeadObject(t.Context(), &sdk_s3.HeadObjectInput{ + Bucket: aws.String("tr-date"), + Key: aws.String("archive.bin"), + }) + require.NoError(t, err) + require.Equal(t, "DEEP_ARCHIVE", string(out.StorageClass)) }, }, { @@ -334,34 +322,23 @@ func TestS3Lifecycle_NoncurrentVersionTransitions(t *testing.T) { err = b.PutBucketLifecycleConfiguration(t.Context(), tt.bucket, tt.lcXML, "") require.NoError(t, err) - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - - go newFastJanitor(b).Run(ctx) - - // Wait for noncurrent version to be transitioned. - require.Eventually(t, func() bool { - out, listErr := b.ListObjectVersions(t.Context(), &sdk_s3.ListObjectVersionsInput{ - Bucket: aws.String(tt.bucket), - }) - if listErr != nil { - return false - } - for _, ver := range out.Versions { - if !aws.ToBool(ver.IsLatest) && string(ver.StorageClass) == "GLACIER" { - return true - } - } - - return false - }, 500*time.Millisecond, 10*time.Millisecond, "noncurrent version must be transitioned to GLACIER") + j := newFastJanitor(b) + j.SweepOnce(t.Context()) + j.SweepOnce(t.Context()) - // Latest version must still be STANDARD. out, err := b.ListObjectVersions(t.Context(), &sdk_s3.ListObjectVersionsInput{ Bucket: aws.String(tt.bucket), }) require.NoError(t, err) + transitioned := false + for _, ver := range out.Versions { + if !aws.ToBool(ver.IsLatest) && string(ver.StorageClass) == "GLACIER" { + transitioned = true + } + } + require.True(t, transitioned, "noncurrent version must be transitioned to GLACIER") + for _, ver := range out.Versions { if aws.ToBool(ver.IsLatest) { assert.NotEqual(t, "GLACIER", string(ver.StorageClass), diff --git a/services/s3/listing.go b/services/s3/listing.go index 8d80827f72..79ba568aaa 100644 --- a/services/s3/listing.go +++ b/services/s3/listing.go @@ -310,6 +310,11 @@ func (b *InMemoryBackend) ListObjectsV2( ctx context.Context, input *s3.ListObjectsV2Input, ) (*s3.ListObjectsV2Output, error) { + delim := aws.ToString(input.Delimiter) + if delim != "" && delim != "/" && b.IsDirectoryBucket(aws.ToString(input.Bucket)) { + return nil, ErrDirectoryBucketDelimiter + } + // Re-use ListObjects logic but handle V2 specific params marker := "" if input.ContinuationToken != nil && *input.ContinuationToken != "" { diff --git a/services/s3/listing_xml_fast.go b/services/s3/listing_xml_fast.go new file mode 100644 index 0000000000..e4c6e58f5f --- /dev/null +++ b/services/s3/listing_xml_fast.go @@ -0,0 +1,202 @@ +package s3 + +import ( + "bytes" + "context" + "net/http" + "strconv" + "unicode/utf8" + + "github.com/blackbirdworks/gopherstack/pkgs/logger" +) + +// Hand-encodes the List(Objects|ObjectsV2) XML body -- encoding/xml reflection +// was ~45% of ListObjectsV2's CPU. Must stay byte-identical to xml.Encoder.Encode. + +// writeXMLElem writes tag containing s, always present (no omitempty). +func writeXMLElem(buf *bytes.Buffer, tag, s string) { + buf.WriteByte('<') + buf.WriteString(tag) + buf.WriteByte('>') + escapeXMLString(buf, s) + buf.WriteString("') +} + +// writeXMLElemOmitEmpty writes tag only when s is non-empty, mirroring an +// `omitempty` struct tag on a string field. +func writeXMLElemOmitEmpty(buf *bytes.Buffer, tag, s string) { + if s == "" { + return + } + writeXMLElem(buf, tag, s) +} + +func writeXMLInt(buf *bytes.Buffer, tag string, n int) { + buf.WriteByte('<') + buf.WriteString(tag) + buf.WriteByte('>') + buf.WriteString(strconv.Itoa(n)) + buf.WriteString("') +} + +func writeXMLInt64(buf *bytes.Buffer, tag string, n int64) { + buf.WriteByte('<') + buf.WriteString(tag) + buf.WriteByte('>') + buf.WriteString(strconv.FormatInt(n, 10)) + buf.WriteString("') +} + +func writeXMLBool(buf *bytes.Buffer, tag string, v bool) { + buf.WriteByte('<') + buf.WriteString(tag) + buf.WriteByte('>') + if v { + buf.WriteString("true") + } else { + buf.WriteString("false") + } + buf.WriteString("') +} + +// escapeXMLString mirrors encoding/xml printer.EscapeString's escaping table, +// so hand-written output stays byte-identical to reflection-based marshaling. +func escapeXMLString(buf *bytes.Buffer, s string) { + last := 0 + for i := 0; i < len(s); { + r, width := utf8.DecodeRuneInString(s[i:]) + i += width + + var esc string + switch r { + case '"': + esc = """ + case '\'': + esc = "'" + case '&': + esc = "&" + case '<': + esc = "<" + case '>': + esc = ">" + case '\t': + esc = " " + case '\n': + esc = " " + case '\r': + esc = " " + default: + if !isValidXMLChar(r) || (r == 0xFFFD && width == 1) { + esc = "�" + + break + } + + continue + } + + buf.WriteString(s[last : i-width]) + buf.WriteString(esc) + last = i + } + buf.WriteString(s[last:]) +} + +// isValidXMLChar mirrors encoding/xml's isInCharacterRange. +func isValidXMLChar(r rune) bool { + return r == 0x09 || r == 0x0A || r == 0x0D || + r >= 0x20 && r <= 0xD7FF || + r >= 0xE000 && r <= 0xFFFD || + r >= 0x10000 && r <= 0x10FFFF +} + +func writeOwnerXML(buf *bytes.Buffer, o *Owner) { + if o == nil { + return + } + buf.WriteString("") + writeXMLElem(buf, "ID", o.ID) + writeXMLElem(buf, "DisplayName", o.DisplayName) + buf.WriteString("") +} + +func writeObjectXML(buf *bytes.Buffer, o *ObjectXML) { + buf.WriteString("") + writeOwnerXML(buf, o.Owner) + writeXMLElem(buf, "Key", o.Key) + writeXMLElem(buf, "LastModified", o.LastModified) + writeXMLElem(buf, "ETag", o.ETag) + writeXMLElem(buf, "StorageClass", o.StorageClass) + writeXMLElemOmitEmpty(buf, "ChecksumAlgorithm", o.ChecksumAlgorithm) + writeXMLInt64(buf, "Size", o.Size) + buf.WriteString("") +} + +func writeCommonPrefixXML(buf *bytes.Buffer, cp *CommonPrefixXML) { + buf.WriteString("") + writeXMLElem(buf, "Prefix", cp.Prefix) + buf.WriteString("") +} + +// writeListBucketV2XML appends the ListObjectsV2 response body to buf. Field +// order and omitempty behavior mirror ListBucketV2Result's xml tags exactly. +func writeListBucketV2XML(buf *bytes.Buffer, r *ListBucketV2Result) { + buf.WriteString("") + writeXMLElemOmitEmpty(buf, "StartAfter", r.StartAfter) + writeXMLElem(buf, "Prefix", r.Prefix) + writeXMLElemOmitEmpty(buf, "Delimiter", r.Delimiter) + writeXMLElemOmitEmpty(buf, "ContinuationToken", r.ContinuationToken) + writeXMLElemOmitEmpty(buf, "NextContinuationToken", r.NextContinuationToken) + writeXMLElem(buf, "Name", r.Name) + writeXMLElemOmitEmpty(buf, "EncodingType", r.EncodingType) + for i := range r.Contents { + writeObjectXML(buf, &r.Contents[i]) + } + for i := range r.CommonPrefixes { + writeCommonPrefixXML(buf, &r.CommonPrefixes[i]) + } + writeXMLInt(buf, "KeyCount", r.KeyCount) + writeXMLInt(buf, "MaxKeys", r.MaxKeys) + writeXMLBool(buf, "IsTruncated", r.IsTruncated) + buf.WriteString("") +} + +// writeListBucketXML appends the ListObjects (v1) response body to buf. Field +// order and omitempty behavior mirror ListBucketResult's xml tags exactly. +func writeListBucketXML(buf *bytes.Buffer, r *ListBucketResult) { + buf.WriteString("") + writeXMLElem(buf, "Name", r.Name) + writeXMLElem(buf, "Prefix", r.Prefix) + writeXMLElemOmitEmpty(buf, "Delimiter", r.Delimiter) + writeXMLElemOmitEmpty(buf, "Marker", r.Marker) + writeXMLElemOmitEmpty(buf, "NextMarker", r.NextMarker) + writeXMLElemOmitEmpty(buf, "EncodingType", r.EncodingType) + for i := range r.Contents { + writeObjectXML(buf, &r.Contents[i]) + } + for i := range r.CommonPrefixes { + writeCommonPrefixXML(buf, &r.CommonPrefixes[i]) + } + writeXMLInt(buf, "MaxKeys", r.MaxKeys) + writeXMLBool(buf, "IsTruncated", r.IsTruncated) + buf.WriteString("") +} + +// writeListXMLResponse writes an already-encoded XML body (header + root +// element) with the same headers httputils.WriteXML sets. +func writeListXMLResponse(ctx context.Context, w http.ResponseWriter, code int, buf *bytes.Buffer) { + w.Header().Set("Content-Type", "application/xml") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(code) + if _, err := buf.WriteTo(w); err != nil { + logger.Load(ctx).ErrorContext(ctx, "failed to write XML response", "error", err) + } +} diff --git a/services/s3/listing_xml_fast_whitebox_test.go b/services/s3/listing_xml_fast_whitebox_test.go new file mode 100644 index 0000000000..b3cb76a4ea --- /dev/null +++ b/services/s3/listing_xml_fast_whitebox_test.go @@ -0,0 +1,556 @@ +package s3 + +import ( + "bytes" + "encoding/xml" + "fmt" + "reflect" + "testing" + + "github.com/stretchr/testify/require" +) + +// oldEncode is the pre-optimization reflection-based path, kept only here as +// the byte-equivalence reference for listing_xml_fast.go's hand-written encoder. +func oldEncode(t *testing.T, payload any) []byte { + t.Helper() + + var buf bytes.Buffer + buf.WriteString(xml.Header) + require.NoError(t, xml.NewEncoder(&buf).Encode(payload)) + + return buf.Bytes() +} + +func testOwner() *Owner { + return &Owner{ID: "gopherstack", DisplayName: "gopherstack"} +} + +// invalidUTF8AndIllegalRunesKey mixes raw invalid bytes, a truncated +// multi-byte sequence, \x0B, U+FFFE, and a lone surrogate encoded as bytes. +const invalidUTF8AndIllegalRunesKey = "a\xff\xfeb\xe4\xb8c\x0bd￾e\xed\xa0\x80f" + +func TestListBucketV2XML_GoldenEquivalence(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + resp ListBucketV2Result + }{ + { + name: "empty_list", + resp: ListBucketV2Result{Name: "b", Prefix: "", MaxKeys: 1000, KeyCount: 0}, + }, + { + name: "escaping_special_chars", + resp: ListBucketV2Result{ + Name: "b", + Contents: []ObjectXML{ + { + Key: `a&bd"e'f`, + LastModified: "2024-01-01T00:00:00Z", + ETag: `"abc123"`, + StorageClass: "STANDARD", + Size: 5, + }, + }, + KeyCount: 1, + MaxKeys: 1000, + }, + }, + { + name: "control_characters", + resp: ListBucketV2Result{ + Name: "b", + Contents: []ObjectXML{ + { + Key: "tab\ttab\nnewline\rcr\x00null\x01soh", + LastModified: "2024-01-01T00:00:00Z", + ETag: "etag", + StorageClass: "STANDARD", + Size: 1, + }, + }, + KeyCount: 1, + MaxKeys: 1000, + }, + }, + { + name: "unicode_keys", + resp: ListBucketV2Result{ + Name: "b", + Contents: []ObjectXML{ + { + Key: "日本語/文件-é-😀", + LastModified: "2024-01-01T00:00:00Z", + ETag: "etag", + StorageClass: "STANDARD", + Size: 2, + }, + }, + KeyCount: 1, + MaxKeys: 1000, + }, + }, + { + name: "invalid_utf8_and_illegal_runes", + resp: ListBucketV2Result{ + Name: "b", + Contents: []ObjectXML{ + { + Key: invalidUTF8AndIllegalRunesKey, + LastModified: "2024-01-01T00:00:00Z", + ETag: "etag", + StorageClass: "STANDARD", + Size: 2, + }, + }, + KeyCount: 1, + MaxKeys: 1000, + }, + }, + { + name: "encoding_type_url", + resp: ListBucketV2Result{ + Name: "b", + Prefix: "a%2Fb", + EncodingType: "url", + Contents: []ObjectXML{ + { + Key: "a%2Fb%2Fkey+with+spaces", + LastModified: "2024-01-01T00:00:00Z", + ETag: "etag", + StorageClass: "STANDARD", + Size: 3, + }, + }, + KeyCount: 1, + MaxKeys: 1000, + }, + }, + { + name: "delimiter_and_common_prefixes", + resp: ListBucketV2Result{ + Name: "b", + Delimiter: "/", + CommonPrefixes: []CommonPrefixXML{ + {Prefix: "dir1/"}, + {Prefix: "dir2/"}, + }, + Contents: []ObjectXML{ + { + Key: "root-key", + LastModified: "2024-01-01T00:00:00Z", + ETag: "etag", + StorageClass: "STANDARD", + Size: 4, + }, + }, + KeyCount: 3, + MaxKeys: 1000, + }, + }, + { + name: "owner_present", + resp: ListBucketV2Result{ + Name: "b", + Contents: []ObjectXML{ + { + Owner: testOwner(), + Key: "key-with-owner", + LastModified: "2024-01-01T00:00:00Z", + ETag: "etag", + StorageClass: "STANDARD", + Size: 6, + }, + }, + KeyCount: 1, + MaxKeys: 1000, + }, + }, + { + name: "owner_absent", + resp: ListBucketV2Result{ + Name: "b", + Contents: []ObjectXML{ + { + Owner: nil, + Key: "key-without-owner", + LastModified: "2024-01-01T00:00:00Z", + ETag: "etag", + StorageClass: "STANDARD", + Size: 7, + }, + }, + KeyCount: 1, + MaxKeys: 1000, + }, + }, + { + name: "checksum_algorithms", + resp: ListBucketV2Result{ + Name: "b", + Contents: []ObjectXML{ + { + Key: "k1", LastModified: "t", ETag: "e1", + StorageClass: "STANDARD", ChecksumAlgorithm: "CRC32", Size: 1, + }, + { + Key: "k2", LastModified: "t", ETag: "e2", + StorageClass: "STANDARD", ChecksumAlgorithm: "SHA256", Size: 2, + }, + { + Key: "k3", LastModified: "t", ETag: "e3", + StorageClass: "STANDARD", ChecksumAlgorithm: "", Size: 3, + }, + }, + KeyCount: 3, + MaxKeys: 1000, + }, + }, + { + name: "truncated_with_continuation_token", + resp: ListBucketV2Result{ + Name: "b", + StartAfter: "start-key", + ContinuationToken: "cont-token", + NextContinuationToken: "next-token", + IsTruncated: true, + Contents: []ObjectXML{ + {Key: "k1", LastModified: "t", ETag: "e1", StorageClass: "STANDARD", Size: 1}, + }, + KeyCount: 1, + MaxKeys: 1, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + want := oldEncode(t, tt.resp) + + var got bytes.Buffer + got.WriteString(xml.Header) + writeListBucketV2XML(&got, &tt.resp) + + require.Equal(t, string(want), got.String()) + }) + } +} + +func TestListBucketXML_GoldenEquivalence(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + resp ListBucketResult + }{ + { + name: "empty_list", + resp: ListBucketResult{Name: "b", Prefix: "", MaxKeys: 1000}, + }, + { + name: "escaping_special_chars", + resp: ListBucketResult{ + Name: "b", + Contents: []ObjectXML{ + { + Owner: testOwner(), + Key: `a&bd"e'f`, + LastModified: "2024-01-01T00:00:00Z", + ETag: `"abc123"`, + StorageClass: "STANDARD", + Size: 5, + }, + }, + MaxKeys: 1000, + }, + }, + { + name: "control_characters_and_unicode", + resp: ListBucketResult{ + Name: "b", + Contents: []ObjectXML{ + { + Owner: testOwner(), + Key: "tab\ttab\nnewline\rcr\x00null-日本語-😀", + LastModified: "2024-01-01T00:00:00Z", + ETag: "etag", + StorageClass: "STANDARD", + Size: 1, + }, + }, + MaxKeys: 1000, + }, + }, + { + name: "invalid_utf8_and_illegal_runes", + resp: ListBucketResult{ + Name: "b", + Contents: []ObjectXML{ + { + Owner: testOwner(), + Key: invalidUTF8AndIllegalRunesKey, + LastModified: "2024-01-01T00:00:00Z", + ETag: "etag", + StorageClass: "STANDARD", + Size: 1, + }, + }, + MaxKeys: 1000, + }, + }, + { + name: "encoding_type_url_with_marker", + resp: ListBucketResult{ + Name: "b", + Prefix: "a%2Fb", + Marker: "marker%2Fkey", + NextMarker: "next%2Fmarker", + EncodingType: "url", + Contents: []ObjectXML{ + { + Owner: testOwner(), + Key: "a%2Fb%2Fkey+with+spaces", + LastModified: "2024-01-01T00:00:00Z", + ETag: "etag", + StorageClass: "STANDARD", + Size: 3, + }, + }, + MaxKeys: 1000, + IsTruncated: true, + }, + }, + { + name: "delimiter_and_common_prefixes", + resp: ListBucketResult{ + Name: "b", + Delimiter: "/", + CommonPrefixes: []CommonPrefixXML{ + {Prefix: "dir1/"}, + {Prefix: "dir2/"}, + }, + Contents: []ObjectXML{ + { + Owner: testOwner(), + Key: "root-key", + LastModified: "2024-01-01T00:00:00Z", + ETag: "etag", + StorageClass: "STANDARD", + Size: 4, + }, + }, + MaxKeys: 1000, + }, + }, + { + name: "checksum_algorithms", + resp: ListBucketResult{ + Name: "b", + Contents: []ObjectXML{ + { + Owner: testOwner(), Key: "k1", LastModified: "t", ETag: "e1", + StorageClass: "STANDARD", ChecksumAlgorithm: "CRC32C", Size: 1, + }, + { + Owner: testOwner(), Key: "k2", LastModified: "t", ETag: "e2", + StorageClass: "STANDARD", ChecksumAlgorithm: "", Size: 2, + }, + }, + MaxKeys: 1000, + }, + }, + { + name: "truncated", + resp: ListBucketResult{ + Name: "b", + NextMarker: "next-key", + IsTruncated: true, + Contents: []ObjectXML{ + {Owner: testOwner(), Key: "k1", LastModified: "t", ETag: "e1", StorageClass: "STANDARD", Size: 1}, + }, + MaxKeys: 1, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + want := oldEncode(t, tt.resp) + + var got bytes.Buffer + got.WriteString(xml.Header) + writeListBucketXML(&got, &tt.resp) + + require.Equal(t, string(want), got.String()) + }) + } +} + +func TestEscapeXMLString_MatchesStdlib(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in string + }{ + {name: "empty", in: ""}, + {name: "ascii", in: "hello-world"}, + {name: "all_special", in: `&<>"'`}, + {name: "control_chars", in: "\t\n\r\x00\x1f"}, + {name: "unicode", in: "日本語😀é"}, + {name: "invalid_replacement_char", in: "a�b"}, + {name: "invalid_utf8_bytes", in: "a\xff\xfeb"}, + {name: "truncated_multibyte_sequence", in: "a\xe4\xb8b"}, + {name: "vertical_tab_illegal_xml_char", in: "a\x0bb"}, + {name: "noncharacter_ufffe", in: "a￾b"}, + {name: "lone_surrogate_as_bytes", in: "a\xed\xa0\x80b"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + type wrapper struct { + XMLName xml.Name `xml:"W"` + V string `xml:"V"` + } + + want := oldEncode(t, wrapper{V: tt.in}) + + var got bytes.Buffer + got.WriteString(xml.Header) + got.WriteString("") + writeXMLElem(&got, "V", tt.in) + got.WriteString("") + + require.Equal(t, string(want), got.String()) + }) + } +} + +// fillNonZero recursively sets every exported, settable field of v to a +// distinct non-zero value, so a filled struct exercises every field. +func fillNonZero(v reflect.Value, seed *int) { + switch v.Kind() { //nolint:exhaustive // only kinds used by the XML response types + case reflect.String: + *seed++ + v.SetString(fmt.Sprintf("v%d", *seed)) + case reflect.Bool: + v.SetBool(true) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + *seed++ + v.SetInt(int64(*seed)) + case reflect.Slice: + elem := reflect.New(v.Type().Elem()).Elem() + fillNonZero(elem, seed) + s := reflect.MakeSlice(v.Type(), 1, 1) + s.Index(0).Set(elem) + v.Set(s) + case reflect.Pointer: + p := reflect.New(v.Type().Elem()) + fillNonZero(p.Elem(), seed) + v.Set(p) + case reflect.Struct: + fillStructFields(v, seed) + } +} + +// fillStructFields skips XMLName: its runtime value never affects a +// tag-named root element (verified against encoding/xml's own behavior). +func fillStructFields(v reflect.Value, seed *int) { + for i := range v.NumField() { + f := v.Type().Field(i) + if f.Name == "XMLName" || !v.Field(i).CanSet() { + continue + } + fillNonZero(v.Field(i), seed) + } +} + +// newFilled builds a fully non-zero-filled *T via reflection, so no field +// can stay accidentally zero/unexercised in the drift-guard test below. +func newFilled[T any]() *T { + v := reflect.New(reflect.TypeOf(*new(T))).Elem() + seed := 0 + fillNonZero(v, &seed) + + return v.Addr().Interface().(*T) //nolint:forcetypeassert // v was constructed from T above +} + +// Fills every field via reflection: a field the hand-written encoder doesn't +// know about shows up here as a byte diff instead of silently vanishing. +func TestListXML_DriftGuard_ReflectiveFill(t *testing.T) { + t.Parallel() + + t.Run("list_bucket_v2_result", func(t *testing.T) { + t.Parallel() + + resp := newFilled[ListBucketV2Result]() + want := oldEncode(t, resp) + + var got bytes.Buffer + got.WriteString(xml.Header) + writeListBucketV2XML(&got, resp) + + require.Equal( + t, + string(want), + got.String(), + "a field on ListBucketV2Result or a nested type changed -- update writeListBucketV2XML in listing_xml_fast.go", + ) + }) + + t.Run("list_bucket_result", func(t *testing.T) { + t.Parallel() + + resp := newFilled[ListBucketResult]() + want := oldEncode(t, resp) + + var got bytes.Buffer + got.WriteString(xml.Header) + writeListBucketXML(&got, resp) + + require.Equal(t, string(want), got.String(), + "a field on ListBucketResult or a nested type changed -- update writeListBucketXML in listing_xml_fast.go") + }) +} + +// assertFieldNames is the fallback drift guard: a field rename/add/remove +// fails loudly here even without the reflective-fill test above. +func assertFieldNames(t *testing.T, typ reflect.Type, want []string) { + t.Helper() + + got := make([]string, 0, typ.NumField()) + for field := range typ.Fields() { + got = append(got, field.Name) + } + + require.Equal(t, want, got, + "%s's fields changed -- update listing_xml_fast.go's XML writer for %s, then update this pinned list", + typ.Name(), typ.Name()) +} + +func TestListXMLStructs_FieldNamesPinned(t *testing.T) { + t.Parallel() + + assertFieldNames(t, reflect.TypeFor[ListBucketV2Result](), []string{ + "XMLName", "StartAfter", "Prefix", "Delimiter", "ContinuationToken", + "NextContinuationToken", "Name", "EncodingType", "Contents", + "CommonPrefixes", "KeyCount", "MaxKeys", "IsTruncated", + }) + assertFieldNames(t, reflect.TypeFor[ListBucketResult](), []string{ + "XMLName", "Name", "Prefix", "Delimiter", "Marker", "NextMarker", + "EncodingType", "Contents", "CommonPrefixes", "MaxKeys", "IsTruncated", + }) + assertFieldNames(t, reflect.TypeFor[ObjectXML](), []string{ + "Owner", "Key", "LastModified", "ETag", "StorageClass", "ChecksumAlgorithm", "Size", + }) + assertFieldNames(t, reflect.TypeFor[Owner](), []string{"ID", "DisplayName"}) + assertFieldNames(t, reflect.TypeFor[CommonPrefixXML](), []string{"Prefix"}) +} diff --git a/services/s3/multipart.go b/services/s3/multipart.go index a189cae024..ebd5167249 100644 --- a/services/s3/multipart.go +++ b/services/s3/multipart.go @@ -150,7 +150,13 @@ func (b *InMemoryBackend) UploadPart( return nil, err } - storedData := bytes.Clone(buf.Bytes()) + // Clone only if PutBuffer will recycle buf; parts are typically well above + // the pool's 64KiB cap and get discarded, so the clone is usually skipped + // (mirrors computeObjectHashes in objects.go). + storedData := buf.Bytes() + if httputils.WillPool(buf) { + storedData = bytes.Clone(storedData) + } etag := hex.EncodeToString(md5Hasher.Sum(nil)) // 2. Validate Content-MD5 from context if present. diff --git a/services/s3/notification_dispatch_test.go b/services/s3/notification_dispatch_test.go index 23344b5409..776b7f245e 100644 --- a/services/s3/notification_dispatch_test.go +++ b/services/s3/notification_dispatch_test.go @@ -7,7 +7,7 @@ import ( "net/http/httptest" "strings" "testing" - "time" + "testing/synctest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -18,219 +18,214 @@ import ( func TestHandler_NotificationDispatch_PutObject(t *testing.T) { t.Parallel() - handler, backend := newTestHandler(t) - mustCreateBucket(t, backend, "notif-put") - - notifXML := `` + - `q1` + - `arn:aws:sqs:us-east-1:000000000000:my-queue` + - `s3:ObjectCreated:*` + - `` - req := httptest.NewRequest( - http.MethodPut, - "/notif-put?notification", - strings.NewReader(notifXML), - ) - rec := httptest.NewRecorder() - serveS3Handler(handler, rec, req) - require.Equal(t, http.StatusOK, rec.Code) - - mock := &mockNotificationDispatcher{} - handler.SetNotificationDispatcher(mock) - - req = httptest.NewRequest(http.MethodPut, "/notif-put/key1", strings.NewReader("hello")) - rec = httptest.NewRecorder() - serveS3Handler(handler, rec, req) - require.Equal(t, http.StatusOK, rec.Code) - - require.Eventually(t, func() bool { + synctest.Test(t, func(t *testing.T) { + handler, backend := newTestHandler(t) + mustCreateBucket(t, backend, "notif-put") + + notifXML := `` + + `q1` + + `arn:aws:sqs:us-east-1:000000000000:my-queue` + + `s3:ObjectCreated:*` + + `` + req := httptest.NewRequest( + http.MethodPut, + "/notif-put?notification", + strings.NewReader(notifXML), + ) + rec := httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + mock := &mockNotificationDispatcher{} + handler.SetNotificationDispatcher(mock) + + req = httptest.NewRequest(http.MethodPut, "/notif-put/key1", strings.NewReader("hello")) + rec = httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + synctest.Wait() + mock.mu.Lock() defer mock.mu.Unlock() - - return len(mock.created) == 1 - }, 200*time.Millisecond, 5*time.Millisecond) - - mock.mu.Lock() - defer mock.mu.Unlock() - assert.Equal(t, "notif-put", mock.created[0].bucket) - assert.Equal(t, "key1", mock.created[0].key) + require.Len(t, mock.created, 1) + assert.Equal(t, "notif-put", mock.created[0].bucket) + assert.Equal(t, "key1", mock.created[0].key) + }) } func TestHandler_NotificationDispatch_DeleteObject(t *testing.T) { t.Parallel() - handler, backend := newTestHandler(t) - mustCreateBucket(t, backend, "notif-del") - mustPutObject(t, backend, "notif-del", "key1", []byte("data")) - - notifXML := `` + - `q1` + - `arn:aws:sqs:us-east-1:000000000000:my-queue` + - `s3:ObjectRemoved:*` + - `` - putNotifReq := httptest.NewRequest( - http.MethodPut, - "/notif-del?notification", - strings.NewReader(notifXML), - ) - rec := httptest.NewRecorder() - serveS3Handler(handler, rec, putNotifReq) - require.Equal(t, http.StatusOK, rec.Code) - - mock := &mockNotificationDispatcher{} - handler.SetNotificationDispatcher(mock) - - req := httptest.NewRequest(http.MethodDelete, "/notif-del/key1", nil) - rec = httptest.NewRecorder() - serveS3Handler(handler, rec, req) - require.Equal(t, http.StatusNoContent, rec.Code) - - require.Eventually(t, func() bool { + synctest.Test(t, func(t *testing.T) { + handler, backend := newTestHandler(t) + mustCreateBucket(t, backend, "notif-del") + mustPutObject(t, backend, "notif-del", "key1", []byte("data")) + + notifXML := `` + + `q1` + + `arn:aws:sqs:us-east-1:000000000000:my-queue` + + `s3:ObjectRemoved:*` + + `` + putNotifReq := httptest.NewRequest( + http.MethodPut, + "/notif-del?notification", + strings.NewReader(notifXML), + ) + rec := httptest.NewRecorder() + serveS3Handler(handler, rec, putNotifReq) + require.Equal(t, http.StatusOK, rec.Code) + + mock := &mockNotificationDispatcher{} + handler.SetNotificationDispatcher(mock) + + req := httptest.NewRequest(http.MethodDelete, "/notif-del/key1", nil) + rec = httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusNoContent, rec.Code) + + synctest.Wait() + mock.mu.Lock() defer mock.mu.Unlock() - - return len(mock.deleted) == 1 - }, 200*time.Millisecond, 5*time.Millisecond) - - mock.mu.Lock() - defer mock.mu.Unlock() - assert.Equal(t, "notif-del", mock.deleted[0].bucket) - assert.Equal(t, "key1", mock.deleted[0].key) + require.Len(t, mock.deleted, 1) + assert.Equal(t, "notif-del", mock.deleted[0].bucket) + assert.Equal(t, "key1", mock.deleted[0].key) + }) } func TestHandler_NotificationDispatch_NoDispatchWithoutConfig(t *testing.T) { t.Parallel() - handler, backend := newTestHandler(t) - mustCreateBucket(t, backend, "no-notif") + synctest.Test(t, func(t *testing.T) { + handler, backend := newTestHandler(t) + mustCreateBucket(t, backend, "no-notif") - mock := &mockNotificationDispatcher{} - handler.SetNotificationDispatcher(mock) + mock := &mockNotificationDispatcher{} + handler.SetNotificationDispatcher(mock) - req := httptest.NewRequest(http.MethodPut, "/no-notif/key1", strings.NewReader("hello")) - rec := httptest.NewRecorder() - serveS3Handler(handler, rec, req) - require.Equal(t, http.StatusOK, rec.Code) + req := httptest.NewRequest(http.MethodPut, "/no-notif/key1", strings.NewReader("hello")) + rec := httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusOK, rec.Code) - time.Sleep(20 * time.Millisecond) - mock.mu.Lock() - defer mock.mu.Unlock() - assert.Empty(t, mock.created) - assert.Empty(t, mock.deleted) + synctest.Wait() + + mock.mu.Lock() + defer mock.mu.Unlock() + assert.Empty(t, mock.created) + assert.Empty(t, mock.deleted) + }) } func TestHandler_NotificationDispatch_CopyObject(t *testing.T) { t.Parallel() - handler, backend := newTestHandler(t) - mustCreateBucket(t, backend, "notif-copy") - mustPutObject(t, backend, "notif-copy", "src-key", []byte("source data")) - - notifXML := `` + - `q1` + - `arn:aws:sqs:us-east-1:000000000000:copy-queue` + - `s3:ObjectCreated:*` + - `` - req := httptest.NewRequest( - http.MethodPut, - "/notif-copy?notification", - strings.NewReader(notifXML), - ) - rec := httptest.NewRecorder() - serveS3Handler(handler, rec, req) - require.Equal(t, http.StatusOK, rec.Code) - - mock := &mockNotificationDispatcher{} - handler.SetNotificationDispatcher(mock) - - req = httptest.NewRequest(http.MethodPut, "/notif-copy/dest-key", nil) - req.Header.Set("X-Amz-Copy-Source", "/notif-copy/src-key") - rec = httptest.NewRecorder() - serveS3Handler(handler, rec, req) - require.Equal(t, http.StatusOK, rec.Code) - - require.Eventually(t, func() bool { + synctest.Test(t, func(t *testing.T) { + handler, backend := newTestHandler(t) + mustCreateBucket(t, backend, "notif-copy") + mustPutObject(t, backend, "notif-copy", "src-key", []byte("source data")) + + notifXML := `` + + `q1` + + `arn:aws:sqs:us-east-1:000000000000:copy-queue` + + `s3:ObjectCreated:*` + + `` + req := httptest.NewRequest( + http.MethodPut, + "/notif-copy?notification", + strings.NewReader(notifXML), + ) + rec := httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + mock := &mockNotificationDispatcher{} + handler.SetNotificationDispatcher(mock) + + req = httptest.NewRequest(http.MethodPut, "/notif-copy/dest-key", nil) + req.Header.Set("X-Amz-Copy-Source", "/notif-copy/src-key") + rec = httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + synctest.Wait() + mock.mu.Lock() defer mock.mu.Unlock() - - return len(mock.created) == 1 - }, 200*time.Millisecond, 5*time.Millisecond) - - mock.mu.Lock() - defer mock.mu.Unlock() - assert.Equal(t, "notif-copy", mock.created[0].bucket) - assert.Equal(t, "dest-key", mock.created[0].key) + require.Len(t, mock.created, 1) + assert.Equal(t, "notif-copy", mock.created[0].bucket) + assert.Equal(t, "dest-key", mock.created[0].key) + }) } func TestHandler_NotificationDispatch_CompleteMultipartUpload(t *testing.T) { t.Parallel() - handler, backend := newTestHandler(t) - mustCreateBucket(t, backend, "notif-mpu") - - notifXML := `` + - `q1` + - `arn:aws:sqs:us-east-1:000000000000:mpu-queue` + - `s3:ObjectCreated:*` + - `` - req := httptest.NewRequest( - http.MethodPut, - "/notif-mpu?notification", - strings.NewReader(notifXML), - ) - rec := httptest.NewRecorder() - serveS3Handler(handler, rec, req) - require.Equal(t, http.StatusOK, rec.Code) - - mock := &mockNotificationDispatcher{} - handler.SetNotificationDispatcher(mock) - - // Start multipart upload. - req = httptest.NewRequest(http.MethodPost, "/notif-mpu/mp-key?uploads", nil) - rec = httptest.NewRecorder() - serveS3Handler(handler, rec, req) - require.Equal(t, http.StatusOK, rec.Code) - - var initResp s3.InitiateMultipartUploadResult - require.NoError(t, xml.NewDecoder(rec.Body).Decode(&initResp)) - uploadID := initResp.UploadID - - // Upload a part. - req = httptest.NewRequest( - http.MethodPut, - "/notif-mpu/mp-key?partNumber=1&uploadId="+uploadID, - strings.NewReader("part1"), - ) - rec = httptest.NewRecorder() - serveS3Handler(handler, rec, req) - require.Equal(t, http.StatusOK, rec.Code) - etag1 := rec.Header().Get("ETag") - - // Complete the upload. - completeXML := fmt.Sprintf( - `1%s`, - etag1, - ) - req = httptest.NewRequest( - http.MethodPost, - "/notif-mpu/mp-key?uploadId="+uploadID, - strings.NewReader(completeXML), - ) - rec = httptest.NewRecorder() - serveS3Handler(handler, rec, req) - require.Equal(t, http.StatusOK, rec.Code) - - require.Eventually(t, func() bool { + synctest.Test(t, func(t *testing.T) { + handler, backend := newTestHandler(t) + mustCreateBucket(t, backend, "notif-mpu") + + notifXML := `` + + `q1` + + `arn:aws:sqs:us-east-1:000000000000:mpu-queue` + + `s3:ObjectCreated:*` + + `` + req := httptest.NewRequest( + http.MethodPut, + "/notif-mpu?notification", + strings.NewReader(notifXML), + ) + rec := httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + mock := &mockNotificationDispatcher{} + handler.SetNotificationDispatcher(mock) + + // Start multipart upload. + req = httptest.NewRequest(http.MethodPost, "/notif-mpu/mp-key?uploads", nil) + rec = httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + var initResp s3.InitiateMultipartUploadResult + require.NoError(t, xml.NewDecoder(rec.Body).Decode(&initResp)) + uploadID := initResp.UploadID + + // Upload a part. + req = httptest.NewRequest( + http.MethodPut, + "/notif-mpu/mp-key?partNumber=1&uploadId="+uploadID, + strings.NewReader("part1"), + ) + rec = httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusOK, rec.Code) + etag1 := rec.Header().Get("ETag") + + // Complete the upload. + completeXML := fmt.Sprintf( + `1%s`, + etag1, + ) + req = httptest.NewRequest( + http.MethodPost, + "/notif-mpu/mp-key?uploadId="+uploadID, + strings.NewReader(completeXML), + ) + rec = httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + synctest.Wait() + mock.mu.Lock() defer mock.mu.Unlock() - - return len(mock.created) == 1 - }, 200*time.Millisecond, 5*time.Millisecond) - - mock.mu.Lock() - defer mock.mu.Unlock() - assert.Equal(t, "notif-mpu", mock.created[0].bucket) - assert.Equal(t, "mp-key", mock.created[0].key) + require.Len(t, mock.created, 1) + assert.Equal(t, "notif-mpu", mock.created[0].bucket) + assert.Equal(t, "mp-key", mock.created[0].key) + }) } // TestHandler_NotificationDispatch_PostObject_EventNameIsPost verifies that a @@ -241,89 +236,85 @@ func TestHandler_NotificationDispatch_CompleteMultipartUpload(t *testing.T) { func TestHandler_NotificationDispatch_PostObject_EventNameIsPost(t *testing.T) { t.Parallel() - handler, backend := newTestHandler(t) - mustCreateBucket(t, backend, "notif-post") - - notifXML := `` + - `q1` + - `arn:aws:sqs:us-east-1:000000000000:post-queue` + - `s3:ObjectCreated:Post` + - `` - req := httptest.NewRequest( - http.MethodPut, - "/notif-post?notification", - strings.NewReader(notifXML), - ) - rec := httptest.NewRecorder() - serveS3Handler(handler, rec, req) - require.Equal(t, http.StatusOK, rec.Code) - - queue := &captureQueue{} - handler.SetNotificationDispatcher( - s3.NewNotificationDispatcher(&s3.NotificationTargets{SQSSender: queue}, "us-east-1"), - ) - - body, contentType := buildPostForm(t, map[string]string{"key": "posted.txt"}, "posted.txt", []byte("hi")) - req = httptest.NewRequest(http.MethodPost, "/notif-post", body) - req.Header.Set("Content-Type", contentType) - rec = httptest.NewRecorder() - serveS3Handler(handler, rec, req) - require.Equal(t, http.StatusNoContent, rec.Code) - - require.Eventually(t, func() bool { + synctest.Test(t, func(t *testing.T) { + handler, backend := newTestHandler(t) + mustCreateBucket(t, backend, "notif-post") + + notifXML := `` + + `q1` + + `arn:aws:sqs:us-east-1:000000000000:post-queue` + + `s3:ObjectCreated:Post` + + `` + req := httptest.NewRequest( + http.MethodPut, + "/notif-post?notification", + strings.NewReader(notifXML), + ) + rec := httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + queue := &captureQueue{} + handler.SetNotificationDispatcher( + s3.NewNotificationDispatcher(&s3.NotificationTargets{SQSSender: queue}, "us-east-1"), + ) + + body, contentType := buildPostForm(t, map[string]string{"key": "posted.txt"}, "posted.txt", []byte("hi")) + req = httptest.NewRequest(http.MethodPost, "/notif-post", body) + req.Header.Set("Content-Type", contentType) + rec = httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusNoContent, rec.Code) + + synctest.Wait() + queue.mu.Lock() defer queue.mu.Unlock() - - return len(queue.messages) == 1 - }, 200*time.Millisecond, 5*time.Millisecond) - - queue.mu.Lock() - defer queue.mu.Unlock() - assert.Contains(t, queue.messages[0], `"eventName":"s3:ObjectCreated:Post"`) + require.Len(t, queue.messages, 1) + assert.Contains(t, queue.messages[0], `"eventName":"s3:ObjectCreated:Post"`) + }) } func TestHandler_NotificationDispatch_DeleteObjects(t *testing.T) { t.Parallel() - handler, backend := newTestHandler(t) - mustCreateBucket(t, backend, "notif-delobj") - mustPutObject(t, backend, "notif-delobj", "key1", []byte("data1")) - mustPutObject(t, backend, "notif-delobj", "key2", []byte("data2")) - - notifXML := `` + - `q1` + - `arn:aws:sqs:us-east-1:000000000000:del-queue` + - `s3:ObjectRemoved:*` + - `` - req := httptest.NewRequest( - http.MethodPut, - "/notif-delobj?notification", - strings.NewReader(notifXML), - ) - rec := httptest.NewRecorder() - serveS3Handler(handler, rec, req) - require.Equal(t, http.StatusOK, rec.Code) - - mock := &mockNotificationDispatcher{} - handler.SetNotificationDispatcher(mock) - - deleteXML := `key1key2` - req = httptest.NewRequest(http.MethodPost, "/notif-delobj?delete", strings.NewReader(deleteXML)) - rec = httptest.NewRecorder() - serveS3Handler(handler, rec, req) - require.Equal(t, http.StatusOK, rec.Code) - - require.Eventually(t, func() bool { + synctest.Test(t, func(t *testing.T) { + handler, backend := newTestHandler(t) + mustCreateBucket(t, backend, "notif-delobj") + mustPutObject(t, backend, "notif-delobj", "key1", []byte("data1")) + mustPutObject(t, backend, "notif-delobj", "key2", []byte("data2")) + + notifXML := `` + + `q1` + + `arn:aws:sqs:us-east-1:000000000000:del-queue` + + `s3:ObjectRemoved:*` + + `` + req := httptest.NewRequest( + http.MethodPut, + "/notif-delobj?notification", + strings.NewReader(notifXML), + ) + rec := httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + mock := &mockNotificationDispatcher{} + handler.SetNotificationDispatcher(mock) + + deleteXML := `key1key2` + req = httptest.NewRequest(http.MethodPost, "/notif-delobj?delete", strings.NewReader(deleteXML)) + rec = httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusOK, rec.Code) + + synctest.Wait() + mock.mu.Lock() defer mock.mu.Unlock() - - return len(mock.deleted) == 2 - }, 200*time.Millisecond, 5*time.Millisecond) - - mock.mu.Lock() - defer mock.mu.Unlock() - assert.Equal(t, "notif-delobj", mock.deleted[0].bucket) - assert.Equal(t, "notif-delobj", mock.deleted[1].bucket) + require.Len(t, mock.deleted, 2) + assert.Equal(t, "notif-delobj", mock.deleted[0].bucket) + assert.Equal(t, "notif-delobj", mock.deleted[1].bucket) + }) } // ---- Object Lock tests ---- diff --git a/services/s3/object_lambda_test.go b/services/s3/object_lambda_test.go index e525cead17..c629784977 100644 --- a/services/s3/object_lambda_test.go +++ b/services/s3/object_lambda_test.go @@ -8,7 +8,6 @@ import ( "strings" "sync" "testing" - "time" "github.com/aws/aws-sdk-go-v2/aws" sdk_s3 "github.com/aws/aws-sdk-go-v2/service/s3" @@ -197,20 +196,14 @@ func TestS3ObjectLambda_ConfigClearedOnBucketDelete(t *testing.T) { serveS3Handler(handler, rec, req) require.Equal(t, http.StatusNoContent, rec.Code) - // Run the janitor so the pending-delete bucket is fully removed from the - // table (DeleteBucket only marks it pending; removal is asynchronous). - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - go s3.NewJanitor(backend, s3.Settings{JanitorInterval: 5 * time.Millisecond}).Run(ctx) + // DeleteBucket only marks the bucket pending; drain it synchronously so + // the table reflects full removal before recreating the bucket. + s3.NewJanitor(backend, s3.Settings{}).DrainPendingBucketsOnce(t.Context()) - // Recreate a bucket with the same name and put a plain object. - require.Eventually(t, func() bool { - req = httptest.NewRequest(http.MethodPut, "/"+bucket, nil) - rec = httptest.NewRecorder() - serveS3Handler(handler, rec, req) - - return rec.Code == http.StatusOK - }, time.Second, 10*time.Millisecond, "recreated bucket should succeed once janitor drains the pending delete") + req = httptest.NewRequest(http.MethodPut, "/"+bucket, nil) + rec = httptest.NewRecorder() + serveS3Handler(handler, rec, req) + require.Equal(t, http.StatusOK, rec.Code) req = httptest.NewRequest( http.MethodPut, diff --git a/services/s3/objects.go b/services/s3/objects.go index a3c443722b..bc0296b90d 100644 --- a/services/s3/objects.go +++ b/services/s3/objects.go @@ -1152,7 +1152,13 @@ func (b *InMemoryBackend) computeObjectHashes( return 0, nil, "", nil, err } - return n, bytes.Clone(buf.Bytes()), hex.EncodeToString(md5Hasher.Sum(nil)), s3Hasher, nil + // Clone only if PutBuffer will recycle buf; oversized buffers are dropped. + data := buf.Bytes() + if httputils.WillPool(buf) { + data = bytes.Clone(data) + } + + return n, data, hex.EncodeToString(md5Hasher.Sum(nil)), s3Hasher, nil } // validateContentMD5 validates the Content-MD5 header from context against the computed etag. diff --git a/services/s3/persistence.go b/services/s3/persistence.go index d01f3f36f4..6a3c554175 100644 --- a/services/s3/persistence.go +++ b/services/s3/persistence.go @@ -166,6 +166,43 @@ func reinitUploadMutexes(uploads []*StoredMultipartUpload) { } } +// storedBucketAlias avoids infinite recursion from StoredBucket.MarshalJSON. +type storedBucketAlias StoredBucket + +// MarshalJSON serialises the bucket under bucket.mu.RLock so Snapshot can't race PutObject/PutBucketWebsite/etc. +// Those mutate bucket fields under bucket.mu without ever taking b.mu (gopherstack-fwd0g). +func (bucket *StoredBucket) MarshalJSON() ([]byte, error) { + bucket.mu.RLock("Snapshot") + defer bucket.mu.RUnlock() + + return json.Marshal((*storedBucketAlias)(bucket)) +} + +// storedObjectAlias avoids infinite recursion from StoredObject.MarshalJSON. +type storedObjectAlias StoredObject + +// MarshalJSON serialises the object under obj.mu.RLock, nested under the +// caller's bucket.mu.RLock -- see StoredBucket.MarshalJSON. +func (obj *StoredObject) MarshalJSON() ([]byte, error) { + obj.mu.RLock("Snapshot") + defer obj.mu.RUnlock() + + return json.Marshal((*storedObjectAlias)(obj)) +} + +// storedMultipartUploadAlias avoids infinite recursion from +// StoredMultipartUpload.MarshalJSON. +type storedMultipartUploadAlias StoredMultipartUpload + +// MarshalJSON serialises the upload under its own mu.RLock so Snapshot can't race storePart/CompleteMultipartUpload. +// Those mutate upload fields under upload.mu without ever taking b.mu. +func (u *StoredMultipartUpload) MarshalJSON() ([]byte, error) { + u.mu.RLock("Snapshot") + defer u.mu.RUnlock() + + return json.Marshal((*storedMultipartUploadAlias)(u)) +} + // Snapshot implements persistence.Persistable by delegating to the backend. func (h *S3Handler) Snapshot(ctx context.Context) []byte { type snapshotter interface { diff --git a/services/s3/persistence_race_test.go b/services/s3/persistence_race_test.go new file mode 100644 index 0000000000..e3309ca9c4 --- /dev/null +++ b/services/s3/persistence_race_test.go @@ -0,0 +1,91 @@ +package s3_test + +import ( + "bytes" + "fmt" + "strconv" + "sync" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + sdk_s3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/s3" +) + +// TestSnapshot_RacesWithObjectAndUploadWrites reproduces gopherstack-fwd0g: Snapshot marshals buckets under b.mu.RLock. +// PutObject/UploadPart mutate stored fields under bucket.mu/obj.mu/upload.mu alone; run with -race. +func TestSnapshot_RacesWithObjectAndUploadWrites(t *testing.T) { + t.Parallel() + + tests := []struct { + mutate func(t *testing.T, b *s3.InMemoryBackend, bucket string) + name string + }{ + { + name: "concurrent_snapshot_and_put_object", + mutate: func(t *testing.T, b *s3.InMemoryBackend, bucket string) { + t.Helper() + + for i := range 100 { + mustPutObject(t, b, bucket, fmt.Sprintf("key-%03d", i), []byte("payload")) + } + }, + }, + { + name: "concurrent_snapshot_and_upload_part", + mutate: func(t *testing.T, b *s3.InMemoryBackend, bucket string) { + t.Helper() + + created, err := b.CreateMultipartUpload(t.Context(), &sdk_s3.CreateMultipartUploadInput{ + Bucket: aws.String(bucket), + Key: aws.String("mp-key"), + }) + require.NoError(t, err) + + for i := int32(1); i <= 100; i++ { + _, uploadErr := b.UploadPart(t.Context(), &sdk_s3.UploadPartInput{ + Bucket: aws.String(bucket), + Key: aws.String("mp-key"), + UploadId: created.UploadId, + PartNumber: aws.Int32(i), + Body: bytes.NewReader([]byte("part-" + strconv.Itoa(int(i)))), + }) + require.NoError(t, uploadErr) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := s3.NewInMemoryBackend(nil) + const bucket = "race-snapshot-bucket" + + mustCreateBucket(t, b, bucket) + + var wg sync.WaitGroup + + stop := make(chan struct{}) + + wg.Go(func() { + for { + select { + case <-stop: + return + default: + } + + _ = b.Snapshot(t.Context()) + } + }) + + tt.mutate(t, b, bucket) + close(stop) + wg.Wait() + }) + } +} diff --git a/services/s3/realclient_bucket_config_and_object_ops_test.go b/services/s3/realclient_bucket_config_and_object_ops_test.go index 93d61938b2..3278572994 100644 --- a/services/s3/realclient_bucket_config_and_object_ops_test.go +++ b/services/s3/realclient_bucket_config_and_object_ops_test.go @@ -1041,10 +1041,7 @@ func TestRealClient_BucketConfigAndObjectOps(t *testing.T) { // so swapping it in here lets a real, unmodified client reach this // op at all. lambdaServerURL := strings.Replace(srv.URL, "127.0.0.1", "localhost", 1) - lambdaFn := &typedWriteGetObjectResponseLambda{ - serverURL: lambdaServerURL, - body: "lambda-typed-client-body", - } + lambdaFn := newTypedWriteGetObjectResponseLambda(lambdaServerURL, "lambda-typed-client-body") handler.SetObjectLambdaConfig(bucket, "arn:aws:lambda:us-east-1:000000000000:function:transformer") handler.SetNotificationDispatcher( s3.NewNotificationDispatcher(&s3.NotificationTargets{LambdaInvoker: lambdaFn}, "us-east-1")) @@ -1061,12 +1058,12 @@ func TestRealClient_BucketConfigAndObjectOps(t *testing.T) { // WriteGetObjectResponse signals the pending channel, which can // race ahead of the lambda invoker's own goroutine finishing its // client.WriteGetObjectResponse call and recording the result -- - // poll rather than read once. - require.Eventually(t, func() bool { - called, _ := lambdaFn.result() - - return called - }, time.Second, time.Millisecond, "the lambda invoker (which drives WriteGetObjectResponse) must run") + // wait on the done signal rather than polling. + select { + case <-lambdaFn.done: + case <-time.After(time.Second): + t.Fatal("the lambda invoker (which drives WriteGetObjectResponse) must run") + } _, wgorErr := lambdaFn.result() require.NoError(t, wgorErr, "the typed WriteGetObjectResponse call itself must succeed") @@ -1092,12 +1089,21 @@ func TestRealClient_BucketConfigAndObjectOps(t *testing.T) { // real client's request encoding for the op. type typedWriteGetObjectResponseLambda struct { wgorErr error + done chan struct{} serverURL string body string mu sync.Mutex called bool } +func newTypedWriteGetObjectResponseLambda(serverURL, body string) *typedWriteGetObjectResponseLambda { + return &typedWriteGetObjectResponseLambda{ + serverURL: serverURL, + body: body, + done: make(chan struct{}), + } +} + // result reports whether InvokeFunction ran and, if so, the error its // typed WriteGetObjectResponse call returned. Guarded by mu since // InvokeFunction runs on a goroutine spawned by handleObjectLambdaGetObject @@ -1116,6 +1122,7 @@ func (l *typedWriteGetObjectResponseLambda) setResult(called bool, err error) { l.called = called l.wgorErr = err + close(l.done) } func (l *typedWriteGetObjectResponseLambda) InvokeFunction( diff --git a/services/s3/sigv4.go b/services/s3/sigv4.go index 4cc79dd498..9ee13064dc 100644 --- a/services/s3/sigv4.go +++ b/services/s3/sigv4.go @@ -79,7 +79,14 @@ func (h *S3Handler) verifyHeaderAuth( return true } - if scope.service != "s3" && scope.service != "s3-object-lambda" { + // "s3express" is the signing name every S3 Express One Zone request uses + // (CreateSession and any Zonal endpoint operation on a directory bucket -- + // s3@v1.111.0 internal/customizations/express_signer.go, + // SetSigV4SigningName("s3express")), not "s3". Rejecting it here is what + // previously turned a real client's directory-bucket CreateSession call + // into a 403 SignatureDoesNotMatch before any actual signature was even + // checked (gopherstack-z2w1a). + if scope.service != "s3" && scope.service != "s3-object-lambda" && scope.service != "s3express" { h.writeSignatureError(ctx, w, r, "Credential should be scoped to correct service: s3.") return false @@ -107,7 +114,12 @@ func (h *S3Handler) verifyHeaderAuth( return false } - if !h.signatureMatches(r, scope) { + secret, ok := h.resolveSigningSecret(ctx, w, r, scope) + if !ok { + return false + } + + if !h.signatureMatches(r, scope, secret) { h.writeSignatureError(ctx, w, r, "The request signature we calculated does not match the signature you provided. "+ "Check your key and signing method.") @@ -118,6 +130,32 @@ func (h *S3Handler) verifyHeaderAuth( return true } +// resolveSigningSecret returns the secret to verify r's signature against: +// h.PresignSecret ordinarily, or a live S3 Express session's own secret key +// when r carries an x-amz-s3session-token. Returns ok=false (having already +// written the error response) when that session token is unknown, mismatched, +// or expired. +func (h *S3Handler) resolveSigningSecret( + ctx context.Context, w http.ResponseWriter, r *http.Request, scope authScope, +) (string, bool) { + sessionToken := r.Header.Get(headerAmzSessionToken) + if sessionToken == "" { + return h.PresignSecret, true + } + + _, secret, ok := h.Backend.ExpressSessionSecret(scope.accessKeyID, sessionToken) + if !ok { + httputils.WriteS3ErrorResponse(ctx, w, r, ErrorResponse{ + Code: "ExpiredToken", + Message: "The provided token has expired.", + }, http.StatusForbidden) + + return "", false + } + + return secret, true +} + // writeSignatureError emits a SignatureDoesNotMatch 403 with the given message. func (h *S3Handler) writeSignatureError( ctx context.Context, w http.ResponseWriter, r *http.Request, msg string, @@ -231,9 +269,16 @@ func parseAmzTime(raw string) (time.Time, bool) { return time.Time{}, false } -// signatureMatches recomputes the SigV4 header signature for r and compares it -// against the client-provided signature in constant time. -func (h *S3Handler) signatureMatches(r *http.Request, scope authScope) bool { +// headerAmzSessionToken is the header an S3 Express One Zone client carries +// its CreateSession token on (s3@v1.111.0 internal/customizations/ +// express_signer.go's headerAmzSessionToken constant). +const headerAmzSessionToken = "X-Amz-S3session-Token" //nolint:gosec // header name, not a credential + +// signatureMatches recomputes the SigV4 header signature for r using secret +// and compares it against the client-provided signature in constant time. +// secret is h.PresignSecret for ordinary requests, or the looked-up S3 +// Express session's own secret key when an x-amz-s3session-token is present. +func (h *S3Handler) signatureMatches(r *http.Request, scope authScope, secret string) bool { canonicalReq := buildHeaderCanonicalRequest(r, scope.signedHeaders) amzDate := r.Header.Get("X-Amz-Date") credentialScope := strings.Join( @@ -246,7 +291,7 @@ func (h *S3Handler) signatureMatches(r *http.Request, scope authScope) bool { hexSHA256(canonicalReq), }, "\n") - signingKey := derivePresignSigningKey(h.PresignSecret, scope.date, scope.region, scope.service) + signingKey := derivePresignSigningKey(secret, scope.date, scope.region, scope.service) expected := hex.EncodeToString(hmacSHA256Bytes(signingKey, stringToSign)) return hmac.Equal([]byte(expected), []byte(scope.signature)) diff --git a/services/s3/sse_crypto.go b/services/s3/sse_crypto.go index 4f60c2c933..51955184e0 100644 --- a/services/s3/sse_crypto.go +++ b/services/s3/sse_crypto.go @@ -78,13 +78,13 @@ func extractCopySourceSSECInfo(r *http.Request) (sseInfo, error) { // sseInfo captures SSE parameters extracted from an HTTP request. type sseInfo struct { // Algorithm is one of "AES256", "aws:kms", "aws:kms:dsse", or "" (none). - Algorithm string + Algorithm string `json:"Algorithm"` // KMSKeyID is the KMS key ID, populated when Algorithm is aws:kms/dsse. - KMSKeyID string + KMSKeyID string `json:"KMSKeyID"` // SSECAlgorithm is "AES256" when SSE-C is requested. - SSECAlgorithm string + SSECAlgorithm string `json:"SSECAlgorithm"` // SSECKeyMD5 is the base64-encoded MD5 of the customer-supplied key. - SSECKeyMD5 string + SSECKeyMD5 string `json:"SSECKeyMD5"` // SSECKeyB64 is the base64-encoded raw customer key. Kept on the // request-scoped sseInfo only — not persisted (json:"-") — so the backend // can encrypt the body on PUT and the GET handler can decrypt when the @@ -93,7 +93,7 @@ type sseInfo struct { // EncryptionContext is the base64-encoded JSON KMS encryption context // (SSEKMSEncryptionContext), round-tripped verbatim — this emulator // doesn't call KMS, so it's opaque AAD here, not decoded/validated. - EncryptionContext string + EncryptionContext string `json:"EncryptionContext"` } // extractSSEInfo reads SSE-* request headers and validates SSE-C when present. diff --git a/services/s3/store.go b/services/s3/store.go index abbfa1de15..cfa641bf2b 100644 --- a/services/s3/store.go +++ b/services/s3/store.go @@ -12,6 +12,7 @@ import ( "github.com/blackbirdworks/gopherstack/pkgs/config" "github.com/blackbirdworks/gopherstack/pkgs/lockmetrics" "github.com/blackbirdworks/gopherstack/pkgs/logger" + "github.com/blackbirdworks/gopherstack/pkgs/safemap" "github.com/blackbirdworks/gopherstack/pkgs/store" "github.com/aws/aws-sdk-go-v2/aws" @@ -96,46 +97,52 @@ func getRegionFromS3Context(ctx context.Context, defaultRegion string) string { } type InMemoryBackend struct { + compressor Compressor + // serviceCtx is the long-lived context for background work (replication). + // Initialised in NewInMemoryBackend so it is always non-nil; overridden by + // SetServiceContext when the handler wires in the real service context. + serviceCtx context.Context + // uploadsByBucket is a secondary index replacing the old + // bucket->uploadID->*StoredMultipartUpload nesting for the "all uploads in + // bucket X" access pattern (ListMultipartUploads, janitor cleanup, + // DeleteBucket cleanup). A caller-supplied uploadID is only valid for the + // bucket it was issued against — see getUpload, which enforces that the + // same way the old b.uploads[bucketName][uploadID] nesting did. + uploadsByBucket *store.Index[StoredMultipartUpload] // registry lets Reset/Snapshot/Restore collapse the buckets/uploads // lifecycle to one call each (registry.ResetAll/SnapshotAll/RestoreAll) // instead of hand-rolled per-map wiring. See pkgs/store's package doc and // the services/sqs pilot (commit 0f09d77c) for the pattern this follows. registry *store.Registry - // buckets is keyed by bucket name (globally unique — see StoredBucket.Region's - // doc comment). This replaces the old region->name->*StoredBucket nesting plus - // the separate bucketIndex name->region map: Table.Get(name) alone now answers - // both "does it exist" and "give me the bucket", and StoredBucket.Region - // carries what bucketIndex used to. - buckets *store.Table[StoredBucket] - // uploads is keyed by UploadID (a random 32-hex-char string — see - // newObjectVersionID — so it is unique across all buckets). uploadsByBucket - // is a secondary index replacing the old bucket->uploadID->*StoredMultipartUpload - // nesting for the "all uploads in bucket X" access pattern (ListMultipartUploads, - // janitor cleanup, DeleteBucket cleanup). A caller-supplied uploadID is only - // valid for the bucket it was issued against — see getUpload, which enforces - // that the same way the old b.uploads[bucketName][uploadID] nesting did. - uploads *store.Table[StoredMultipartUpload] - uploadsByBucket *store.Index[StoredMultipartUpload] // tags is intentionally left as a plain map (not a store.Table): its key is // a composite "bucket/key/versionID" string that is not a pure function of // the stored value (a bare []types.Tag has no identity field of its own) — // the same reason services/ec2's store_setup.go leaves e.g. // vpcPeeringOptions/instanceIMDSOptions unconverted. - tags map[string][]types.Tag - mu *lockmetrics.RWMutex - compressor Compressor - // serviceCtx is the long-lived context for background work (replication). - // Initialised in NewInMemoryBackend so it is always non-nil; overridden by - // SetServiceContext when the handler wires in the real service context. - serviceCtx context.Context + tags map[string][]types.Tag + mu *lockmetrics.RWMutex + // uploads is keyed by UploadID (a random 32-hex-char string — see + // newObjectVersionID — so it is unique across all buckets). + uploads *store.Table[StoredMultipartUpload] + // buckets is keyed by bucket name (globally unique — see StoredBucket.Region's + // doc comment). This replaces the old region->name->*StoredBucket nesting plus + // the separate bucketIndex name->region map: Table.Get(name) alone now answers + // both "does it exist" and "give me the bucket", and StoredBucket.Region + // carries what bucketIndex used to. + buckets *store.Table[StoredBucket] serviceCancel context.CancelFunc - defaultRegion string - // serviceCtxMu guards serviceCtx and serviceCancel. - serviceCtxMu sync.RWMutex + // expressSessions holds live S3 Express CreateSession credentials, keyed + // by AccessKeyID. Isolated single-map state with its own TTL sweep (see + // express_session.go) -- not registered with b.registry, since session + // credentials are deliberately not persisted across a snapshot/restore. + expressSessions *safemap.Map[string, expressSession] + defaultRegion string // replicationWg tracks all in-flight replication goroutines. // DrainReplicationGoroutines blocks until they all finish. replicationWg sync.WaitGroup compressionMinBytes int + // serviceCtxMu guards serviceCtx and serviceCancel. + serviceCtxMu sync.RWMutex // skipMultipartSizeCheck disables the 5 MiB minimum part size check during // CompleteMultipartUpload. This is intended for use in unit tests only. skipMultipartSizeCheck bool @@ -221,6 +228,7 @@ func NewInMemoryBackend(compressor Compressor) *InMemoryBackend { mu: lockmetrics.New("s3"), serviceCtx: ctx, serviceCancel: cancel, + expressSessions: safemap.New[string, expressSession]("s3.expressSessions"), } } diff --git a/services/s3/transition_default_min_object_size_test.go b/services/s3/transition_default_min_object_size_test.go index 72518e5ab4..2002689501 100644 --- a/services/s3/transition_default_min_object_size_test.go +++ b/services/s3/transition_default_min_object_size_test.go @@ -2,7 +2,6 @@ package s3_test import ( "bytes" - "context" "testing" "time" @@ -65,7 +64,7 @@ func TestTransitionDefaultMinimumObjectSize(t *testing.T) { `, verify: func(t *testing.T, b *s3.InMemoryBackend, bucket string) { t.Helper() - requireStorageClassEventually(t, b, bucket, "big.bin", "GLACIER") + requireStorageClassNow(t, b, bucket, "big.bin", "GLACIER") }, }, { @@ -81,7 +80,7 @@ func TestTransitionDefaultMinimumObjectSize(t *testing.T) { `, verify: func(t *testing.T, b *s3.InMemoryBackend, bucket string) { t.Helper() - requireStorageClassEventually(t, b, bucket, "small.txt", "GLACIER") + requireStorageClassNow(t, b, bucket, "small.txt", "GLACIER") }, }, { @@ -113,7 +112,7 @@ func TestTransitionDefaultMinimumObjectSize(t *testing.T) { `, verify: func(t *testing.T, b *s3.InMemoryBackend, bucket string) { t.Helper() - requireStorageClassEventually(t, b, bucket, "small.txt", "GLACIER") + requireStorageClassNow(t, b, bucket, "small.txt", "GLACIER") }, }, } @@ -133,10 +132,10 @@ func TestTransitionDefaultMinimumObjectSize(t *testing.T) { err := b.PutBucketLifecycleConfiguration(t.Context(), bucket, tt.lcXML, tt.transitionDefaultMinObjectSize) require.NoError(t, err) - ctx, cancel := context.WithCancel(t.Context()) - defer cancel() - - go newFastJanitor(b).Run(ctx) + j := newFastJanitor(b) + for range 3 { + j.SweepOnce(t.Context()) + } tt.verify(t, b, bucket) }) @@ -184,27 +183,22 @@ func TestTransitionDefaultMinimumObjectSize_Echoed(t *testing.T) { ) } -func requireStorageClassEventually(t *testing.T, b *s3.InMemoryBackend, bucket, key, want string) { +func requireStorageClassNow(t *testing.T, b *s3.InMemoryBackend, bucket, key, want string) { t.Helper() - require.Eventually(t, func() bool { - out, err := b.HeadObject(t.Context(), &sdk_s3.HeadObjectInput{ - Bucket: aws.String(bucket), - Key: aws.String(key), - }) - - return err == nil && string(out.StorageClass) == want - }, 500*time.Millisecond, 10*time.Millisecond, "object %s must reach storage class %s", key, want) + out, err := b.HeadObject(t.Context(), &sdk_s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + require.NoError(t, err) + require.Equal(t, want, string(out.StorageClass)) } -// requireStorageClassStable asserts the object's storage class never -// transitions across a window long enough for the fast test janitor to have -// swept it multiple times. +// requireStorageClassStable asserts the object's storage class survived +// repeated janitor sweeps (called synchronously by the caller) unchanged. func requireStorageClassStable(t *testing.T, b *s3.InMemoryBackend, bucket, key string) { t.Helper() - time.Sleep(100 * time.Millisecond) - out, err := b.HeadObject(t.Context(), &sdk_s3.HeadObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), diff --git a/services/s3control/access_grants.go b/services/s3control/access_grants.go index 78e3ba0486..d190001e20 100644 --- a/services/s3control/access_grants.go +++ b/services/s3control/access_grants.go @@ -118,7 +118,9 @@ func (b *InMemoryBackend) ListAccessGrantsInstances(accountID string) []*AccessG return nil } - return []*AccessGrantsInstance{inst} + cp := *inst + + return []*AccessGrantsInstance{&cp} } func (b *InMemoryBackend) GetAccessGrantsInstance(accountID string) (*AccessGrantsInstance, error) { @@ -130,7 +132,9 @@ func (b *InMemoryBackend) GetAccessGrantsInstance(accountID string) (*AccessGran return nil, awserr.New("AccessGrantsInstanceNotExistsError", awserr.ErrNotFound) } - return inst, nil + cp := *inst + + return &cp, nil } // errAccessGrantsInstanceNotEmpty is returned when DeleteAccessGrantsInstance @@ -231,7 +235,9 @@ func (b *InMemoryBackend) GetAccessGrantsInstanceForPrefix( } _ = prefix - return inst, nil + cp := *inst + + return &cp, nil } // ---- Access Grants CRUD ---- @@ -342,7 +348,9 @@ func (b *InMemoryBackend) GetAccessGrantsLocation( return nil, awserr.New("NoSuchAccessGrantsLocation", awserr.ErrNotFound) } - return loc, nil + cp := *loc + + return &cp, nil } // errAccessGrantsLocationNotEmpty is returned when DeleteAccessGrantsLocation @@ -399,8 +407,9 @@ func (b *InMemoryBackend) UpdateAccessGrantsLocation( return nil, awserr.New("NoSuchAccessGrantsLocation", awserr.ErrNotFound) } loc.IAMRoleArn = iamRoleArn + cp := *loc - return loc, nil + return &cp, nil } // ListAccessGrantsLocations returns all locations for an account. diff --git a/services/s3control/access_grants_race_test.go b/services/s3control/access_grants_race_test.go new file mode 100644 index 0000000000..332b44a439 --- /dev/null +++ b/services/s3control/access_grants_race_test.go @@ -0,0 +1,139 @@ +package s3control_test + +import ( + "sync" + "testing" + + s3control "github.com/blackbirdworks/gopherstack/services/s3control" +) + +// TestAccessGrantsInstanceConcurrentWithAssociate proves the instance +// getters must not hand back the live pointer Associate...IdentityCenter mutates. +func TestAccessGrantsInstanceConcurrentWithAssociate(t *testing.T) { + t.Parallel() + + tests := []struct { + reader func(b *s3control.InMemoryBackend, accountID string) + name string + }{ + { + name: "GetAccessGrantsInstance races associate", + reader: func(b *s3control.InMemoryBackend, accountID string) { + inst, err := b.GetAccessGrantsInstance(accountID) + if err != nil { + return + } + + _ = inst.IdentityCenterArn + }, + }, + { + name: "ListAccessGrantsInstances races associate", + reader: func(b *s3control.InMemoryBackend, accountID string) { + for _, inst := range b.ListAccessGrantsInstances(accountID) { + _ = inst.IdentityCenterArn + } + }, + }, + { + name: "GetAccessGrantsInstanceForPrefix races associate", + reader: func(b *s3control.InMemoryBackend, accountID string) { + inst, err := b.GetAccessGrantsInstanceForPrefix(accountID, "prefix") + if err != nil { + return + } + + _ = inst.IdentityCenterArn + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := s3control.NewInMemoryBackend() + const accountID = "000000000000" + b.CreateAccessGrantsInstance(accountID, "") + + const iterations = 300 + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + for range iterations { + tt.reader(b, accountID) + } + }() + + go func() { + defer wg.Done() + + for range iterations { + b.AssociateAccessGrantsIdentityCenter(accountID, "arn:aws:sso:::instance/ssoins-1") + } + }() + + wg.Wait() + }) + } +} + +// TestAccessGrantsLocationConcurrentWithUpdate proves GetAccessGrantsLocation +// must not hand back the live pointer UpdateAccessGrantsLocation mutates. +func TestAccessGrantsLocationConcurrentWithUpdate(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + }{ + {name: "GetAccessGrantsLocation races update"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := s3control.NewInMemoryBackend() + const accountID = "000000000000" + b.CreateAccessGrantsInstance(accountID, "") + + loc := b.CreateAccessGrantsLocation(accountID, "s3://", "arn:aws:iam::000000000000:role/initial") + + const iterations = 300 + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + for range iterations { + got, err := b.GetAccessGrantsLocation(accountID, loc.AccessGrantsLocationID) + if err != nil { + continue + } + + _ = got.IAMRoleArn + } + }() + + go func() { + defer wg.Done() + + for range iterations { + _, _ = b.UpdateAccessGrantsLocation( + accountID, + loc.AccessGrantsLocationID, + "arn:aws:iam::000000000000:role/updated", + ) + } + }() + + wg.Wait() + }) + } +} diff --git a/services/sagemaker/handler_notebook_instances_test.go b/services/sagemaker/handler_notebook_instances_test.go index c6ff552b2d..1ea6b4e57f 100644 --- a/services/sagemaker/handler_notebook_instances_test.go +++ b/services/sagemaker/handler_notebook_instances_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -109,25 +110,28 @@ func TestHandler_DeleteNotebookInstance_NotStopped(t *testing.T) { func TestHandler_NotebookInstance_EventuallyInService(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - doSageMakerRequest(t, h, "CreateNotebookInstance", map[string]any{ - "NotebookInstanceName": "async-notebook", - "InstanceType": "ml.t2.medium", - "RoleArn": "arn:aws:iam::000000000000:role/notebook-role", - }) + doSageMakerRequest(t, h, "CreateNotebookInstance", map[string]any{ + "NotebookInstanceName": "async-notebook", + "InstanceType": "ml.t2.medium", + "RoleArn": "arn:aws:iam::000000000000:role/notebook-role", + }) - // Wait for async status transition. - time.Sleep(300 * time.Millisecond) + // Wait for async status transition. + time.Sleep(300 * time.Millisecond) + synctest.Wait() - recDesc := doSageMakerRequest(t, h, "DescribeNotebookInstance", map[string]any{ - "NotebookInstanceName": "async-notebook", - }) - assert.Equal(t, http.StatusOK, recDesc.Code) + recDesc := doSageMakerRequest(t, h, "DescribeNotebookInstance", map[string]any{ + "NotebookInstanceName": "async-notebook", + }) + assert.Equal(t, http.StatusOK, recDesc.Code) - var descOut map[string]any - require.NoError(t, json.Unmarshal(recDesc.Body.Bytes(), &descOut)) - assert.NotEmpty(t, descOut["NotebookInstanceStatus"]) + var descOut map[string]any + require.NoError(t, json.Unmarshal(recDesc.Body.Bytes(), &descOut)) + assert.NotEmpty(t, descOut["NotebookInstanceStatus"]) + }) } // TestUpdateNotebookInstance_RequiresStoppedState verifies that updating a notebook diff --git a/services/sagemaker/handler_processing_jobs_test.go b/services/sagemaker/handler_processing_jobs_test.go index 134e0c88fe..65da75075a 100644 --- a/services/sagemaker/handler_processing_jobs_test.go +++ b/services/sagemaker/handler_processing_jobs_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -16,29 +17,36 @@ import ( func TestHandler_DeleteProcessingJob(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - doSageMakerRequest(t, h, "CreateProcessingJob", map[string]any{ - "ProcessingJobName": "del-pj", - "RoleArn": "arn:aws:iam::000000000000:role/test", - "AppSpecification": map[string]any{"ImageUri": "img:latest"}, - "ProcessingResources": map[string]any{ - "ClusterConfig": map[string]any{"InstanceType": "ml.m5.large", "InstanceCount": 1, "VolumeSizeInGB": 10}, - }, - }) + doSageMakerRequest(t, h, "CreateProcessingJob", map[string]any{ + "ProcessingJobName": "del-pj", + "RoleArn": "arn:aws:iam::000000000000:role/test", + "AppSpecification": map[string]any{"ImageUri": "img:latest"}, + "ProcessingResources": map[string]any{ + "ClusterConfig": map[string]any{ + "InstanceType": "ml.m5.large", + "InstanceCount": 1, + "VolumeSizeInGB": 10, + }, + }, + }) - // Cannot delete while still InProgress. - recEarly := doSageMakerRequest(t, h, "DeleteProcessingJob", map[string]any{"ProcessingJobName": "del-pj"}) - assert.Equal(t, http.StatusBadRequest, recEarly.Code) + // Cannot delete while still InProgress. + recEarly := doSageMakerRequest(t, h, "DeleteProcessingJob", map[string]any{"ProcessingJobName": "del-pj"}) + assert.Equal(t, http.StatusBadRequest, recEarly.Code) - // Wait for the simulated job to reach a terminal state. - time.Sleep(400 * time.Millisecond) + // Wait for the simulated job to reach a terminal state. + time.Sleep(400 * time.Millisecond) + synctest.Wait() - recDelete := doSageMakerRequest(t, h, "DeleteProcessingJob", map[string]any{"ProcessingJobName": "del-pj"}) - require.Equal(t, http.StatusOK, recDelete.Code) + recDelete := doSageMakerRequest(t, h, "DeleteProcessingJob", map[string]any{"ProcessingJobName": "del-pj"}) + require.Equal(t, http.StatusOK, recDelete.Code) - recDescribe := doSageMakerRequest(t, h, "DescribeProcessingJob", map[string]any{"ProcessingJobName": "del-pj"}) - assert.Equal(t, http.StatusBadRequest, recDescribe.Code) + recDescribe := doSageMakerRequest(t, h, "DescribeProcessingJob", map[string]any{"ProcessingJobName": "del-pj"}) + assert.Equal(t, http.StatusBadRequest, recDescribe.Code) + }) } func TestHandler_DeleteProcessingJob_NotFound(t *testing.T) { diff --git a/services/sagemaker/handler_training_jobs_test.go b/services/sagemaker/handler_training_jobs_test.go index cc17c918dd..7efd37761f 100644 --- a/services/sagemaker/handler_training_jobs_test.go +++ b/services/sagemaker/handler_training_jobs_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -16,6 +17,14 @@ import ( func TestHandler_TrainingJobLifecycle(t *testing.T) { t.Parallel() + synctest.Test(t, func(t *testing.T) { + testHandlerTrainingJobLifecycle(t) + }) +} + +func testHandlerTrainingJobLifecycle(t *testing.T) { + t.Helper() + h := newTestHandler(t) // Create training job. @@ -85,6 +94,14 @@ func TestHandler_TrainingJobLifecycle(t *testing.T) { func TestHandler_DeleteTrainingJob_InProgress(t *testing.T) { t.Parallel() + synctest.Test(t, func(t *testing.T) { + testHandlerDeleteTrainingJobInProgress(t) + }) +} + +func testHandlerDeleteTrainingJobInProgress(t *testing.T) { + t.Helper() + h := newTestHandler(t) doSageMakerRequest(t, h, "CreateTrainingJob", map[string]any{ diff --git a/services/sagemaker/handler_transform_jobs_test.go b/services/sagemaker/handler_transform_jobs_test.go index a07f61d8f3..edf1955d24 100644 --- a/services/sagemaker/handler_transform_jobs_test.go +++ b/services/sagemaker/handler_transform_jobs_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -18,68 +19,71 @@ import ( func TestHandler_TransformJobLifecycle(t *testing.T) { t.Parallel() - h := newTestHandler(t) + synctest.Test(t, func(t *testing.T) { + h := newTestHandler(t) - doSageMakerRequest(t, h, "CreateModel", map[string]any{"ModelName": "my-model"}) + doSageMakerRequest(t, h, "CreateModel", map[string]any{"ModelName": "my-model"}) - // Create - rec := doSageMakerRequest(t, h, "CreateTransformJob", map[string]any{ - "TransformJobName": "my-transform", - "ModelName": "my-model", - "TransformInput": map[string]any{ - "DataSource": map[string]any{ - "S3DataSource": map[string]any{ - "S3Uri": "s3://bucket/input", - "S3DataType": "S3Prefix", + // Create + rec := doSageMakerRequest(t, h, "CreateTransformJob", map[string]any{ + "TransformJobName": "my-transform", + "ModelName": "my-model", + "TransformInput": map[string]any{ + "DataSource": map[string]any{ + "S3DataSource": map[string]any{ + "S3Uri": "s3://bucket/input", + "S3DataType": "S3Prefix", + }, }, + "ContentType": "text/csv", }, - "ContentType": "text/csv", - }, - "TransformOutput": map[string]any{ - "S3OutputPath": "s3://bucket/output", - }, - "TransformResources": map[string]any{ - "InstanceType": "ml.m5.large", - "InstanceCount": 1, - }, - "BatchStrategy": "MultiRecord", - "Environment": map[string]string{"KEY": "VALUE"}, - }) - assert.Equal(t, http.StatusOK, rec.Code) - - var createResp map[string]string - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &createResp)) - assert.NotEmpty(t, createResp["TransformJobArn"]) - - // Describe — InProgress initially - rec = doSageMakerRequest(t, h, "DescribeTransformJob", map[string]any{ - "TransformJobName": "my-transform", - }) - assert.Equal(t, http.StatusOK, rec.Code) - - var descResp map[string]any - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - assert.Equal(t, "my-transform", descResp["TransformJobName"]) - assert.Equal(t, "my-model", descResp["ModelName"]) - assert.Equal(t, "InProgress", descResp["TransformJobStatus"]) - assert.Equal(t, "MultiRecord", descResp["BatchStrategy"]) - - // List - rec = doSageMakerRequest(t, h, "ListTransformJobs", map[string]any{}) - assert.Equal(t, http.StatusOK, rec.Code) + "TransformOutput": map[string]any{ + "S3OutputPath": "s3://bucket/output", + }, + "TransformResources": map[string]any{ + "InstanceType": "ml.m5.large", + "InstanceCount": 1, + }, + "BatchStrategy": "MultiRecord", + "Environment": map[string]string{"KEY": "VALUE"}, + }) + assert.Equal(t, http.StatusOK, rec.Code) - var listResp map[string]any - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &listResp)) - summaries := listResp["TransformJobSummaries"].([]any) - assert.Len(t, summaries, 1) + var createResp map[string]string + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &createResp)) + assert.NotEmpty(t, createResp["TransformJobArn"]) - // Wait for completion - time.Sleep(400 * time.Millisecond) - rec = doSageMakerRequest(t, h, "DescribeTransformJob", map[string]any{ - "TransformJobName": "my-transform", + // Describe — InProgress initially + rec = doSageMakerRequest(t, h, "DescribeTransformJob", map[string]any{ + "TransformJobName": "my-transform", + }) + assert.Equal(t, http.StatusOK, rec.Code) + + var descResp map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + assert.Equal(t, "my-transform", descResp["TransformJobName"]) + assert.Equal(t, "my-model", descResp["ModelName"]) + assert.Equal(t, "InProgress", descResp["TransformJobStatus"]) + assert.Equal(t, "MultiRecord", descResp["BatchStrategy"]) + + // List + rec = doSageMakerRequest(t, h, "ListTransformJobs", map[string]any{}) + assert.Equal(t, http.StatusOK, rec.Code) + + var listResp map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &listResp)) + summaries := listResp["TransformJobSummaries"].([]any) + assert.Len(t, summaries, 1) + + // Wait for completion + time.Sleep(400 * time.Millisecond) + synctest.Wait() + rec = doSageMakerRequest(t, h, "DescribeTransformJob", map[string]any{ + "TransformJobName": "my-transform", + }) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) + assert.Equal(t, "Completed", descResp["TransformJobStatus"]) }) - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &descResp)) - assert.Equal(t, "Completed", descResp["TransformJobStatus"]) } func TestHandler_TransformJob_NotFound(t *testing.T) { diff --git a/services/scheduler/handler.go b/services/scheduler/handler.go index 12a3166d50..702b247a03 100644 --- a/services/scheduler/handler.go +++ b/services/scheduler/handler.go @@ -9,6 +9,7 @@ import ( "net/url" "sort" "strings" + "sync/atomic" "time" "github.com/labstack/echo/v5" @@ -116,6 +117,8 @@ type Handler struct { // ClientToken so a lost-response retry replays the original result instead of // failing with ConflictException on the now-existing name. See idempotency.go. idempotency *safemap.Map[string, idempotentResult] + // idempotencyInsertsSinceSweep paces maybeEvictExpiredIdempotency. + idempotencyInsertsSinceSweep atomic.Int64 } // Runner returns the internal runner for cross-service wiring. diff --git a/services/scheduler/idempotency.go b/services/scheduler/idempotency.go index 09fe7d4046..0ce1e41676 100644 --- a/services/scheduler/idempotency.go +++ b/services/scheduler/idempotency.go @@ -10,6 +10,13 @@ import "time" // caching results indefinitely. const clientTokenTTL = 5 * time.Minute +// idempotencyEvictThreshold: cache size that arms the expired-entry sweep; +// unreplayed tokens otherwise live until process exit. +const idempotencyEvictThreshold = 256 + +// idempotencyEvictSweepInterval: inserts between sweeps once armed. +const idempotencyEvictSweepInterval = 64 + // idempotentResult is a cached successful Create*'s ARN, keyed by clientTokenKey. type idempotentResult struct { expiresAt time.Time @@ -58,4 +65,33 @@ func (h *Handler) storeIdempotent(key, arn string) { } h.idempotency.Set(key, idempotentResult{arn: arn, expiresAt: time.Now().Add(clientTokenTTL)}) + h.maybeEvictExpiredIdempotency() +} + +// maybeEvictExpiredIdempotency drops expired entries once the cache is large. +func (h *Handler) maybeEvictExpiredIdempotency() { + if h.idempotency.Len() < idempotencyEvictThreshold { + return + } + + if h.idempotencyInsertsSinceSweep.Add(1) < idempotencyEvictSweepInterval { + return + } + + h.idempotencyInsertsSinceSweep.Store(0) + + now := time.Now() + + var expired []string + h.idempotency.Range(func(key string, res idempotentResult) bool { + if now.After(res.expiresAt) { + expired = append(expired, key) + } + + return true + }) + + for _, key := range expired { + h.idempotency.Delete(key) + } } diff --git a/services/scheduler/schedules_test.go b/services/scheduler/schedules_test.go index 0f3a050d9e..a0001b10dc 100644 --- a/services/scheduler/schedules_test.go +++ b/services/scheduler/schedules_test.go @@ -7,6 +7,7 @@ import ( "net/http" "strings" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -1286,31 +1287,33 @@ func TestUpdateSchedule_NotFound(t *testing.T) { func TestUpdateSchedule_UpdatesLastModificationDate(t *testing.T) { t.Parallel() - b := scheduler.NewInMemoryBackend("000000000000", "us-east-1") - h := scheduler.NewHandler(b) + synctest.Test(t, func(t *testing.T) { + b := scheduler.NewInMemoryBackend("000000000000", "us-east-1") + h := scheduler.NewHandler(b) - createScheduleViaHandler(t, h, "upd-sched", "", "rate(1 minute)") + createScheduleViaHandler(t, h, "upd-sched", "", "rate(1 minute)") - s1, err := b.GetSchedule(context.Background(), "upd-sched", "") - require.NoError(t, err) + s1, err := b.GetSchedule(context.Background(), "upd-sched", "") + require.NoError(t, err) - // Advance time enough to guarantee LastModificationDate changes. - time.Sleep(1100 * time.Millisecond) + // Advance time enough to guarantee LastModificationDate changes. + time.Sleep(1100 * time.Millisecond) - doSchedulerRequest(t, h, "UpdateSchedule", map[string]any{ - "Name": "upd-sched", - "ScheduleExpression": "rate(2 minutes)", - "Target": map[string]string{"Arn": "arn:a", "RoleArn": "arn:r"}, - "FlexibleTimeWindow": map[string]string{"Mode": "OFF"}, - "State": "ENABLED", - }) + doSchedulerRequest(t, h, "UpdateSchedule", map[string]any{ + "Name": "upd-sched", + "ScheduleExpression": "rate(2 minutes)", + "Target": map[string]string{"Arn": "arn:a", "RoleArn": "arn:r"}, + "FlexibleTimeWindow": map[string]string{"Mode": "OFF"}, + "State": "ENABLED", + }) - s2, err := b.GetSchedule(context.Background(), "upd-sched", "") - require.NoError(t, err) + s2, err := b.GetSchedule(context.Background(), "upd-sched", "") + require.NoError(t, err) - assert.True(t, s2.LastModificationDate.After(s1.LastModificationDate), - "LastModificationDate should advance after UpdateSchedule") - assert.Equal(t, "rate(2 minutes)", s2.ScheduleExpression) + assert.True(t, s2.LastModificationDate.After(s1.LastModificationDate), + "LastModificationDate should advance after UpdateSchedule") + assert.Equal(t, "rate(2 minutes)", s2.ScheduleExpression) + }) } func TestUpdateSchedule_ValidatesState(t *testing.T) { diff --git a/services/scheduler/whitebox_test.go b/services/scheduler/whitebox_test.go index fea89febc0..82c2fd1574 100644 --- a/services/scheduler/whitebox_test.go +++ b/services/scheduler/whitebox_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "testing" @@ -148,6 +149,54 @@ func TestScheduler_Runner_LocCacheEviction(t *testing.T) { assert.Equal(t, 0, locCacheLen(runner), "stale timezone cache entries should be evicted") } +// TestScheduler_IdempotencyEviction proves storeIdempotent opportunistically +// sweeps expired entries once the cache grows past idempotencyEvictThreshold, +// so a Create* call whose ClientToken is never replayed does not sit in the +// cache forever (only lookupIdempotent pruned before this fix, and only for +// the exact key it was asked about). +func TestScheduler_IdempotencyEviction(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + seedExpired int + inserts int + wantSwept bool + }{ + {name: "below threshold keeps expired", seedExpired: 1, inserts: 1}, + { + name: "threshold and sweep interval evicts expired", + seedExpired: idempotencyEvictThreshold + 16, + inserts: idempotencyEvictSweepInterval, + wantSwept: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + h := NewHandler(NewInMemoryBackend("000000000000", "us-east-1")) + past := time.Now().Add(-time.Hour) + + for i := range tt.seedExpired { + h.idempotency.Set(fmt.Sprintf("expired-%d", i), idempotentResult{arn: "arn:expired", expiresAt: past}) + } + + for i := range tt.inserts { + h.storeIdempotent(fmt.Sprintf("live-%d", i), "arn:live") + } + + _, stillPresent := h.idempotency.Get("expired-0") + if tt.wantSwept { + assert.False(t, stillPresent, "expired entry should have been swept") + } else { + assert.True(t, stillPresent, "expired entry should remain below the eviction threshold") + } + }) + } +} + type whiteboxLambdaInvoker struct{} func (*whiteboxLambdaInvoker) InvokeFunction(_ context.Context, _, _ string, _ []byte) ([]byte, int, error) { diff --git a/services/secretsmanager/PARITY.md b/services/secretsmanager/PARITY.md index f1e8a80a3f..6da5fc750c 100644 --- a/services/secretsmanager/PARITY.md +++ b/services/secretsmanager/PARITY.md @@ -106,6 +106,24 @@ leaks: {status: fixed, note: "Found a real data race: ListSecrets/ListSecretVers ## Notes +- **2026-09-26 (gopherstack-lr8qu, rotation window)**: `RotationRules.Duration` was stored + but never validated or used — rotation.go fired exactly at the cron/rate boundary with no + window concept. Per rotate-secrets_schedule.html ("Secrets Manager rotates your secret at + any time during the rotation window"), firing at the window START is a valid deterministic + choice, kept as-is; fixed the actual gaps: (1) `rate()` schedules now align their window + start per docs — day-based to midnight UTC, hour-based to the top of the hour — instead of + literally `lastRotated + interval` at an arbitrary time of day (`nextRotationOccurrence`, + rotation.go); cron() was already hour-aligned. (2) `Duration` is now validated: format/length + per `API_RotationRulesType.html` (`[0-9]+h`, length 2-3), and "must not extend into the next + rotation window or the next UTC day" against the schedule's window limit + (`scheduleWindowLimit`/`cronHourlyWindowLimit`). (3) `AutomaticallyAfterDays` and + `ScheduleExpression` are now mutually exclusive per the same doc page (previously + unenforced). No persisted-field or wire-shape changes — `DescribeSecret`'s `RotationRules`/ + `NextRotationDate` echo unchanged shapes, just correct values. New tests: + `rotation_window_test.go` (table-driven validation + window-alignment cases, one + `synctest`-based end-to-end firing test). `go test -race -count=3 + ./services/secretsmanager/...` and `golangci-lint run ./services/secretsmanager/...` clean. + - **2026-09-19 (gopherstack-1x2u0 leak-audit follow-up)**: retrofitted the ~340 test call sites constructing `InMemoryBackend` to register `t.Cleanup(b.StopRotationScheduler)` (safe/idempotent even when the scheduler was diff --git a/services/secretsmanager/listsecretversionids_test.go b/services/secretsmanager/listsecretversionids_test.go index 68f8d1ba00..c959855911 100644 --- a/services/secretsmanager/listsecretversionids_test.go +++ b/services/secretsmanager/listsecretversionids_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "strings" "testing" + "testing/synctest" "time" "github.com/labstack/echo/v5" @@ -108,32 +109,34 @@ func TestListSecretVersionIds_IncludeDeprecated(t *testing.T) { func TestListSecretVersionIds_SortedNewestFirst(t *testing.T) { t.Parallel() - b := secretsmanager.NewInMemoryBackend() - t.Cleanup(b.StopRotationScheduler) - _, err := b.CreateSecret(context.Background(), &secretsmanager.CreateSecretInput{ - Name: "lvid-sort", - SecretString: "v1", - ClientRequestToken: "v1", - }) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := secretsmanager.NewInMemoryBackend() + t.Cleanup(b.StopRotationScheduler) + _, err := b.CreateSecret(context.Background(), &secretsmanager.CreateSecretInput{ + Name: "lvid-sort", + SecretString: "v1", + ClientRequestToken: "v1", + }) + require.NoError(t, err) - time.Sleep(2 * time.Millisecond) + time.Sleep(2 * time.Millisecond) - _, err = b.PutSecretValue(context.Background(), &secretsmanager.PutSecretValueInput{ - SecretID: "lvid-sort", - SecretString: "v2", - ClientRequestToken: "v2", - }) - require.NoError(t, err) + _, err = b.PutSecretValue(context.Background(), &secretsmanager.PutSecretValueInput{ + SecretID: "lvid-sort", + SecretString: "v2", + ClientRequestToken: "v2", + }) + require.NoError(t, err) - out, err := b.ListSecretVersionIDs( - context.Background(), - &secretsmanager.ListSecretVersionIDsInput{SecretID: "lvid-sort"}, - ) - require.NoError(t, err) - require.Len(t, out.Versions, 2) - // Newest (v2 = AWSCURRENT) should be first - assert.Equal(t, "v2", out.Versions[0].VersionID) + out, err := b.ListSecretVersionIDs( + context.Background(), + &secretsmanager.ListSecretVersionIDsInput{SecretID: "lvid-sort"}, + ) + require.NoError(t, err) + require.Len(t, out.Versions, 2) + // Newest (v2 = AWSCURRENT) should be first + assert.Equal(t, "v2", out.Versions[0].VersionID) + }) } func TestListSecretVersionIds_NotFound(t *testing.T) { diff --git a/services/secretsmanager/models.go b/services/secretsmanager/models.go index 698c27e048..6bbe7aa262 100644 --- a/services/secretsmanager/models.go +++ b/services/secretsmanager/models.go @@ -85,7 +85,8 @@ type ExternalSecretRotationMetadataItem struct { type RotationRulesType struct { // AutomaticallyAfterDays rotates the secret after this many days. AutomaticallyAfterDays *int64 `json:"AutomaticallyAfterDays,omitempty"` - // Duration is an optional ISO-8601 duration window for rotation. + // Duration is the rotation window length, formatted "h" (e.g. "3h"). + // Optional; validateRotationRules enforces its format and range. Duration string `json:"Duration,omitempty"` // ScheduleExpression is an optional cron/rate expression for rotation scheduling. ScheduleExpression string `json:"ScheduleExpression,omitempty"` diff --git a/services/secretsmanager/rotatesecret_test.go b/services/secretsmanager/rotatesecret_test.go index b34be1204c..c12929cdf6 100644 --- a/services/secretsmanager/rotatesecret_test.go +++ b/services/secretsmanager/rotatesecret_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "strings" "testing" + "testing/synctest" "time" "github.com/labstack/echo/v5" @@ -678,55 +679,65 @@ func TestRotateSecret_InvalidDays(t *testing.T) { // RotateSecret cron scheduling // --------------------------------------------------------------------------- -// TestRotateSecret_CronScheduleTriggersRotation verifies that setting a -// ScheduleExpression with a cron expression enables automatic background rotation. +// TestRotateSecret_CronScheduleTriggersRotation checks the scheduler rotates on a cron; +// RotateImmediately=false keeps RotateSecret from rotating first. func TestRotateSecret_CronScheduleTriggersRotation(t *testing.T) { t.Parallel() - b := secretsmanager.NewInMemoryBackend() - t.Cleanup(b.StopRotationScheduler) - _, err := b.CreateSecret(context.Background(), &secretsmanager.CreateSecretInput{ - Name: "cron-sched-secret", - SecretString: "initial", - }) - require.NoError(t, err) - - before, err := b.GetSecretValue( - context.Background(), - &secretsmanager.GetSecretValueInput{SecretID: "cron-sched-secret"}, - ) - require.NoError(t, err) - - // Use a cron that fires every minute to trigger fast in tests. - _, err = b.RotateSecret(context.Background(), &secretsmanager.RotateSecretInput{ - SecretID: "cron-sched-secret", - RotationLambdaARN: testLambdaARN, - RotationRules: &secretsmanager.RotationRulesType{ - ScheduleExpression: "cron(* * * * ? *)", - }, - }) - require.NoError(t, err) - - deadline := time.Now().Add(5 * time.Second) - rotated := false + synctest.Test(t, func(t *testing.T) { + b := secretsmanager.NewInMemoryBackend() + t.Cleanup(b.StopRotationScheduler) + _, err := b.CreateSecret(context.Background(), &secretsmanager.CreateSecretInput{ + Name: "cron-sched-secret", + SecretString: "initial", + }) + require.NoError(t, err) - for time.Now().Before(deadline) { - current, currentErr := b.GetSecretValue( + before, err := b.GetSecretValue( context.Background(), &secretsmanager.GetSecretValueInput{SecretID: "cron-sched-secret"}, ) - require.NoError(t, currentErr) + require.NoError(t, err) + + rotateImmediately := false + _, err = b.RotateSecret(context.Background(), &secretsmanager.RotateSecretInput{ + SecretID: "cron-sched-secret", + RotationLambdaARN: testLambdaARN, + RotateImmediately: &rotateImmediately, + RotationRules: &secretsmanager.RotationRulesType{ + ScheduleExpression: "cron(* * * * ? *)", + }, + }) + require.NoError(t, err) - if current.VersionID != before.VersionID { - rotated = true + afterCall, err := b.GetSecretValue( + context.Background(), + &secretsmanager.GetSecretValueInput{SecretID: "cron-sched-secret"}, + ) + require.NoError(t, err) + assert.Equal(t, before.VersionID, afterCall.VersionID, + "RotateImmediately=false must not rotate synchronously") - break - } + // The next cron boundary is at most 60s away; advance past it and let the + // scheduler goroutine finish. + time.Sleep(65 * time.Second) + synctest.Wait() - time.Sleep(200 * time.Millisecond) - } + after, err := b.GetSecretValue( + context.Background(), + &secretsmanager.GetSecretValueInput{SecretID: "cron-sched-secret"}, + ) + require.NoError(t, err) + assert.NotEqual(t, before.VersionID, after.VersionID, + "cron-scheduled rotation must fire once the virtual clock passes the boundary") - assert.True(t, rotated, "cron-scheduled rotation must fire within 5 seconds") + desc, err := b.DescribeSecret( + context.Background(), + &secretsmanager.DescribeSecretInput{SecretID: "cron-sched-secret"}, + ) + require.NoError(t, err) + assert.NotNil(t, desc.LastRotatedDate) + }) } // TestRotateSecret_ScheduleExpressionPersisted verifies that a cron ScheduleExpression diff --git a/services/secretsmanager/rotation.go b/services/secretsmanager/rotation.go index ee1a0595d2..25c5dd1670 100644 --- a/services/secretsmanager/rotation.go +++ b/services/secretsmanager/rotation.go @@ -15,6 +15,12 @@ const ( rotationSchedulerInterval = time.Second // hoursPerDay is the number of hours in a day, used for day-granularity truncation. hoursPerDay = 24 + // rateUnitHour/rateUnitDay (+ plurals) are rate() expression unit words, deduplicated + // across rotationInterval/scheduleWindowLimit/rotationRateUnit. + rateUnitHour = "hour" + rateUnitHours = "hours" + rateUnitDay = "day" + rateUnitDays = "days" ) // pendingRotation describes a Lambda-backed rotation awaiting its step invocations. @@ -25,8 +31,8 @@ type pendingRotation struct { lambdaARN string } -// computeNextRotationDate returns the predicted next rotation timestamp for a secret, or nil if -// rotation is not configured or cannot be computed. +// computeNextRotationDate returns the secret's next rotation timestamp, or nil if rotation +// isn't configured. Reports the window start, since gopherstack fires there (rotate-secrets_schedule.html). func computeNextRotationDate(secret *Secret) *float64 { if !secret.RotationEnabled || secret.RotationRules == nil { return nil @@ -43,25 +49,69 @@ func computeNextRotationDate(secret *Secret) *float64 { baseTime := time.Unix(0, int64(*base*float64(time.Second))) - if isCronExpression(secret.RotationRules.ScheduleExpression) { - next, ok := nextCronTime(secret.RotationRules.ScheduleExpression, baseTime) - if !ok { - return nil - } + next, ok := nextRotationOccurrence(secret.RotationRules, baseTime) + if !ok { + return nil + } + + nextFloat := UnixTimeFloat(next) + + return &nextFloat +} - nextFloat := UnixTimeFloat(next) +// rotationRateUnit reports a rate() ScheduleExpression's unit ("hour" or "day"), or "" for +// AutomaticallyAfterDays, cron(), or this repo's test-only second/minute rate units. +func rotationRateUnit(rules *RotationRulesType) string { + const rateExpressionParts = 2 - return &nextFloat + if rules == nil || rules.AutomaticallyAfterDays != nil { + return "" } - interval, ok := rotationInterval(secret.RotationRules) + expr := strings.TrimSpace(rules.ScheduleExpression) + if !strings.HasPrefix(expr, "rate(") || !strings.HasSuffix(expr, ")") { + return "" + } + + parts := strings.Fields(strings.TrimSuffix(strings.TrimPrefix(expr, "rate("), ")")) + if len(parts) != rateExpressionParts { + return "" + } + + switch parts[1] { + case rateUnitHour, rateUnitHours: + return rateUnitHour + case rateUnitDay, rateUnitDays: + return rateUnitDay + default: + return "" + } +} + +// nextRotationOccurrence returns the next rotation window start after base, per +// rotate-secrets_schedule.html (rate(days) aligns to midnight UTC; cron() is pre-aligned). +func nextRotationOccurrence(rules *RotationRulesType, base time.Time) (time.Time, bool) { + if isCronExpression(rules.ScheduleExpression) { + return nextCronTime(rules.ScheduleExpression, base) + } + + interval, ok := rotationInterval(rules) if !ok { - return nil + return time.Time{}, false } - nextFloat := UnixTimeFloat(baseTime.Add(interval)) + candidate := base.Add(interval) - return &nextFloat + // AWS schedules evaluate in UTC, but base may carry a local Location (e.g. time.Unix). + // Date/hour components are read from candidate.UTC() to keep the alignment correct. + switch u := candidate.UTC(); rotationRateUnit(rules) { + case rateUnitDay: + return time.Date(u.Year(), u.Month(), u.Day(), 0, 0, 0, 0, time.UTC), true + case rateUnitHour: + return time.Date(u.Year(), u.Month(), u.Day(), u.Hour(), 0, 0, 0, time.UTC), true + default: + return candidate, true + } } // RotateSecret creates a new version of the secret (rotation stub). @@ -369,11 +419,131 @@ func validateRotationRules(rules *RotationRulesType) error { maxRotationDays, ) } + + // AutomaticallyAfterDays and ScheduleExpression are mutually exclusive + // (API_RotationRulesType.html). + if rules.ScheduleExpression != "" { + return fmt.Errorf( + "%w: RotationRules must set AutomaticallyAfterDays or ScheduleExpression, not both", + ErrInvalidParameter, + ) + } + } + + if rules.Duration == "" { + return nil + } + + durationHours, err := parseRotationDuration(rules.Duration) + if err != nil { + return err + } + + // A Duration must not extend into the next rotation window or the next UTC day + // (API_RotationRulesType.html). + if limit, ok := scheduleWindowLimit(rules.ScheduleExpression); ok && durationHours > limit { + return fmt.Errorf( + "%w: Duration %s must not extend into the next rotation window or the next UTC day", + ErrInvalidParameter, rules.Duration, + ) } return nil } +// parseRotationDuration validates and parses a RotationRules.Duration string. Per +// API_RotationRulesType.html: pattern "[0-9]+h", length 2-3 (i.e. 1-2 digit hours). +func parseRotationDuration(s string) (int, error) { + const minLen, maxLen = 2, 3 + + invalid := func() error { + return fmt.Errorf( + "%w: Duration %q must match pattern [0-9]+h with length %d-%d (e.g. \"3h\")", + ErrInvalidParameter, s, minLen, maxLen, + ) + } + + if len(s) < minLen || len(s) > maxLen || !strings.HasSuffix(s, "h") { + return 0, invalid() + } + + digits := strings.TrimSuffix(s, "h") + for _, r := range digits { + if r < '0' || r > '9' { + return 0, invalid() + } + } + + hours, err := strconv.Atoi(digits) + if err != nil || hours <= 0 { + return 0, invalid() + } + + return hours, nil +} + +// scheduleWindowLimit returns the max valid Duration, in hours, for a ScheduleExpression +// (rotate-secrets_schedule.html). ok is false for schedules with no real-AWS window. +func scheduleWindowLimit(expr string) (int, bool) { + const rateExpressionParts = 2 + + expr = strings.TrimSpace(expr) + + switch { + case isCronExpression(expr): + cf, err := parseCronExpr(expr) + if err != nil { + return 0, false + } + + if len(cf.hours) > 1 { + return cronHourlyWindowLimit(cf.hours), true + } + + return hoursPerDay - cf.hours[0], true + + case strings.HasPrefix(expr, "rate(") && strings.HasSuffix(expr, ")"): + parts := strings.Fields(strings.TrimSuffix(strings.TrimPrefix(expr, "rate("), ")")) + if len(parts) != rateExpressionParts { + return 0, false + } + + n, err := strconv.Atoi(parts[0]) + if err != nil || n <= 0 { + return 0, false + } + + switch parts[1] { + case rateUnitHour, rateUnitHours: + return n, true + case rateUnitDay, rateUnitDays: + return hoursPerDay, true + default: + return 0, false + } + + default: + return 0, false + } +} + +// cronHourlyWindowLimit returns the smallest gap, in hours and wrapping past midnight, between +// an hours-based cron's Hours values -- the max Duration allowed (rotate-secrets_schedule.html). +func cronHourlyWindowLimit(hours []int) int { + minGap := hoursPerDay + for i := 1; i < len(hours); i++ { + if gap := hours[i] - hours[i-1]; gap < minGap { + minGap = gap + } + } + + if wrapGap := hoursPerDay - hours[len(hours)-1] + hours[0]; wrapGap < minGap { + minGap = wrapGap + } + + return minGap +} + // CancelRotateSecret cancels an in-progress rotation by removing the AWSPENDING staging label. func (b *InMemoryBackend) CancelRotateSecret( ctx context.Context, input *CancelRotateSecretInput, @@ -545,21 +715,12 @@ func rotationDue(rules *RotationRulesType, now time.Time, base *float64) bool { return false } - if isCronExpression(rules.ScheduleExpression) { - next, ok := nextCronTime(rules.ScheduleExpression, baseTime) - if !ok { - return false - } - - return !now.Before(next) - } - - interval, ok := rotationInterval(rules) + next, ok := nextRotationOccurrence(rules, baseTime) if !ok { return false } - return now.Sub(baseTime) >= interval + return !now.Before(next) } func rotationInterval(rules *RotationRulesType) (time.Duration, bool) { @@ -597,9 +758,9 @@ func rotationInterval(rules *RotationRulesType) (time.Duration, bool) { return time.Duration(n) * time.Second, true case "minute", "minutes": return time.Duration(n) * time.Minute, true - case "hour", "hours": + case rateUnitHour, rateUnitHours: return time.Duration(n) * time.Hour, true - case "day", "days": + case rateUnitDay, rateUnitDays: return time.Duration(n) * 24 * time.Hour, true default: return 0, false diff --git a/services/secretsmanager/rotation_window_test.go b/services/secretsmanager/rotation_window_test.go new file mode 100644 index 0000000000..7e9e7d445c --- /dev/null +++ b/services/secretsmanager/rotation_window_test.go @@ -0,0 +1,225 @@ +package secretsmanager_test + +import ( + "context" + "testing" + "testing/synctest" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/secretsmanager" +) + +// TestRotateSecret_RotationRulesValidation exercises Duration format/range validation and the +// AutomaticallyAfterDays/ScheduleExpression mutual exclusivity (API_RotationRulesType.html). +func TestRotateSecret_RotationRulesValidation(t *testing.T) { + t.Parallel() + + sevenDays := int64(7) + + tests := []struct { + rules *secretsmanager.RotationRulesType + name string + wantErr bool + }{ + { + name: "duration_fits_hourly_rate_window", + rules: &secretsmanager.RotationRulesType{ScheduleExpression: "rate(4 hours)", Duration: "1h"}, + }, + { + name: "duration_fits_daily_cron_window", + rules: &secretsmanager.RotationRulesType{ScheduleExpression: "cron(0 8 * * ? *)", Duration: "3h"}, + }, + { + name: "duration_equal_to_hourly_window_limit_allowed", + rules: &secretsmanager.RotationRulesType{ScheduleExpression: "rate(4 hours)", Duration: "4h"}, + }, + { + name: "duration_missing_h_suffix", + rules: &secretsmanager.RotationRulesType{ScheduleExpression: "rate(1 day)", Duration: "3"}, + wantErr: true, + }, + { + name: "duration_non_numeric", + rules: &secretsmanager.RotationRulesType{ScheduleExpression: "rate(1 day)", Duration: "abh"}, + wantErr: true, + }, + { + name: "duration_zero_hours", + rules: &secretsmanager.RotationRulesType{ScheduleExpression: "rate(1 day)", Duration: "0h"}, + wantErr: true, + }, + { + name: "duration_too_long_for_length_constraint", + rules: &secretsmanager.RotationRulesType{ScheduleExpression: "rate(1 day)", Duration: "100h"}, + wantErr: true, + }, + { + name: "duration_exceeds_hourly_rate_window", + rules: &secretsmanager.RotationRulesType{ScheduleExpression: "rate(1 hour)", Duration: "2h"}, + wantErr: true, + }, + { + name: "duration_extends_past_daily_cron_window", + rules: &secretsmanager.RotationRulesType{ScheduleExpression: "cron(0 10 * * ? *)", Duration: "20h"}, + wantErr: true, + }, + { + name: "duration_not_validated_against_window_without_scheduleexpression", + rules: &secretsmanager.RotationRulesType{ + AutomaticallyAfterDays: &sevenDays, + Duration: "50h", + }, + }, + { + name: "automaticallyafterdays_and_scheduleexpression_mutually_exclusive", + rules: &secretsmanager.RotationRulesType{ + AutomaticallyAfterDays: &sevenDays, + ScheduleExpression: "rate(1 day)", + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := secretsmanager.NewInMemoryBackend() + t.Cleanup(b.StopRotationScheduler) + ctx := context.Background() + + secretName := "rules-" + tt.name + _, err := b.CreateSecret(ctx, &secretsmanager.CreateSecretInput{ + Name: secretName, + SecretString: "v", + }) + require.NoError(t, err) + + rotateImmediately := false + _, err = b.RotateSecret(ctx, &secretsmanager.RotateSecretInput{ + SecretID: secretName, + RotationLambdaARN: testLambdaARN, + RotationRules: tt.rules, + RotateImmediately: &rotateImmediately, + }) + + if tt.wantErr { + require.ErrorIs(t, err, secretsmanager.ErrInvalidParameter) + + return + } + + require.NoError(t, err) + }) + } +} + +// TestComputeNextRotationDate_WindowAlignment verifies rate() window starts align to midnight +// UTC (days) or the top of the hour (hours), per rotate-secrets_schedule.html. +func TestComputeNextRotationDate_WindowAlignment(t *testing.T) { + t.Parallel() + + base := time.Date(2024, 3, 15, 15, 30, 0, 0, time.UTC) + + tests := []struct { + want time.Time + rules *secretsmanager.RotationRulesType + name string + }{ + { + name: "daily_rate_aligns_to_midnight_utc", + rules: &secretsmanager.RotationRulesType{ScheduleExpression: "rate(2 days)"}, + want: time.Date(2024, 3, 17, 0, 0, 0, 0, time.UTC), + }, + { + name: "hourly_rate_aligns_to_the_hour", + rules: &secretsmanager.RotationRulesType{ScheduleExpression: "rate(6 hours)"}, + want: time.Date(2024, 3, 15, 21, 0, 0, 0, time.UTC), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := secretsmanager.NewInMemoryBackend() + t.Cleanup(b.StopRotationScheduler) + b.SetNowForTest(func() time.Time { return base }) + t.Cleanup(func() { b.SetNowForTest(time.Now) }) + + ctx := context.Background() + secretName := "align-" + tt.name + _, err := b.CreateSecret(ctx, &secretsmanager.CreateSecretInput{ + Name: secretName, + SecretString: "v", + }) + require.NoError(t, err) + + rotateImmediately := false + _, err = b.RotateSecret(ctx, &secretsmanager.RotateSecretInput{ + SecretID: secretName, + RotationLambdaARN: testLambdaARN, + RotationRules: tt.rules, + RotateImmediately: &rotateImmediately, + }) + require.NoError(t, err) + + desc, err := b.DescribeSecret(ctx, &secretsmanager.DescribeSecretInput{SecretID: secretName}) + require.NoError(t, err) + require.NotNil(t, desc.NextRotationDate) + assert.InDelta(t, secretsmanager.UnixTimeFloat(tt.want), *desc.NextRotationDate, 1) + }) + } +} + +// TestRotateSecret_RateScheduleFiresAtAlignedWindowStart proves the scheduler fires at the +// aligned window start, not the literal unaligned base+interval instant. +func TestRotateSecret_RateScheduleFiresAtAlignedWindowStart(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + b := secretsmanager.NewInMemoryBackend() + t.Cleanup(b.StopRotationScheduler) + ctx := context.Background() + + now0 := time.Now().UTC() + _, err := b.CreateSecret(ctx, &secretsmanager.CreateSecretInput{ + Name: "rate-align-fire", + SecretString: "initial", + }) + require.NoError(t, err) + + before, err := b.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{SecretID: "rate-align-fire"}) + require.NoError(t, err) + + rotateImmediately := false + _, err = b.RotateSecret(ctx, &secretsmanager.RotateSecretInput{ + SecretID: "rate-align-fire", + RotationLambdaARN: testLambdaARN, + RotateImmediately: &rotateImmediately, + RotationRules: &secretsmanager.RotationRulesType{ScheduleExpression: "rate(2 hours)"}, + }) + require.NoError(t, err) + + candidate := now0.Add(2 * time.Hour) + aligned := time.Date( + candidate.Year(), candidate.Month(), candidate.Day(), candidate.Hour(), 0, 0, 0, time.UTC, + ) + + time.Sleep(aligned.Sub(now0) + time.Second) + synctest.Wait() + + after, err := b.GetSecretValue(ctx, &secretsmanager.GetSecretValueInput{SecretID: "rate-align-fire"}) + require.NoError(t, err) + assert.NotEqual(t, before.VersionID, after.VersionID, + "rotation must fire once the virtual clock passes the aligned window start") + + desc, err := b.DescribeSecret(ctx, &secretsmanager.DescribeSecretInput{SecretID: "rate-align-fire"}) + require.NoError(t, err) + require.NotNil(t, desc.LastRotatedDate) + assert.InDelta(t, secretsmanager.UnixTimeFloat(aligned), *desc.LastRotatedDate, 2) + }) +} diff --git a/services/secretsmanager/scheduler_shutdown_test.go b/services/secretsmanager/scheduler_shutdown_test.go index 9f1c68982e..d546164333 100644 --- a/services/secretsmanager/scheduler_shutdown_test.go +++ b/services/secretsmanager/scheduler_shutdown_test.go @@ -2,6 +2,7 @@ package secretsmanager //nolint:testpackage // existing issue. import ( "testing" + "testing/synctest" "time" ) @@ -54,31 +55,34 @@ func TestStopRotationScheduler(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { t.Parallel() - _ = t.Context() - b := NewInMemoryBackend() - t.Cleanup(b.StopRotationScheduler) + synctest.Test(t, func(t *testing.T) { + _ = t.Context() - if tc.start { - b.ensureRotationScheduler() - // Give the goroutine a moment to be scheduled. - time.Sleep(5 * time.Millisecond) - } + b := NewInMemoryBackend() + t.Cleanup(b.StopRotationScheduler) - assertStopsPromptly(t, 2*time.Second, b.StopRotationScheduler) + if tc.start { + b.ensureRotationScheduler() + // Give the goroutine a moment to be scheduled. + time.Sleep(5 * time.Millisecond) + } - if tc.stopTwice { - // A second stop must not panic (close-of-closed-channel) and - // must remain a no-op. assertStopsPromptly(t, 2*time.Second, b.StopRotationScheduler) - } - // The stop channel must be closed (loop is guaranteed unblocked). - select { - case <-b.schedulerStop: - default: - t.Fatal("schedulerStop channel was not closed after StopRotationScheduler") - } + if tc.stopTwice { + // A second stop must not panic (close-of-closed-channel) and + // must remain a no-op. + assertStopsPromptly(t, 2*time.Second, b.StopRotationScheduler) + } + + // The stop channel must be closed (loop is guaranteed unblocked). + select { + case <-b.schedulerStop: + default: + t.Fatal("schedulerStop channel was not closed after StopRotationScheduler") + } + }) }) } } diff --git a/services/serverlessrepo/cloud_formation.go b/services/serverlessrepo/cloud_formation.go index fd75472001..e72ff88b21 100644 --- a/services/serverlessrepo/cloud_formation.go +++ b/services/serverlessrepo/cloud_formation.go @@ -2,9 +2,10 @@ package serverlessrepo import ( "fmt" - "strconv" "time" + "github.com/google/uuid" + "github.com/blackbirdworks/gopherstack/pkgs/arn" ) @@ -43,7 +44,9 @@ func (b *InMemoryBackend) CreateCloudFormationTemplate( } now := time.Now() - templateID := fmt.Sprintf("%s-%d", appName, now.UnixNano()) + // TemplateId is a bare UUID (confirmed pattern on CreateCloudFormationTemplateOutput, + // aws-sdk-go-v2/service/serverlessapplicationrepository), not appName-prefixed. + templateID := uuid.NewString() t := &CloudFormationTemplate{ ApplicationID: app.ApplicationID, TemplateID: templateID, @@ -142,7 +145,9 @@ func (b *InMemoryBackend) CreateCloudFormationChangeSetWithOptions( } } - suffix := strconv.FormatInt(time.Now().UnixNano(), 10) + // suffix mirrors real CloudFormation's UUID stack-ID suffix + // (arn:...:stack/{name}/{uuid}). + suffix := uuid.NewString() csName := changeSetName if csName == "" { diff --git a/services/serverlessrepo/id_generation_test.go b/services/serverlessrepo/id_generation_test.go new file mode 100644 index 0000000000..eafe1b0253 --- /dev/null +++ b/services/serverlessrepo/id_generation_test.go @@ -0,0 +1,76 @@ +package serverlessrepo_test + +import ( + "regexp" + "testing" + "testing/synctest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/serverlessrepo" +) + +// uuidPattern locks in the fix for CreateCloudFormationTemplate's TemplateId +// and CreateCloudFormationChangeSet's StackId suffix, which used to collide under synctest. +var uuidPattern = regexp.MustCompile( + `^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`, +) + +func TestServerlessRepoBackend_IDs_Unique(t *testing.T) { + t.Parallel() + + tests := []struct { + create func(t *testing.T, b *serverlessrepo.InMemoryBackend) string + name string + }{ + { + name: "cloud_formation_template", + create: func(t *testing.T, b *serverlessrepo.InMemoryBackend) string { + t.Helper() + + tmpl, err := b.CreateCloudFormationTemplate("my-app", "1.0.0") + require.NoError(t, err) + + return tmpl.TemplateID + }, + }, + { + name: "cloud_formation_change_set_stack_id", + create: func(t *testing.T, b *serverlessrepo.InMemoryBackend) string { + t.Helper() + + cs, err := b.CreateCloudFormationChangeSet("my-app", "my-stack", "", "1.0.0") + require.NoError(t, err) + + // StackID is an ARN "...:stack/{stackName}/{uuid}"; extract the + // trailing UUID suffix. + idx := len(cs.StackID) - uuidLen + require.GreaterOrEqual(t, idx, 0) + + return cs.StackID[idx:] + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + b := serverlessrepo.NewInMemoryBackend("000000000000", "us-east-1") + _, err := b.CreateApplication("my-app", "desc", "author", "", "1.0.0", nil, "", "", "") + require.NoError(t, err) + + id1 := tt.create(t, b) + id2 := tt.create(t, b) + + assert.NotEqual(t, id1, id2, "two resources created back-to-back must get distinct IDs") + assert.Regexp(t, uuidPattern, id1) + assert.Regexp(t, uuidPattern, id2) + }) + }) + } +} + +const uuidLen = 36 diff --git a/services/serverlessrepo/models.go b/services/serverlessrepo/models.go index 7fa9d313fb..bf6299503d 100644 --- a/services/serverlessrepo/models.go +++ b/services/serverlessrepo/models.go @@ -93,11 +93,11 @@ type CloudFormationTemplate struct { Status string `json:"status"` TemplateURL string `json:"templateUrl,omitempty"` // AppName identifies the owning application. TemplateID is already - // globally unique (it is generated as "-"), so it - // remains the store.Table[CloudFormationTemplate] primary key (see - // store_setup.go); AppName exists purely to drive the additive "byApp" - // secondary index used for DeleteApplication's cascade delete. It is not - // part of the Serverless Application Repository wire API, hence json:"-". + // globally unique (a UUID), so it remains the + // store.Table[CloudFormationTemplate] primary key (see store_setup.go); + // AppName exists purely to drive the additive "byApp" secondary index + // used for DeleteApplication's cascade delete. It is not part of the + // Serverless Application Repository wire API, hence json:"-". AppName string `json:"-"` } diff --git a/services/ses/janitor_test.go b/services/ses/janitor_test.go index dc8035cf7a..1a96da6f98 100644 --- a/services/ses/janitor_test.go +++ b/services/ses/janitor_test.go @@ -3,6 +3,7 @@ package ses_test import ( "context" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -208,25 +209,27 @@ func TestJanitor_Run_CancelContext(t *testing.T) { func TestSESJanitor_SweepExpiredEmails(t *testing.T) { t.Parallel() - b := ses.NewInMemoryBackend() - b.SetEmailTTL(time.Millisecond) // very short TTL - require.NoError(t, b.VerifyEmailIdentity("j@test.com")) + synctest.Test(t, func(t *testing.T) { + b := ses.NewInMemoryBackend() + b.SetEmailTTL(time.Millisecond) // very short TTL + require.NoError(t, b.VerifyEmailIdentity("j@test.com")) - _, err := b.SendEmail(ses.SendEmailInput{ - From: "j@test.com", To: []string{"to@test.com"}, Subject: "s", BodyText: "b", - }) - require.NoError(t, err) + _, err := b.SendEmail(ses.SendEmailInput{ + From: "j@test.com", To: []string{"to@test.com"}, Subject: "s", BodyText: "b", + }) + require.NoError(t, err) - require.Equal(t, 1, b.EmailCount()) + require.Equal(t, 1, b.EmailCount()) - // Wait for TTL to expire then sweep. - time.Sleep(5 * time.Millisecond) + // Wait for TTL to expire then sweep. + time.Sleep(5 * time.Millisecond) - j := ses.NewJanitor(b, 0) - j.SweepOnce(t.Context()) + j := ses.NewJanitor(b, 0) + j.SweepOnce(t.Context()) - assert.Equal(t, 0, b.EmailCount()) - assert.Equal(t, 0, b.EmailsByIDCount()) + assert.Equal(t, 0, b.EmailCount()) + assert.Equal(t, 0, b.EmailsByIDCount()) + }) } func TestSESJanitor_SweepNoExpired(t *testing.T) { diff --git a/services/ses/persistence_test.go b/services/ses/persistence_test.go index 4a499daeaa..35abdcfe91 100644 --- a/services/ses/persistence_test.go +++ b/services/ses/persistence_test.go @@ -2,6 +2,7 @@ package ses_test import ( "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -513,29 +514,31 @@ func TestSESPersistence_TemplatesAndConfigSetsRoundTrip(t *testing.T) { func TestSESPersistence_RestorePrunesExpiredEmails(t *testing.T) { t.Parallel() - original := ses.NewInMemoryBackend() - require.NoError(t, original.VerifyEmailIdentity("prune@test.com")) + synctest.Test(t, func(t *testing.T) { + original := ses.NewInMemoryBackend() + require.NoError(t, original.VerifyEmailIdentity("prune@test.com")) - _, err := original.SendEmail(ses.SendEmailInput{ - From: "prune@test.com", To: []string{"to@test.com"}, Subject: "keep", BodyText: "b", - }) - require.NoError(t, err) + _, err := original.SendEmail(ses.SendEmailInput{ + From: "prune@test.com", To: []string{"to@test.com"}, Subject: "keep", BodyText: "b", + }) + require.NoError(t, err) - snap := original.Snapshot(t.Context()) - require.NotNil(t, snap) + snap := original.Snapshot(t.Context()) + require.NotNil(t, snap) - // Restore into a backend with a very short TTL so the snapshot email is - // considered expired at restore time. - fresh := ses.NewInMemoryBackend() - fresh.SetEmailTTL(time.Nanosecond) // instant expiry + // Restore into a backend with a very short TTL so the snapshot email is + // considered expired at restore time. + fresh := ses.NewInMemoryBackend() + fresh.SetEmailTTL(time.Nanosecond) // instant expiry - time.Sleep(time.Millisecond) // ensure TTL has passed + time.Sleep(time.Millisecond) // ensure TTL has passed - require.NoError(t, fresh.Restore(t.Context(), snap)) + require.NoError(t, fresh.Restore(t.Context(), snap)) - // The expired email must have been pruned. - assert.Equal(t, 0, fresh.EmailCount()) - assert.Equal(t, 0, fresh.EmailsByIDCount()) + // The expired email must have been pruned. + assert.Equal(t, 0, fresh.EmailCount()) + assert.Equal(t, 0, fresh.EmailsByIDCount()) + }) } func TestSESPersistence_RestoreCapsToBound(t *testing.T) { diff --git a/services/sns/PARITY.md b/services/sns/PARITY.md index c248d1606e..816b148874 100644 --- a/services/sns/PARITY.md +++ b/services/sns/PARITY.md @@ -42,7 +42,7 @@ ops: ListOriginationNumbers: {wire: fixed, errors: ok, state: ok, persist: ok, note: "AWS has no public create API; empty by default, SeedOriginationNumber for tests. FIXED 2026-08-14 (gopherstack-3tpf structural diff): XMLOriginationPhone (the domain model itself, not just a DTO) was entirely missing CreatedAt and Status, two real members of types.PhoneNumberInformation (types/types.go:82-103) confirmed present in the actual awsAwsquery_deserializeDocumentPhoneNumberInformation wire decoder (deserializers.go:7950) -- a real client always decoded a nil CreatedAt and empty Status regardless of what SeedOriginationNumber supplied. Added both fields (CreatedAt *time.Time xml:CreatedAt,omitempty; Status string xml:Status,omitempty, matching the cloudformation *time.Time-for-omitempty convention). Verified via TestListOriginationNumbers_CreatedAtAndStatusWireRoundTrip driving the real aws-sdk-go-v2 SNS client; hand-reverted the struct fields (test unchanged) and confirmed the revert does not just fail the assertion but fails to COMPILE (\"unknown field Status/CreatedAt in struct literal\"), the strongest possible confirmation the fields were structurally absent, not merely unwired."} TagResource/UntagResource/ListTagsForResource: {wire: ok, errors: ok, state: ok, persist: ok, note: "pkgs/tags-backed. VERIFIED CLEAN (wrapper-key sweep, 2026-08-29): checked for the stepfunctions-class bug (a Tags field typed as a Go map when the SDK sends an array, or vice versa). sns@v1.42.4 serializers.go:3862-3867/3893-3898 confirm TagResource.Tags serializes as Tags.member.N.Key/Value (awsAwsquery_serializeDocumentTagList, array element name 'member') and UntagResource.TagKeys as TagKeys.member.N (awsAwsquery_serializeDocumentTagKeyList) — handler_tags.go's parseSNSTagsFromForm/parseSNSTagKeysFromForm already parse exactly these wrapper names. Confirmed via TestTagResourceFamily_SDKRoundTrip (tag_resource_sdk_test.go) driving the real SDK client."} families: - filter_policy_matching: {status: ok, note: "prefix/suffix/equals-ignore-case/anything-but(+nested)/exists/numeric(6 ops)/wildcard/cidr/$or, MessageBody vs MessageAttributes scope, String.Array expansion, 150-condition cap, 256KiB size cap, 5-key-per-policy cap (fixed this pass, was unenforced), FilterPolicyLimitExceeded 200/topic+10,000/account quota (fixed this pass, was unenforced and the error sentinel/code did not exist at all) — field-diffed against docs.aws.amazon.com/sns/latest/dg/subscription-filter-policy-constraints.html and API_Subscribe.html Errors table"} + filter_policy_matching: {status: ok, note: "prefix/suffix/equals-ignore-case/anything-but(+nested)/exists/numeric(6 ops)/wildcard/cidr/$or, MessageBody vs MessageAttributes scope, String.Array expansion, 150-condition cap, 256KiB size cap, 5-key-per-policy cap (fixed this pass, was unenforced), FilterPolicyLimitExceeded 200/topic+10,000/account quota (fixed this pass, was unenforced and the error sentinel/code did not exist at all) — field-diffed against docs.aws.amazon.com/sns/latest/dg/subscription-filter-policy-constraints.html and API_Subscribe.html Errors table. 2026-09-26: re-verified every string.value-matching.html operator (exact/anything-but+prefix+suffix+wildcard/equals-ignore-case/cidr/prefix/suffix/wildcard) is implemented and covered; the one real gap found was a malformed cidr operand (bad IP/CIDR syntax) silently never matching instead of being rejected at Subscribe/SetSubscriptionAttributes time like the numeric operand shape already was — fixed via validateCIDROperand, same eager-validation pattern as validateNumericOperands. Added TestRealClient_FilterPolicyCIDR_SQSDelivery, an end-to-end real-SDK SNS->SQS test proving only the in-CIDR publish reaches the subscribed queue."} fifo_topics: {status: ok, note: "MessageGroupId required, ContentBasedDeduplication (SHA-256 body digest) vs explicit MessageDeduplicationId mutually exclusive, 5-min dedup window with bounded+swept map, 20-digit zero-padded monotonic SequenceNumber per topic, PublishBatch per-entry dedup"} delivery_lambda_firehose_sms_application: {status: ok, note: "fixed this pass: (1) Lambda envelope now carries the real per-publish Timestamp/Signature/SigningCertURL/UnsubscribeURL instead of a fabricated random-UUID signature and empty cert/unsub URLs; (2) Firehose now respects RawMessageDelivery (envelopes as JSON when false, matching AWS default, previously always sent the bare message); DLQ redrive on failure now forwards the same body that was attempted"} replay_policy_archive: {status: ok, note: "fans out through the same per-protocol delivery functions Publish uses (SQS via the emitter, Lambda/Firehose via their delivery functions). fixed this pass (bd: gopherstack-bz6), re-verified against docs.aws.amazon.com/sns/latest/dg/fifo-message-archiving-replay.html and message-archiving-and-replay-topic-owner.html ('Amazon SNS message archiving and replay is only available for application-to-application (A2A) FIFO topics'): ArchivePolicy is now rejected (InvalidParameter) on non-FIFO topics at both CreateTopic and SetTopicAttributes; ReplayPolicy is now rejected (InvalidParameter) unless the subscription's topic is FIFO and its protocol is sqs/lambda/firehose. Previously ArchivePolicy/ReplayPolicy were accepted on any topic and fanned out to any protocol (HTTP/email/sms/application), which is not real AWS behavior — standard topics have no archive/replay mechanism at all, and SMS/Application/HTTP/HTTPS are A2P protocols never eligible even on a FIFO topic"} diff --git a/services/sns/archive_test.go b/services/sns/archive_test.go index 13b90c6918..6904cf17a7 100644 --- a/services/sns/archive_test.go +++ b/services/sns/archive_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -191,48 +192,50 @@ func TestReplayPolicyValidAccepted(t *testing.T) { func TestReplayPolicyTriggersLambdaReplay(t *testing.T) { t.Parallel() - b := newTestBackend(t) - lambda := &mockLambdaInvoker{} - b.SetLambdaBackend(lambda) - - tp, err := b.CreateTopic("replay-lambda-topic.fifo", map[string]string{ - "ArchivePolicy": `{"MessageRetentionPeriod":30}`, - }) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := newTestBackend(t) + lambda := &mockLambdaInvoker{} + b.SetLambdaBackend(lambda) - // Publish messages before subscribing. - pastTime := time.Now().UTC().Add(-time.Hour) - for i := range 3 { - _, err = b.Publish(tp.TopicArn, fmt.Sprintf("archived-%d", i), "", "", nil) + tp, err := b.CreateTopic("replay-lambda-topic.fifo", map[string]string{ + "ArchivePolicy": `{"MessageRetentionPeriod":30}`, + }) require.NoError(t, err) - } - // Subscribe AFTER the messages were published. - sub, err := b.Subscribe( - tp.TopicArn, "lambda", "arn:aws:lambda:us-east-1:000000000000:function:replay-fn", "", - ) - require.NoError(t, err) + // Publish messages before subscribing. + pastTime := time.Now().UTC().Add(-time.Hour) + for i := range 3 { + _, err = b.Publish(tp.TopicArn, fmt.Sprintf("archived-%d", i), "", "", nil) + require.NoError(t, err) + } - // Set ReplayPolicy to replay from before the archived messages. - replayFrom := pastTime.Format(time.RFC3339) - err = b.SetSubscriptionAttributes(sub.SubscriptionArn, "ReplayPolicy", - fmt.Sprintf(`{"replayFromTimestamp":"%s"}`, replayFrom)) - require.NoError(t, err) + // Subscribe AFTER the messages were published. + sub, err := b.Subscribe( + tp.TopicArn, "lambda", "arn:aws:lambda:us-east-1:000000000000:function:replay-fn", "", + ) + require.NoError(t, err) - // Expect all 3 archived messages to be replayed. - require.Eventually(t, func() bool { return lambda.Count() == 3 }, - 3*time.Second, 10*time.Millisecond, "not all archived messages were replayed") - - // Verify all archived messages were replayed, in original publish order. - for i, invocation := range lambda.All() { - var envelope map[string]any - require.NoError(t, json.Unmarshal(invocation.Payload, &envelope)) - records, _ := envelope["Records"].([]any) - require.Len(t, records, 1) - record, _ := records[0].(map[string]any) - snsData, _ := record["Sns"].(map[string]any) - assert.Equal(t, fmt.Sprintf("archived-%d", i), snsData["Message"]) - } + // Set ReplayPolicy to replay from before the archived messages. + replayFrom := pastTime.Format(time.RFC3339) + err = b.SetSubscriptionAttributes(sub.SubscriptionArn, "ReplayPolicy", + fmt.Sprintf(`{"replayFromTimestamp":"%s"}`, replayFrom)) + require.NoError(t, err) + + // Wait for the async replay goroutine to deliver all 3 archived messages. + synctest.Wait() + require.Equal(t, 3, lambda.Count(), "not all archived messages were replayed") + + // Verify all archived messages were replayed, in original publish order. + for i, invocation := range lambda.All() { + var envelope map[string]any + require.NoError(t, json.Unmarshal(invocation.Payload, &envelope)) + records, _ := envelope["Records"].([]any) + require.Len(t, records, 1) + record, _ := records[0].(map[string]any) + snsData, _ := record["Sns"].(map[string]any) + assert.Equal(t, fmt.Sprintf("archived-%d", i), snsData["Message"]) + } + }) } // TestReplayPolicyFutureTimestampReplaysNothing verifies that a @@ -240,32 +243,34 @@ func TestReplayPolicyTriggersLambdaReplay(t *testing.T) { func TestReplayPolicyFutureTimestampReplaysNothing(t *testing.T) { t.Parallel() - b := newTestBackend(t) - lambda := &mockLambdaInvoker{} - b.SetLambdaBackend(lambda) + synctest.Test(t, func(t *testing.T) { + b := newTestBackend(t) + lambda := &mockLambdaInvoker{} + b.SetLambdaBackend(lambda) - tp, err := b.CreateTopic("replay-future-topic.fifo", map[string]string{ - "ArchivePolicy": `{"MessageRetentionPeriod":30}`, - }) - require.NoError(t, err) + tp, err := b.CreateTopic("replay-future-topic.fifo", map[string]string{ + "ArchivePolicy": `{"MessageRetentionPeriod":30}`, + }) + require.NoError(t, err) - _, err = b.Publish(tp.TopicArn, "past-message", "", "", nil) - require.NoError(t, err) + _, err = b.Publish(tp.TopicArn, "past-message", "", "", nil) + require.NoError(t, err) - sub, err := b.Subscribe( - tp.TopicArn, "lambda", "arn:aws:lambda:us-east-1:000000000000:function:future-fn", "", - ) - require.NoError(t, err) + sub, err := b.Subscribe( + tp.TopicArn, "lambda", "arn:aws:lambda:us-east-1:000000000000:function:future-fn", "", + ) + require.NoError(t, err) - // ReplayFromTimestamp is in the future → no messages match. - futureTS := time.Now().UTC().Add(24 * time.Hour).Format(time.RFC3339) - err = b.SetSubscriptionAttributes(sub.SubscriptionArn, "ReplayPolicy", - fmt.Sprintf(`{"replayFromTimestamp":"%s"}`, futureTS)) - require.NoError(t, err) + // ReplayFromTimestamp is in the future → no messages match. + futureTS := time.Now().UTC().Add(24 * time.Hour).Format(time.RFC3339) + err = b.SetSubscriptionAttributes(sub.SubscriptionArn, "ReplayPolicy", + fmt.Sprintf(`{"replayFromTimestamp":"%s"}`, futureTS)) + require.NoError(t, err) - // Wait briefly; no invocation should arrive. - time.Sleep(400 * time.Millisecond) - assert.Equal(t, 0, lambda.Count(), "no message should be replayed with a future replayFromTimestamp") + // Let the async replay goroutine finish finding nothing to replay. + synctest.Wait() + assert.Equal(t, 0, lambda.Count(), "no message should be replayed with a future replayFromTimestamp") + }) } // TestReplayPolicyDeliversToA2AProtocols verifies that a subscription's @@ -307,8 +312,8 @@ func TestReplayPolicyDeliversToA2AProtocols(t *testing.T) { endpoint: "arn:aws:lambda:us-east-1:123456789012:function:replay-fn", verify: func(t *testing.T) { t.Helper() - require.Eventually(t, func() bool { return lambda.Count() == 1 }, - 2*time.Second, 10*time.Millisecond, "lambda function was never invoked") + synctest.Wait() + require.Equal(t, 1, lambda.Count(), "lambda function was never invoked") var envelope map[string]any require.NoError(t, json.Unmarshal(lambda.Last().Payload, &envelope)) @@ -336,8 +341,8 @@ func TestReplayPolicyDeliversToA2AProtocols(t *testing.T) { endpoint: "arn:aws:firehose:us-east-1:123456789012:deliverystream/" + streamName, verify: func(t *testing.T) { t.Helper() - require.Eventually(t, func() bool { return len(firehose.RecordsFor(streamName)) == 1 }, - 2*time.Second, 10*time.Millisecond, "firehose stream received no record") + synctest.Wait() + require.Len(t, firehose.RecordsFor(streamName), 1, "firehose stream received no record") var envelope map[string]any require.NoError(t, json.Unmarshal(firehose.RecordsFor(streamName)[0], &envelope)) @@ -352,29 +357,31 @@ func TestReplayPolicyDeliversToA2AProtocols(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - // Each subtest builds its own isolated backend, topic, and subscription. - b := newTestBackend(t) - tp, err := b.CreateTopic("replay-fanout-"+tc.name+".fifo", map[string]string{ - "ArchivePolicy": `{"MessageRetentionPeriod":30}`, - }) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + // Each subtest builds its own isolated backend, topic, and subscription. + b := newTestBackend(t) + tp, err := b.CreateTopic("replay-fanout-"+tc.name+".fifo", map[string]string{ + "ArchivePolicy": `{"MessageRetentionPeriod":30}`, + }) + require.NoError(t, err) - // Publish before subscribing so the message lands only in the archive. - pastTime := time.Now().UTC().Add(-time.Hour) - _, err = b.Publish(tp.TopicArn, archivedMessage, "", "", nil) - require.NoError(t, err) + // Publish before subscribing so the message lands only in the archive. + pastTime := time.Now().UTC().Add(-time.Hour) + _, err = b.Publish(tp.TopicArn, archivedMessage, "", "", nil) + require.NoError(t, err) - res := tc.setup(t, b) + res := tc.setup(t, b) - sub, err := b.Subscribe(tp.TopicArn, tc.proto, res.endpoint, "") - require.NoError(t, err) + sub, err := b.Subscribe(tp.TopicArn, tc.proto, res.endpoint, "") + require.NoError(t, err) - replayFrom := pastTime.Format(time.RFC3339) - err = b.SetSubscriptionAttributes(sub.SubscriptionArn, "ReplayPolicy", - fmt.Sprintf(`{"replayFromTimestamp":"%s"}`, replayFrom)) - require.NoError(t, err) + replayFrom := pastTime.Format(time.RFC3339) + err = b.SetSubscriptionAttributes(sub.SubscriptionArn, "ReplayPolicy", + fmt.Sprintf(`{"replayFromTimestamp":"%s"}`, replayFrom)) + require.NoError(t, err) - res.verify(t) + res.verify(t) + }) }) } } diff --git a/services/sns/filter_policy.go b/services/sns/filter_policy.go index 5155764696..f313d3bb96 100644 --- a/services/sns/filter_policy.go +++ b/services/sns/filter_policy.go @@ -3,6 +3,7 @@ package sns import ( "encoding/json" "fmt" + "net" "strconv" "strings" ) @@ -15,8 +16,9 @@ import ( // - Total attribute conditions ≤ maxFilterPolicyConditions (150). // - Object-condition operator names are restricted to the AWS-supported set // (`prefix`, `suffix`, `equals-ignore-case`, `anything-but`, `exists`, -// `numeric`). +// `numeric`, `wildcard`, `cidr`). // - Numeric operand shape (operator/number pairs) is well-formed. +// - CIDR operand is a valid IPv4/IPv6 address or CIDR block. // // Nesting depth (for nested-object filter policies) is not yet enforced — // issue #1679 item 13. @@ -217,14 +219,51 @@ func validateConditionShapes(key string, conditions []json.RawMessage) error { } } - numericRaw, ok := obj["numeric"] - if !ok { - continue + if numericRaw, ok := obj["numeric"]; ok { + if err := validateNumericOperands(key, numericRaw); err != nil { + return err + } + } + + if cidrRaw, ok := obj["cidr"]; ok { + if err := validateCIDROperand(key, cidrRaw); err != nil { + return err + } } + } + + return nil +} + +// validateCIDROperand enforces that a "cidr" condition operand is a string +// containing a valid IPv4/IPv6 address (bare host route) or CIDR block, +// rejecting it eagerly at Subscribe/SetSubscriptionAttributes time rather +// than letting it silently never match at evaluation (matchCIDR). +func validateCIDROperand(key string, raw json.RawMessage) error { + var operand string + if err := json.Unmarshal(raw, &operand); err != nil { + return fmt.Errorf( + "%w: FilterPolicy attribute %q cidr operand must be a string", + ErrInvalidParameter, key, + ) + } - if err := validateNumericOperands(key, numericRaw); err != nil { - return err + if strings.Contains(operand, "/") { + if _, _, err := net.ParseCIDR(operand); err != nil { + return fmt.Errorf( + "%w: FilterPolicy attribute %q cidr operand %q is not a valid CIDR block", + ErrInvalidParameter, key, operand, + ) } + + return nil + } + + if net.ParseIP(operand) == nil { + return fmt.Errorf( + "%w: FilterPolicy attribute %q cidr operand %q is not a valid IP address", + ErrInvalidParameter, key, operand, + ) } return nil diff --git a/services/sns/filter_policy_cidr_e2e_test.go b/services/sns/filter_policy_cidr_e2e_test.go new file mode 100644 index 0000000000..4c0a2efb9f --- /dev/null +++ b/services/sns/filter_policy_cidr_e2e_test.go @@ -0,0 +1,128 @@ +package sns_test + +import ( + "net/http/httptest" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awscfg "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + snssdk "github.com/aws/aws-sdk-go-v2/service/sns" + snstypes "github.com/aws/aws-sdk-go-v2/service/sns/types" + sqssdk "github.com/aws/aws-sdk-go-v2/service/sqs" + sqstypes "github.com/aws/aws-sdk-go-v2/service/sqs/types" + "github.com/labstack/echo/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/pkgs/events" + "github.com/blackbirdworks/gopherstack/pkgs/service" + "github.com/blackbirdworks/gopherstack/services/sns" + "github.com/blackbirdworks/gopherstack/services/sqs" +) + +// newTestSQSClient stands up the real aws-sdk-go-v2 SQS client against an +// httptest server running backend's Handler, mirroring newTestSNSClient. +func newTestSQSClient(t *testing.T, backend *sqs.InMemoryBackend) *sqssdk.Client { + t.Helper() + + e := echo.New() + registry := service.NewRegistry() + require.NoError(t, registry.Register(sqs.NewHandler(backend))) + e.Use(service.NewServiceRouter(registry).RouteHandler()) + + srv := httptest.NewServer(e) + t.Cleanup(srv.Close) + + cfg, err := awscfg.LoadDefaultConfig( + t.Context(), + awscfg.WithRegion("us-east-1"), + awscfg.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider("test", "test", ""), + ), + ) + require.NoError(t, err) + + return sqssdk.NewFromConfig(cfg, func(o *sqssdk.Options) { + o.BaseEndpoint = aws.String(srv.URL) + }) +} + +// TestRealClient_FilterPolicyCIDR_SQSDelivery drives the "cidr" FilterPolicy +// operator end-to-end through the real aws-sdk-go-v2 SNS and SQS clients, +// wired the same way production does (SNS publish emitter -> SQS +// SubscribeToSNS). An SQS queue subscribed with a source_ip cidr filter must +// receive only the publish whose source_ip attribute falls inside the block. +func TestRealClient_FilterPolicyCIDR_SQSDelivery(t *testing.T) { + t.Parallel() + + snsBackend := sns.NewInMemoryBackend() + sqsBackend := sqs.NewInMemoryBackend() + t.Cleanup(sqsBackend.Close) + + emitter := events.NewInMemoryEmitter[*events.SNSPublishedEvent]() + snsBackend.SetPublishEmitter(emitter) + sqsBackend.SubscribeToSNS(emitter) + + snsClient := newTestSNSClient(t, sns.NewHandler(snsBackend)) + sqsClient := newTestSQSClient(t, sqsBackend) + ctx := t.Context() + + topicOut, err := snsClient.CreateTopic(ctx, &snssdk.CreateTopicInput{ + Name: aws.String("cidr-filter-topic"), + }) + require.NoError(t, err) + topicArn := aws.ToString(topicOut.TopicArn) + + queueOut, err := sqsClient.CreateQueue(ctx, &sqssdk.CreateQueueInput{ + QueueName: aws.String("cidr-filter-queue"), + }) + require.NoError(t, err) + queueURL := aws.ToString(queueOut.QueueUrl) + + attrOut, err := sqsClient.GetQueueAttributes(ctx, &sqssdk.GetQueueAttributesInput{ + QueueUrl: aws.String(queueURL), + AttributeNames: []sqstypes.QueueAttributeName{sqstypes.QueueAttributeNameQueueArn}, + }) + require.NoError(t, err) + queueArn := attrOut.Attributes["QueueArn"] + + _, err = snsClient.Subscribe(ctx, &snssdk.SubscribeInput{ + TopicArn: aws.String(topicArn), + Protocol: aws.String("sqs"), + Endpoint: aws.String(queueArn), + Attributes: map[string]string{ + "FilterPolicy": `{"source_ip":[{"cidr":"10.0.0.0/24"}]}`, + "RawMessageDelivery": "true", + }, + }) + require.NoError(t, err) + + publish := func(sourceIP string) { + t.Helper() + + _, pubErr := snsClient.Publish(ctx, &snssdk.PublishInput{ + TopicArn: aws.String(topicArn), + Message: aws.String("from-" + sourceIP), + MessageAttributes: map[string]snstypes.MessageAttributeValue{ + "source_ip": { + DataType: aws.String("String"), + StringValue: aws.String(sourceIP), + }, + }, + }) + require.NoError(t, pubErr) + } + + publish("172.16.0.5") // outside the /24 block; must be filtered out + publish("10.0.0.42") // inside the /24 block; must be delivered + + recvOut, err := sqsClient.ReceiveMessage(ctx, &sqssdk.ReceiveMessageInput{ + QueueUrl: aws.String(queueURL), + MaxNumberOfMessages: 10, + WaitTimeSeconds: 1, + }) + require.NoError(t, err) + require.Len(t, recvOut.Messages, 1, "only the in-CIDR publish should reach the queue") + assert.Equal(t, "from-10.0.0.42", aws.ToString(recvOut.Messages[0].Body)) +} diff --git a/services/sns/filter_policy_test.go b/services/sns/filter_policy_test.go index 44f4bab29e..bd9977d4e2 100644 --- a/services/sns/filter_policy_test.go +++ b/services/sns/filter_policy_test.go @@ -545,6 +545,33 @@ func TestSNS_FilterPolicyValidation(t *testing.T) { name: "accepts_equals_ignore_case_operator", filterPolicy: `{"region":[{"equals-ignore-case":"us-east-1"}]}`, }, + { + name: "rejects_malformed_cidr_block", + filterPolicy: `{"source_ip":[{"cidr":"10.0.0.0/999"}]}`, + wantErr: "not a valid CIDR block", + }, + { + name: "rejects_non_ip_cidr_operand", + filterPolicy: `{"source_ip":[{"cidr":"not-an-ip"}]}`, + wantErr: "not a valid IP address", + }, + { + name: "rejects_non_string_cidr_operand", + filterPolicy: `{"source_ip":[{"cidr":10}]}`, + wantErr: "must be a string", + }, + { + name: "accepts_valid_cidr_block", + filterPolicy: `{"source_ip":[{"cidr":"10.0.0.0/24"}]}`, + }, + { + name: "accepts_valid_bare_ip_cidr", + filterPolicy: `{"source_ip":[{"cidr":"192.168.1.1"}]}`, + }, + { + name: "accepts_valid_ipv6_cidr", + filterPolicy: `{"source_ip":[{"cidr":"2001:db8::/32"}]}`, + }, } for _, tt := range tests { diff --git a/services/sqs/PARITY.md b/services/sqs/PARITY.md index 82d86c704e..fbdea03ec7 100644 --- a/services/sqs/PARITY.md +++ b/services/sqs/PARITY.md @@ -37,7 +37,7 @@ families: message_attribute_md5: {status: ok, note: "computeMD5OfMessageAttributes matches the AWS wire algorithm exactly: sorted names, 4-byte-BE-length-prefixed name/dataType/value, 1-byte transport type (1=String|Number, 2=Binary); subset-MD5 on filtered receive re-hashes only when the returned set is a strict subset, reuses the send-time digest otherwise"} fifo_dedup: {status: ok, note: "explicit MessageDeduplicationId vs ContentBasedDeduplication (SHA-256 of body, NOT MD5) correctly mutually validated; 5-minute window; DeduplicationScope=queue|messageGroup key scoping; bounded map (100k) with oldest-expiry eviction + janitor sweep"} fifo_ordering: {status: ok, note: "fixed this pass (see ReceiveMessage/ChangeMessageVisibility above): requeueMessage now reinserts by SequenceNumber (fixed-width zero-padded decimal, so lexicographic sort == numeric sort) instead of appending to the tail, preserving strict per-MessageGroupId ordering across visibility resets. Confirmed correct behavior (not a bug): only one message per group may be in-flight at a time — this matches real AWS FIFO semantics, not an over-restriction"} - fifo_throughput_limit: {status: partial, note: "Fixed this pass (gopherstack-qgh): FifoThroughputLimit=perQueue (the AWS default, applied whenever the attribute is unset) previously had no rate limiter at all. checkFIFOPerQueueRateLimit now enforces the same 300 TPS sliding-1s-window as checkFIFOPerGroupRateLimit, keyed by queue instead of by group, selected in preflightFIFOSend by the queue's effective FifoThroughputLimit attribute; exceeding it now returns the real RequestThrottled exception (aws-sdk-go-v2/service/sqs@v1.51.0 types/errors.go:1141-1151, https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-quotas.html#quotas-requests), not the OverLimit code checkFIFOPerGroupRateLimit uses (that code's own doc comment scopes it to ReceiveMessage in-flight / AddPermission permission-count limits, not send-rate throttling). Still partial: like the pre-existing per-group limiter, this covers only SendMessage/SendMessageBatch (both funnel through preflightFIFOSend) at the flat 300 TPS figure; AWS's real perQueue budget is a per-operation-type matrix (SendMessage/ReceiveMessage/DeleteMessage each with their own 300 unbatched / 3000 batched budget) that is not modeled — see gaps. The backend's clock is now a seam (InMemoryBackend.nowFunc / SetNowFunc) rather than bare time.Now(), so both FIFO rate limiters are deterministically testable — see families.fifo_throughput_limit tests in fifo_throughput_test.go. Also fixed (already covered elsewhere in ops.SetQueueAttributes): the FifoThroughputLimit=perMessageGroupId / DeduplicationScope=messageGroup pairing rule is enforced against effective (not just same-call) state"} + fifo_throughput_limit: {status: ok, note: "Fixed 2026-09-26 (gopherstack, FIFO throughput quota matrix): the flat 300 TPS SendMessage-only limiter is replaced by a per-API-method budget model matching AWS's documented quotas (https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/quotas-messages.html#quotas-throughput and https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/high-throughput-fifo.html): SendMessage, ReceiveMessage, and DeleteMessage each get their own independent 300-calls/sec + 3,000-messages/sec (via batching) sliding-1s-window budget (checkFIFOThroughput, fifo.go), scoped queue-wide under the default FifoThroughputLimit=perQueue or per MessageGroupId under perMessageGroupId (fifoThroughputScopeKey). SendMessageBatch/DeleteMessageBatch now consume exactly one call-slot per distinct scope touched (grouping entries by scope key -- computeFIFOSendThrottling/computeFIFODeleteThrottling) plus one message-slot per entry, instead of one call-slot per entry (the prior batch-accounting bug: a batch of 10 was throttling around the 30th call instead of the 300th). ReceiveMessage is deliberately queue-scoped in both FifoThroughputLimit modes -- unlike Send/Delete it carries no MessageGroupId the caller can select, so perMessageGroupId's per-partition budget isn't request-observable at the API surface, and the check must run once per API call (not once per pollReceive's ~1s internal recheck, an implementation detail) before messages are picked, using the requested MaxNumberOfMessages rather than the actual count returned (no rollback if picking were to fail after the fact). Throttled calls return the real RequestThrottled exception unchanged (aws-sdk-go-v2/service/sqs@v1.51.0 types/errors.go:1141-1151; verified end-to-end against the typed SDK client). Throttling remains unconditionally on for every FIFO queue (matching its pre-existing always-on behavior -- there was never an opt-in gate to preserve) and is a no-op for standard queues and for all non-FIFO traffic. See families.fifo_throughput_limit tests in fifo_throughput_test.go."} visibility_timeout_and_inflight: {status: ok, note: "12h (43200s) max validated on both ChangeMessageVisibility and now ReceiveMessage (backend-level, was JSON-only before this pass); in-flight caps 120k standard / 20k FIFO -> OverLimit; sweepInFlight/pickVisibleMessages single-pass janitor+receive-path cleanup"} dlq_redrive: {status: fixed, note: "fixed this pass: RedriveAllowPolicy (allowAll/denyAll/byQueue+sourceQueueArns) was shape-validated by validateRedriveAllowPolicy but never enforced — any source queue could point RedrivePolicy at any DLQ regardless of the DLQ's declared permission. checkRedriveAllowPolicy now enforces it in applyRedrivePolicy (shared by CreateQueue/SetQueueAttributes/Restore). maxReceiveCount routing (tryRouteToDLQ), DLQ must be same region + same FIFO-ness, StartMessageMoveTask default-destination-by-RedrivePolicy all verified correct"} delay_queues: {status: ok, note: "queue-level DelaySeconds + per-message DelaySeconds (message wins), FIFO rejects per-message delay, delayedCount maintained incrementally for O(1) GetQueueAttributes"} @@ -47,7 +47,6 @@ families: persistence: {status: fixed, note: "fixed this pass: (1) hasActivity (janitor skip-idle-queue flag) was never restored, so a restored non-FIFO queue with pending messages was silently invisible to the background janitor until an unrelated SendMessage touched it again; (2) fifoSeqCounter was not persisted, so SequenceNumber could regress/duplicate for a FIFO queue that already had messages sent before a snapshot/restore; (3) lastPurgedAt (PurgeQueue 60s cooldown) was not persisted, resetting the cooldown on every restart. This pass added: the new QueueDeletedRecently cooldown map (b.recentlyDeleted) is persisted as a new top-level backendSnapshot.RecentlyDeleted field (region/name -> unix-milli, no version bump needed since it's an additive omitempty field), following the same rationale as lastPurgedAt — otherwise a restore immediately followed by CreateQueue would silently drop the 60s wait-before-recreate rule for a queue deleted just before the snapshot"} gaps: [] items_still_open: - - "FifoThroughputLimit=perQueue's SendMessage/SendMessageBatch budget is now enforced (see families.fifo_throughput_limit, gopherstack-qgh) at a flat 300 TPS, matching the pre-existing per-group limiter's fidelity. Still not modeled: AWS's perQueue budget is actually per-operation-type (ReceiveMessage and DeleteMessage each have their own separate 300 unbatched / 3000 batched budget) — only the send path is rate-limited here. Implementing the full matrix would need per-operation counters on ReceiveMessage/DeleteMessage too; deferred as lower-value (SendMessage is the path most likely to matter for burst-load testing) rather than invented without a concrete need driving the other two." - "KMS SSE (SqsManagedSseEnabled/KmsMasterKeyId/KmsDataKeyReusePeriodSeconds) are accepted, range/shape-validated, and round-trip through GetQueueAttributes, but no actual encryption is modeled (expected for this class of emulator; would require cross-service KMS integration — out of scope for services/sqs/)." - "SqsManagedSseEnabled/KmsMasterKeyId mutual exclusion is documented (\"Only one server-side encryption option is supported per queue (for example, SSE-KMS or SSE-SQS)\", aws-sdk-go-v2/service/sqs@v1.46.4 api_op_SetQueueAttributes.go:139-141) but NOT enforced: a queue can have SqsManagedSseEnabled=true (the CreateQueue default, buildDefaultAttributes) and a non-empty KmsMasterKeyId simultaneously, and both are stored/echoed. Evaluated this pass (gopherstack-gcpg) and deliberately left unenforced: unlike the FifoThroughputLimit/DeduplicationScope pairing (which has an explicit \"allowed only when\" sentence), this line is advisory/descriptive and neither the SDK doc comments nor the AWS console-configuration guide (checked via web fetch) specify the API-level enforcement mechanics — reject vs. auto-clear vs. last-key-wins. Also: the existing test suite (TestSSE_KmsMasterKeyId_Configurable, TestSSE_KMS_SetViaSetQueueAttributes, TestKMSAttrsConfigurable, TestSSE_Idempotency_SameKMSKey) already exercises KmsMasterKeyId being set against the default SqsManagedSseEnabled=true and expects success, so a guessed enforcement rule risks the same invented-behavior mistake flagged for the throughput limiter. Real encryption is out of scope regardless (see gap above); if this gets revisited, resolve the reject-vs-auto-clear question against a live AWS account or an authoritative source first." - "2026-08-14 (gopherstack-3tpf): independently re-confirmed via a mechanical struct-field diff (cmd/structfielddiff), not by re-reading this file's prior claims -- all 23 ops, every Input/Output/nested struct (BatchResultErrorEntry, Message/MessageAttributeValue, ListMessageMoveTasksResultEntry, etc.) diffed field-by-field against aws-sdk-go-v2/service/sqs@v1.46.4. Zero new gaps: every real field this SDK declares has a matching gopherstack field. Confirmed MessageAttributeValue.BinaryListValues/StringListValues are correctly absent (SDK doc comment: 'Not implemented. Reserved for future use.', types/types.go:226,232 -- known noise, not a gap). No REST/header-bound members exist for this service (JSON + Query protocols only, no header bindings in serializers.go). No code changes made to this service this pass." @@ -61,6 +60,16 @@ leaks: {status: clean, note: "fixed this pass: restoreQueueFromSnapshot now seed ## Notes +### 2026-09-26 FIFO throughput quota matrix + +Closed the last items_still_open entry for families.fifo_throughput_limit: the +flat 300 TPS SendMessage-only limiter is now a per-API-method model (SendMessage, +ReceiveMessage, DeleteMessage each with their own 300-calls/sec + 3,000-messages/sec +budget), with FifoThroughputLimit choosing queue-wide vs per-MessageGroupId scope, +and batch calls (SendMessageBatch/DeleteMessageBatch) correctly consuming one +call-slot per scope touched instead of one per entry. See families.fifo_throughput_limit +above for the full writeup and doc citations. + ### 2026-09-24 terraform-coverage sweep (codeartifact-timestream-and-messaging) JSON-protocol errors never set X-Amzn-Query-Error (real AWS SQS does), breaking diff --git a/services/sqs/README.md b/services/sqs/README.md index 567f7bca3d..5a887d353f 100644 --- a/services/sqs/README.md +++ b/services/sqs/README.md @@ -8,14 +8,13 @@ | Metric | Value | | --- | --- | | PARITY entries audited | 20 (19 ok, 1 partial) | -| Feature families | 11 (10 ok, 1 partial) | -| Known gaps | 4 | +| Feature families | 11 (11 ok) | +| Known gaps | 3 | | Deferred items | 4 | | Resource leaks | clean | ### Known gaps -- FifoThroughputLimit=perQueue's SendMessage/SendMessageBatch budget is now enforced (see families.fifo_throughput_limit, gopherstack-qgh) at a flat 300 TPS, matching the pre-existing per-group limiter's fidelity. Still not modeled: AWS's perQueue budget is actually per-operation-type (ReceiveMessage and DeleteMessage each have their own separate 300 unbatched / 3000 batched budget) — only the send path is rate-limited here. Implementing the full matrix would need per-operation counters on ReceiveMessage/DeleteMessage too; deferred as lower-value (SendMessage is the path most likely to matter for burst-load testing) rather than invented without a concrete need driving the other two. - KMS SSE (SqsManagedSseEnabled/KmsMasterKeyId/KmsDataKeyReusePeriodSeconds) are accepted, range/shape-validated, and round-trip through GetQueueAttributes, but no actual encryption is modeled (expected for this class of emulator; would require cross-service KMS integration — out of scope for services/sqs/). - SqsManagedSseEnabled/KmsMasterKeyId mutual exclusion is documented ("Only one server-side encryption option is supported per queue (for example, SSE-KMS or SSE-SQS)", aws-sdk-go-v2/service/sqs@v1.46.4 api_op_SetQueueAttributes.go:139-141) but NOT enforced: a queue can have SqsManagedSseEnabled=true (the CreateQueue default, buildDefaultAttributes) and a non-empty KmsMasterKeyId simultaneously, and both are stored/echoed. Evaluated this pass (gopherstack-gcpg) and deliberately left unenforced: unlike the FifoThroughputLimit/DeduplicationScope pairing (which has an explicit "allowed only when" sentence), this line is advisory/descriptive and neither the SDK doc comments nor the AWS console-configuration guide (checked via web fetch) specify the API-level enforcement mechanics — reject vs. auto-clear vs. last-key-wins. Also: the existing test suite (TestSSE_KmsMasterKeyId_Configurable, TestSSE_KMS_SetViaSetQueueAttributes, TestKMSAttrsConfigurable, TestSSE_Idempotency_SameKMSKey) already exercises KmsMasterKeyId being set against the default SqsManagedSseEnabled=true and expects success, so a guessed enforcement rule risks the same invented-behavior mistake flagged for the throughput limiter. Real encryption is out of scope regardless (see gap above); if this gets revisited, resolve the reject-vs-auto-clear question against a live AWS account or an authoritative source first. - 2026-08-14 (gopherstack-3tpf): independently re-confirmed via a mechanical struct-field diff (cmd/structfielddiff), not by re-reading this file's prior claims -- all 23 ops, every Input/Output/nested struct (BatchResultErrorEntry, Message/MessageAttributeValue, ListMessageMoveTasksResultEntry, etc.) diffed field-by-field against aws-sdk-go-v2/service/sqs@v1.46.4. Zero new gaps: every real field this SDK declares has a matching gopherstack field. Confirmed MessageAttributeValue.BinaryListValues/StringListValues are correctly absent (SDK doc comment: 'Not implemented. Reserved for future use.', types/types.go:226,232 -- known noise, not a gap). No REST/header-bound members exist for this service (JSON + Query protocols only, no header bindings in serializers.go). No code changes made to this service this pass. diff --git a/services/sqs/bench_protocol_test.go b/services/sqs/bench_protocol_test.go new file mode 100644 index 0000000000..59280edf57 --- /dev/null +++ b/services/sqs/bench_protocol_test.go @@ -0,0 +1,425 @@ +package sqs_test + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "sync" + "testing" + "time" + + "github.com/labstack/echo/v5" + + "github.com/blackbirdworks/gopherstack/services/sqs" +) + +// tickClock advances virtual time by step on every read, keeping FIFO +// throughput windows (300 calls/sec, 3000 msgs/sec) from throttling +// benchmark iterations without any real sleep. +type tickClock struct { + now time.Time + step time.Duration + mu sync.Mutex +} + +func newTickClock(step time.Duration) *tickClock { + return &tickClock{now: time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC), step: step} +} + +func (c *tickClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + + c.now = c.now.Add(c.step) + + return c.now +} + +func benchJSONRequest(b *testing.B, h *sqs.Handler, action string, body []byte) { + b.Helper() + + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/x-amz-json-1.0") + req.Header.Set("X-Amz-Target", "AmazonSQS."+action) + + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + + if err := h.Handler()(c); err != nil { + b.Fatalf("%s: %v", action, err) + } +} + +// benchQueryRequest sends a pre-encoded Query-protocol body. Callers encode +// vals.Encode() once outside the timed loop so the benchmark measures the +// server's parse/handle cost, not client-side re-encoding on every iteration. +func benchQueryRequest(b *testing.B, h *sqs.Handler, action string, body []byte) { + b.Helper() + + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + + if err := h.Handler()(c); err != nil { + b.Fatalf("query %s: %v", action, err) + } +} + +func jsonSendBody(b *testing.B, qURL string) []byte { + b.Helper() + + body, err := json.Marshal(map[string]any{ + "QueueUrl": qURL, + "MessageBody": "benchmark payload of representative length for SQS SendMessage", + "MessageAttributes": map[string]any{ + "attr-one": map[string]any{"DataType": "String", "StringValue": "value-one"}, + "attr-two": map[string]any{"DataType": "Number", "StringValue": "42"}, + }, + }) + if err != nil { + b.Fatalf("marshal send body: %v", err) + } + + return body +} + +func BenchmarkJSONSendMessage(b *testing.B) { + backend := newBenchBackend(b) + h := sqs.NewHandler(backend) + qURL := benchCreateQueue(b, backend, "bench-json-send-q") + body := jsonSendBody(b, qURL) + + b.ReportAllocs() + + for b.Loop() { + benchJSONRequest(b, h, "SendMessage", body) + } +} + +func BenchmarkQuerySendMessage(b *testing.B) { + backend := newBenchBackend(b) + h := sqs.NewHandler(backend) + qURL := benchCreateQueue(b, backend, "bench-query-send-q") + + vals := url.Values{ + "Action": {"SendMessage"}, + "QueueUrl": {qURL}, + "MessageBody": {"benchmark payload of representative length for SQS SendMessage"}, + "MessageAttribute.1.Name": {"attr-one"}, + "MessageAttribute.1.Value.DataType": {"String"}, + "MessageAttribute.1.Value.StringValue": {"value-one"}, + "MessageAttribute.2.Name": {"attr-two"}, + "MessageAttribute.2.Value.DataType": {"Number"}, + "MessageAttribute.2.Value.StringValue": {"42"}, + } + body := []byte(vals.Encode()) + + b.ReportAllocs() + + for b.Loop() { + benchQueryRequest(b, h, "SendMessage", body) + } +} + +func BenchmarkJSONSendMessageBatch10(b *testing.B) { + backend := newBenchBackend(b) + h := sqs.NewHandler(backend) + qURL := benchCreateQueue(b, backend, "bench-json-batch-q") + + entries := make([]map[string]any, 10) + for i := range entries { + entries[i] = map[string]any{ + "Id": strconv.Itoa(i), + "MessageBody": "batch payload " + strconv.Itoa(i), + "MessageAttributes": map[string]any{ + "attr-one": map[string]any{"DataType": "String", "StringValue": "value-one"}, + }, + } + } + + body, err := json.Marshal(map[string]any{"QueueUrl": qURL, "Entries": entries}) + if err != nil { + b.Fatalf("marshal batch body: %v", err) + } + + b.ReportAllocs() + + for b.Loop() { + benchJSONRequest(b, h, "SendMessageBatch", body) + } +} + +func BenchmarkQuerySendMessageBatch10(b *testing.B) { + backend := newBenchBackend(b) + h := sqs.NewHandler(backend) + qURL := benchCreateQueue(b, backend, "bench-query-batch-q") + + vals := url.Values{"Action": {"SendMessageBatch"}, "QueueUrl": {qURL}} + for i := 1; i <= 10; i++ { + n := strconv.Itoa(i) + prefix := "SendMessageBatchRequestEntry." + n + "." + vals.Set(prefix+"Id", n) + vals.Set(prefix+"MessageBody", "batch payload "+n) + vals.Set(prefix+"MessageAttribute.1.Name", "attr-one") + vals.Set(prefix+"MessageAttribute.1.Value.DataType", "String") + vals.Set(prefix+"MessageAttribute.1.Value.StringValue", "value-one") + } + body := []byte(vals.Encode()) + + b.ReportAllocs() + + for b.Loop() { + benchQueryRequest(b, h, "SendMessageBatch", body) + } +} + +const benchReceiveDepth = 10000 + +func setupReceiveDepthQueue(b *testing.B, backend *sqs.InMemoryBackend, name string) string { + b.Helper() + + qURL := benchCreateQueue(b, backend, name) + benchSendN(b, backend, qURL, benchReceiveDepth) + + return qURL +} + +func BenchmarkJSONReceiveMessage10_Depth10000(b *testing.B) { + backend := newBenchBackend(b) + h := sqs.NewHandler(backend) + qURL := setupReceiveDepthQueue(b, backend, "bench-json-recv-q") + + body, err := json.Marshal(map[string]any{"QueueUrl": qURL, "MaxNumberOfMessages": 10}) + if err != nil { + b.Fatalf("marshal receive body: %v", err) + } + + b.ReportAllocs() + + for b.Loop() { + benchJSONRequest(b, h, "ReceiveMessage", body) + } +} + +func BenchmarkQueryReceiveMessage10_Depth10000(b *testing.B) { + backend := newBenchBackend(b) + h := sqs.NewHandler(backend) + qURL := setupReceiveDepthQueue(b, backend, "bench-query-recv-q") + + vals := url.Values{ + "Action": {"ReceiveMessage"}, + "QueueUrl": {qURL}, + "MaxNumberOfMessages": {"10"}, + "AttributeName.1": {"All"}, + } + body := []byte(vals.Encode()) + + b.ReportAllocs() + + for b.Loop() { + benchQueryRequest(b, h, "ReceiveMessage", body) + } +} + +// setupFIFOManyGroups creates a FIFO queue with numGroups groups of +// perGroup messages each, sent under a virtual clock so the 300 calls/sec +// FIFO throughput budget never throttles the benchmark itself. +func setupFIFOManyGroups(b *testing.B, backend *sqs.InMemoryBackend, name string, numGroups, perGroup int) string { + b.Helper() + + out, err := backend.CreateQueue(&sqs.CreateQueueInput{ + QueueName: name + ".fifo", + Endpoint: testEndpoint, + Attributes: map[string]string{ + "FifoQueue": "true", + "ContentBasedDeduplication": "true", + }, + }) + if err != nil { + b.Fatalf("CreateQueue: %v", err) + } + + clock := newTickClock(4 * time.Millisecond) + sqs.SetNowFunc(backend, clock.Now) + + for g := range numGroups { + group := "group-" + strconv.Itoa(g) + for i := range perGroup { + _, sendErr := backend.SendMessage(&sqs.SendMessageInput{ + QueueURL: out.QueueURL, + MessageBody: "fifo body " + strconv.Itoa(g) + "-" + strconv.Itoa(i), + MessageGroupID: group, + }) + if sendErr != nil { + b.Fatalf("SendMessage: %v", sendErr) + } + } + } + + return out.QueueURL +} + +func BenchmarkJSONReceiveMessage10_FIFOManyGroups(b *testing.B) { + backend := newBenchBackend(b) + h := sqs.NewHandler(backend) + qURL := setupFIFOManyGroups(b, backend, "bench-json-fifo-recv-q", 1000, 10) + + body, err := json.Marshal(map[string]any{"QueueUrl": qURL, "MaxNumberOfMessages": 10}) + if err != nil { + b.Fatalf("marshal receive body: %v", err) + } + + b.ReportAllocs() + + for b.Loop() { + benchJSONRequest(b, h, "ReceiveMessage", body) + } +} + +func BenchmarkQueryReceiveMessage10_FIFOManyGroups(b *testing.B) { + backend := newBenchBackend(b) + h := sqs.NewHandler(backend) + qURL := setupFIFOManyGroups(b, backend, "bench-query-fifo-recv-q", 1000, 10) + + vals := url.Values{ + "Action": {"ReceiveMessage"}, + "QueueUrl": {qURL}, + "MaxNumberOfMessages": {"10"}, + "AttributeName.1": {"All"}, + } + body := []byte(vals.Encode()) + + b.ReportAllocs() + + for b.Loop() { + benchQueryRequest(b, h, "ReceiveMessage", body) + } +} + +// setupInFlightHandles receives depth/10 batches of 10 messages under a long +// visibility timeout (direct backend calls, no HTTP) and returns all handles. +func setupInFlightHandles(b *testing.B, backend *sqs.InMemoryBackend, qURL string, depth int) []string { + b.Helper() + + handles := make([]string, 0, depth) + for len(handles) < depth { + recv, err := backend.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: qURL, + MaxNumberOfMessages: 10, + VisibilityTimeout: 3600, + }) + if err != nil || len(recv.Messages) == 0 { + b.Fatalf("ReceiveMessage: %v (got %d)", err, len(recv.Messages)) + } + handles = append(handles, receiptHandles(recv.Messages)...) + } + + return handles +} + +func BenchmarkJSONDeleteMessage(b *testing.B) { + backend := newBenchBackend(b) + h := sqs.NewHandler(backend) + qURL := setupReceiveDepthQueue(b, backend, "bench-json-delete-q") + handles := setupInFlightHandles(b, backend, qURL, benchReceiveDepth) + + bodies := make([][]byte, len(handles)) + for i, rh := range handles { + body, err := json.Marshal(map[string]any{"QueueUrl": qURL, "ReceiptHandle": rh}) + if err != nil { + b.Fatalf("marshal delete body: %v", err) + } + bodies[i] = body + } + + b.ReportAllocs() + + for i := 0; b.Loop(); i++ { + benchJSONRequest(b, h, "DeleteMessage", bodies[i%len(bodies)]) + } +} + +func BenchmarkQueryDeleteMessage(b *testing.B) { + backend := newBenchBackend(b) + h := sqs.NewHandler(backend) + qURL := setupReceiveDepthQueue(b, backend, "bench-query-delete-q") + handles := setupInFlightHandles(b, backend, qURL, benchReceiveDepth) + + bodies := make([][]byte, len(handles)) + for i, rh := range handles { + vals := url.Values{"Action": {"DeleteMessage"}, "QueueUrl": {qURL}, "ReceiptHandle": {rh}} + bodies[i] = []byte(vals.Encode()) + } + + b.ReportAllocs() + + for i := 0; b.Loop(); i++ { + benchQueryRequest(b, h, "DeleteMessage", bodies[i%len(bodies)]) + } +} + +func BenchmarkJSONDeleteMessageBatch10(b *testing.B) { + backend := newBenchBackend(b) + h := sqs.NewHandler(backend) + qURL := benchCreateQueue(b, backend, "bench-json-delbatch-q") + + b.ReportAllocs() + + for b.Loop() { + b.StopTimer() + benchSendN(b, backend, qURL, 10) + recv, err := backend.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: qURL, MaxNumberOfMessages: 10, VisibilityTimeout: 300, + }) + if err != nil || len(recv.Messages) != 10 { + b.Fatalf("ReceiveMessage: %v (got %d)", err, len(recv.Messages)) + } + entries := make([]map[string]any, 10) + for i, m := range recv.Messages { + entries[i] = map[string]any{"Id": strconv.Itoa(i), "ReceiptHandle": m.ReceiptHandle} + } + body, err := json.Marshal(map[string]any{"QueueUrl": qURL, "Entries": entries}) + if err != nil { + b.Fatalf("marshal delete batch body: %v", err) + } + b.StartTimer() + + benchJSONRequest(b, h, "DeleteMessageBatch", body) + } +} + +func BenchmarkQueryDeleteMessageBatch10(b *testing.B) { + backend := newBenchBackend(b) + h := sqs.NewHandler(backend) + qURL := benchCreateQueue(b, backend, "bench-query-delbatch-q") + + b.ReportAllocs() + + for b.Loop() { + b.StopTimer() + benchSendN(b, backend, qURL, 10) + recv, err := backend.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: qURL, MaxNumberOfMessages: 10, VisibilityTimeout: 300, + }) + if err != nil || len(recv.Messages) != 10 { + b.Fatalf("ReceiveMessage: %v (got %d)", err, len(recv.Messages)) + } + vals := url.Values{"Action": {"DeleteMessageBatch"}, "QueueUrl": {qURL}} + for i, m := range recv.Messages { + n := strconv.Itoa(i + 1) + vals.Set("DeleteMessageBatchRequestEntry."+n+".Id", strconv.Itoa(i)) + vals.Set("DeleteMessageBatchRequestEntry."+n+".ReceiptHandle", m.ReceiptHandle) + } + body := []byte(vals.Encode()) + b.StartTimer() + + benchQueryRequest(b, h, "DeleteMessageBatch", body) + } +} diff --git a/services/sqs/clock_injection_test.go b/services/sqs/clock_injection_test.go new file mode 100644 index 0000000000..5058a32dcf --- /dev/null +++ b/services/sqs/clock_injection_test.go @@ -0,0 +1,228 @@ +package sqs_test + +import ( + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/sqs" +) + +// fakeClock is a mutex-guarded, manually-advanced time source injected into +// InMemoryBackend via sqs.SetNowFunc, proving timing decisions read the +// backend's single clock rather than calling time.Now directly. +type fakeClock struct { + now time.Time + mu sync.Mutex +} + +func newFakeClock(start time.Time) *fakeClock { + return &fakeClock{now: start} +} + +func (c *fakeClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + + return c.now +} + +func (c *fakeClock) Advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + + c.now = c.now.Add(d) +} + +func newClockedBackend(t *testing.T) (*sqs.InMemoryBackend, *fakeClock) { + t.Helper() + + b := sqs.NewInMemoryBackend() + t.Cleanup(b.Close) + + clock := newFakeClock(time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)) + sqs.SetNowFunc(b, clock.Now) + + return b, clock +} + +func TestClockInjection_VisibilityTimeoutExpiry(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + advance time.Duration + wantVisible bool + }{ + {name: "before_timeout_stays_inflight", advance: 4 * time.Second, wantVisible: false}, + {name: "after_timeout_returns_to_queue", advance: 6 * time.Second, wantVisible: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b, clock := newClockedBackend(t) + qURL := createTestQueue(t, b, "vis-"+tt.name) + + _, err := b.SendMessage(&sqs.SendMessageInput{QueueURL: qURL, MessageBody: "body"}) + require.NoError(t, err) + + out, err := b.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: qURL, + MaxNumberOfMessages: 1, + VisibilityTimeout: 5, + }) + require.NoError(t, err) + require.Len(t, out.Messages, 1) + + clock.Advance(tt.advance) + + out2, err := b.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: qURL, + MaxNumberOfMessages: 1, + }) + require.NoError(t, err) + + if tt.wantVisible { + assert.Len(t, out2.Messages, 1, "message should be returned to the queue after visibility expiry") + } else { + assert.Empty(t, out2.Messages, "message should still be in-flight") + } + }) + } +} + +func TestClockInjection_RetentionExpiry(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + advance time.Duration + wantMessage bool + }{ + {name: "before_retention_message_stays", advance: 4 * time.Second, wantMessage: true}, + {name: "after_retention_message_dropped", advance: 6 * time.Second, wantMessage: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b, clock := newClockedBackend(t) + qURL := createTestQueue(t, b, "ret-"+tt.name) + b.SetRetentionForTest(qURL, 5) + + _, err := b.SendMessage(&sqs.SendMessageInput{QueueURL: qURL, MessageBody: "body"}) + require.NoError(t, err) + + clock.Advance(tt.advance) + + out, err := b.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: qURL, + MaxNumberOfMessages: 1, + }) + require.NoError(t, err) + + if tt.wantMessage { + assert.Len(t, out.Messages, 1) + } else { + assert.Empty(t, out.Messages, "message should be dropped after retention expiry") + } + }) + } +} + +func TestClockInjection_DelaySecondsHidesMessage(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + advance time.Duration + wantVisible bool + }{ + {name: "before_delay_hidden", advance: 4 * time.Second, wantVisible: false}, + {name: "after_delay_visible", advance: 6 * time.Second, wantVisible: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b, clock := newClockedBackend(t) + qURL := createTestQueue(t, b, "delay-"+tt.name) + + _, err := b.SendMessage(&sqs.SendMessageInput{ + QueueURL: qURL, + MessageBody: "body", + DelaySeconds: 5, + }) + require.NoError(t, err) + + clock.Advance(tt.advance) + + out, err := b.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: qURL, + MaxNumberOfMessages: 1, + }) + require.NoError(t, err) + + if tt.wantVisible { + assert.Len(t, out.Messages, 1) + } else { + assert.Empty(t, out.Messages, "message should still be delayed") + } + }) + } +} + +func TestClockInjection_DedupWindowExpiryAllowsResend(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + advance time.Duration + wantSameMsgID bool + }{ + {name: "within_window_returns_original", advance: 1 * time.Second, wantSameMsgID: true}, + {name: "after_window_accepts_resend", advance: 301 * time.Second, wantSameMsgID: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b, clock := newClockedBackend(t) + qURL := createTestQueue(t, b, "dedup-"+tt.name+".fifo") + + out1, err := b.SendMessage(&sqs.SendMessageInput{ + QueueURL: qURL, + MessageBody: "body", + MessageGroupID: "group", + MessageDeduplicationID: "dedup-1", + }) + require.NoError(t, err) + + clock.Advance(tt.advance) + + out2, err := b.SendMessage(&sqs.SendMessageInput{ + QueueURL: qURL, + MessageBody: "body", + MessageGroupID: "group", + MessageDeduplicationID: "dedup-1", + }) + require.NoError(t, err) + + if tt.wantSameMsgID { + assert.Equal(t, out1.MessageID, out2.MessageID, + "duplicate within the dedup window should return the original message ID") + } else { + assert.NotEqual(t, out1.MessageID, out2.MessageID, + "resend after the dedup window expired should be treated as a new message") + } + }) + } +} diff --git a/services/sqs/dead_letter.go b/services/sqs/dead_letter.go index 97836d314e..f116d3a471 100644 --- a/services/sqs/dead_letter.go +++ b/services/sqs/dead_letter.go @@ -75,7 +75,7 @@ func applyRedrivePolicy(q *Queue, attrs map[string]string, backend *InMemoryBack q.MaxReceiveCount = int(count) q.dlq = dlq - now := time.Now() + now := backend.now() q.mu.Lock() defer q.mu.Unlock() diff --git a/services/sqs/fifo.go b/services/sqs/fifo.go index 55814a20cb..912caaf86c 100644 --- a/services/sqs/fifo.go +++ b/services/sqs/fifo.go @@ -2,68 +2,86 @@ package sqs import "time" -// checkFIFOPerGroupRateLimit enforces the 300 TPS per-message-group AWS limit -// for FIFO queues running with FifoThroughputLimit=perMessageGroupId. -// -// Maintains a sliding 1-second window per group, pruning timestamps older -// than the window on each call. Returns ErrOverLimit when the window is -// already full; otherwise appends the new send and returns nil. -// -// Caller must hold b.mu (write). Allocates the per-queue map lazily. -func checkFIFOPerGroupRateLimit(q *Queue, group string, now time.Time) error { - if group == "" { - return nil - } +// fifoAPIMethod is one of the three FIFO API actions AWS meters +// independently: SendMessage, ReceiveMessage, DeleteMessage. +type fifoAPIMethod int + +const ( + fifoMethodSend fifoAPIMethod = iota + fifoMethodReceive + fifoMethodDelete +) + +// fifoThroughputKey is one independent budget: an API method plus a scope +// ("" for queue-wide, or a MessageGroupId under perMessageGroupId). +type fifoThroughputKey struct { + scopeKey string + method fifoAPIMethod +} + +// fifoRateWindow is a 1-second sliding window of call and message timestamps +// for one fifoThroughputKey (AWS: 300 calls/sec, 3,000 messages/sec batched). +type fifoRateWindow struct { + calls []time.Time + messages []time.Time +} + +const ( + fifoCallsPerSecond = 300 + fifoMessagesPerSecond = 3000 +) - if q.fifoSendTimes == nil { - q.fifoSendTimes = make(map[string][]time.Time) +// fifoThroughputScopeKey returns groupID under perMessageGroupId, else "" for +// the AWS-default queue-wide scope (unset attribute included). +func fifoThroughputScopeKey(q *Queue, groupID string) string { + if q.Attributes[attrFifoThroughputLimit] == fifoThroughputLimitPerMessageGroupID { + return groupID } - cutoff := now.Add(-time.Second) - prev := q.fifoSendTimes[group] - kept := prev[:0] - for _, t := range prev { + return "" +} + +// pruneRateWindow drops timestamps at or before cutoff, reusing times' +// backing array. +func pruneRateWindow(times []time.Time, cutoff time.Time) []time.Time { + kept := times[:0] + for _, t := range times { if t.After(cutoff) { kept = append(kept, t) } } - if len(kept) >= fifoPerGroupTPS { - q.fifoSendTimes[group] = kept + return kept +} - return ErrOverLimit +// checkFIFOThroughput consumes one call slot + msgCount message slots for +// (method, scopeKey), or returns ErrRequestThrottled leaving both unchanged. +// Caller must hold q.mu; now must be b.now(), not time.Now(), for determinism. +func checkFIFOThroughput(q *Queue, method fifoAPIMethod, scopeKey string, msgCount int, now time.Time) error { + if q.fifoThroughput == nil { + q.fifoThroughput = make(map[fifoThroughputKey]*fifoRateWindow) } - q.fifoSendTimes[group] = append(kept, now) + key := fifoThroughputKey{method: method, scopeKey: scopeKey} - return nil -} + w := q.fifoThroughput[key] + if w == nil { + w = &fifoRateWindow{} + q.fifoThroughput[key] = w + } -// checkFIFOPerQueueRateLimit enforces the AWS-documented queue-wide 300 TPS -// send rate for FIFO queues at FifoThroughputLimit=perQueue — the AWS -// default, applied whenever the attribute is unset or explicitly "perQueue". -// Same sliding-1s-window mechanism as checkFIFOPerGroupRateLimit, keyed by -// the queue as a whole instead of by message group. -// -// Caller must hold q.mu (write). now must come from the backend's clock -// (InMemoryBackend.now), not time.Now() directly, so tests can drive the -// window deterministically without real sleeps. -func checkFIFOPerQueueRateLimit(q *Queue, now time.Time) error { cutoff := now.Add(-time.Second) - prev := q.fifoSendTimesQueue - kept := prev[:0] - for _, t := range prev { - if t.After(cutoff) { - kept = append(kept, t) - } - } - q.fifoSendTimesQueue = kept + w.calls = pruneRateWindow(w.calls, cutoff) + w.messages = pruneRateWindow(w.messages, cutoff) - if len(q.fifoSendTimesQueue) >= fifoPerQueueTPS { + if len(w.calls) >= fifoCallsPerSecond || len(w.messages)+msgCount > fifoMessagesPerSecond { return ErrRequestThrottled } - q.fifoSendTimesQueue = append(q.fifoSendTimesQueue, now) + w.calls = append(w.calls, now) + for range msgCount { + w.messages = append(w.messages, now) + } return nil } @@ -102,31 +120,25 @@ type fifoPreflight struct { Handled bool } -// preflightFIFOSend runs the FIFO-only preconditions a SendMessage must -// satisfy before the message is constructed: parameter validation, per-group -// throughput limiting, and content-based deduplication. -// -// Caller must already hold b.mu (write). +// preflightFIFOSend validates FIFO params, throughput-limits, then dedups. +// checkThroughput is false when the batch caller already reserved the +// budget (computeFIFOSendThrottling). Caller must hold q.mu. func preflightFIFOSend( q *Queue, input *SendMessageInput, md5Body, sha256Body string, + checkThroughput bool, now time.Time, ) fifoPreflight { if err := validateFIFOParams(input, q); err != nil { return fifoPreflight{Err: err, Handled: true} } - // Unset FifoThroughputLimit defaults to perQueue (models.go's - // buildDefaultAttributes never stamps it), so only the explicit - // perMessageGroupId value takes the per-group path; everything else - // (including "") gets the queue-wide limiter. - if q.Attributes[attrFifoThroughputLimit] == fifoThroughputLimitPerMessageGroupID { - if err := checkFIFOPerGroupRateLimit(q, input.MessageGroupID, now); err != nil { + if checkThroughput { + scopeKey := fifoThroughputScopeKey(q, input.MessageGroupID) + if err := checkFIFOThroughput(q, fifoMethodSend, scopeKey, 1, now); err != nil { return fifoPreflight{Err: err, Handled: true} } - } else if err := checkFIFOPerQueueRateLimit(q, now); err != nil { - return fifoPreflight{Err: err, Handled: true} } if out, dup := checkDedup( @@ -165,6 +177,66 @@ func validateFIFOParams(input *SendMessageInput, q *Queue) error { return nil } +// computeFIFOSendThrottling groups entries by throughput scope, consuming each +// scope's budget once per group (not per entry). Caller must hold q.mu. +func computeFIFOSendThrottling(q *Queue, entries []SendMessageBatchEntry, now time.Time) []bool { + groups := make(map[string][]int) + + for i, entry := range entries { + params := &SendMessageInput{ + MessageGroupID: entry.MessageGroupID, + MessageDeduplicationID: entry.MessageDeduplicationID, + DelaySeconds: entry.DelaySeconds, + } + if validateFIFOParams(params, q) != nil { + continue + } + + key := fifoThroughputScopeKey(q, entry.MessageGroupID) + groups[key] = append(groups[key], i) + } + + throttled := make([]bool, len(entries)) + + for key, idxs := range groups { + if checkFIFOThroughput(q, fifoMethodSend, key, len(idxs), now) != nil { + for _, i := range idxs { + throttled[i] = true + } + } + } + + return throttled +} + +// computeFIFODeleteThrottling is DeleteMessageBatch's analog of +// computeFIFOSendThrottling, scoping by each handle's in-flight MessageGroupId. +func computeFIFODeleteThrottling(q *Queue, entries []DeleteMessageBatchEntry, now time.Time) []bool { + groups := make(map[string][]int) + + for i, entry := range entries { + inf, found := q.inFlightByHandle[entry.ReceiptHandle] + if !found { + continue + } + + key := fifoThroughputScopeKey(q, inf.Msg.MessageGroupID) + groups[key] = append(groups[key], i) + } + + throttled := make([]bool, len(entries)) + + for key, idxs := range groups { + if checkFIFOThroughput(q, fifoMethodDelete, key, len(idxs), now) != nil { + for _, i := range idxs { + throttled[i] = true + } + } + } + + return throttled +} + // dedupKey returns the deduplication map key, respecting the queue's // DeduplicationScope attribute. When scope is "queue" (queue-wide), only the // effective dedup ID is used as the key. The default scope is "messageGroup", diff --git a/services/sqs/fifo_throughput_test.go b/services/sqs/fifo_throughput_test.go index 1534f870b9..306575e4db 100644 --- a/services/sqs/fifo_throughput_test.go +++ b/services/sqs/fifo_throughput_test.go @@ -67,7 +67,7 @@ func TestFIFOThroughputLimit_PerQueueDefault_301stThrottled(t *testing.T) { t.Parallel() client, backend := newFIFOThroughputTestServer(t) - sqs.SetNowFunc(backend, fixedThroughputTestTime) + sqs.SetNowFunc(backend, newFixedThroughputClock()) ctx := t.Context() @@ -109,7 +109,7 @@ func TestFIFOThroughputLimit_PerMessageGroupId_TwoGroupsAt300_NotThrottled(t *te t.Parallel() client, backend := newFIFOThroughputTestServer(t) - sqs.SetNowFunc(backend, fixedThroughputTestTime) + sqs.SetNowFunc(backend, newFixedThroughputClock()) ctx := t.Context() @@ -137,10 +137,266 @@ func TestFIFOThroughputLimit_PerMessageGroupId_TwoGroupsAt300_NotThrottled(t *te } } -// fixedThroughputTestTime pins the backend clock to one instant so a whole -// test run happens in a single 1-second rate-limit window, decoupling the -// assertions from real wall-clock timing (see the no-time.Sleep-in-tests -// convention). -func fixedThroughputTestTime() time.Time { - return time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) +// newFixedThroughputClock freezes now() at creation time (near-real, not an +// arbitrary date, since receiveOnce's retention sweep uses the real clock). +func newFixedThroughputClock() func() time.Time { + now := time.Now() + + return func() time.Time { return now } +} + +// newFIFOThroughputBackend drives the backend directly (no HTTP), for tests +// making hundreds of calls where SDK/httptest overhead would be unwieldy. +func newFIFOThroughputBackend(t *testing.T) *sqs.InMemoryBackend { + t.Helper() + + backend := sqs.NewInMemoryBackend() + t.Cleanup(backend.Close) + sqs.SetNowFunc(backend, newFixedThroughputClock()) + + return backend +} + +// throughputBatchSize mirrors SendMessageBatch/ReceiveMessage's own 10-entry +// AWS batch cap. +const throughputBatchSize = 10 + +// sendAndReceiveDistinctGroups sends and receives n messages, one per distinct +// group, so nothing blocks on FIFO's one-in-flight-per-group limit. +func sendAndReceiveDistinctGroups(t *testing.T, b *sqs.InMemoryBackend, qURL string, n int) []string { + t.Helper() + + for i := 0; i < n; i += throughputBatchSize { + end := min(i+throughputBatchSize, n) + + entries := make([]sqs.SendMessageBatchEntry, 0, end-i) + for j := i; j < end; j++ { + entries = append(entries, sqs.SendMessageBatchEntry{ + ID: fmt.Sprintf("id-%d", j), + MessageBody: fmt.Sprintf("msg-%d", j), + MessageGroupID: fmt.Sprintf("group-%d", j), + MessageDeduplicationID: fmt.Sprintf("dedup-%d", j), + }) + } + + out, err := b.SendMessageBatch(&sqs.SendMessageBatchInput{QueueURL: qURL, Entries: entries}) + require.NoError(t, err) + require.Empty(t, out.Failed) + } + + handles := make([]string, 0, n) + for len(handles) < n { + out, err := b.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: qURL, + MaxNumberOfMessages: throughputBatchSize, + VisibilityTimeout: sqs.NoVisibilityTimeout, + }) + require.NoError(t, err) + require.NotEmpty(t, out.Messages, "expected more messages available to receive") + + for _, m := range out.Messages { + handles = append(handles, m.ReceiptHandle) + } + } + + return handles +} + +// TestFIFOThroughputLimit_IndependentPerMethodBudgets confirms SendMessage, +// ReceiveMessage, and DeleteMessage each get their own 300-calls/sec budget. +func TestFIFOThroughputLimit_IndependentPerMethodBudgets(t *testing.T) { + t.Parallel() + + b := newFIFOThroughputBackend(t) + + out, err := b.CreateQueue(&sqs.CreateQueueInput{ + QueueName: "independent-method-budgets.fifo", + Endpoint: testEndpoint, + }) + require.NoError(t, err) + + qURL := out.QueueURL + + for i := range 300 { + _, sendErr := b.SendMessage(&sqs.SendMessageInput{ + QueueURL: qURL, + MessageBody: fmt.Sprintf("msg-%d", i), + MessageGroupID: fmt.Sprintf("group-%d", i), + MessageDeduplicationID: fmt.Sprintf("dedup-%d", i), + }) + require.NoError(t, sendErr, "send %d of 300 must succeed", i) + } + + _, err = b.SendMessage(&sqs.SendMessageInput{ + QueueURL: qURL, + MessageBody: "overflow", + MessageGroupID: "group-overflow", + MessageDeduplicationID: "dedup-overflow", + }) + require.ErrorIs(t, err, sqs.ErrRequestThrottled, "SendMessage's own budget must now be exhausted") + + // ReceiveMessage has its own independent budget: exhausting SendMessage + // above must not throttle it. + recvOut, err := b.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: qURL, + MaxNumberOfMessages: 5, + VisibilityTimeout: sqs.NoVisibilityTimeout, + }) + require.NoError(t, err, "ReceiveMessage must not be affected by SendMessage's exhausted budget") + require.Len(t, recvOut.Messages, 5) + + // Likewise DeleteMessage. + for i, msg := range recvOut.Messages { + delErr := b.DeleteMessage(&sqs.DeleteMessageInput{QueueURL: qURL, ReceiptHandle: msg.ReceiptHandle}) + require.NoError(t, delErr, "delete %d must not be affected by SendMessage's exhausted budget", i) + } +} + +// TestFIFOThroughputLimit_SendMessageBatch_CallBudgetCountsOncePerCall: 300 +// batches of 10 (3,000 messages) must succeed; a per-entry (not per-call) +// bug would throttle around the 30th batch instead of the 301st. +func TestFIFOThroughputLimit_SendMessageBatch_CallBudgetCountsOncePerCall(t *testing.T) { + t.Parallel() + + b := newFIFOThroughputBackend(t) + + out, err := b.CreateQueue(&sqs.CreateQueueInput{ + QueueName: "batch-call-budget.fifo", + Endpoint: testEndpoint, + }) + require.NoError(t, err) + + qURL := out.QueueURL + + for i := range 300 { + entries := make([]sqs.SendMessageBatchEntry, throughputBatchSize) + for j := range throughputBatchSize { + n := i*throughputBatchSize + j + entries[j] = sqs.SendMessageBatchEntry{ + ID: fmt.Sprintf("id-%d", n), + MessageBody: fmt.Sprintf("msg-%d", n), + MessageGroupID: fmt.Sprintf("group-%d", n), + MessageDeduplicationID: fmt.Sprintf("dedup-%d", n), + } + } + + batchOut, batchErr := b.SendMessageBatch(&sqs.SendMessageBatchInput{QueueURL: qURL, Entries: entries}) + require.NoError(t, batchErr, "batch %d of 300 must succeed", i) + require.Empty(t, batchOut.Failed, "batch %d of 300: all 10 entries must succeed", i) + } + + overflowOut, err := b.SendMessageBatch(&sqs.SendMessageBatchInput{ + QueueURL: qURL, + Entries: []sqs.SendMessageBatchEntry{{ + ID: "overflow", + MessageBody: "overflow", + MessageGroupID: "group-overflow", + MessageDeduplicationID: "dedup-overflow", + }}, + }) + require.NoError(t, err, "SendMessageBatch succeeds at the transport level even when every entry fails") + require.Len(t, overflowOut.Failed, 1) + require.Equal(t, sqs.ErrRequestThrottled.Error(), overflowOut.Failed[0].Code) +} + +// TestFIFOThroughputLimit_ReceiveMessage_301stThrottled: an empty receive +// still counts as one API call, so no messages need to exist. +func TestFIFOThroughputLimit_ReceiveMessage_301stThrottled(t *testing.T) { + t.Parallel() + + b := newFIFOThroughputBackend(t) + + out, err := b.CreateQueue(&sqs.CreateQueueInput{ + QueueName: "receive-budget-throttle.fifo", + Endpoint: testEndpoint, + }) + require.NoError(t, err) + + qURL := out.QueueURL + + for i := range 300 { + _, recvErr := b.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: qURL, + MaxNumberOfMessages: 1, + VisibilityTimeout: sqs.NoVisibilityTimeout, + }) + require.NoError(t, recvErr, "receive %d of 300 must succeed", i) + } + + _, err = b.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: qURL, + MaxNumberOfMessages: 1, + VisibilityTimeout: sqs.NoVisibilityTimeout, + }) + require.ErrorIs(t, err, sqs.ErrRequestThrottled, "the 301st ReceiveMessage must be throttled") +} + +// TestFIFOThroughputLimit_DeleteMessage_ScopeSelection: perQueue shares one +// budget across all 301 distinct-group deletes (throttles at #301); +// perMessageGroupId gives each group its own (all 301 succeed). +func TestFIFOThroughputLimit_DeleteMessage_ScopeSelection(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + fifoThroughputLimit string + wantAllSucceed bool + }{ + { + name: "perqueue shares one budget across every group", + fifoThroughputLimit: "perQueue", + wantAllSucceed: false, + }, + { + name: "permessagegroupid gives each group its own budget", + fifoThroughputLimit: "perMessageGroupId", + wantAllSucceed: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + b := newFIFOThroughputBackend(t) + + attrs := map[string]string{"FifoThroughputLimit": tc.fifoThroughputLimit} + if tc.fifoThroughputLimit == "perMessageGroupId" { + attrs["DeduplicationScope"] = "messageGroup" + } + + out, err := b.CreateQueue(&sqs.CreateQueueInput{ + QueueName: "delete-scope-" + tc.fifoThroughputLimit + ".fifo", + Endpoint: testEndpoint, + Attributes: attrs, + }) + require.NoError(t, err) + + qURL := out.QueueURL + + const total = 301 + + handles := sendAndReceiveDistinctGroups(t, b, qURL, total) + + failedAt := -1 + + for i, handle := range handles { + delErr := b.DeleteMessage(&sqs.DeleteMessageInput{QueueURL: qURL, ReceiptHandle: handle}) + if delErr != nil { + require.ErrorIs(t, delErr, sqs.ErrRequestThrottled, "delete %d failed with an unexpected error", i) + + failedAt = i + + break + } + } + + if tc.wantAllSucceed { + require.Equal(t, -1, failedAt, "every delete must succeed: each group makes only one call") + } else { + require.Equal(t, 300, failedAt, + "the shared queue-wide budget must throttle exactly the 301st delete, regardless of grouping") + } + }) + } } diff --git a/services/sqs/janitor.go b/services/sqs/janitor.go index 9c310956ad..79e739c871 100644 --- a/services/sqs/janitor.go +++ b/services/sqs/janitor.go @@ -48,7 +48,7 @@ func (j *Janitor) Run(ctx context.Context) { // It delegates to InMemoryBackend.pruneState so the handler and internal janitor share one code path. func (j *Janitor) sweepExpiredMessages(ctx context.Context) { before := j.Backend.totalMessages() - j.Backend.pruneState(time.Now()) + j.Backend.pruneState(j.Backend.now()) after := j.Backend.totalMessages() if purged := before - after; purged > 0 { @@ -102,7 +102,7 @@ func (b *InMemoryBackend) runJanitor() { case <-b.janitorStop: return case <-ticker.C: - b.pruneState(time.Now()) + b.pruneState(b.now()) } } } diff --git a/services/sqs/message_move_tasks.go b/services/sqs/message_move_tasks.go index 9e8b712d29..3d17111f59 100644 --- a/services/sqs/message_move_tasks.go +++ b/services/sqs/message_move_tasks.go @@ -101,8 +101,7 @@ func (b *InMemoryBackend) findDefaultMoveDestinationLocked(dlqARN string) (strin // approximateQueueDepthLocked returns the approximate number of visible messages in the queue with the given name. // Must be called with b.mu held (either read or write). -func approximateQueueDepthLocked(q *Queue) int64 { - now := time.Now() +func approximateQueueDepthLocked(q *Queue, now time.Time) int64 { visible := 0 for _, msg := range q.messages { @@ -150,6 +149,8 @@ func (b *InMemoryBackend) startMessageMoveTaskLocked( b.mu.Lock("StartMessageMoveTask") defer b.mu.Unlock() + now := b.now() + // Check for existing running task on the same source ARN (AWS realism). // We check task status while holding both b.mu and t.mu to ensure the // status snapshot is consistent with the subsequent task insertion. @@ -186,7 +187,7 @@ func (b *InMemoryBackend) startMessageMoveTaskLocked( // Snapshot queue depth under the lock so the estimate is consistent. srcQueue, _ := b.lookupQueueByURL("", srcURL) - totalCount := approximateQueueDepthLocked(srcQueue) + totalCount := approximateQueueDepthLocked(srcQueue, now) taskHandle := uuid.NewString() @@ -199,7 +200,7 @@ func (b *InMemoryBackend) startMessageMoveTaskLocked( destArn: destArn, status: MoveTaskStatusRunning, maxPerSec: input.MaxNumberOfMessagesPerSecond, - startedAt: time.Now().UnixMilli(), + startedAt: now.UnixMilli(), totalCount: totalCount, } diff --git a/services/sqs/message_visibility.go b/services/sqs/message_visibility.go index 91353421f1..01c65dd970 100644 --- a/services/sqs/message_visibility.go +++ b/services/sqs/message_visibility.go @@ -24,16 +24,21 @@ func resolveVisibilityTimeout(requested int, q *Queue) int { // buildBlockedGroups returns the set of FIFO message group IDs that currently // have at least one in-flight message. Messages in a blocked group must not be // delivered until all earlier in-flight messages for that group are deleted, -// ensuring strict per-group ordering. -func buildBlockedGroups(inflight []*InFlightMessage) map[string]bool { - blocked := make(map[string]bool) - for _, inf := range inflight { +// ensuring strict per-group ordering. Returns q.blockedGroupsScratch, reused. +func buildBlockedGroups(q *Queue) map[string]bool { + if q.blockedGroupsScratch == nil { + q.blockedGroupsScratch = make(map[string]bool, len(q.inFlightMessages)) + } else { + clear(q.blockedGroupsScratch) + } + + for _, inf := range q.inFlightMessages { if inf.Msg.MessageGroupID != "" { - blocked[inf.Msg.MessageGroupID] = true + q.blockedGroupsScratch[inf.Msg.MessageGroupID] = true } } - return blocked + return q.blockedGroupsScratch } // prepareAndPickMessages consolidates reQueueExpired, expireRetainedMessages, @@ -301,7 +306,7 @@ func prepareAndPickMessages( // Pass 2: sweep q.messages (original + re-queued from Pass 1) in-place. var blockedGroups map[string]bool if q.IsFIFO { - blockedGroups = buildBlockedGroups(q.inFlightMessages) + blockedGroups = buildBlockedGroups(q) } var result []*Message @@ -343,14 +348,14 @@ func (b *InMemoryBackend) ChangeMessageVisibility(input *ChangeMessageVisibility q.mu.Lock() defer q.mu.Unlock() - return changeVisibility(q, input.ReceiptHandle, input.VisibilityTimeout) + return changeVisibility(q, input.ReceiptHandle, input.VisibilityTimeout, b.now()) } // changeVisibility updates the VisibleAt time for an in-flight message by receipt handle. // When visibilityTimeout is 0 the message is immediately returned to the visible queue, // matching the AWS behaviour where a zero timeout makes a message immediately available. // Caller must hold q.mu. -func changeVisibility(q *Queue, receiptHandle string, visibilityTimeout int) error { +func changeVisibility(q *Queue, receiptHandle string, visibilityTimeout int, now time.Time) error { // Use inFlightByHandle for lookup; fall back to linear scan if map not populated // (e.g., restored from snapshot before #56 was applied). inf, found := q.inFlightByHandle[receiptHandle] @@ -372,7 +377,6 @@ func changeVisibility(q *Queue, receiptHandle string, visibilityTimeout int) err if visibilityTimeout == 0 { // Move back to the visible queue immediately. - now := time.Now() inf.Msg.VisibleAt = now if !tryRouteToDLQ(q, inf.Msg, now) { requeueMessage(q, inf.Msg) @@ -389,7 +393,7 @@ func changeVisibility(q *Queue, receiptHandle string, visibilityTimeout int) err return nil } - inf.VisibleAt = time.Now().Add(time.Duration(visibilityTimeout) * time.Second) + inf.VisibleAt = now.Add(time.Duration(visibilityTimeout) * time.Second) return nil } @@ -421,6 +425,7 @@ func (b *InMemoryBackend) ChangeMessageVisibilityBatch( defer q.mu.Unlock() out := &ChangeMessageVisibilityBatchOutput{} + now := b.now() for _, entry := range input.Entries { if entry.VisibilityTimeout < 0 || entry.VisibilityTimeout > maxVisibilityTimeoutSeconds { @@ -434,7 +439,7 @@ func (b *InMemoryBackend) ChangeMessageVisibilityBatch( continue } - if err := changeVisibility(q, entry.ReceiptHandle, entry.VisibilityTimeout); err != nil { + if err := changeVisibility(q, entry.ReceiptHandle, entry.VisibilityTimeout, now); err != nil { out.Failed = append(out.Failed, BatchErrorEntry{ ID: entry.ID, Code: "MessageNotInflight", diff --git a/services/sqs/messages.go b/services/sqs/messages.go index 67ff1457d0..7828f0291b 100644 --- a/services/sqs/messages.go +++ b/services/sqs/messages.go @@ -61,7 +61,7 @@ func (b *InMemoryBackend) SendMessage(input *SendMessageInput) (*SendMessageOutp q.mu.Lock() defer q.mu.Unlock() - out, err := sendMessageLocked(q, input, md5Body, sha256Body, md5Attrs, md5SysAttrs, msgID, b.now()) + out, err := sendMessageLocked(q, input, md5Body, sha256Body, md5Attrs, md5SysAttrs, msgID, true, b.now()) if err != nil { return nil, err } @@ -71,13 +71,13 @@ func (b *InMemoryBackend) SendMessage(input *SendMessageInput) (*SendMessageOutp return out, nil } -// sendMessageLocked appends one message to an already-locked queue. -// md5Body, sha256Body, md5Attrs, and msgID must be pre-computed by the caller. -// Caller must hold q.mu (#55). Used by both SendMessage and SendMessageBatch (#58). +// sendMessageLocked appends one message to an already-locked queue (#55/#58). +// checkThroughput is false when the batch caller already reserved the budget. func sendMessageLocked( q *Queue, input *SendMessageInput, md5Body, sha256Body, md5Attrs, md5SysAttrs, msgID string, + checkThroughput bool, now time.Time, ) (*SendMessageOutput, error) { // SendMessage's top-level entry point already checks these three (empty @@ -105,7 +105,7 @@ func sendMessageLocked( } if q.IsFIFO { - if pre := preflightFIFOSend(q, input, md5Body, sha256Body, now); pre.Handled { + if pre := preflightFIFOSend(q, input, md5Body, sha256Body, checkThroughput, now); pre.Handled { return pre.Output, pre.Err } } @@ -327,6 +327,10 @@ func (b *InMemoryBackend) ReceiveMessage( return nil, err } + if err := b.checkReceiveThroughput(input); err != nil { + return nil, err + } + waitSecs := b.resolveWaitSeconds(input.QueueURL, input.WaitTimeSeconds) name := queueNameFromInput(input.QueueURL) @@ -348,12 +352,37 @@ func (b *InMemoryBackend) ReceiveMessage( return b.pollReceive(name, input, waitSecs) } +// checkReceiveThroughput runs once per API call, not per pollReceive recheck. +// Always queue-scoped (no MessageGroupId here); reserves MaxNumberOfMessages. +func (b *InMemoryBackend) checkReceiveThroughput(input *ReceiveMessageInput) error { + b.mu.RLock("checkReceiveThroughput") + q, ok := b.lookupQueueByName(input.Region, queueNameFromInput(input.QueueURL)) + b.mu.RUnlock() + + if !ok || !q.IsFIFO { + return nil + } + + maxMessages := input.MaxNumberOfMessages + if maxMessages <= 0 { + maxMessages = 1 + } + if maxMessages > maxBatchSize { + maxMessages = maxBatchSize + } + + q.mu.Lock() + defer q.mu.Unlock() + + return checkFIFOThroughput(q, fifoMethodReceive, "", maxMessages, b.now()) +} + func (b *InMemoryBackend) pollReceive( name string, input *ReceiveMessageInput, waitSecs int, ) (*ReceiveMessageOutput, error) { - deadline := time.Now().Add(time.Duration(waitSecs) * time.Second) + deadline := b.now().Add(time.Duration(waitSecs) * time.Second) const recheckInterval = time.Second @@ -373,7 +402,7 @@ func (b *InMemoryBackend) pollReceive( return &ReceiveMessageOutput{Messages: msgs}, nil } - remaining := time.Until(deadline) + remaining := deadline.Sub(b.now()) if remaining <= 0 { return &ReceiveMessageOutput{}, nil } @@ -446,11 +475,12 @@ func (b *InMemoryBackend) receiveOnce( q.mu.Lock() defer q.mu.Unlock() - now := time.Now() + now := b.now() // #54: single-pass prepareAndPickMessages replaces the four-pass sequence. + // Dedup pruning is left to the janitor + checkDedup/storeDedup's lazy + // per-key expiry: ReceiveMessage never reads q.DeduplicationIDs. if q.IsFIFO { - pruneDedup(q, now) pruneReceiveAttempts(q, now) // FIFO exactly-once retry: if the caller repeats with the same @@ -536,6 +566,13 @@ func (b *InMemoryBackend) DeleteMessage(input *DeleteMessageInput) error { return ErrReceiptHandleInvalid } + if q.IsFIFO { + scopeKey := fifoThroughputScopeKey(q, inf.Msg.MessageGroupID) + if err := checkFIFOThroughput(q, fifoMethodDelete, scopeKey, 1, b.now()); err != nil { + return err + } + } + delete(q.inFlightByHandle, input.ReceiptHandle) removeInFlight(q, inf) @@ -606,15 +643,28 @@ type batchEntryPrep struct { // processSendMessageBatchEntries iterates over batch entries (already lock-held on q), // delegates to sendMessageLocked, and accumulates Successful/Failed results. +// throttled[i] true skips straight to a RequestThrottled failure for entry i. func processSendMessageBatchEntries( q *Queue, input *SendMessageBatchInput, preps []batchEntryPrep, + throttled []bool, now time.Time, ) *SendMessageBatchOutput { out := &SendMessageBatchOutput{} for i, entry := range input.Entries { + if throttled[i] { + out.Failed = append(out.Failed, BatchResultErrorEntry{ + ID: entry.ID, + Code: ErrRequestThrottled.Error(), + Message: ErrRequestThrottled.Error(), + SenderFault: true, + }) + + continue + } + p := preps[i] sendOut, err := sendMessageLocked(q, &SendMessageInput{ QueueURL: input.QueueURL, @@ -625,7 +675,7 @@ func processSendMessageBatchEntries( DelaySeconds: entry.DelaySeconds, MessageAttributes: entry.MessageAttributes, MessageSystemAttributes: entry.MessageSystemAttributes, - }, p.md5Body, p.sha256Body, p.md5Attrs, p.md5SysAttrs, p.msgID, now) + }, p.md5Body, p.sha256Body, p.md5Attrs, p.md5SysAttrs, p.msgID, false, now) if err != nil { out.Failed = append(out.Failed, BatchResultErrorEntry{ ID: entry.ID, @@ -724,9 +774,14 @@ func (b *InMemoryBackend) SendMessageBatch( q.mu.Lock() defer q.mu.Unlock() + throttled := make([]bool, len(input.Entries)) + if q.IsFIFO { + throttled = computeFIFOSendThrottling(q, input.Entries, now) + } + // Process entries in input order; append results directly so Successful and // Failed slices already match the original entry order without sorting. - out := processSendMessageBatchEntries(q, input, preps, now) + out := processSendMessageBatchEntries(q, input, preps, throttled, now) b.emitMetric("NumberOfMessagesSent", float64(len(out.Successful))) @@ -734,6 +789,8 @@ func (b *InMemoryBackend) SendMessageBatch( } // DeleteMessageBatch deletes a batch of messages from the specified queue. +// Holds q.mu for the whole batch (not per-entry) so throughput can be +// reserved once per batch via computeFIFODeleteThrottling. func (b *InMemoryBackend) DeleteMessageBatch( input *DeleteMessageBatchInput, ) (*DeleteMessageBatchOutput, error) { @@ -748,37 +805,52 @@ func (b *InMemoryBackend) DeleteMessageBatch( // AWS returns QueueDoesNotExist at the batch level (not per-entry) when the // target queue does not exist. - var queueExists bool - - func() { - b.mu.RLock("DeleteMessageBatch.queueCheck") - defer b.mu.RUnlock() - - _, queueExists = b.lookupQueueByName(input.Region, queueNameFromInput(input.QueueURL)) - }() + b.mu.RLock("DeleteMessageBatch") + q, queueExists := b.lookupQueueByName(input.Region, queueNameFromInput(input.QueueURL)) + b.mu.RUnlock() if !queueExists { return nil, ErrQueueNotFound } + q.mu.Lock() + defer q.mu.Unlock() + + throttled := make([]bool, len(input.Entries)) + if q.IsFIFO { + throttled = computeFIFODeleteThrottling(q, input.Entries, b.now()) + } + out := &DeleteMessageBatchOutput{} - for _, entry := range input.Entries { - err := b.DeleteMessage(&DeleteMessageInput{ - QueueURL: input.QueueURL, - ReceiptHandle: entry.ReceiptHandle, - }) - if err != nil { + for i, entry := range input.Entries { + if throttled[i] { out.Failed = append(out.Failed, BatchResultErrorEntry{ ID: entry.ID, - Code: err.Error(), - Message: err.Error(), + Code: ErrRequestThrottled.Error(), + Message: ErrRequestThrottled.Error(), SenderFault: true, }) continue } + inf, found := q.inFlightByHandle[entry.ReceiptHandle] + if !found { + out.Failed = append(out.Failed, BatchResultErrorEntry{ + ID: entry.ID, + Code: ErrReceiptHandleInvalid.Error(), + Message: ErrReceiptHandleInvalid.Error(), + SenderFault: true, + }) + + continue + } + + delete(q.inFlightByHandle, entry.ReceiptHandle) + removeInFlight(q, inf) + b.emitMetric("NumberOfMessagesDeleted", 1) + out.Successful = append(out.Successful, DeleteMessageBatchResultEntry{ID: entry.ID}) } diff --git a/services/sqs/models.go b/services/sqs/models.go index d326981660..37cbcdf3a9 100644 --- a/services/sqs/models.go +++ b/services/sqs/models.go @@ -185,22 +185,21 @@ type Queue struct { deduplicationMsgIDs map[string]string Attributes map[string]string Permissions map[string]*QueuePermissionEntry - fifoSendTimes map[string][]time.Time - receiveAttempts map[string]*receiveAttemptEntry + // fifoThroughput holds one budget window per (API method, scope); see fifo.go. + fifoThroughput map[fifoThroughputKey]*fifoRateWindow + receiveAttempts map[string]*receiveAttemptEntry // inFlightByHandle indexes in-flight messages by receipt handle for O(1) delete (#56). inFlightByHandle map[string]*InFlightMessage - Tags *tags.Tags - DeduplicationIDs map[string]time.Time - dlq *Queue - Name string - URL string - Region string - messages []*Message - inFlightMessages []*InFlightMessage - // fifoSendTimesQueue is the sliding-1s-window send-time log for - // checkFIFOPerQueueRateLimit, mirroring fifoSendTimes but keyed by the - // whole queue instead of by message group (FifoThroughputLimit=perQueue). - fifoSendTimesQueue []time.Time + // blockedGroupsScratch is cleared and reused by each FIFO receive; guarded by mu. + blockedGroupsScratch map[string]bool + Tags *tags.Tags + DeduplicationIDs map[string]time.Time + dlq *Queue + Name string + URL string + Region string + messages []*Message + inFlightMessages []*InFlightMessage // mu guards queue-level state independently of the backend-global mu (#55). mu sync.Mutex fifoSeqCounter uint64 @@ -216,19 +215,6 @@ type Queue struct { IsFIFO bool } -// fifoPerGroupTPS is the AWS-documented per-message-group send rate when -// FifoThroughputLimit=perMessageGroupId. SDKs receiving more than this on a -// single group get OverLimit and back off. -const fifoPerGroupTPS = 300 - -// fifoPerQueueTPS is the AWS-documented queue-wide send rate for FIFO queues -// running with the default FifoThroughputLimit=perQueue: 300 TPS per API -// action without batching (SendMessage, ReceiveMessage, and DeleteMessage -// budgets are separate; only SendMessage is enforced here — see -// checkFIFOPerQueueRateLimit). -// https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-quotas.html#quotas-requests -const fifoPerQueueTPS = 300 - // QueueInfo holds the immutable-after-creation fields of a queue, returned by ListAll. type QueueInfo struct { Name string diff --git a/services/sqs/persistence.go b/services/sqs/persistence.go index 0cd71df3b3..5bcb009084 100644 --- a/services/sqs/persistence.go +++ b/services/sqs/persistence.go @@ -24,7 +24,7 @@ const sqsSnapshotVersion = 2 // separate DTO (rather than JSON tags directly on Queue) because Queue also // carries live, non-serialisable state — an open notify channel, a mutex, a // self-referential dlq pointer rebuilt post-restore from RedrivePolicy, and -// short-lived caches (fifoSendTimes, fifoSendTimesQueue, receiveAttempts) — that must never be +// short-lived caches (fifoThroughput, receiveAttempts) — that must never be // part of an on-disk snapshot. type queueSnapshot struct { DeduplicationIDs map[string]time.Time `json:"deduplicationIDs"` @@ -99,33 +99,24 @@ type backendSnapshot struct { Version int `json:"version"` } -// Snapshot serialises the backend state to JSON. -// It implements persistence.Persistable. -func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { - b.mu.RLock("Snapshot") - defer b.mu.RUnlock() +// marshalQueuesLocked marshals each queue's DTO to JSON while q.mu is held, then reassembles the JSON array. +// queues must already be sorted, as [store.Table.Snapshot] returns them, to keep the wire format unchanged. +func marshalQueuesLocked(queues []*Queue) (json.RawMessage, error) { + if len(queues) == 0 { + return json.RawMessage("null"), nil + } - // Build a throwaway DTO registry purely to reuse store's deterministic, - // type-erased JSON encoding (store.Registry.SnapshotAll) instead of - // hand-rolling the marshal step. This is intentionally separate from the - // live b.registry: Queue/moveTaskState carry fields (channels, mutexes, - // cancel funcs, a dlq back-pointer) that cannot round-trip through JSON, - // and only terminal move tasks are meant to survive a restart — a direct - // snapshot of the live tables could not express either constraint. - dtoReg := store.NewRegistry() - queueDTOs := store.Register(dtoReg, "queues", store.New(queueSnapshotKey)) - moveDTOs := store.Register(dtoReg, "moveTasks", store.New(moveTaskSnapshotKey)) + parts := make([]json.RawMessage, len(queues)) - for _, q := range b.queues.Snapshot() { + for i, q := range queues { q.mu.Lock() + var lastPurgedAtMillis int64 if !q.lastPurgedAt.IsZero() { lastPurgedAtMillis = q.lastPurgedAt.UnixMilli() } - fifoSeqCounter := q.fifoSeqCounter - q.mu.Unlock() - queueDTOs.Put(&queueSnapshot{ + data, err := json.Marshal(&queueSnapshot{ DeduplicationIDs: q.DeduplicationIDs, Attributes: q.Attributes, Tags: q.Tags, @@ -138,11 +129,40 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { Region: q.Region, MaxReceiveCount: q.MaxReceiveCount, IsFIFO: q.IsFIFO, - FifoSeqCounter: fifoSeqCounter, + FifoSeqCounter: q.fifoSeqCounter, LastPurgedAtUnixMilli: lastPurgedAtMillis, }) + + q.mu.Unlock() + + if err != nil { + return nil, err + } + + parts[i] = data } + return json.Marshal(parts) +} + +// Snapshot serialises the backend state to JSON. +// It implements persistence.Persistable. +func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { + b.mu.RLock("Snapshot") + defer b.mu.RUnlock() + + // Queues are marshaled while q.mu is held (see marshalQueuesLocked), not via the DTO-registry pattern below. + // SendMessage/ReceiveMessage mutate Queue and *Message fields under q.mu alone, never b.mu (gopherstack-fwd0g). + queuesJSON, err := marshalQueuesLocked(b.queues.Snapshot()) + if err != nil { + logger.Load(ctx).WarnContext(ctx, "sqs: snapshot table marshal failed", "error", err) + + return nil + } + + dtoReg := store.NewRegistry() + moveDTOs := store.Register(dtoReg, "moveTasks", store.New(moveTaskSnapshotKey)) + // Persist terminal move tasks (COMPLETED/CANCELLED/FAILED) so task history // survives restarts. RUNNING tasks are skipped because the goroutine cannot // be resumed, and CANCELLING is a transient state that resolves to CANCELLED. @@ -184,6 +204,7 @@ func (b *InMemoryBackend) Snapshot(ctx context.Context) []byte { return nil } + tables["queues"] = queuesJSON var recentlyDeleted map[string]int64 if len(b.recentlyDeleted) > 0 { @@ -256,7 +277,7 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { region = b.effectiveRegion("") } - liveQueues = append(liveQueues, restoreQueueFromSnapshot(qs, region)) + liveQueues = append(liveQueues, restoreQueueFromSnapshot(qs, region, b.now())) } b.queues.Restore(liveQueues) @@ -295,7 +316,7 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { // Restore the ErrQueueDeletedRecently cooldown, dropping any entry whose // 60-second window has already elapsed since it was snapshotted so the // map doesn't carry stale cooldowns forward indefinitely. - now := time.Now() + now := b.now() recentlyDeleted := make(map[string]time.Time, len(snap.RecentlyDeleted)) for key, deletedAtMillis := range snap.RecentlyDeleted { @@ -311,7 +332,7 @@ func (b *InMemoryBackend) Restore(ctx context.Context, data []byte) error { } // restoreQueueFromSnapshot rebuilds a Queue from its persisted snapshot. -func restoreQueueFromSnapshot(qs *queueSnapshot, region string) *Queue { +func restoreQueueFromSnapshot(qs *queueSnapshot, region string, now time.Time) *Queue { if qs.DeduplicationIDs == nil { qs.DeduplicationIDs = make(map[string]time.Time) } @@ -334,7 +355,6 @@ func restoreQueueFromSnapshot(qs *queueSnapshot, region string) *Queue { inf.sliceIdx = i } - now := time.Now() delayedCount := 0 for _, msg := range qs.Messages { diff --git a/services/sqs/persistence_race_test.go b/services/sqs/persistence_race_test.go new file mode 100644 index 0000000000..5612781290 --- /dev/null +++ b/services/sqs/persistence_race_test.go @@ -0,0 +1,91 @@ +package sqs_test + +import ( + "fmt" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/sqs" +) + +// TestSnapshot_RacesWithSendReceiveMessage reproduces gopherstack-fwd0g: Snapshot read Queue fields outside q.mu. +// SendMessage/ReceiveMessage mutate those fields, and the *Message values they point at, under q.mu; run with -race. +func TestSnapshot_RacesWithSendReceiveMessage(t *testing.T) { + t.Parallel() + + tests := []struct { + mutate func(t *testing.T, b *sqs.InMemoryBackend, queueURL string) + name string + }{ + { + name: "concurrent_snapshot_and_send_message", + mutate: func(t *testing.T, b *sqs.InMemoryBackend, queueURL string) { + t.Helper() + + for i := range 200 { + _, err := b.SendMessage(&sqs.SendMessageInput{ + QueueURL: queueURL, + MessageBody: fmt.Sprintf("body-%03d", i), + }) + require.NoError(t, err) + } + }, + }, + { + name: "concurrent_snapshot_and_receive_message", + mutate: func(t *testing.T, b *sqs.InMemoryBackend, queueURL string) { + t.Helper() + + for i := range 200 { + _, err := b.SendMessage(&sqs.SendMessageInput{ + QueueURL: queueURL, + MessageBody: fmt.Sprintf("body-%03d", i), + }) + require.NoError(t, err) + } + + for range 200 { + _, err := b.ReceiveMessage(&sqs.ReceiveMessageInput{ + QueueURL: queueURL, + MaxNumberOfMessages: 1, + }) + require.NoError(t, err) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := sqs.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + t.Cleanup(b.Close) + + out, err := b.CreateQueue(&sqs.CreateQueueInput{QueueName: "race-snapshot-queue"}) + require.NoError(t, err) + + var wg sync.WaitGroup + + stop := make(chan struct{}) + + wg.Go(func() { + for { + select { + case <-stop: + return + default: + } + + _ = b.Snapshot(t.Context()) + } + }) + + tt.mutate(t, b, out.QueueURL) + close(stop) + wg.Wait() + }) + } +} diff --git a/services/sqs/query.go b/services/sqs/query.go index caca36b460..a27923f61f 100644 --- a/services/sqs/query.go +++ b/services/sqs/query.go @@ -3,7 +3,6 @@ package sqs import ( "encoding/xml" "errors" - "fmt" "net/http" "net/url" "strconv" @@ -202,17 +201,27 @@ func (h *Handler) queueURLEndpoint(r *http.Request) string { return r.Host } +// numberedParam reads "prefix.N" or "prefix.N.suffix"; avoids fmt.Sprintf in +// per-entry loops. +func numberedParam(vals url.Values, prefix string, n int, suffix string) string { + if suffix == "" { + return vals.Get(prefix + "." + strconv.Itoa(n)) + } + + return vals.Get(prefix + "." + strconv.Itoa(n) + "." + suffix) +} + // parseQueryAttrMap parses numbered Attribute.N.Name / Attribute.N.Value pairs. func parseQueryAttrMap(vals url.Values) map[string]string { attrs := make(map[string]string) for i := 1; i <= maxParseIterations; i++ { - name := vals.Get(fmt.Sprintf("Attribute.%d.Name", i)) + name := numberedParam(vals, "Attribute", i, "Name") if name == "" { break } - attrs[name] = vals.Get(fmt.Sprintf("Attribute.%d.Value", i)) + attrs[name] = numberedParam(vals, "Attribute", i, "Value") } return attrs @@ -223,12 +232,12 @@ func parseQueryTagMap(vals url.Values) map[string]string { tagMap := make(map[string]string) for i := 1; i <= maxParseIterations; i++ { - key := vals.Get(fmt.Sprintf("Tag.%d.Key", i)) + key := numberedParam(vals, "Tag", i, "Key") if key == "" { break } - tagMap[key] = vals.Get(fmt.Sprintf("Tag.%d.Value", i)) + tagMap[key] = numberedParam(vals, "Tag", i, "Value") } return tagMap @@ -239,7 +248,7 @@ func parseQueryList(vals url.Values, prefix string) []string { var result []string for i := 1; i <= maxParseIterations; i++ { - v := vals.Get(fmt.Sprintf("%s.%d", prefix, i)) + v := numberedParam(vals, prefix, i, "") if v == "" { break } diff --git a/services/sqs/query_message_visibility.go b/services/sqs/query_message_visibility.go index 8a5389467b..84a135b5d1 100644 --- a/services/sqs/query_message_visibility.go +++ b/services/sqs/query_message_visibility.go @@ -1,7 +1,6 @@ package sqs import ( - "fmt" "net/http" "net/url" "strconv" @@ -11,16 +10,18 @@ import ( func parseQueryChangeBatchEntries(vals url.Values) []ChangeMessageVisibilityBatchRequestEntry { var entries []ChangeMessageVisibilityBatchRequestEntry + const prefix = "ChangeMessageVisibilityBatchRequestEntry" + for i := 1; i <= maxParseIterations; i++ { - id := vals.Get(fmt.Sprintf("ChangeMessageVisibilityBatchRequestEntry.%d.Id", i)) + id := numberedParam(vals, prefix, i, "Id") if id == "" { break } - vt, _ := strconv.Atoi(vals.Get(fmt.Sprintf("ChangeMessageVisibilityBatchRequestEntry.%d.VisibilityTimeout", i))) + vt, _ := strconv.Atoi(numberedParam(vals, prefix, i, "VisibilityTimeout")) entries = append(entries, ChangeMessageVisibilityBatchRequestEntry{ ID: id, - ReceiptHandle: vals.Get(fmt.Sprintf("ChangeMessageVisibilityBatchRequestEntry.%d.ReceiptHandle", i)), + ReceiptHandle: numberedParam(vals, prefix, i, "ReceiptHandle"), VisibilityTimeout: vt, }) } diff --git a/services/sqs/query_messages.go b/services/sqs/query_messages.go index a952be844d..d144929981 100644 --- a/services/sqs/query_messages.go +++ b/services/sqs/query_messages.go @@ -2,7 +2,6 @@ package sqs import ( "encoding/base64" - "fmt" "net/http" "net/url" "sort" @@ -15,17 +14,17 @@ func parseQueryMsgAttr(vals url.Values) map[string]MessageAttributeValue { attrs := make(map[string]MessageAttributeValue) for i := 1; i <= maxParseIterations; i++ { - name := vals.Get(fmt.Sprintf("MessageAttribute.%d.Name", i)) + name := numberedParam(vals, "MessageAttribute", i, "Name") if name == "" { break } attr := MessageAttributeValue{ - DataType: vals.Get(fmt.Sprintf("MessageAttribute.%d.Value.DataType", i)), - StringValue: vals.Get(fmt.Sprintf("MessageAttribute.%d.Value.StringValue", i)), + DataType: numberedParam(vals, "MessageAttribute", i, "Value.DataType"), + StringValue: numberedParam(vals, "MessageAttribute", i, "Value.StringValue"), } - if b64 := vals.Get(fmt.Sprintf("MessageAttribute.%d.Value.BinaryValue", i)); b64 != "" { + if b64 := numberedParam(vals, "MessageAttribute", i, "Value.BinaryValue"); b64 != "" { decoded, decErr := decodeMsgAttrBinary(b64) if decErr == nil { attr.BinaryValue = decoded @@ -57,20 +56,20 @@ func decodeMsgAttrBinary(encoded string) ([]byte, error) { // SendMessageBatchRequestEntry.{entryIdx}.MessageAttribute.{j}.Value.BinaryValue func parseQueryBatchMsgAttrs(vals url.Values, entryIdx int) map[string]MessageAttributeValue { attrs := make(map[string]MessageAttributeValue) - prefix := fmt.Sprintf("SendMessageBatchRequestEntry.%d.MessageAttribute", entryIdx) + prefix := "SendMessageBatchRequestEntry." + strconv.Itoa(entryIdx) + ".MessageAttribute" for j := 1; j <= maxParseIterations; j++ { - name := vals.Get(fmt.Sprintf("%s.%d.Name", prefix, j)) + name := numberedParam(vals, prefix, j, "Name") if name == "" { break } attr := MessageAttributeValue{ - DataType: vals.Get(fmt.Sprintf("%s.%d.Value.DataType", prefix, j)), - StringValue: vals.Get(fmt.Sprintf("%s.%d.Value.StringValue", prefix, j)), + DataType: numberedParam(vals, prefix, j, "Value.DataType"), + StringValue: numberedParam(vals, prefix, j, "Value.StringValue"), } - if b64 := vals.Get(fmt.Sprintf("%s.%d.Value.BinaryValue", prefix, j)); b64 != "" { + if b64 := numberedParam(vals, prefix, j, "Value.BinaryValue"); b64 != "" { decoded, decErr := decodeMsgAttrBinary(b64) if decErr == nil { attr.BinaryValue = decoded @@ -91,19 +90,21 @@ func parseQueryBatchMsgAttrs(vals url.Values, entryIdx int) map[string]MessageAt func parseQuerySendBatchEntries(vals url.Values) []SendMessageBatchEntry { var entries []SendMessageBatchEntry + const prefix = "SendMessageBatchRequestEntry" + for i := 1; i <= maxParseIterations; i++ { - id := vals.Get(fmt.Sprintf("SendMessageBatchRequestEntry.%d.Id", i)) + id := numberedParam(vals, prefix, i, "Id") if id == "" { break } - delay, _ := strconv.Atoi(vals.Get(fmt.Sprintf("SendMessageBatchRequestEntry.%d.DelaySeconds", i))) + delay, _ := strconv.Atoi(numberedParam(vals, prefix, i, "DelaySeconds")) entries = append(entries, SendMessageBatchEntry{ ID: id, - MessageBody: vals.Get(fmt.Sprintf("SendMessageBatchRequestEntry.%d.MessageBody", i)), + MessageBody: numberedParam(vals, prefix, i, "MessageBody"), DelaySeconds: delay, - MessageGroupID: vals.Get(fmt.Sprintf("SendMessageBatchRequestEntry.%d.MessageGroupId", i)), - MessageDeduplicationID: vals.Get(fmt.Sprintf("SendMessageBatchRequestEntry.%d.MessageDeduplicationId", i)), + MessageGroupID: numberedParam(vals, prefix, i, "MessageGroupId"), + MessageDeduplicationID: numberedParam(vals, prefix, i, "MessageDeduplicationId"), MessageAttributes: parseQueryBatchMsgAttrs(vals, i), }) } @@ -115,15 +116,17 @@ func parseQuerySendBatchEntries(vals url.Values) []SendMessageBatchEntry { func parseQueryDeleteBatchEntries(vals url.Values) []DeleteMessageBatchEntry { var entries []DeleteMessageBatchEntry + const prefix = "DeleteMessageBatchRequestEntry" + for i := 1; i <= maxParseIterations; i++ { - id := vals.Get(fmt.Sprintf("DeleteMessageBatchRequestEntry.%d.Id", i)) + id := numberedParam(vals, prefix, i, "Id") if id == "" { break } entries = append(entries, DeleteMessageBatchEntry{ ID: id, - ReceiptHandle: vals.Get(fmt.Sprintf("DeleteMessageBatchRequestEntry.%d.ReceiptHandle", i)), + ReceiptHandle: numberedParam(vals, prefix, i, "ReceiptHandle"), }) } diff --git a/services/sqs/query_tags.go b/services/sqs/query_tags.go index c0163a8eca..b3612f4964 100644 --- a/services/sqs/query_tags.go +++ b/services/sqs/query_tags.go @@ -1,7 +1,6 @@ package sqs import ( - "fmt" "net/http" "net/url" "sort" @@ -18,12 +17,12 @@ func parseQueryTagMembers(vals url.Values) map[string]string { tagMap := make(map[string]string) for i := 1; i <= maxParseIterations; i++ { - key := vals.Get(fmt.Sprintf("Tag.%d.Key", i)) + key := numberedParam(vals, "Tag", i, "Key") if key == "" { break } - tagMap[key] = vals.Get(fmt.Sprintf("Tag.%d.Value", i)) + tagMap[key] = numberedParam(vals, "Tag", i, "Value") } if len(tagMap) == 0 { diff --git a/services/sqs/queue_attributes.go b/services/sqs/queue_attributes.go index 1e32b01f96..51fd419c3d 100644 --- a/services/sqs/queue_attributes.go +++ b/services/sqs/queue_attributes.go @@ -4,7 +4,6 @@ import ( "encoding/json" "slices" "strconv" - "time" ) // GetQueueAttributes returns queue attributes, computing dynamic ones on the fly. @@ -98,7 +97,7 @@ func (b *InMemoryBackend) SetQueueAttributes(input *SetQueueAttributesInput) err mergeQueueAttributes(q.Attributes, input.Attributes) - q.Attributes[attrLastModifiedTimestamp] = strconv.FormatInt(time.Now().Unix(), 10) + q.Attributes[attrLastModifiedTimestamp] = strconv.FormatInt(b.now().Unix(), 10) return nil } diff --git a/services/sqs/queues.go b/services/sqs/queues.go index 8af5a09abb..c7be4adbaf 100644 --- a/services/sqs/queues.go +++ b/services/sqs/queues.go @@ -76,8 +76,8 @@ func validateQueueName(name string) error { } // buildDefaultAttributes initialises the attribute map for a new queue. -func buildDefaultAttributes(queueName, accountID, region string, isFIFO bool) map[string]string { - now := strconv.FormatInt(time.Now().Unix(), 10) +func buildDefaultAttributes(queueName, accountID, region string, isFIFO bool, nowTime time.Time) map[string]string { + now := strconv.FormatInt(nowTime.Unix(), 10) queueARN := arn.Build("sqs", region, accountID, queueName) attrs := map[string]string{ @@ -139,11 +139,13 @@ func (b *InMemoryBackend) CreateQueue(input *CreateQueueInput) (*CreateQueueOutp return &CreateQueueOutput{QueueURL: q.URL}, nil } - if err := b.checkQueueDeletedRecently(region, input.QueueName, time.Now()); err != nil { + now := b.now() + + if err := b.checkQueueDeletedRecently(region, input.QueueName, now); err != nil { return nil, err } - attrs := buildDefaultAttributes(input.QueueName, b.accountID, region, isFIFO) + attrs := buildDefaultAttributes(input.QueueName, b.accountID, region, isFIFO, now) mergeQueueAttributes(attrs, input.Attributes) @@ -220,7 +222,7 @@ func (b *InMemoryBackend) DeleteQueue(input *DeleteQueueInput) error { q.Tags.Close() } - b.recentlyDeleted[queueKey(q.Region, q.Name)] = time.Now() + b.recentlyDeleted[queueKey(q.Region, q.Name)] = b.now() b.queues.Delete(queueKey(q.Region, q.Name)) @@ -284,9 +286,11 @@ func (b *InMemoryBackend) PurgeQueue(input *PurgeQueueInput) error { return ErrQueueNotFound } + now := b.now() + // AWS enforces a 60-second cooldown between PurgeQueue calls on the same queue. // b.mu is already held (write-locked above), so this read is safe. - if !q.lastPurgedAt.IsZero() && time.Since(q.lastPurgedAt) < purgeCooldownSecs*time.Second { + if !q.lastPurgedAt.IsZero() && now.Sub(q.lastPurgedAt) < purgeCooldownSecs*time.Second { return ErrPurgeQueueInProgress } @@ -294,7 +298,7 @@ func (b *InMemoryBackend) PurgeQueue(input *PurgeQueueInput) error { q.inFlightMessages = nil q.inFlightByHandle = make(map[string]*InFlightMessage) q.delayedCount = 0 - q.lastPurgedAt = time.Now() + q.lastPurgedAt = now // For FIFO queues, purging messages also resets the deduplication state so // that producers can re-send messages with the same deduplication IDs. diff --git a/services/sqs/sns_delivery.go b/services/sqs/sns_delivery.go index f538f6c52b..802bd2ff2d 100644 --- a/services/sqs/sns_delivery.go +++ b/services/sqs/sns_delivery.go @@ -192,6 +192,8 @@ func parseQueueARNOrURL(endpoint string) (string, string) { func buildSNSEnvelope(ev *events.SNSPublishedEvent, _ string) string { ts := ev.Timestamp if ts == "" { + // Cosmetic payload fallback only, not compared against any internal + // deadline/expiry, so it stays on wall-clock time rather than b.now(). ts = time.Now().UTC().Format(time.RFC3339) } diff --git a/services/sqs/store.go b/services/sqs/store.go index b59a0d514d..3afde4bded 100644 --- a/services/sqs/store.go +++ b/services/sqs/store.go @@ -44,9 +44,9 @@ type InMemoryBackend struct { janitorStop chan struct{} mu *lockmetrics.RWMutex // nowFunc is the backend's time source for FIFO throughput rate limiting - // (see checkFIFOPerQueueRateLimit / checkFIFOPerGroupRateLimit), overridable - // in tests via export_test.go's SetNowFunc for deterministic windows without - // real sleeps. Defaults to time.Now. + // (see checkFIFOThroughput), overridable in tests via export_test.go's + // SetNowFunc for deterministic windows without real sleeps. Defaults to + // time.Now. nowFunc func() time.Time // recentlyDeleted maps a queueKey(region, name) to the time DeleteQueue was // called for it, so CreateQueue can enforce AWS's 60-second diff --git a/services/ssm/PARITY.md b/services/ssm/PARITY.md index ef8f53ef71..dfe5d531ad 100644 --- a/services/ssm/PARITY.md +++ b/services/ssm/PARITY.md @@ -460,10 +460,6 @@ items_still_open: family uses, over-projecting fields real AWS's narrower types.Association response never carries -- not a wire break (a real client discards unknown keys), disclosed rather than hand-syncing a second narrower type against the same store." - - "UpdateAssociation merges omitted fields instead of nulling them per its own doc - comment's replace semantics -- fixing this needs UpdateAssociationInput's scalar fields - switched to pointers to distinguish omitted from explicitly-cleared, which would ripple - through every existing merge-semantics test in associations_test.go." - "StartAutomationExecutionInput's AlarmConfiguration/ClientToken/Tags/TargetLocations/ TargetMaps/TargetParameterName/Targets remain unmodeled (this backend runs one synchronous single-account/region execution, nothing for multi-target fan-out to plug @@ -473,8 +469,7 @@ items_still_open: AlarmConfiguration/ClientToken/LoggingInfo/TaskInvocationParameters/TaskParameters remain unmodeled -- TaskInvocationParameters is a real 4-variant union (RunCommand/Automation/StepFunctions/Lambda) this backend's shallow task model has - nothing to plug into. UpdateMaintenanceWindowTaskInput.Replace is also unmodeled -- - this backend always merges, same class as UpdateAssociation's replace-semantics gap." + nothing to plug into." - "GetMaintenanceWindowExecutionTaskInvocationOutput.Parameters (the actual command/automation parameters used for one invocation) is unmodeled -- this backend has no per-invocation parameter snapshot, only task-level defaults." @@ -483,9 +478,7 @@ items_still_open: the real per-Property map-key convention for the untyped []map[string]string output can't be verified from the pinned SDK source, so fixing it risks fabricating a differently-wrong shape." - - "UpdatePatchBaselineInput.Replace is unmodeled, same class as UpdateAssociation's - replace-semantics gap (needs pointer fields, would ripple through merge-semantics - tests); CreatePatchBaselineInput.ClientToken (idempotency) is low-value and unmodeled." + - "CreatePatchBaselineInput.ClientToken (idempotency) is low-value and unmodeled." - "GetDeployablePatchSnapshotForInstanceInput.BaselineOverride is unmodeled -- this backend's snapshot response is already synthetic, so honoring a second, non-registered baseline needs real effective-patch computation this backend doesn't have." @@ -507,20 +500,6 @@ items_still_open: execution preview never resolves document content by version, and neither preview output type echoes the version back on the real wire either, so there is no observable point to prove this against." - - "Commands/command invocations (SendCommand) are evicted via the janitor's - existing sweepExpiredCommands, but its window (commandExpirySecs, default 1h) - ties to the real, wire-visible ExpiresAfter/Timeout field (aws-sdk-go-v2/ - service/ssm@v1.77.0 types/types.go:1221-1226: 'ExpiresAfter is calculated - based on the total timeout for the overall command'), not to AWS's separately- - documented 30-day command-history retention (docs.aws.amazon.com/systems- - manager/latest/userguide/running-commands.html, 'Execution history - retention': 'The history of each command is available for up to 30 days'). - Raising commandExpirySecs to 30 days would fix retention but would also - silently wrong the ExpiresAfter wire value (no test currently locks in its - Timeout-derived semantics, but it is a real, client-visible field); the two - concepts need decoupling (a separate terminal-status-gated history sweep, - independent of ExpiresAfter) rather than reusing one field for both. Found - 2026-09-24, bd 1x2u0-adjacent sweep; not fixed this pass." - "ListOpsItemEvents' OpsItemEventSummary.DetailType and ListOpsItemRelatedItems' OpsItemRelatedItem.CreatedBy/LastModifiedBy/LastModifiedTime (found 2026-09-18, list-summary-shapes sweep) remain unmodeled -- DetailType has no real backing concept @@ -545,6 +524,61 @@ leaks: {status: clean, note: "Janitor (janitor.go) is the only background gorout ## Notes +### 2026-09-26: items_still_open burn-down (merge-vs-replace + command history retention) + +Three items closed. (1) UpdateAssociation merged omitted optional fields instead of +nulling them; api_op_UpdateAssociation.go states the opposite verbatim ("the system +removes all optional parameters from the request and overwrites the association with +null values for those parameters. This is by design."). No signature change was needed +-- UpdateAssociationInput's fields were already pointers/nilable except +ComplianceSeverity/SyncCompliance/ApplyOnlyAtCronInterval, which the real SDK also +models as non-pointer (AWS itself can't distinguish "omitted" from "explicit zero +value" for those three either), so an unconditional assign in +applyAssociationCoreUpdates/applyAssociationExtendedUpdates (associations.go) matches +AWS's own limitation exactly. TestUpdateAssociation_ReplacesOmittedFields_RealClient +(replace_semantics_test.go) proves the null-out via a real client; +testAssociationMaxConcurrencyPreserved (update_omitted_members_preserve_state_test.go) +had ratified the old merge behavior and was removed, since UpdateAssociation is the one +op in that file's suite that does NOT preserve omitted fields. (2) UpdatePatchBaseline's +Replace field was entirely unmodeled -- added (models_patch_baselines.go), with +replacePatchBaselineUpdate (patch_baselines.go) implementing "If True, then all fields +that are required by the CreatePatchBaseline operation are also required for this API +request. Optional fields that aren't specified are set to null" (BaselineId is already +required; CreatePatchBaseline's only other required field is Name, so Replace=true now +requires Name too). Default (Replace unset/false) behavior is unchanged (merge, matching +"Fields not specified in the request are left unchanged"). Proven by +TestUpdatePatchBaseline_Replace_RealClient. (3) Command-history retention: commands were +only ever evicted via ExpiresAfter (commandExpirySecs, default 1h, tied to the real +Timeout-derived wire field), never via AWS's separately-documented 30-day command-history +retention (running-commands.html). Added a `terminalAt` field on Command (set by +completeCommand/CancelCommand, not a wire member, same non-persisted-across-restore +convention as the existing `completeAfter`) and a new, independent janitor sweep +(sweepExpiredCommandHistory, janitor.go) gated on it via a new +commandHistoryRetentionSecs backend field (default 30 days, overridable via +WithCommandHistoryRetention like the existing WithCommandTTL). Proven by +TestJanitor_SweepsExpiredCommandHistory_RealClient. Not fixed, left with a reason: the +generic Filters/Aggregators/caller-identity/scheduler/CloudWatch-alarm items, which need +unmodeled subsystems. + +### 2026-09-26 (follow-up): UpdateMaintenanceWindowTask/-Target Replace semantics + +Closed the remaining UpdateMaintenanceWindowTaskInput.Replace item. Per +api_op_UpdateMaintenanceWindowTask.go: "If you set Replace to true, then all fields +required by the RegisterTaskWithMaintenanceWindow operation are required for this +request. Optional fields that aren't specified are set to null." WindowId/WindowTaskId +are already always-required; of Register's other required fields (TaskArn, TaskType, +WindowId), only TaskArn also appears on UpdateMaintenanceWindowTaskInput (TaskType can't +be changed per the op's own doc comment), so Replace=true now requires TaskArn. +replaceMaintenanceWindowTaskUpdate (maintenance_window.go) nulls every other unspecified +optional field. Also implemented the sibling UpdateMaintenanceWindowTargetInput.Replace +(same doc pattern, sourced from RegisterTargetWithMaintenanceWindow's required fields: +Targets is the only one also present on Update, so Replace=true requires Targets). +Default (Replace unset/false) merge behavior is unchanged. Neither op documents an error +code for a missing required field under Replace (checked +API_UpdateMaintenanceWindowTask.html/API_UpdateMaintenanceWindowTarget.html -- both list +only DoesNotExistException/InternalServerError); ValidationException used, consistent +with UpdatePatchBaseline's Replace path. + ### 2026-09-19 (terraform-coverage sweep, ssm-and-backup) CreatePatchBaseline left ApprovalRules/GlobalFilters nil (crashed terraform-provider-aws's @@ -1623,3 +1657,29 @@ additive-only. Added `leak_main_test.go`. Janitor StartWorker test call sites already cancel their ctx via `context.WithCancel(t.Context())`. `go test -race -count=2` clean. + +## 2026-09-26 de-stub sweep: fake-success "stub compat" paths + +`UpdateMaintenanceWindowTarget`/`UpdateMaintenanceWindowTask` fabricated a +200 success (echoing the request IDs back) for a non-existent +`WindowTargetId`/`WindowTaskId` instead of the real `DoesNotExistException` +both ops' own deserializers model. `DisassociateOpsItemRelatedItem` did the +same for an unknown `OpsItemId`/`AssociationId`, now `OpsItemNotFoundException` +/ new `OpsItemRelatedItemAssociationNotFoundException` (`errors.go`). All +three also now reject an empty required ID with `ValidationException` +instead of silently proceeding. `GetMaintenanceWindowTask`'s doc comment +was stale (code already validated/errored correctly) -- corrected, no +behavior change. + +Test coverage: `error_path_sweep_test.go` -- two new table-driven real +`aws-sdk-go-v2` client tests for the maintenance-window ops (not-found via +`errors.As(*ssmtypes.DoesNotExistException)`, empty-ID via +`smithy.APIError.ErrorCode() == "ValidationException"`), one for +`DisassociateOpsItemRelatedItem`. Updated `maintenance_window_test.go`'s +two stub-ratifying tests to assert the new 400/DoesNotExistException +instead of 200. + +Gates: `gofmt -l`, `go build ./...`, `go vet ./services/ssm/...`, +`go test -race -count=1 ./services/ssm/...`, `golangci-lint run +./services/ssm/...` (0 issues), `go run ./cmd/parityfmtcheck -dir services` +all clean. diff --git a/services/ssm/README.md b/services/ssm/README.md index e00c23e2dc..a5ab8aef11 100644 --- a/services/ssm/README.md +++ b/services/ssm/README.md @@ -9,7 +9,7 @@ | --- | --- | | PARITY entries audited | 105 (104 ok, 1 gap) | | Feature families | 21 (21 ok) | -| Known gaps | 31 | +| Known gaps | 29 | | Deferred items | 0 | | Resource leaks | clean | @@ -32,19 +32,17 @@ - "Association/AssociationDescription's AlarmConfiguration/TriggeredAlarms need CloudWatch-alarm infra this backend lacks; TargetLocations/TargetMaps are alternate multi-account/key-value targeting schemes this backend's Targets-only model doesn't support; ScheduleOffset/LastExecutionDate/LastSuccessfulExecutionDate need a real scheduler (associations run synchronously on demand, not on a cron loop)." - "DescribeAssociationInput.AssociationVersion is accepted-and-ignored -- this backend keeps only the current version of an association (no version-history store)." - "ListAssociations marshals the same internal Association record every other op in this family uses, over-projecting fields real AWS's narrower types.Association response never carries -- not a wire break (a real client discards unknown keys), disclosed rather than hand-syncing a second narrower type against the same store." -- "UpdateAssociation merges omitted fields instead of nulling them per its own doc comment's replace semantics -- fixing this needs UpdateAssociationInput's scalar fields switched to pointers to distinguish omitted from explicitly-cleared, which would ripple through every existing merge-semantics test in associations_test.go." - "StartAutomationExecutionInput's AlarmConfiguration/ClientToken/Tags/TargetLocations/ TargetMaps/TargetParameterName/Targets remain unmodeled (this backend runs one synchronous single-account/region execution, nothing for multi-target fan-out to plug into); SendAutomationSignal's Payload is stored but not consulted since this backend has no per-step Waiting/InProgress state (every step goes straight to Success)." -- "RegisterTaskWithMaintenanceWindowInput/UpdateMaintenanceWindowTaskInput's AlarmConfiguration/ClientToken/LoggingInfo/TaskInvocationParameters/TaskParameters remain unmodeled -- TaskInvocationParameters is a real 4-variant union (RunCommand/Automation/StepFunctions/Lambda) this backend's shallow task model has nothing to plug into. UpdateMaintenanceWindowTaskInput.Replace is also unmodeled -- this backend always merges, same class as UpdateAssociation's replace-semantics gap." +- "RegisterTaskWithMaintenanceWindowInput/UpdateMaintenanceWindowTaskInput's AlarmConfiguration/ClientToken/LoggingInfo/TaskInvocationParameters/TaskParameters remain unmodeled -- TaskInvocationParameters is a real 4-variant union (RunCommand/Automation/StepFunctions/Lambda) this backend's shallow task model has nothing to plug into." - "GetMaintenanceWindowExecutionTaskInvocationOutput.Parameters (the actual command/automation parameters used for one invocation) is unmodeled -- this backend has no per-invocation parameter snapshot, only task-level defaults." - "DescribePatchPropertiesOutput.Properties aggregates baseline name/OS pairs instead of listing distinct catalogue values of the requested Property, per its own doc comment -- the real per-Property map-key convention for the untyped []map[string]string output can't be verified from the pinned SDK source, so fixing it risks fabricating a differently-wrong shape." -- "UpdatePatchBaselineInput.Replace is unmodeled, same class as UpdateAssociation's replace-semantics gap (needs pointer fields, would ripple through merge-semantics tests); CreatePatchBaselineInput.ClientToken (idempotency) is low-value and unmodeled." +- CreatePatchBaselineInput.ClientToken (idempotency) is low-value and unmodeled. - "GetDeployablePatchSnapshotForInstanceInput.BaselineOverride is unmodeled -- this backend's snapshot response is already synthetic, so honoring a second, non-registered baseline needs real effective-patch computation this backend doesn't have." - "DescribePatchGroupStateOutput is missing 6 real *int32 members (InstancesWithAvailableSecurityUpdates and 5 others) -- these need per-instance security-update-specific and pending-reboot compliance tracking InstancePatchState doesn't carry (only FailedCount/InstalledCount/MissingCount)." - "DescribeAvailablePatches' PATCH_ID filter key remains unhonored -- real AWS's Patch.Id is a distinct opaque identifier from the KB number/Name this synthetic catalogue already models, and fabricating one would invent data with nothing real to verify it against." - "documentMatchesFilters' DocumentKeyValuesFilter Owner key ('Self' vs. other accounts) is unmodeled -- this backend has no caller-identity infra to resolve 'Self' against, same disclosed-gap class as ServiceSetting.LastModifiedUser." - "ListCommandInvocationsInput.Details is declared but inert -- real AWS only populates CommandInvocation.CommandPlugins (per-plugin status/output) when Details=true, and this backend has no CommandPlugin type or per-plugin execution state." - "StartExecutionPreviewInput.DocumentVersion is declared but inert -- this backend's execution preview never resolves document content by version, and neither preview output type echoes the version back on the real wire either, so there is no observable point to prove this against." -- "Commands/command invocations (SendCommand) are evicted via the janitor's existing sweepExpiredCommands, but its window (commandExpirySecs, default 1h) ties to the real, wire-visible ExpiresAfter/Timeout field (aws-sdk-go-v2/ service/ssm@v1.77.0 types/types.go:1221-1226: 'ExpiresAfter is calculated based on the total timeout for the overall command'), not to AWS's separately- documented 30-day command-history retention (docs.aws.amazon.com/systems- manager/latest/userguide/running-commands.html, 'Execution history retention': 'The history of each command is available for up to 30 days'). Raising commandExpirySecs to 30 days would fix retention but would also silently wrong the ExpiresAfter wire value (no test currently locks in its Timeout-derived semantics, but it is a real, client-visible field); the two concepts need decoupling (a separate terminal-status-gated history sweep, independent of ExpiresAfter) rather than reusing one field for both. Found 2026-09-24, bd 1x2u0-adjacent sweep; not fixed this pass." - "ListOpsItemEvents' OpsItemEventSummary.DetailType and ListOpsItemRelatedItems' OpsItemRelatedItem.CreatedBy/LastModifiedBy/LastModifiedTime (found 2026-09-18, list-summary-shapes sweep) remain unmodeled -- DetailType has no real backing concept in this backend's two hardcoded create/update event records to source a value from without fabricating one, and CreatedBy/LastModifiedBy need the same caller-identity infra ServiceSetting.LastModifiedUser lacks; a related item also has no update path so LastModifiedTime would just duplicate CreatedTime. OpsItemRelatedItem.OpsItemId and CreatedTime themselves WERE fixed this pass (previously dropped despite being trivially sourced from the request/creation time)." ## More diff --git a/services/ssm/associations.go b/services/ssm/associations.go index 1266d0fceb..243c447967 100644 --- a/services/ssm/associations.go +++ b/services/ssm/associations.go @@ -638,77 +638,29 @@ func (b *InMemoryBackend) ListAssociations( return &ListAssociationsOutputFull{Associations: page, NextToken: next}, nil } -// applyAssociationCoreUpdates applies UpdateAssociationInput's original -// (pre-extended-fields) settable properties to assoc in place. +// applyAssociationCoreUpdates replaces (not merges) assoc's original +// settable properties: AWS nulls every omitted optional field (api_op_UpdateAssociation.go). func applyAssociationCoreUpdates(assoc *Association, input *UpdateAssociationInput) { - if input.AssociationName != nil { - assoc.AssociationName = *input.AssociationName - } - - if input.DocumentVersion != nil { - assoc.DocumentVersion = *input.DocumentVersion - } - - if input.Parameters != nil { - assoc.Parameters = copyAssocParameters(input.Parameters) - } - - if input.Targets != nil { - assoc.Targets = copyAssocTargets(input.Targets) - } + assoc.AssociationName = ptrconv.String(input.AssociationName) + assoc.DocumentVersion = ptrconv.String(input.DocumentVersion) + assoc.Parameters = copyAssocParameters(input.Parameters) + assoc.Targets = copyAssocTargets(input.Targets) } -// applyAssociationExtendedUpdates applies the State Manager fields added -// alongside CreateAssociationInput (ApplyOnlyAtCronInterval/ -// AssociationDispatchAssumeRole/AutomationTargetParameterName/CalendarNames/ -// ComplianceSeverity/Duration/MaxConcurrency/MaxErrors/OutputLocation/ -// ScheduleExpression/SyncCompliance) to assoc in place. Split out of -// UpdateAssociation to keep its cyclomatic complexity under the package -// limit. +// applyAssociationExtendedUpdates applies the State Manager fields the same +// way: replace, not merge -- see applyAssociationCoreUpdates. func applyAssociationExtendedUpdates(assoc *Association, input *UpdateAssociationInput) { - if input.ApplyOnlyAtCronInterval { - assoc.ApplyOnlyAtCronInterval = input.ApplyOnlyAtCronInterval - } - - if input.AssociationDispatchAssumeRole != nil { - assoc.AssociationDispatchAssumeRole = *input.AssociationDispatchAssumeRole - } - - if input.AutomationTargetParameterName != nil { - assoc.AutomationTargetParameterName = *input.AutomationTargetParameterName - } - - if input.CalendarNames != nil { - assoc.CalendarNames = append([]string(nil), input.CalendarNames...) - } - - if input.ComplianceSeverity != "" { - assoc.ComplianceSeverity = input.ComplianceSeverity - } - - if input.Duration != nil { - assoc.Duration = input.Duration - } - - if input.MaxConcurrency != nil { - assoc.MaxConcurrency = *input.MaxConcurrency - } - - if input.MaxErrors != nil { - assoc.MaxErrors = *input.MaxErrors - } - - if input.OutputLocation != nil { - assoc.OutputLocation = copyAssocOutputLocation(input.OutputLocation) - } - - if input.ScheduleExpression != nil { - assoc.ScheduleExpression = *input.ScheduleExpression - } - - if input.SyncCompliance != "" { - assoc.SyncCompliance = input.SyncCompliance - } + assoc.ApplyOnlyAtCronInterval = input.ApplyOnlyAtCronInterval + assoc.AssociationDispatchAssumeRole = ptrconv.String(input.AssociationDispatchAssumeRole) + assoc.AutomationTargetParameterName = ptrconv.String(input.AutomationTargetParameterName) + assoc.CalendarNames = append([]string(nil), input.CalendarNames...) + assoc.ComplianceSeverity = input.ComplianceSeverity + assoc.Duration = input.Duration + assoc.MaxConcurrency = ptrconv.String(input.MaxConcurrency) + assoc.MaxErrors = ptrconv.String(input.MaxErrors) + assoc.OutputLocation = copyAssocOutputLocation(input.OutputLocation) + assoc.ScheduleExpression = ptrconv.String(input.ScheduleExpression) + assoc.SyncCompliance = input.SyncCompliance } // UpdateAssociation updates an existing association. diff --git a/services/ssm/commands.go b/services/ssm/commands.go index c289862208..8b4c73309b 100644 --- a/services/ssm/commands.go +++ b/services/ssm/commands.go @@ -315,6 +315,7 @@ func (b *InMemoryBackend) completeCommand(region, cmdID string) { cmd.Status = overall cmd.StatusDetails = overall cmd.completeAfter = 0 + cmd.terminalAt = completionTime cmdTable.Put(&cmd) } @@ -582,6 +583,7 @@ func (b *InMemoryBackend) CancelCommand( if allCancelled { cmd := *cmdPtr cmd.Status = commandStatusCancelled + cmd.terminalAt = UnixTimeFloat(time.Now()) cmdTable.Put(&cmd) } diff --git a/services/ssm/error_path_sweep_test.go b/services/ssm/error_path_sweep_test.go index c9bf9e86b6..0fae3bb706 100644 --- a/services/ssm/error_path_sweep_test.go +++ b/services/ssm/error_path_sweep_test.go @@ -1,12 +1,15 @@ package ssm_test import ( + "context" "errors" "testing" "github.com/aws/aws-sdk-go-v2/aws" ssmsdk "github.com/aws/aws-sdk-go-v2/service/ssm" ssmtypes "github.com/aws/aws-sdk-go-v2/service/ssm/types" + smithy "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/blackbirdworks/gopherstack/services/ssm" @@ -208,3 +211,178 @@ func TestPutParameter_InvalidKMSKey_RealClient(t *testing.T) { var ik *ssmtypes.InvalidKeyId require.ErrorAs(t, err, &ik, "expected a real InvalidKeyId from the SDK deserializer") } + +// TestMaintenanceWindowUpdate_NotFound_RealClient: unknown IDs return DoesNotExistException. +func TestMaintenanceWindowUpdate_NotFound_RealClient(t *testing.T) { + t.Parallel() + + tests := []struct { + call func(ctx context.Context, client *ssmsdk.Client) error + name string + }{ + { + name: "UpdateMaintenanceWindowTarget", + call: func(ctx context.Context, client *ssmsdk.Client) error { + _, err := client.UpdateMaintenanceWindowTarget(ctx, &ssmsdk.UpdateMaintenanceWindowTargetInput{ + WindowId: aws.String("mw-0123456789abcdef0"), + WindowTargetId: aws.String("wt-0123456789abcdef0"), + }) + + return err + }, + }, + { + name: "UpdateMaintenanceWindowTask", + call: func(ctx context.Context, client *ssmsdk.Client) error { + _, err := client.UpdateMaintenanceWindowTask(ctx, &ssmsdk.UpdateMaintenanceWindowTaskInput{ + WindowId: aws.String("mw-0123456789abcdef0"), + WindowTaskId: aws.String("task-0123456789abcdef0"), + }) + + return err + }, + }, + { + name: "GetMaintenanceWindowTask", + call: func(ctx context.Context, client *ssmsdk.Client) error { + _, err := client.GetMaintenanceWindowTask(ctx, &ssmsdk.GetMaintenanceWindowTaskInput{ + WindowId: aws.String("mw-0123456789abcdef0"), + WindowTaskId: aws.String("task-0123456789abcdef0"), + }) + + return err + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + client := newTestSSMClient(t, ssm.NewHandler(ssm.NewInMemoryBackend())) + + err := tt.call(t.Context(), client) + require.Error(t, err) + + var dne *ssmtypes.DoesNotExistException + require.ErrorAs(t, err, &dne, "expected a real DoesNotExistException from the SDK deserializer") + }) + } +} + +// TestMaintenanceWindowUpdate_EmptyID_ValidationException: empty IDs reach the wire +// (SDK only rejects nil) and must fail server-side. +func TestMaintenanceWindowUpdate_EmptyID_ValidationException(t *testing.T) { + t.Parallel() + + tests := []struct { + call func(ctx context.Context, client *ssmsdk.Client) error + name string + }{ + { + name: "UpdateMaintenanceWindowTarget_empty_WindowTargetId", + call: func(ctx context.Context, client *ssmsdk.Client) error { + _, err := client.UpdateMaintenanceWindowTarget(ctx, &ssmsdk.UpdateMaintenanceWindowTargetInput{ + WindowId: aws.String("mw-0123456789abcdef0"), + WindowTargetId: aws.String(""), + }) + + return err + }, + }, + { + name: "UpdateMaintenanceWindowTask_empty_WindowTaskId", + call: func(ctx context.Context, client *ssmsdk.Client) error { + _, err := client.UpdateMaintenanceWindowTask(ctx, &ssmsdk.UpdateMaintenanceWindowTaskInput{ + WindowId: aws.String("mw-0123456789abcdef0"), + WindowTaskId: aws.String(""), + }) + + return err + }, + }, + { + name: "GetMaintenanceWindowTask_empty_WindowTaskId", + call: func(ctx context.Context, client *ssmsdk.Client) error { + _, err := client.GetMaintenanceWindowTask(ctx, &ssmsdk.GetMaintenanceWindowTaskInput{ + WindowId: aws.String("mw-0123456789abcdef0"), + WindowTaskId: aws.String(""), + }) + + return err + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + client := newTestSSMClient(t, ssm.NewHandler(ssm.NewInMemoryBackend())) + + err := tt.call(t.Context(), client) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ValidationException", apiErr.ErrorCode()) + }) + } +} + +// TestDisassociateOpsItemRelatedItem_NotFound_RealClient: unknown OpsItem or association +// returns its not-found error. +func TestDisassociateOpsItemRelatedItem_NotFound_RealClient(t *testing.T) { + t.Parallel() + + tests := []struct { + setup func(t *testing.T, client *ssmsdk.Client) string + name string + wantCode string + }{ + { + name: "unknown_ops_item", + setup: func(t *testing.T, _ *ssmsdk.Client) string { + t.Helper() + + return "oi-does-not-exist" + }, + wantCode: "OpsItemNotFoundException", + }, + { + name: "unknown_association", + setup: func(t *testing.T, client *ssmsdk.Client) string { + t.Helper() + + created, err := client.CreateOpsItem(t.Context(), &ssmsdk.CreateOpsItemInput{ + Title: aws.String("disassociate-not-found-test"), + Source: aws.String("EC2"), + Description: aws.String("desc"), + }) + require.NoError(t, err) + + return aws.ToString(created.OpsItemId) + }, + wantCode: "OpsItemRelatedItemAssociationNotFoundException", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + client := newTestSSMClient(t, ssm.NewHandler(ssm.NewInMemoryBackend())) + opsItemID := tt.setup(t, client) + + _, err := client.DisassociateOpsItemRelatedItem(t.Context(), &ssmsdk.DisassociateOpsItemRelatedItemInput{ + OpsItemId: aws.String(opsItemID), + AssociationId: aws.String("assoc-does-not-exist"), + }) + require.Error(t, err) + + var apiErr smithy.APIError + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, tt.wantCode, apiErr.ErrorCode()) + }) + } +} diff --git a/services/ssm/errors.go b/services/ssm/errors.go index a6568e003d..1c6906f157 100644 --- a/services/ssm/errors.go +++ b/services/ssm/errors.go @@ -44,6 +44,11 @@ var ( // still referenced by a registered task. TargetInUseException is the // real declared exception (ssm@v1.77.0 types/errors.go). ErrMaintenanceWindowTargetInUse = errors.New("TargetInUseException") + // ErrOpsItemRelatedItemAssociationNotFound is returned by + // DisassociateOpsItemRelatedItem when AssociationId doesn't match any + // related item on the OpsItem. OpsItemRelatedItemAssociationNotFoundException + // is the op's own declared exception (ssm@v1.77.0 deserializers.go). + ErrOpsItemRelatedItemAssociationNotFound = errors.New("OpsItemRelatedItemAssociationNotFoundException") ) var ( ErrResourceDataSyncNotFound = errors.New("ResourceDataSyncNotFoundException") diff --git a/services/ssm/handler.go b/services/ssm/handler.go index 5f5d464a2e..2eac258772 100644 --- a/services/ssm/handler.go +++ b/services/ssm/handler.go @@ -368,6 +368,8 @@ func classifySSMOpsError(reqErr error) (string, int, bool) { return "OpsMetadataNotFoundException", statusCode, true case errors.Is(reqErr, ErrOpsMetadataAlreadyExists): return "OpsMetadataAlreadyExistsException", statusCode, true + case errors.Is(reqErr, ErrOpsItemRelatedItemAssociationNotFound): + return "OpsItemRelatedItemAssociationNotFoundException", statusCode, true default: return "", 0, false } diff --git a/services/ssm/janitor.go b/services/ssm/janitor.go index d35674ceb3..2ecac01c27 100644 --- a/services/ssm/janitor.go +++ b/services/ssm/janitor.go @@ -40,6 +40,7 @@ func NewJanitor(backend *InMemoryBackend, interval time.Duration) *Janitor { func (j *Janitor) Run(ctx context.Context) { g := worker.NewGroup(ctx, "ssm") g.Ticker("CommandSweeper", j.Interval, j.TaskTimeout, j.sweepExpiredCommands) + g.Ticker("CommandHistorySweeper", j.Interval, j.TaskTimeout, j.sweepExpiredCommandHistory) g.Ticker("ParameterExpirer", j.Interval, j.TaskTimeout, j.sweepExpiredParameters) g.Ticker("SessionSweeper", j.Interval, j.TaskTimeout, j.sweepTerminatedSessions) g.Ticker("ParameterPolicyNotifier", j.Interval, j.TaskTimeout, j.sweepParameterPolicyNotifications) @@ -57,6 +58,7 @@ func (j *Janitor) Run(ctx context.Context) { // policy-notification dedupe state) is deleted. func (j *Janitor) SweepOnce(ctx context.Context) { j.sweepExpiredCommands(ctx) + j.sweepExpiredCommandHistory(ctx) j.sweepParameterPolicyNotifications(ctx) j.sweepExpiredParameters(ctx) j.sweepTerminatedSessions(ctx) @@ -191,6 +193,54 @@ func (j *Janitor) sweepExpiredCommands(ctx context.Context) { } } +// sweepExpiredCommandHistory evicts terminal commands whose completion is +// older than commandHistoryRetentionSecs, independent of ExpiresAfter. +func (j *Janitor) sweepExpiredCommandHistory(ctx context.Context) { + b := j.Backend + now := UnixTimeFloat(time.Now()) + cutoff := now - b.commandHistoryRetentionSecs + + b.mu.Lock("SSMJanitorCommandHistory") + + type expiredCmd struct { + region string + id string + } + var expired []expiredCmd + + for region, commands := range b.commands { + b.materializeCommandsLocked(region, now) + + for _, cmd := range commands.All() { + if cmd.terminalAt > 0 && cmd.terminalAt < cutoff { + expired = append(expired, expiredCmd{region: region, id: cmd.CommandID}) + } + } + } + + regions := make(map[string]struct{}, len(expired)) + for _, e := range expired { + b.commands[e.region].Delete(e.id) + delete(b.commandInvocations[e.region], e.id) + regions[e.region] = struct{}{} + } + + for region := range regions { + cleanupEmptyInnerMap(b.commandInvocations, region) + } + + b.mu.Unlock() + + count := len(expired) + + telemetry.RecordWorkerItems("ssm", "CommandHistorySweeper", count) + telemetry.RecordWorkerTask("ssm", "CommandHistorySweeper", "success") + + if count > 0 { + logger.Load(ctx).InfoContext(ctx, "SSM janitor: expired command history evicted", "count", count) + } +} + // parameterExpirationPolicy is the JSON shape of an Expiration policy attached // to an SSM parameter via PutParameter.Policies. // AWS policy text format: diff --git a/services/ssm/maintenance_window.go b/services/ssm/maintenance_window.go index 4dc90c4db9..5fc46e8b12 100644 --- a/services/ssm/maintenance_window.go +++ b/services/ssm/maintenance_window.go @@ -1064,7 +1064,6 @@ func (b *InMemoryBackend) DeleteMaintenanceWindow( } // GetMaintenanceWindowTask retrieves a task by WindowId and WindowTaskId. -// Returns an empty task when WindowTaskID is empty (stub compat). func (b *InMemoryBackend) GetMaintenanceWindowTask( ctx context.Context, input *GetMaintenanceWindowTaskInput, @@ -1187,12 +1186,48 @@ func windowTargetMatchesFilters(registered []WindowTarget, requested []WindowTar return false } +// mergeMaintenanceWindowTargetUpdate applies Replace=false semantics: only +// fields the caller set are modified. +func mergeMaintenanceWindowTargetUpdate(target *MaintenanceWindowTarget, input *UpdateMaintenanceWindowTargetInput) { + if input.OwnerInfo != nil { + target.OwnerInfo = *input.OwnerInfo + } + + if input.Name != nil { + target.Name = *input.Name + } + + if input.Description != nil { + target.Description = *input.Description + } + + if len(input.Targets) > 0 { + target.Targets = input.Targets + } +} + +// replaceMaintenanceWindowTargetUpdate applies Replace=true: omitted fields are nulled. +func replaceMaintenanceWindowTargetUpdate(target *MaintenanceWindowTarget, input *UpdateMaintenanceWindowTargetInput) { + target.OwnerInfo = ptrconv.String(input.OwnerInfo) + target.Name = ptrconv.String(input.Name) + target.Description = ptrconv.String(input.Description) + target.Targets = input.Targets +} + // UpdateMaintenanceWindowTarget updates target fields. -// Returns an empty response when the target is not found (stub compat for empty ID). func (b *InMemoryBackend) UpdateMaintenanceWindowTarget( ctx context.Context, input *UpdateMaintenanceWindowTargetInput, ) (*UpdateMaintenanceWindowTargetOutput, error) { + if input.WindowID == "" || input.WindowTargetID == "" { + return nil, fmt.Errorf("%w: WindowId and WindowTargetId are required", ErrValidationException) + } + + replace := ptrconv.Bool(input.Replace) + if replace && len(input.Targets) == 0 { + return nil, fmt.Errorf("%w: Targets is required when Replace is true", ErrValidationException) + } + region := getRegion(ctx) b.mu.Lock("UpdateMaintenanceWindowTarget") defer b.mu.Unlock() @@ -1200,30 +1235,15 @@ func (b *InMemoryBackend) UpdateMaintenanceWindowTarget( store := b.maintenanceWindowTargetsStore(region) targetPtr, exists := store.Get(input.WindowTargetID) if !exists || targetPtr.WindowID != input.WindowID { - // Return a no-op success rather than error to preserve stub compat for - // callers that send non-existent IDs (e.g. the simple stub coverage test). - return &UpdateMaintenanceWindowTargetOutput{ - WindowID: input.WindowID, - WindowTargetID: input.WindowTargetID, - }, nil + return nil, ErrMaintenanceWindowNotFound } target := *targetPtr - if input.OwnerInfo != nil { - target.OwnerInfo = *input.OwnerInfo - } - - if input.Name != nil { - target.Name = *input.Name - } - - if input.Description != nil { - target.Description = *input.Description - } - - if len(input.Targets) > 0 { - target.Targets = input.Targets + if replace { + replaceMaintenanceWindowTargetUpdate(&target, input) + } else { + mergeMaintenanceWindowTargetUpdate(&target, input) } store.Put(&target) @@ -1238,35 +1258,9 @@ func (b *InMemoryBackend) UpdateMaintenanceWindowTarget( }, nil } -// UpdateMaintenanceWindowTask updates task fields. -// Returns a no-op success when the task is not found (stub compat for non-existent IDs). -func (b *InMemoryBackend) UpdateMaintenanceWindowTask( - ctx context.Context, - input *UpdateMaintenanceWindowTaskInput, -) (*UpdateMaintenanceWindowTaskOutput, error) { - if err := validateMaxConcurrency(ptrconv.String(input.MaxConcurrency)); err != nil { - return nil, err - } - - if err := validateMaxErrors(ptrconv.String(input.MaxErrors)); err != nil { - return nil, err - } - - region := getRegion(ctx) - b.mu.Lock("UpdateMaintenanceWindowTask") - defer b.mu.Unlock() - - store := b.maintenanceWindowTasksStore(region) - taskPtr, exists := store.Get(input.WindowTaskID) - if !exists || taskPtr.WindowID != input.WindowID { - return &UpdateMaintenanceWindowTaskOutput{ - WindowID: input.WindowID, - WindowTaskID: input.WindowTaskID, - }, nil - } - - task := *taskPtr - +// mergeMaintenanceWindowTaskUpdate applies Replace=false semantics: only +// fields the caller set are modified. +func mergeMaintenanceWindowTaskUpdate(task *MaintenanceWindowTask, input *UpdateMaintenanceWindowTaskInput) { if input.TaskArn != nil { task.TaskArn = *input.TaskArn } @@ -1302,6 +1296,64 @@ func (b *InMemoryBackend) UpdateMaintenanceWindowTask( if len(input.Targets) > 0 { task.Targets = input.Targets } +} + +// replaceMaintenanceWindowTaskUpdate applies Replace=true: omitted fields are nulled. +func replaceMaintenanceWindowTaskUpdate(task *MaintenanceWindowTask, input *UpdateMaintenanceWindowTaskInput) { + task.TaskArn = ptrconv.String(input.TaskArn) + task.Name = ptrconv.String(input.Name) + task.Description = ptrconv.String(input.Description) + task.ServiceRoleArn = ptrconv.String(input.ServiceRoleArn) + task.MaxConcurrency = ptrconv.String(input.MaxConcurrency) + task.MaxErrors = ptrconv.String(input.MaxErrors) + task.CutoffBehavior = input.CutoffBehavior + task.Targets = input.Targets + + task.Priority = 0 + if input.Priority != nil { + task.Priority = *input.Priority + } +} + +// UpdateMaintenanceWindowTask updates task fields. +func (b *InMemoryBackend) UpdateMaintenanceWindowTask( + ctx context.Context, + input *UpdateMaintenanceWindowTaskInput, +) (*UpdateMaintenanceWindowTaskOutput, error) { + if input.WindowID == "" || input.WindowTaskID == "" { + return nil, fmt.Errorf("%w: WindowId and WindowTaskId are required", ErrValidationException) + } + + if err := validateMaxConcurrency(ptrconv.String(input.MaxConcurrency)); err != nil { + return nil, err + } + + if err := validateMaxErrors(ptrconv.String(input.MaxErrors)); err != nil { + return nil, err + } + + replace := ptrconv.Bool(input.Replace) + if replace && ptrconv.String(input.TaskArn) == "" { + return nil, fmt.Errorf("%w: TaskArn is required when Replace is true", ErrValidationException) + } + + region := getRegion(ctx) + b.mu.Lock("UpdateMaintenanceWindowTask") + defer b.mu.Unlock() + + store := b.maintenanceWindowTasksStore(region) + taskPtr, exists := store.Get(input.WindowTaskID) + if !exists || taskPtr.WindowID != input.WindowID { + return nil, ErrMaintenanceWindowNotFound + } + + task := *taskPtr + + if replace { + replaceMaintenanceWindowTaskUpdate(&task, input) + } else { + mergeMaintenanceWindowTaskUpdate(&task, input) + } store.Put(&task) diff --git a/services/ssm/maintenance_window_test.go b/services/ssm/maintenance_window_test.go index d7e2efbfdf..f371bf910f 100644 --- a/services/ssm/maintenance_window_test.go +++ b/services/ssm/maintenance_window_test.go @@ -59,8 +59,9 @@ func TestStubOps_DescribeMaintenanceWindows(t *testing.T) { assert.Equal(t, http.StatusOK, rec.Code) } -// TestStubOps_UpdateMaintenanceWindowTarget exercises that stub. -func TestStubOps_UpdateMaintenanceWindowTarget(t *testing.T) { +// TestUpdateMaintenanceWindowTarget_NotFound verifies a non-existent target +// returns the real DoesNotExistException instead of a fabricated success. +func TestUpdateMaintenanceWindowTarget_NotFound(t *testing.T) { t.Parallel() h, _ := newTestHandler(t) @@ -70,11 +71,13 @@ func TestStubOps_UpdateMaintenanceWindowTarget(t *testing.T) { "UpdateMaintenanceWindowTarget", `{"WindowId":"mw-1234","WindowTargetId":"tgt-1234"}`, ) - assert.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, http.StatusBadRequest, rec.Code) + assertBodyContains(t, rec, "DoesNotExistException") } -// TestStubOps_UpdateMaintenanceWindowTask exercises that stub. -func TestStubOps_UpdateMaintenanceWindowTask(t *testing.T) { +// TestUpdateMaintenanceWindowTask_NotFound verifies a non-existent task +// returns the real DoesNotExistException instead of a fabricated success. +func TestUpdateMaintenanceWindowTask_NotFound(t *testing.T) { t.Parallel() h, _ := newTestHandler(t) @@ -84,7 +87,8 @@ func TestStubOps_UpdateMaintenanceWindowTask(t *testing.T) { "UpdateMaintenanceWindowTask", `{"WindowId":"mw-1234","WindowTaskId":"task-1234"}`, ) - assert.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, http.StatusBadRequest, rec.Code) + assertBodyContains(t, rec, "DoesNotExistException") } // TestSSMHandler_ChaosOps verifies the chaos interface methods compile and return. diff --git a/services/ssm/models_commands.go b/services/ssm/models_commands.go index 3958875e35..3048d0f6b9 100644 --- a/services/ssm/models_commands.go +++ b/services/ssm/models_commands.go @@ -33,6 +33,9 @@ type Command struct { // command lazily transitions to its terminal status. Zero means the command // completes on the next read (or was created without an exec delay). completeAfter float64 + // terminalAt is when this command went terminal; drives the janitor's + // command-history sweep, independent of ExpiresAfter. Not a wire field. + terminalAt float64 } // CommandInvocation represents the invocation of a command on an instance. diff --git a/services/ssm/models_maintenance_window.go b/services/ssm/models_maintenance_window.go index 8d81960cbd..9d250630d8 100644 --- a/services/ssm/models_maintenance_window.go +++ b/services/ssm/models_maintenance_window.go @@ -518,6 +518,7 @@ type UpdateMaintenanceWindowTargetInput struct { OwnerInfo *string `json:"OwnerInformation,omitempty"` Name *string `json:"Name,omitempty"` Description *string `json:"Description,omitempty"` + Replace *bool `json:"Replace,omitempty"` Targets []WindowTarget `json:"Targets,omitempty"` } @@ -544,6 +545,7 @@ type UpdateMaintenanceWindowTaskInput struct { MaxConcurrency *string `json:"MaxConcurrency,omitempty"` MaxErrors *string `json:"MaxErrors,omitempty"` CutoffBehavior string `json:"CutoffBehavior,omitempty"` + Replace *bool `json:"Replace,omitempty"` Targets []WindowTarget `json:"Targets,omitempty"` } diff --git a/services/ssm/models_patch_baselines.go b/services/ssm/models_patch_baselines.go index 96f4e84f6d..c32bbba2d4 100644 --- a/services/ssm/models_patch_baselines.go +++ b/services/ssm/models_patch_baselines.go @@ -114,6 +114,7 @@ type UpdatePatchBaselineInput struct { ApprovalRules *PatchRuleGroup `json:"ApprovalRules,omitempty"` GlobalFilters *PatchFilterGroup `json:"GlobalFilters,omitempty"` ApprovedPatchesEnableNonSecurity *bool `json:"ApprovedPatchesEnableNonSecurity,omitempty"` + Replace *bool `json:"Replace,omitempty"` BaselineID string `json:"BaselineId"` Name *string `json:"Name,omitempty"` Description *string `json:"Description,omitempty"` diff --git a/services/ssm/ops_items.go b/services/ssm/ops_items.go index ffbbcb2cdc..27a8a14a8e 100644 --- a/services/ssm/ops_items.go +++ b/services/ssm/ops_items.go @@ -604,7 +604,6 @@ func (b *InMemoryBackend) DeleteOpsItem( } // DisassociateOpsItemRelatedItem removes a related item from an OpsItem. -// Returns success if the OpsItem does not exist (stub compat for empty ID). func (b *InMemoryBackend) DisassociateOpsItemRelatedItem( ctx context.Context, input *DisassociateOpsItemRelatedItemInput, @@ -617,20 +616,31 @@ func (b *InMemoryBackend) DisassociateOpsItemRelatedItem( b.mu.Lock("DisassociateOpsItemRelatedItem") defer b.mu.Unlock() + if !b.opsItemsStore(region).Has(input.OpsItemID) { + return nil, ErrOpsItemNotFound + } + store := b.opsItemRelatedItemsStore(region) items, exists := store[input.OpsItemID] if !exists { - // No-op if OpsItem doesn't have any related items. - return &DisassociateOpsItemRelatedItemOutput{}, nil + return nil, ErrOpsItemRelatedItemAssociationNotFound } + found := false filtered := items[:0] + for _, item := range items { if item.AssociationID != input.AssociationID { filtered = append(filtered, item) + } else { + found = true } } + if !found { + return nil, ErrOpsItemRelatedItemAssociationNotFound + } + store[input.OpsItemID] = filtered return &DisassociateOpsItemRelatedItemOutput{}, nil diff --git a/services/ssm/patch_baselines.go b/services/ssm/patch_baselines.go index ebc45cdad1..8531839deb 100644 --- a/services/ssm/patch_baselines.go +++ b/services/ssm/patch_baselines.go @@ -11,6 +11,7 @@ import ( "github.com/google/uuid" + "github.com/blackbirdworks/gopherstack/pkgs/ptrconv" "github.com/blackbirdworks/gopherstack/pkgs/store" ) @@ -574,31 +575,9 @@ func validateUpdatePatchBaselineInput(input *UpdatePatchBaselineInput) error { return validateApprovalRules(input.ApprovalRules) } -// UpdatePatchBaseline updates a patch baseline. -func (b *InMemoryBackend) UpdatePatchBaseline( - ctx context.Context, - input *UpdatePatchBaselineInput, -) (*UpdatePatchBaselineOutput, error) { - if input.BaselineID == "" { - return nil, fmt.Errorf("%w: BaselineId is required", ErrValidationException) - } - - if err := validateUpdatePatchBaselineInput(input); err != nil { - return nil, err - } - - region := getRegion(ctx) - b.mu.Lock("UpdatePatchBaseline") - defer b.mu.Unlock() - - baselines := b.patchBaselinesStore(region) - blPtr, exists := baselines.Get(input.BaselineID) - if !exists { - return nil, ErrPatchBaselineNotFound - } - - bl := *blPtr - +// mergePatchBaselineUpdate applies only the fields present in input (AWS +// default: "Fields not specified in the request are left unchanged"). +func mergePatchBaselineUpdate(bl *PatchBaseline, input *UpdatePatchBaselineInput) { if input.Name != nil { bl.Name = *input.Name } @@ -642,6 +621,59 @@ func (b *InMemoryBackend) UpdatePatchBaseline( if input.ApprovedPatchesEnableNonSecurity != nil { bl.ApprovedPatchesEnableNonSecurity = input.ApprovedPatchesEnableNonSecurity } +} + +// replacePatchBaselineUpdate applies Replace=true: every optional field is +// assigned unconditionally, nulling out ones the caller omitted. +func replacePatchBaselineUpdate(bl *PatchBaseline, input *UpdatePatchBaselineInput) { + bl.Name = *input.Name + bl.Description = ptrconv.String(input.Description) + bl.ApprovedPatches = input.ApprovedPatches + bl.RejectedPatches = input.RejectedPatches + bl.ApprovedPatchesComplianceLevel = input.ApprovedPatchesComplianceLevel + bl.AvailableSecurityUpdatesComplianceStatus = input.AvailableSecurityUpdatesComplianceStatus + bl.RejectedPatchesAction = input.RejectedPatchesAction + bl.ApprovalRules = input.ApprovalRules + bl.GlobalFilters = input.GlobalFilters + bl.Sources = input.Sources + bl.ApprovedPatchesEnableNonSecurity = input.ApprovedPatchesEnableNonSecurity +} + +// UpdatePatchBaseline updates a patch baseline. +func (b *InMemoryBackend) UpdatePatchBaseline( + ctx context.Context, + input *UpdatePatchBaselineInput, +) (*UpdatePatchBaselineOutput, error) { + if input.BaselineID == "" { + return nil, fmt.Errorf("%w: BaselineId is required", ErrValidationException) + } + + if err := validateUpdatePatchBaselineInput(input); err != nil { + return nil, err + } + + replace := ptrconv.Bool(input.Replace) + if replace && input.Name == nil { + return nil, fmt.Errorf("%w: Name is required when Replace is true", ErrValidationException) + } + + region := getRegion(ctx) + b.mu.Lock("UpdatePatchBaseline") + defer b.mu.Unlock() + + baselines := b.patchBaselinesStore(region) + blPtr, exists := baselines.Get(input.BaselineID) + if !exists { + return nil, ErrPatchBaselineNotFound + } + + bl := *blPtr + + if replace { + replacePatchBaselineUpdate(&bl, input) + } else { + mergePatchBaselineUpdate(&bl, input) + } bl.ModifiedDate = UnixTimeFloat(timeNow()) baselines.Put(&bl) diff --git a/services/ssm/replace_semantics_test.go b/services/ssm/replace_semantics_test.go new file mode 100644 index 0000000000..d42bd7dc0d --- /dev/null +++ b/services/ssm/replace_semantics_test.go @@ -0,0 +1,332 @@ +package ssm_test + +import ( + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + ssmsdk "github.com/aws/aws-sdk-go-v2/service/ssm" + ssmtypes "github.com/aws/aws-sdk-go-v2/service/ssm/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/ssm" +) + +// TestUpdateAssociation_ReplacesOmittedFields_RealClient covers +// api_op_UpdateAssociation.go: an omitted optional field must be nulled, not merged. +func TestUpdateAssociation_ReplacesOmittedFields_RealClient(t *testing.T) { + t.Parallel() + + backend := ssm.NewInMemoryBackend() + client := newTestSSMClient(t, ssm.NewHandler(backend)) + ctx := t.Context() + + created, err := client.CreateAssociation(ctx, &ssmsdk.CreateAssociationInput{ + Name: aws.String("AWS-RunShellScript"), + AssociationName: aws.String("original-name"), + DocumentVersion: aws.String("1"), + Targets: []ssmtypes.Target{ + {Key: aws.String("tag:Env"), Values: []string{"prod"}}, + }, + Parameters: map[string][]string{"commands": {"echo hi"}}, + MaxConcurrency: aws.String("50%"), + }) + require.NoError(t, err) + + assocID := created.AssociationDescription.AssociationId + + updated, err := client.UpdateAssociation(ctx, &ssmsdk.UpdateAssociationInput{ + AssociationId: assocID, + ComplianceSeverity: ssmtypes.AssociationComplianceSeverityCritical, + }) + require.NoError(t, err) + + desc := updated.AssociationDescription + assert.Equal(t, ssmtypes.AssociationComplianceSeverityCritical, desc.ComplianceSeverity) + assert.Nil(t, desc.AssociationName, "AssociationName omitted from the update must be nulled") + assert.Nil(t, desc.DocumentVersion, "DocumentVersion omitted from the update must be nulled") + assert.Empty(t, desc.Targets, "Targets omitted from the update must be nulled") + assert.Empty(t, desc.Parameters, "Parameters omitted from the update must be nulled") + assert.Nil(t, desc.MaxConcurrency, "MaxConcurrency omitted from the update must be nulled") +} + +// TestUpdatePatchBaseline_Replace_RealClient covers UpdatePatchBaseline's +// Replace parameter (api_op_UpdatePatchBaseline.go): true nulls omitted fields, false merges. +func TestUpdatePatchBaseline_Replace_RealClient(t *testing.T) { + t.Parallel() + + tests := []struct { + update func(baselineID *string) *ssmsdk.UpdatePatchBaselineInput + check func(t *testing.T, updated *ssmsdk.UpdatePatchBaselineOutput, err error) + name string + }{ + { + name: "replace_true_requires_name", + update: func(baselineID *string) *ssmsdk.UpdatePatchBaselineInput { + return &ssmsdk.UpdatePatchBaselineInput{BaselineId: baselineID, Replace: aws.Bool(true)} + }, + check: func(t *testing.T, _ *ssmsdk.UpdatePatchBaselineOutput, err error) { + t.Helper() + require.Error(t, err) + }, + }, + { + name: "replace_true_nulls_omitted_fields", + update: func(baselineID *string) *ssmsdk.UpdatePatchBaselineInput { + return &ssmsdk.UpdatePatchBaselineInput{ + BaselineId: baselineID, + Name: aws.String("replace-semantics-baseline"), + Replace: aws.Bool(true), + } + }, + check: func(t *testing.T, updated *ssmsdk.UpdatePatchBaselineOutput, err error) { + t.Helper() + require.NoError(t, err) + assert.Empty(t, updated.Description, "Description omitted under Replace=true must be nulled") + assert.Empty(t, updated.ApprovedPatches, "ApprovedPatches omitted under Replace=true must be nulled") + }, + }, + { + name: "replace_false_merges_omitted_fields", + update: func(baselineID *string) *ssmsdk.UpdatePatchBaselineInput { + return &ssmsdk.UpdatePatchBaselineInput{BaselineId: baselineID} + }, + check: func(t *testing.T, updated *ssmsdk.UpdatePatchBaselineOutput, err error) { + t.Helper() + require.NoError(t, err) + assert.Equal(t, "original description", aws.ToString(updated.Description)) + assert.Equal(t, []string{"KB123456"}, updated.ApprovedPatches) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := ssm.NewInMemoryBackend() + client := newTestSSMClient(t, ssm.NewHandler(backend)) + ctx := t.Context() + + created, err := client.CreatePatchBaseline(ctx, &ssmsdk.CreatePatchBaselineInput{ + Name: aws.String("replace-semantics-baseline"), + OperatingSystem: ssmtypes.OperatingSystemAmazonLinux2, + Description: aws.String("original description"), + ApprovedPatches: []string{"KB123456"}, + }) + require.NoError(t, err) + + updated, err := client.UpdatePatchBaseline(ctx, tc.update(created.BaselineId)) + tc.check(t, updated, err) + }) + } +} + +// TestUpdateMaintenanceWindowTarget_Replace_RealClient: Replace=true requires fields and +// nulls omitted ones; false merges (api_op_UpdateMaintenanceWindowTarget.go). +func TestUpdateMaintenanceWindowTarget_Replace_RealClient(t *testing.T) { + t.Parallel() + + tests := []struct { + update func(windowID, targetID *string) *ssmsdk.UpdateMaintenanceWindowTargetInput + check func(t *testing.T, updated *ssmsdk.UpdateMaintenanceWindowTargetOutput, err error) + name string + }{ + { + name: "replace_true_requires_targets", + update: func(windowID, targetID *string) *ssmsdk.UpdateMaintenanceWindowTargetInput { + return &ssmsdk.UpdateMaintenanceWindowTargetInput{ + WindowId: windowID, WindowTargetId: targetID, Replace: aws.Bool(true), + } + }, + check: func(t *testing.T, _ *ssmsdk.UpdateMaintenanceWindowTargetOutput, err error) { + t.Helper() + require.Error(t, err) + }, + }, + { + name: "replace_true_nulls_omitted_fields", + update: func(windowID, targetID *string) *ssmsdk.UpdateMaintenanceWindowTargetInput { + return &ssmsdk.UpdateMaintenanceWindowTargetInput{ + WindowId: windowID, + WindowTargetId: targetID, + Targets: []ssmtypes.Target{ + {Key: aws.String("InstanceIds"), Values: []string{"i-2222222222222222"}}, + }, + Replace: aws.Bool(true), + } + }, + check: func(t *testing.T, updated *ssmsdk.UpdateMaintenanceWindowTargetOutput, err error) { + t.Helper() + require.NoError(t, err) + assert.Empty(t, updated.Name, "Name omitted under Replace=true must be nulled") + assert.Empty(t, updated.OwnerInformation, "OwnerInformation omitted under Replace=true must be nulled") + }, + }, + { + name: "replace_false_merges_omitted_fields", + update: func(windowID, targetID *string) *ssmsdk.UpdateMaintenanceWindowTargetInput { + return &ssmsdk.UpdateMaintenanceWindowTargetInput{WindowId: windowID, WindowTargetId: targetID} + }, + check: func(t *testing.T, updated *ssmsdk.UpdateMaintenanceWindowTargetOutput, err error) { + t.Helper() + require.NoError(t, err) + assert.Equal(t, "original-name", aws.ToString(updated.Name)) + assert.Equal(t, "original-owner", aws.ToString(updated.OwnerInformation)) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := ssm.NewInMemoryBackend() + client := newTestSSMClient(t, ssm.NewHandler(backend)) + ctx := t.Context() + + mw, err := client.CreateMaintenanceWindow(ctx, &ssmsdk.CreateMaintenanceWindowInput{ + Name: aws.String("replace-semantics-window"), + Schedule: aws.String("cron(0 9 ? * MON *)"), + Duration: aws.Int32(2), + Cutoff: 1, + }) + require.NoError(t, err) + + registerInput := &ssmsdk.RegisterTargetWithMaintenanceWindowInput{ + WindowId: mw.WindowId, + ResourceType: ssmtypes.MaintenanceWindowResourceTypeInstance, + Targets: []ssmtypes.Target{ + {Key: aws.String("InstanceIds"), Values: []string{"i-1111111111111111"}}, + }, + Name: aws.String("original-name"), + OwnerInformation: aws.String("original-owner"), + } + + target, err := client.RegisterTargetWithMaintenanceWindow(ctx, registerInput) + require.NoError(t, err) + + updated, err := client.UpdateMaintenanceWindowTarget(ctx, tc.update(mw.WindowId, target.WindowTargetId)) + tc.check(t, updated, err) + }) + } +} + +// TestUpdateMaintenanceWindowTask_Replace_RealClient: Replace=true requires fields and +// nulls omitted ones; false merges (api_op_UpdateMaintenanceWindowTask.go). +func TestUpdateMaintenanceWindowTask_Replace_RealClient(t *testing.T) { + t.Parallel() + + tests := []struct { + update func(windowID, taskID *string) *ssmsdk.UpdateMaintenanceWindowTaskInput + check func(t *testing.T, updated *ssmsdk.UpdateMaintenanceWindowTaskOutput, err error) + name string + }{ + { + name: "replace_true_requires_task_arn", + update: func(windowID, taskID *string) *ssmsdk.UpdateMaintenanceWindowTaskInput { + return &ssmsdk.UpdateMaintenanceWindowTaskInput{ + WindowId: windowID, WindowTaskId: taskID, Replace: aws.Bool(true), + } + }, + check: func(t *testing.T, _ *ssmsdk.UpdateMaintenanceWindowTaskOutput, err error) { + t.Helper() + require.Error(t, err) + }, + }, + { + name: "replace_true_nulls_omitted_fields", + update: func(windowID, taskID *string) *ssmsdk.UpdateMaintenanceWindowTaskInput { + return &ssmsdk.UpdateMaintenanceWindowTaskInput{ + WindowId: windowID, + WindowTaskId: taskID, + TaskArn: aws.String("AWS-RunShellScript"), + Replace: aws.Bool(true), + } + }, + check: func(t *testing.T, updated *ssmsdk.UpdateMaintenanceWindowTaskOutput, err error) { + t.Helper() + require.NoError(t, err) + assert.Empty(t, updated.Name, "Name omitted under Replace=true must be nulled") + assert.Empty(t, updated.ServiceRoleArn, "ServiceRoleArn omitted under Replace=true must be nulled") + assert.Empty(t, updated.Priority, "Priority omitted under Replace=true must be nulled") + }, + }, + { + name: "replace_false_merges_omitted_fields", + update: func(windowID, taskID *string) *ssmsdk.UpdateMaintenanceWindowTaskInput { + return &ssmsdk.UpdateMaintenanceWindowTaskInput{WindowId: windowID, WindowTaskId: taskID} + }, + check: func(t *testing.T, updated *ssmsdk.UpdateMaintenanceWindowTaskOutput, err error) { + t.Helper() + require.NoError(t, err) + assert.Equal(t, "original-name", aws.ToString(updated.Name)) + assert.Equal(t, "arn:aws:iam::123456789012:role/OriginalRole", aws.ToString(updated.ServiceRoleArn)) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + backend := ssm.NewInMemoryBackend() + client := newTestSSMClient(t, ssm.NewHandler(backend)) + ctx := t.Context() + + mw, err := client.CreateMaintenanceWindow(ctx, &ssmsdk.CreateMaintenanceWindowInput{ + Name: aws.String("replace-semantics-window"), + Schedule: aws.String("cron(0 9 ? * MON *)"), + Duration: aws.Int32(2), + Cutoff: 1, + }) + require.NoError(t, err) + + task, err := client.RegisterTaskWithMaintenanceWindow(ctx, &ssmsdk.RegisterTaskWithMaintenanceWindowInput{ + WindowId: mw.WindowId, + TaskArn: aws.String("AWS-RunPowerShellScript"), + TaskType: ssmtypes.MaintenanceWindowTaskTypeRunCommand, + Name: aws.String("original-name"), + ServiceRoleArn: aws.String("arn:aws:iam::123456789012:role/OriginalRole"), + }) + require.NoError(t, err) + + updated, err := client.UpdateMaintenanceWindowTask(ctx, tc.update(mw.WindowId, task.WindowTaskId)) + tc.check(t, updated, err) + }) + } +} + +// TestJanitor_SweepsExpiredCommandHistory_RealClient covers the janitor's +// command-history sweep, which retains a terminal command independently of ExpiresAfter. +func TestJanitor_SweepsExpiredCommandHistory_RealClient(t *testing.T) { + t.Parallel() + + backend := ssm.NewInMemoryBackend().WithCommandHistoryRetention(10 * time.Millisecond) + client := newTestSSMClient(t, ssm.NewHandler(backend)) + ctx := t.Context() + + sent, sendErr := client.SendCommand(ctx, &ssmsdk.SendCommandInput{ + DocumentName: aws.String("AWS-RunShellScript"), + InstanceIds: []string{"i-1111"}, + }) + require.NoError(t, sendErr) + + cmdID := sent.Command.CommandId + + j := ssm.NewJanitor(backend, time.Minute) + + require.Eventually(t, func() bool { + j.SweepOnce(ctx) + + listOut, listErr := client.ListCommands(ctx, &ssmsdk.ListCommandsInput{CommandId: cmdID}) + require.NoError(t, listErr) + + return len(listOut.Commands) == 0 + }, 2*time.Second, 5*time.Millisecond) + + invOut, invErr := client.ListCommandInvocations(ctx, &ssmsdk.ListCommandInvocationsInput{CommandId: cmdID}) + require.NoError(t, invErr) + assert.Empty(t, invOut.CommandInvocations) +} diff --git a/services/ssm/store.go b/services/ssm/store.go index 16d21b98de..71164711ff 100644 --- a/services/ssm/store.go +++ b/services/ssm/store.go @@ -24,6 +24,9 @@ const ( // defaultCommandExpirySecs is the default TTL for SSM commands in seconds (1 hour). // AWS SSM commands expire after 1 hour by default. defaultCommandExpirySecs = 3600 + // defaultCommandHistoryRetentionSecs is the janitor's command-history + // window (30 days, running-commands.html), independent of ExpiresAfter. + defaultCommandHistoryRetentionSecs = 30 * 24 * 60 * 60 // maxHistoryCap is the maximum number of history entries retained per parameter. // Older entries beyond this cap are evicted to prevent unbounded growth. maxHistoryCap = 100 @@ -45,106 +48,108 @@ type KMSEncryptor interface { // InMemoryBackend implements StorageBackend using a concurrency-safe map. type InMemoryBackend struct { - kms KMSEncryptor - gcm cipher.AEAD - parameterPolicyNotifier ParameterPolicyNotifier - registry *store.Registry - parameters map[string]*store.Table[Parameter] - maintenanceWindows map[string]*store.Table[MaintenanceWindow] - maintenanceWindowTargets map[string]*store.Table[MaintenanceWindowTarget] - maintenanceWindowTasks map[string]*store.Table[MaintenanceWindowTask] - sessions map[string]*store.Table[Session] - accessRequests map[string]*store.Table[AccessRequest] - patchGroupToBaseline map[string]map[string]string - tags map[string]map[string]*tags.Tags - associations map[string]*store.Table[Association] - documentVersions map[string]map[string][]DocumentVersion - documentPermissions map[string]map[string][]string - documentSharedVersions map[string]map[string]map[string]string - commands map[string]*store.Table[Command] - commandInvocations map[string]map[string][]CommandInvocation - history map[string]map[string][]ParameterHistory - resourceDataSyncs map[string]*store.Table[ResourceDataSync] - documents map[string]*store.Table[Document] - opsItems map[string]*store.Table[OpsItem] - opsItemRelatedItems map[string]map[string][]OpsItemRelatedItem - opsMetadata map[string]*store.Table[OpsMetadata] - compliance map[string]map[string][]ComplianceItem - activations map[string]*store.Table[Activation] - cloudConnectors map[string]*store.Table[CloudConnector] - inventory map[string]map[string][]InventoryItem - associationExecutions map[string]map[string][]AssociationExecution - automationExecutions map[string]*store.Table[AutomationExecution] - serviceSettings map[string]*store.Table[ServiceSetting] - resourcePolicies map[string]map[string][]*ResourcePolicy - executionPreviews map[string]*store.Table[ExecutionPreview] - instancePatchStates map[string]*store.Table[InstancePatchState] - instancePatches map[string]map[string][]PatchComplianceData - instanceProperties map[string]*store.Table[InstanceProperty] - availablePatches map[string][]Patch - mu *lockmetrics.RWMutex - inventoryDeletions map[string][]InventoryDeletion - miscResourceTags map[string]map[string]map[string]string - resourceIDToOpsMetadataArn map[string]map[string]string - opsItemEvents map[string][]OpsItemEventSummary - parameterLabels map[string]map[string]map[int64][]string - associationExecTargets map[string]map[string][]AssociationExecutionTarget - patchBaselines map[string]*store.Table[PatchBaseline] - notifiedParameterPolicies map[string]map[string]map[string]struct{} - automationExecDelaySecs float64 - commandExecDelaySecs float64 - commandExpirySecs float64 - tableMu sync.Mutex + kms KMSEncryptor + gcm cipher.AEAD + parameterPolicyNotifier ParameterPolicyNotifier + registry *store.Registry + parameters map[string]*store.Table[Parameter] + maintenanceWindows map[string]*store.Table[MaintenanceWindow] + maintenanceWindowTargets map[string]*store.Table[MaintenanceWindowTarget] + maintenanceWindowTasks map[string]*store.Table[MaintenanceWindowTask] + sessions map[string]*store.Table[Session] + accessRequests map[string]*store.Table[AccessRequest] + patchGroupToBaseline map[string]map[string]string + tags map[string]map[string]*tags.Tags + associations map[string]*store.Table[Association] + documentVersions map[string]map[string][]DocumentVersion + documentPermissions map[string]map[string][]string + documentSharedVersions map[string]map[string]map[string]string + commands map[string]*store.Table[Command] + commandInvocations map[string]map[string][]CommandInvocation + history map[string]map[string][]ParameterHistory + resourceDataSyncs map[string]*store.Table[ResourceDataSync] + documents map[string]*store.Table[Document] + opsItems map[string]*store.Table[OpsItem] + opsItemRelatedItems map[string]map[string][]OpsItemRelatedItem + opsMetadata map[string]*store.Table[OpsMetadata] + compliance map[string]map[string][]ComplianceItem + activations map[string]*store.Table[Activation] + cloudConnectors map[string]*store.Table[CloudConnector] + inventory map[string]map[string][]InventoryItem + associationExecutions map[string]map[string][]AssociationExecution + automationExecutions map[string]*store.Table[AutomationExecution] + serviceSettings map[string]*store.Table[ServiceSetting] + resourcePolicies map[string]map[string][]*ResourcePolicy + executionPreviews map[string]*store.Table[ExecutionPreview] + instancePatchStates map[string]*store.Table[InstancePatchState] + instancePatches map[string]map[string][]PatchComplianceData + instanceProperties map[string]*store.Table[InstanceProperty] + availablePatches map[string][]Patch + mu *lockmetrics.RWMutex + inventoryDeletions map[string][]InventoryDeletion + miscResourceTags map[string]map[string]map[string]string + resourceIDToOpsMetadataArn map[string]map[string]string + opsItemEvents map[string][]OpsItemEventSummary + parameterLabels map[string]map[string]map[int64][]string + associationExecTargets map[string]map[string][]AssociationExecutionTarget + patchBaselines map[string]*store.Table[PatchBaseline] + notifiedParameterPolicies map[string]map[string]map[string]struct{} + automationExecDelaySecs float64 + commandExecDelaySecs float64 + commandExpirySecs float64 + commandHistoryRetentionSecs float64 + tableMu sync.Mutex } // NewInMemoryBackend creates a new empty InMemoryBackend. func NewInMemoryBackend() *InMemoryBackend { b := &InMemoryBackend{ - gcm: newInstanceGCM(), - registry: store.NewRegistry(), - parameters: make(map[string]*store.Table[Parameter]), - history: make(map[string]map[string][]ParameterHistory), - tags: make(map[string]map[string]*tags.Tags), - documents: make(map[string]*store.Table[Document]), - documentVersions: make(map[string]map[string][]DocumentVersion), - documentPermissions: make(map[string]map[string][]string), - documentSharedVersions: make(map[string]map[string]map[string]string), - commands: make(map[string]*store.Table[Command]), - commandInvocations: make(map[string]map[string][]CommandInvocation), - activations: make(map[string]*store.Table[Activation]), - cloudConnectors: make(map[string]*store.Table[CloudConnector]), - associations: make(map[string]*store.Table[Association]), - maintenanceWindows: make(map[string]*store.Table[MaintenanceWindow]), - maintenanceWindowTargets: make(map[string]*store.Table[MaintenanceWindowTarget]), - maintenanceWindowTasks: make(map[string]*store.Table[MaintenanceWindowTask]), - sessions: make(map[string]*store.Table[Session]), - accessRequests: make(map[string]*store.Table[AccessRequest]), - patchGroupToBaseline: make(map[string]map[string]string), - opsItems: make(map[string]*store.Table[OpsItem]), - opsItemRelatedItems: make(map[string]map[string][]OpsItemRelatedItem), - opsMetadata: make(map[string]*store.Table[OpsMetadata]), - patchBaselines: make(map[string]*store.Table[PatchBaseline]), - inventory: make(map[string]map[string][]InventoryItem), - compliance: make(map[string]map[string][]ComplianceItem), - resourceDataSyncs: make(map[string]*store.Table[ResourceDataSync]), - parameterLabels: make(map[string]map[string]map[int64][]string), - automationExecutions: make(map[string]*store.Table[AutomationExecution]), - serviceSettings: make(map[string]*store.Table[ServiceSetting]), - resourcePolicies: make(map[string]map[string][]*ResourcePolicy), - executionPreviews: make(map[string]*store.Table[ExecutionPreview]), - instancePatchStates: make(map[string]*store.Table[InstancePatchState]), - instancePatches: make(map[string]map[string][]PatchComplianceData), - instanceProperties: make(map[string]*store.Table[InstanceProperty]), - availablePatches: make(map[string][]Patch), - commandExpirySecs: defaultCommandExpirySecs, - mu: lockmetrics.New("ssm"), - resourceIDToOpsMetadataArn: make(map[string]map[string]string), - miscResourceTags: make(map[string]map[string]map[string]string), - opsItemEvents: make(map[string][]OpsItemEventSummary), - associationExecutions: make(map[string]map[string][]AssociationExecution), - associationExecTargets: make(map[string]map[string][]AssociationExecutionTarget), - inventoryDeletions: make(map[string][]InventoryDeletion), - notifiedParameterPolicies: make(map[string]map[string]map[string]struct{}), + gcm: newInstanceGCM(), + registry: store.NewRegistry(), + parameters: make(map[string]*store.Table[Parameter]), + history: make(map[string]map[string][]ParameterHistory), + tags: make(map[string]map[string]*tags.Tags), + documents: make(map[string]*store.Table[Document]), + documentVersions: make(map[string]map[string][]DocumentVersion), + documentPermissions: make(map[string]map[string][]string), + documentSharedVersions: make(map[string]map[string]map[string]string), + commands: make(map[string]*store.Table[Command]), + commandInvocations: make(map[string]map[string][]CommandInvocation), + activations: make(map[string]*store.Table[Activation]), + cloudConnectors: make(map[string]*store.Table[CloudConnector]), + associations: make(map[string]*store.Table[Association]), + maintenanceWindows: make(map[string]*store.Table[MaintenanceWindow]), + maintenanceWindowTargets: make(map[string]*store.Table[MaintenanceWindowTarget]), + maintenanceWindowTasks: make(map[string]*store.Table[MaintenanceWindowTask]), + sessions: make(map[string]*store.Table[Session]), + accessRequests: make(map[string]*store.Table[AccessRequest]), + patchGroupToBaseline: make(map[string]map[string]string), + opsItems: make(map[string]*store.Table[OpsItem]), + opsItemRelatedItems: make(map[string]map[string][]OpsItemRelatedItem), + opsMetadata: make(map[string]*store.Table[OpsMetadata]), + patchBaselines: make(map[string]*store.Table[PatchBaseline]), + inventory: make(map[string]map[string][]InventoryItem), + compliance: make(map[string]map[string][]ComplianceItem), + resourceDataSyncs: make(map[string]*store.Table[ResourceDataSync]), + parameterLabels: make(map[string]map[string]map[int64][]string), + automationExecutions: make(map[string]*store.Table[AutomationExecution]), + serviceSettings: make(map[string]*store.Table[ServiceSetting]), + resourcePolicies: make(map[string]map[string][]*ResourcePolicy), + executionPreviews: make(map[string]*store.Table[ExecutionPreview]), + instancePatchStates: make(map[string]*store.Table[InstancePatchState]), + instancePatches: make(map[string]map[string][]PatchComplianceData), + instanceProperties: make(map[string]*store.Table[InstanceProperty]), + availablePatches: make(map[string][]Patch), + commandExpirySecs: defaultCommandExpirySecs, + commandHistoryRetentionSecs: defaultCommandHistoryRetentionSecs, + mu: lockmetrics.New("ssm"), + resourceIDToOpsMetadataArn: make(map[string]map[string]string), + miscResourceTags: make(map[string]map[string]map[string]string), + opsItemEvents: make(map[string][]OpsItemEventSummary), + associationExecutions: make(map[string]map[string][]AssociationExecution), + associationExecTargets: make(map[string]map[string][]AssociationExecutionTarget), + inventoryDeletions: make(map[string][]InventoryDeletion), + notifiedParameterPolicies: make(map[string]map[string]map[string]struct{}), } b.registerDefaultDocuments(defaultRegion) @@ -171,6 +176,16 @@ func (b *InMemoryBackend) WithCommandTTL(d time.Duration) *InMemoryBackend { return b } +// WithCommandHistoryRetention sets the janitor's command-history window, +// independent of WithCommandTTL. A zero or negative value keeps the default. +func (b *InMemoryBackend) WithCommandHistoryRetention(d time.Duration) *InMemoryBackend { + if d > 0 { + b.commandHistoryRetentionSecs = d.Seconds() + } + + return b +} + // WithCommandExecDelay sets how long a SendCommand invocation stays in the // InProgress state before completing. The default of zero means commands // complete synchronously (fast). A positive delay makes the InProgress window diff --git a/services/ssm/update_omitted_members_preserve_state_test.go b/services/ssm/update_omitted_members_preserve_state_test.go index 6c286e707e..5ff6f35710 100644 --- a/services/ssm/update_omitted_members_preserve_state_test.go +++ b/services/ssm/update_omitted_members_preserve_state_test.go @@ -19,6 +19,9 @@ import ( // omitted a field silently blanked it instead of leaving the stored value // alone. Each case creates a resource, sets a field, then sends a second // update that omits it and asserts the earlier value survived. +// +// UpdateAssociation is deliberately not a case here: AWS nulls its omitted +// fields instead. See TestUpdateAssociation_ReplacesOmittedFields_RealClient. func TestUpdate_OmittedMembersPreserveState(t *testing.T) { t.Parallel() @@ -26,7 +29,6 @@ func TestUpdate_OmittedMembersPreserveState(t *testing.T) { run func(t *testing.T) name string }{ - {name: "association_max_concurrency", run: testAssociationMaxConcurrencyPreserved}, {name: "cloud_connector_display_name", run: testCloudConnectorDisplayNamePreserved}, {name: "document_display_name", run: testDocumentDisplayNamePreserved}, {name: "maintenance_window_description_and_zero_cutoff", run: testMaintenanceWindowFieldsPreserved}, @@ -44,38 +46,6 @@ func TestUpdate_OmittedMembersPreserveState(t *testing.T) { } } -func testAssociationMaxConcurrencyPreserved(t *testing.T) { - t.Helper() - - backend := ssm.NewInMemoryBackend() - client := newTestSSMClient(t, ssm.NewHandler(backend)) - ctx := t.Context() - - created, err := client.CreateAssociation(ctx, &ssmsdk.CreateAssociationInput{ - Name: aws.String("AWS-RunShellScript"), - InstanceId: aws.String("i-omit-1"), - }) - require.NoError(t, err) - - assocID := created.AssociationDescription.AssociationId - - withVal, err := client.UpdateAssociation(ctx, &ssmsdk.UpdateAssociationInput{ - AssociationId: assocID, - MaxConcurrency: aws.String("50%"), - }) - require.NoError(t, err) - assert.Equal(t, "50%", aws.ToString(withVal.AssociationDescription.MaxConcurrency)) - - withoutVal, err := client.UpdateAssociation(ctx, &ssmsdk.UpdateAssociationInput{ - AssociationId: assocID, - ScheduleExpression: aws.String("rate(1 day)"), - }) - require.NoError(t, err) - assert.Equal(t, "50%", aws.ToString(withoutVal.AssociationDescription.MaxConcurrency), - "MaxConcurrency must survive an update that omits it") - assert.Equal(t, "rate(1 day)", aws.ToString(withoutVal.AssociationDescription.ScheduleExpression)) -} - func testCloudConnectorDisplayNamePreserved(t *testing.T) { t.Helper() diff --git a/services/ssoadmin/instances.go b/services/ssoadmin/instances.go index bf6d4e239f..61cf4066c5 100644 --- a/services/ssoadmin/instances.go +++ b/services/ssoadmin/instances.go @@ -78,6 +78,7 @@ func (b *InMemoryBackend) ListInstances() []*Instance { inst.Status = instanceStatusActive } cp := *inst + cp.Tags = maps.Clone(inst.Tags) list = append(list, &cp) } diff --git a/services/stepfunctions/PARITY.md b/services/stepfunctions/PARITY.md index 80612ddd52..2dacbd021e 100644 --- a/services/stepfunctions/PARITY.md +++ b/services/stepfunctions/PARITY.md @@ -394,23 +394,93 @@ families: (or a legacy Iterator-style Map, which has no ProcessorConfig at all) takes the pre-existing runMapTasks path unconditionally. - DISCLOSED, not modeled (deliberately out of this pass's scope): - ResultWriter's per-item S3 export records (exportMapResults, - asl/result_writer.go) still omit ExecutionArn/Name/StartDate/StopDate - even though real child executions now exist to source them from -- - wiring that through was judged not to "fall out cheaply" (it would - need exportMapResults, which only sees results/errs, to also see the - per-item child Execution records) and was left for a future pass - rather than attempted here. WriterConfig (Transformation/OutputType) - remains parsed but unapplied, unchanged from the prior pass. A - DISTRIBUTED Map Run's parent MapRun *resource* record (as opposed to - its child Execution records, which do persist -- see Execution. - MapRunArn/ItemCount in persistence.go) is still not part of - backendSnapshot at all -- a pre-existing gap predating this pass - (versions/aliases/mapRuns have never been persisted here), so a - restored backend loses DescribeMapRun/ListMapRuns/ - ListExecutions(mapRunArn=...) access to a Map Run whose children - otherwise survive the restore intact. + FIXED 2026-09-26 (WriterConfig sweep), correcting the prior pass's + "DISCLOSED, not modeled" note below: ResultWriter's per-item S3 export + records omitting ExecutionArn/Name/StartDate/StopDate, and + WriterConfig (Transformation/OutputType) being parsed but unapplied, + are both fixed. asl.DistributedMapRunner.RunDistributedMapItem now + returns a DistributedMapItemResult (Output plus ExecutionArn/Name/ + StartDate/StopDate) instead of a bare `any`, threaded through + runDistributedMapTasks into a new `meta []DistributedMapItemResult` + slice that exportMapResults uses to populate Transformation: NONE + records -- populated only for DISTRIBUTED Map items (real child + Executions exist to source it from); INLINE Map iterations still + correctly leave those fields empty, having no such resource. + WriterConfig.Transformation (NONE: full metadata record with + JSON-stringified Input/Output, matching a real DescribeExecution; + COMPACT: raw per-item output; FLATTEN: COMPACT plus splicing any + array output into the outer array) and OutputType (JSON: array; + JSONL: newline-delimited, no enclosing array) are now applied to the + S3-exported SUCCEEDED_n.json/FAILED_n.json files AND to the no-export + preview output (ResultWriter with WriterConfig but no Resource/ + Parameters, AWS's documented "preview the formatted output" shape). + Per AWS's documented note ("If a child workflow execution fails, Step + Functions returns its execution result unchanged"), a FAILED item's + record is always the full NONE-shaped record regardless of + Transformation -- verified via + TestDistributedMapResultWriter_FailedItemsKeepFullRecord. Verified + against input-output-resultwriter.html for the exact Transformation/ + OutputType semantics; see + TestDistributedMapResultWriter_TransformationOutputType (all 6 + Transformation x OutputType combinations, real SDK client + wired + in-process S3) and TestDistributedMapResultWriter_ + DistributedChildIdentity. A DISTRIBUTED Map Run's parent MapRun + *resource* record (as opposed to its child Execution records, which + do persist -- see Execution.MapRunArn/ItemCount in persistence.go) is + still not part of backendSnapshot at all -- a pre-existing gap + predating this pass (versions/aliases/mapRuns have never been + persisted here), so a restored backend loses DescribeMapRun/ + ListMapRuns/ListExecutions(mapRunArn=...) access to a Map Run whose + children otherwise survive the restore intact. + + 2026-09-26 (WriterConfig sweep, new finding, not fixed this pass): + re-reading input-output-itemreader.html surfaced that ItemReader only + ever supports Resource=arn:aws:states:::s3:getObject with InputType + JSON/JSON Lines/CSV against a single object -- Resource= + arn:aws:states:::s3:listObjectsV2 (bucket/prefix metadata iteration, + optionally with Transformation=LOAD_AND_FLATTEN), InputType=MANIFEST + (ManifestType ATHENA_DATA/S3_INVENTORY), and InputType=PARQUET are + all real, documented ItemReader shapes with no code path here at all + -- resolveItemsFromReader never inspects ItemReader.Resource, and + decodeReaderItems' InputType switch has no MANIFEST/PARQUET case. + This was never previously documented in this file (grepped: no prior + mention of ListObjectsV2/ManifestType/PARQUET anywhere in this + PARITY.md's history). Not attempted that pass. + + FIXED 2026-09-26 (ItemReader Resource sweep), closing most of the + above: resolveItemsFromReader now switches on ItemReader.Resource. + arn:aws:states:::s3:listObjectsV2 lists the bucket/prefix (a new + asl.S3ListReader interface, paginating via s3Adapter. + ListObjectsV2Items against services/s3.StorageBackend.ListObjectsV2) + and returns one item per object -- {Etag,Key,LastModified,Size, + StorageClass}, LastModified as epoch seconds via pkgs/awstime.Epoch, + matching the docs' example shape exactly. ReaderConfig.Transformation + LOAD_AND_FLATTEN (new field) instead reads and decodes each listed + object's content per InputType (JSON/JSONL/CSV; zero-byte + trailing-slash "folder" keys are skipped, since they have no content + to decode) and flattens every object's items into one array, per + "Processing nested data sets" in the docs. ReaderConfig.ManifestType + (new field) S3_INVENTORY -- and the legacy bare InputType=MANIFEST, + which the docs' own example uses without ManifestType set, treated as + identical -- reads a manifest.json (fileSchema, files[].key), fetches + each listed CSV data file (gzip-decompressed when the key ends + .gz, via compress/gzip), and decodes it with fileSchema's + comma-separated column names as CSV headers, matching the docs' + worked example's field names and values (TestItemReader_S3Manifest). + All ItemReader + failures (S3 NoSuchBucket/NoSuchKey, unsupported Resource, unsupported + ManifestType/InputType) are now wrapped as a States.ItemReaderFailed + FailError instead of falling back to the error's raw Go string as the + Catch-match code -- AWS's own documented predefined error name for + this failure class, so a Map state's Catch can now match it + specifically instead of only via States.ALL. ManifestType=ATHENA_DATA + and InputType=PARQUET are explicitly rejected with dedicated sentinel + errors (ErrAthenaManifestUnsupported/ErrParquetUnsupported) rather + than silently mis-parsed -- see the narrowed items_still_open entry. + Verified via TestItemReader_S3ListObjectsV2/TestItemReader_S3Manifest/ + TestItemReader_S3GetObject_Errors, all driven through the real + aws-sdk-go-v2 sfn client with objects seeded in the in-process S3 + backend. asl_parallel: status: ok note: "Unchanged this pass." @@ -452,7 +522,7 @@ families: filter_semantics: {status: ok, note: "gopherstack-uox6 (value-semantics sweep, 2026-08-30): this service establishes no prior sweep of this kind. First, its protocol: aws-sdk-go-v2/service/sfn@v1.45.4's types package has NO Filter struct at all (grep of types/types.go) -- this API surface has almost no server-side filtering. The one real filter is ListExecutionsInput.StatusFilter (types.ExecutionStatus, a single-value equality field, not a list), applied at executions.go:643 via an exact bucket lookup -- no documented modifier to get wrong. Everything else this service's ~14 hand-rolled 'match' helpers implement is Amazon States Language Choice-state comparators (asl/executor.go), which decide whether a state's input satisfies a rule, not an SDK list filter, but the same right-field-wrong-algorithm risk applies: evaluateChoiceRule's And/Or/Not (correct all/any/negate), IsPresent/IsNull/IsString/IsNumeric/IsBoolean/IsTimestamp (each compares a computed bool against *rule.IsX with ==, correctly honoring both true and false rather than only checking truthiness), and the String/Numeric/Boolean/Timestamp -Equals/-LessThan/-GreaterThan/-LessThanEquals/-GreaterThanEquals families (each Path and literal variant) were all read and are correct. stringMatchesPattern/globMatch (StringMatches) is the one genuine wildcard comparator in this family -- verified against the ASL spec's documented semantics (its own doc comment: '*' matches zero or more chars, backslash escapes the next character, anchored both ends) via a real two-pointer backtracking implementation; correct, including the escape case. No bugs found -- clean verdict."} gaps: [] items_still_open: - - "Map Distributed Map ResultWriter's WriterConfig (Transformation/OutputType) is parsed but not applied, only the plain S3-export shape; per-item result records still omit ExecutionArn/Name/StartDate/StopDate (bd: gopherstack-8j8). Real child Execution records now exist for DISTRIBUTED Map (bd: gopherstack-zov6, this pass) but exportMapResults was deliberately not wired to source those fields from them -- disclosed, not modeled, see asl_map family note." + - "2026-09-26 (ItemReader gap-closure sweep), narrowed further: CSVDelimiter (COMMA default/PIPE/SEMICOLON/SPACE/TAB, ReaderConfig field, applied to both the plain s3:getObject CSV path and S3_INVENTORY manifest data files) and ItemsPointer (RFC 6901 JSON Pointer selecting a nested array within a JSON InputType file, e.g. '/data/items') are now implemented -- see the 2026-09-26 ItemReader gap-closure sweep note. CSVHeaderLocation (FIRST_ROW/GIVEN+CSVHeaders) and MaxItems/MaxItemsPath were already correctly wired before this pass (TestDecodeReaderItems, TestExecutor_ItemReaderMaxItemsPath) and needed no change. Two real gaps remain, both explicitly disclosed via distinct sentinel errors rather than silently mis-decoding: (1) ManifestType=ATHENA_DATA (asl.ErrAthenaManifestUnsupported) -- input-output-itemreader.html describes the manifest only as 'a structured CSV list of the data files', which is not precise enough to implement against confidently (Athena's own UNLOAD manifest format elsewhere is JSON, not CSV), and the doc's $states.context.Map.Item.Source addition for this mode is unmodeled too; (2) InputType=PARQUET (asl.ErrParquetUnsupported) -- no pure-Go Parquet reader dependency exists in go.mod, and adding one is out of scope (explicitly disallowed for this pass too). No bd filed yet for either." - "STALE, corrected this pass (bd: gopherstack-zov6): this line previously read 'Map ItemProcessor.ProcessorConfig.Mode (INLINE/DISTRIBUTED) not parsed/validated (bd: gopherstack-8im)' -- Mode/ExecutionType parsing and validation (parser.go) were already done before this pass; what was actually missing was Mode being acted on. FIXED: a DISTRIBUTED Map state now spawns a real child Execution per item/batch instead of running inline (see asl_map family note). Genuinely still open: DescribeMapRun/ListMapRuns/ListExecutions(mapRunArn=...) lose access to a Map Run after a backend restore, because the MapRun *resource* table (unlike its child Execution records, which do persist) has never been part of backendSnapshot -- a pre-existing gap, not introduced this pass." - "STALE, corrected 2026-09-11 (bd: gopherstack-1sf): StartExecutionInput has no ClientRequestToken member in the real SDK, so there was nothing to model there. FIXED: EXPRESS name reuse is now immediate (uniqueness check skipped for EXPRESS), and STANDARD reuse of a still-RUNNING execution's name with matching Input now returns that execution (idempotent) instead of erroring; differing Input or a closed execution still conflicts. See the StartExecution note above and Test_StartExecution_NameReuseSemantics." - "StartExecution's STANDARD name-reuse conflict does not expire: AWS allows reusing a closed execution's name 90 days after it closes, but this emulator conflicts on any existing name regardless of how long it has been closed (no notion of elapsed wall-clock time since close) -- disclosed, not modeled (bd: gopherstack-1sf)" @@ -467,6 +537,158 @@ leaks: {status: clean, note: "StopExecution/DeleteStateMachine cancel the execut ## Notes +### 2026-09-26 ItemReader gap-closure sweep (CSVDelimiter, ItemsPointer) + +Follow-up to the ItemReader Resource sweep below, which flagged CSVDelimiter +and ItemsPointer as newly-discovered, still-unimplemented ReaderConfig +fields. Re-read input-output-itemreader.html for both fields' exact +semantics and implemented: + +- **`ReaderConfig.CSVDelimiter`** (new field): `COMMA` (default), `PIPE`, + `SEMICOLON`, `SPACE`, `TAB`, case-insensitive; an unrecognized value is a + `States.ItemReaderFailed` error rather than silently falling back to + comma. Wired into `decodeCSVItems` via `csv.Reader.Comma`, and threaded + through to `S3_INVENTORY` manifest data files too (the docs: "You can + specify this field when InputType is CSV or MANIFEST") -- previously + `resolveS3InventoryManifest` built its own `fileCfg` with no way to carry + the outer `ReaderConfig`'s delimiter through, so it's now passed the full + `cfg` and copies `CSVDelimiter` onto `fileCfg`. +- **`ReaderConfig.ItemsPointer`** (new field): an RFC 6901 JSON Pointer + (`/data/items`, forward-slash-separated, numeric array indices, `~1`/`~0` + escapes) selecting a nested array within a JSON `InputType` file, per the + docs' example (`{"data": {"items": [...]}}` -> `"/data/items"`). Resolving + to anything other than a JSON array (an object, scalar, or a path that + doesn't exist) is a `States.ItemReaderFailed` error. Only applies to + `InputType: JSON` (or omitted, its default); `JSONL`/`CSV` are unaffected + and still auto-detect as before when `ItemsPointer` is unset. + +Verified CSVHeaderLocation (`FIRST_ROW`/`GIVEN`+`CSVHeaders`) and +MaxItems/MaxItemsPath were already correctly implemented and tested +(`TestDecodeReaderItems`'s `csv_first_row_header`/`csv_given_headers`/ +`csv_max_items_truncates` cases, `TestExecutor_ItemReaderMaxItemsPath`) -- +no changes needed there. + +InputType=PARQUET and ManifestType=ATHENA_DATA remain unimplemented, +unchanged from the prior sweep: no pure-Go Parquet reader dependency exists +in `go.mod` and adding one was out of scope (explicitly disallowed for this +pass), and ATHENA_DATA's manifest format isn't documented precisely enough +to implement against confidently. Both still fail with their existing +dedicated sentinel errors (`ErrParquetUnsupported`/ +`ErrAthenaManifestUnsupported`), not silently. + +New table-driven cases in `TestDecodeReaderItems` +(`asl/intrinsics_extras_test.go`): CSV with `PIPE`/lowercase `semicolon`/ +`TAB`/`SPACE` delimiters, an unsupported delimiter error, `ItemsPointer` +selecting a nested array (including a path segment that indexes into an +array), and error cases (points at a non-array, path doesn't exist, path +doesn't start with `/`). New SDK-roundtrip tests: a `CSVDelimiter: PIPE` +case added to `TestItemReader_S3Manifest` (delimiter carried through to the +manifest's data file), and a new `TestItemReader_ItemsPointer` +(`item_reader_s3_resource_test.go`, nested-array selection and the +not-an-array failure, both driven through the real +`aws-sdk-go-v2/service/sfn` client with the in-process S3 backend). + +Gates green: `gofmt`, `go build ./...`, `go vet ./services/stepfunctions/...`, +`go test -race -count=1` (this package), `golangci-lint run` (0 findings), +`go test ./pkgs/persistence/`, `go run ./cmd/parityfmtcheck -dir services`. +No `go.mod`/`go.sum` changes. + +### 2026-09-26 ItemReader Resource sweep (listObjectsV2, MANIFEST, LOAD_AND_FLATTEN) + +Closed most of the `items_still_open` gap the WriterConfig sweep below found +in the same session: ItemReader ignored `ItemReader.Resource` entirely and +only ever did a single `s3:getObject` decoded as JSON/JSON Lines/CSV. Read +input-output-itemreader.html end to end for the exact item shapes and error +behavior. Implemented: + +- **Resource `arn:aws:states:::s3:listObjectsV2`**: a new `asl.S3ListReader` + interface (`ListObjectsV2Items`), implemented by the existing `s3Adapter` + against `services/s3.StorageBackend.ListObjectsV2`, paginating via + `ContinuationToken` until exhausted. Default mode returns one item per + object: `{"Etag","Key","LastModified","Size","StorageClass"}`, matching + the docs' example exactly (`LastModified` as epoch seconds via + `pkgs/awstime.Epoch`). +- **`ReaderConfig.Transformation: LOAD_AND_FLATTEN`** (new field): reads and + decodes each listed object's content per `InputType` (JSON/JSONL/CSV) and + flattens every object's records into one item array, per the docs' + "Processing nested data sets" section. Zero-byte keys ending in `/` (S3 + console folder placeholders) are skipped, since they have no content. +- **`ReaderConfig.ManifestType: S3_INVENTORY`** (new field), and the legacy + bare `InputType: MANIFEST` the docs' own worked example uses without + `ManifestType` set (treated identically): reads a `manifest.json` + (`fileSchema`, `files[].key`), fetches each listed CSV data file + (gzip-decompressed when the key ends `.gz`), and decodes it using + `fileSchema`'s comma-separated column names as CSV headers. +- **Error behavior**: every ItemReader failure (missing bucket/key, + unsupported `Resource`, unsupported `ManifestType`/`InputType`) is now + wrapped as a `States.ItemReaderFailed` `FailError` -- AWS's own documented + predefined error name for this failure class -- instead of leaking the + raw Go error string as the Catch-match code. A Map state's `Catch` can now + match `States.ItemReaderFailed` specifically, not only via `States.ALL`. + +Not implemented, each behind a dedicated sentinel error rather than a silent +stub (see the narrowed `items_still_open` entry and the `asl_map` family +note): `ManifestType: ATHENA_DATA` (the docs describe its manifest only as +"a structured CSV list of the data files", which isn't precise enough to +implement against confidently -- Athena's own UNLOAD manifest format is +documented elsewhere as JSON, not CSV -- and `$states.context.Map.Item.Source` +is unmodeled too); `InputType: PARQUET` (no pure-Go Parquet reader dependency +exists in `go.mod`, and this pass does not add one, per instructions). +`CSVDelimiter` and `ItemsPointer` remain unparsed (`ReaderConfig` has no +fields for either) -- discovered while reading the docs for this pass but +out of the four originally-recorded gaps, so left as-is and disclosed above +rather than silently addressed. STALE, corrected same-day by the 2026-09-26 +ItemReader gap-closure sweep above: both are now implemented. + +New table-driven tests, driven through a real `aws-sdk-go-v2/service/sfn` +client over `httptest` with objects seeded in the in-process S3 backend +(`item_reader_s3_resource_test.go`): `TestItemReader_S3ListObjectsV2` +(metadata mode, LOAD_AND_FLATTEN JSON, LOAD_AND_FLATTEN CSV, missing-bucket +error), `TestItemReader_S3Manifest` (S3_INVENTORY with a gzip data file, the +legacy `InputType: MANIFEST` alias, and the ATHENA_DATA gap), and +`TestItemReader_S3GetObject_Errors` (missing key, the PARQUET gap). + +Gates green: `gofmt`, `go build ./...`, `go vet ./services/stepfunctions/...`, +`go test -race -count=1` (this package), `golangci-lint run` (0 findings), +`go run ./cmd/parityfmtcheck -dir services`. No `go.mod`/`go.sum` changes. +`go test ./pkgs/persistence/` fails on this branch, but only on a +pre-existing `services/sqs` snapshot-version-guard finding from a different, +concurrently-in-progress change to that package -- unrelated to this sweep +and `services/sqs` was not touched here. + +### 2026-09-26 Distributed Map ResultWriter WriterConfig sweep + +Implemented ResultWriter.WriterConfig (Transformation: NONE/COMPACT/FLATTEN, +OutputType: JSON/JSONL), the item this file's own `items_still_open` named +as the open gap, per input-output-resultwriter.html. Also threaded real +DISTRIBUTED Map child-execution identity (ExecutionArn/Name/StartDate/ +StopDate) into NONE-transformation records via a new +`asl.DistributedMapItemResult` return type on `DistributedMapRunner. +RunDistributedMapItem` (previously a bare `any`) -- the other half of the +same gap, closing gopherstack-8j8's remaining scope. See the `asl_map` +family note for the full before/after and the new +`TestDistributedMapResultWriter_*` tests (table-driven over all 6 +Transformation x OutputType combinations, plus FAILED-item and DISTRIBUTED- +identity cases, all driven through the real aws-sdk-go-v2 sfn client with +the in-process S3 backend wired). + +Also read input-output-itemreader.html and input-output-itembatcher.html +end to end per this sweep's brief. ItemBatcher and ToleratedFailureCount/ +ToleratedFailurePercentage (and their `*Path` siblings) were already +correctly implemented -- no changes needed there. Found, but did not fix, +that ItemReader has never supported `Resource: arn:aws:states::: +s3:listObjectsV2`, `InputType: MANIFEST`, or `InputType: PARQUET` -- only +`s3:getObject` with JSON/JSON Lines/CSV. This was not previously documented +anywhere in this file; recorded in `items_still_open` and the `asl_map` +family note rather than attempted, since ListObjectsV2 needs a wire-shape +citation this pass didn't chase down and MANIFEST/PARQUET are meaningfully +larger builds (a manifest-driven GetObject fan-out; a binary columnar +decoder) than fit this pass's scope. + +Gates green: `gofmt`, `go build ./...`, `go vet`, `go test -race` (this +package), `golangci-lint run` (0 findings), `go test ./pkgs/persistence/`, +`cmd/parityfmtcheck`. No `go.mod`/`go.sum` changes. + ### 2026-09-24 perf sweep ListExecutions/ListExecutionsByMapRun value-copied+sorted every matching diff --git a/services/stepfunctions/README.md b/services/stepfunctions/README.md index 73bfd03589..f74ab245ef 100644 --- a/services/stepfunctions/README.md +++ b/services/stepfunctions/README.md @@ -15,7 +15,7 @@ ### Known gaps -- Map Distributed Map ResultWriter's WriterConfig (Transformation/OutputType) is parsed but not applied, only the plain S3-export shape; per-item result records still omit ExecutionArn/Name/StartDate/StopDate (bd: gopherstack-8j8). Real child Execution records now exist for DISTRIBUTED Map (bd: gopherstack-zov6, this pass) but exportMapResults was deliberately not wired to source those fields from them -- disclosed, not modeled, see asl_map family note. +- 2026-09-26 (ItemReader gap-closure sweep), narrowed further: CSVDelimiter (COMMA default/PIPE/SEMICOLON/SPACE/TAB, ReaderConfig field, applied to both the plain s3:getObject CSV path and S3_INVENTORY manifest data files) and ItemsPointer (RFC 6901 JSON Pointer selecting a nested array within a JSON InputType file, e.g. '/data/items') are now implemented -- see the 2026-09-26 ItemReader gap-closure sweep note. CSVHeaderLocation (FIRST_ROW/GIVEN+CSVHeaders) and MaxItems/MaxItemsPath were already correctly wired before this pass (TestDecodeReaderItems, TestExecutor_ItemReaderMaxItemsPath) and needed no change. Two real gaps remain, both explicitly disclosed via distinct sentinel errors rather than silently mis-decoding: (1) ManifestType=ATHENA_DATA (asl.ErrAthenaManifestUnsupported) -- input-output-itemreader.html describes the manifest only as 'a structured CSV list of the data files', which is not precise enough to implement against confidently (Athena's own UNLOAD manifest format elsewhere is JSON, not CSV), and the doc's $states.context.Map.Item.Source addition for this mode is unmodeled too; (2) InputType=PARQUET (asl.ErrParquetUnsupported) -- no pure-Go Parquet reader dependency exists in go.mod, and adding one is out of scope (explicitly disallowed for this pass too). No bd filed yet for either. - STALE, corrected this pass (bd: gopherstack-zov6): this line previously read 'Map ItemProcessor.ProcessorConfig.Mode (INLINE/DISTRIBUTED) not parsed/validated (bd: gopherstack-8im)' -- Mode/ExecutionType parsing and validation (parser.go) were already done before this pass; what was actually missing was Mode being acted on. FIXED: a DISTRIBUTED Map state now spawns a real child Execution per item/batch instead of running inline (see asl_map family note). Genuinely still open: DescribeMapRun/ListMapRuns/ListExecutions(mapRunArn=...) lose access to a Map Run after a backend restore, because the MapRun *resource* table (unlike its child Execution records, which do persist) has never been part of backendSnapshot -- a pre-existing gap, not introduced this pass. - STALE, corrected 2026-09-11 (bd: gopherstack-1sf): StartExecutionInput has no ClientRequestToken member in the real SDK, so there was nothing to model there. FIXED: EXPRESS name reuse is now immediate (uniqueness check skipped for EXPRESS), and STANDARD reuse of a still-RUNNING execution's name with matching Input now returns that execution (idempotent) instead of erroring; differing Input or a closed execution still conflicts. See the StartExecution note above and Test_StartExecution_NameReuseSemantics. - StartExecution's STANDARD name-reuse conflict does not expire: AWS allows reusing a closed execution's name 90 days after it closes, but this emulator conflicts on any existing name regardless of how long it has been closed (no notion of elapsed wall-clock time since close) -- disclosed, not modeled (bd: gopherstack-1sf) diff --git a/services/stepfunctions/asl/distributed_map.go b/services/stepfunctions/asl/distributed_map.go index 0668ec5854..5dbf443002 100644 --- a/services/stepfunctions/asl/distributed_map.go +++ b/services/stepfunctions/asl/distributed_map.go @@ -14,6 +14,21 @@ import ( // Mode defaults to INLINE when ProcessorConfig is omitted or Mode is "". const processorModeDistributed = "DISTRIBUTED" +// DistributedMapItemResult is what running one Distributed Map item (or +// ItemBatcher batch) as a real child execution contributes back: its +// output, plus enough of the child's own identity for ResultWriter's +// Transformation: NONE metadata (ExecutionArn, Name, StartDate, StopDate -- +// AWS docs: input-output-resultwriter.html). Populated regardless of +// whether the child succeeded or failed, so a FAILED item's record can +// still carry its own ExecutionArn/Name/dates. +type DistributedMapItemResult struct { + Output any + ExecutionArn string + Name string + StartDate float64 + StopDate float64 +} + // DistributedMapRunner spawns a real child state-machine execution for one // Distributed Map item (or ItemBatcher batch) and blocks until it reaches a // terminal state, returning its output the way an INLINE iteration's @@ -28,7 +43,7 @@ type DistributedMapRunner interface { iterator *StateMachine, idx int, item any, - ) (any, error) + ) (DistributedMapItemResult, error) } // SetDistributedMapRunner configures the backend hook that spawns real child @@ -57,6 +72,7 @@ func (e *Executor) runDistributedMapTasks( items []any, results []any, errs []error, + meta []DistributedMapItemResult, concurrency int, ) { sem := semaphore.NewWeighted(int64(concurrency)) @@ -68,7 +84,7 @@ func (e *Executor) runDistributedMapTasks( } e.spawnDistributedMapTask( - ctx, executionARN, mapRunARN, stateName, iterator, i, item, results, errs, sem, &wg, + ctx, executionARN, mapRunARN, stateName, iterator, i, item, results, errs, meta, sem, &wg, ) } @@ -83,6 +99,7 @@ func (e *Executor) spawnDistributedMapTask( item any, results []any, errs []error, + meta []DistributedMapItemResult, sem *semaphore.Weighted, wg *sync.WaitGroup, ) { @@ -95,12 +112,14 @@ func (e *Executor) spawnDistributedMapTask( out, err := e.distributedMapRunner.RunDistributedMapItem( ctx, executionARN, mapRunARN, stateName, iterator, idx, item, ) + meta[idx] = out + if err != nil { errs[idx] = err return } - results[idx] = out + results[idx] = out.Output }) } diff --git a/services/stepfunctions/asl/executor.go b/services/stepfunctions/asl/executor.go index 47e46a124c..5c1abdfe44 100644 --- a/services/stepfunctions/asl/executor.go +++ b/services/stepfunctions/asl/executor.go @@ -1,6 +1,8 @@ package asl import ( + "bytes" + "compress/gzip" "context" cryptorand "crypto/rand" "encoding/base64" @@ -8,6 +10,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "maps" "math" "math/rand/v2" @@ -19,6 +22,8 @@ import ( "time" "golang.org/x/sync/semaphore" + + "github.com/blackbirdworks/gopherstack/pkgs/awstime" ) // ErrExecutionFailed is returned when a Fail state is reached. @@ -81,8 +86,19 @@ const ( errCodeStatesTimeout = "States.Timeout" errCodeStatesTaskFailed = "States.TaskFailed" errCodeStatesExceedToleratedFailureThreshold = "States.ExceedToleratedFailureThreshold" + errCodeStatesItemReaderFailed = "States.ItemReaderFailed" +) + +const ( + itemReaderResourceGetObject = "arn:aws:states:::s3:getObject" + itemReaderResourceListObjectsV2 = "arn:aws:states:::s3:listObjectsV2" ) +// aslNullLiteral is ASL's "null" string: a ResultPath sentinel, an +// intrinsic-function literal, and (in result_writer.go) a JSON marshal +// fallback -- three unrelated meanings that happen to share this text. +const aslNullLiteral = "null" + // Sentinel errors for Map state tolerated-failure threshold resolution. var ( ErrToleratedFailureCountNotNumber = errors.New("ToleratedFailureCountPath: value is not a number") @@ -117,6 +133,25 @@ type S3Reader interface { GetObjectBytes(ctx context.Context, bucket, key string) ([]byte, error) } +// S3ObjectItem is one object's metadata, as returned by S3's ListObjectsV2 +// and consumed by a Map state ItemReader whose Resource is s3:listObjectsV2 +// (AWS docs: input-output-itemreader.html). +type S3ObjectItem struct { + LastModified time.Time + Key string + ETag string + StorageClass string + Size int64 +} + +// S3ListReader lists objects in an S3 bucket/prefix, for a Map state +// ItemReader whose Resource is arn:aws:states:::s3:listObjectsV2. Optional: +// implemented by the same adapter as S3Reader, but checked separately so an +// S3Reader that predates this capability (e.g. a test double) still compiles. +type S3ListReader interface { + ListObjectsV2Items(ctx context.Context, bucket, prefix string) ([]S3ObjectItem, error) +} + // S3Writer writes objects to S3 for a Distributed Map state's ResultWriter. type S3Writer interface { PutObjectBytes(ctx context.Context, bucket, key string, data []byte) error @@ -1894,6 +1929,7 @@ func (e *Executor) runMapItemsAndFinalize( ) (any, error) { results := make([]any, len(items)) errs := make([]error, len(items)) + meta := make([]DistributedMapItemResult, len(items)) maxConcurrency, err := e.resolveMaxConcurrency(state, mapInput) if err != nil { @@ -1908,7 +1944,18 @@ func (e *Executor) runMapItemsAndFinalize( } if isDistributedMapIterator(iterator) && e.distributedMapRunner != nil { - e.runDistributedMapTasks(ctx, executionARN, mapRunARN, stateName, iterator, items, results, errs, concurrency) + e.runDistributedMapTasks( + ctx, + executionARN, + mapRunARN, + stateName, + iterator, + items, + results, + errs, + meta, + concurrency, + ) } else { e.runMapTasks(ctx, executionARN, iterator, items, results, errs, concurrency) } @@ -1917,7 +1964,9 @@ func (e *Executor) runMapItemsAndFinalize( resultsWritten := 0 if finalErr == nil && state.ResultWriter != nil { - out, resultsWritten, finalErr = e.exportMapResults(ctx, state, stateName, mapRunARN, items, results, errs) + out, resultsWritten, finalErr = e.exportMapResults( + ctx, state, stateName, mapRunARN, items, results, errs, meta, e.execMeta.StateMachineArn, + ) } if e.mapRunNotifier != nil && mapRunARN != "" { @@ -2131,6 +2180,26 @@ func (e *Executor) getMapIterator(state *State) (*StateMachine, error) { // ErrS3ReaderNotConfigured is returned when ItemReader requires S3 but no S3Reader is set. var ErrS3ReaderNotConfigured = errors.New("S3 reader not configured for Map state ItemReader") +// ErrS3ListReaderNotConfigured is returned when an ItemReader's Resource is +// s3:listObjectsV2 but the configured S3Reader doesn't implement S3ListReader. +var ErrS3ListReaderNotConfigured = errors.New("S3 list reader not configured for Map state ItemReader") + +// ErrItemReaderUnsupportedResource is returned for an ItemReader.Resource +// this emulator doesn't recognize. +var ErrItemReaderUnsupportedResource = errors.New("ItemReader: unsupported Resource") + +// ErrAthenaManifestUnsupported is returned for ReaderConfig.ManifestType +// ATHENA_DATA, which this emulator doesn't implement -- see PARITY.md. +var ErrAthenaManifestUnsupported = errors.New( + "ItemReader: ManifestType ATHENA_DATA is not supported by this emulator", +) + +// ErrParquetUnsupported is returned for InputType PARQUET, which this +// emulator doesn't decode (no pure-Go Parquet reader dependency) -- see PARITY.md. +var ErrParquetUnsupported = errors.New( + "ItemReader: InputType PARQUET is not supported by this emulator", +) + // ErrItemReaderInvalidData is returned when ItemReader S3 object cannot be parsed as items. var ErrItemReaderInvalidData = errors.New( "ItemReader: unable to parse S3 object as JSON array or JSON lines", @@ -2189,9 +2258,34 @@ func (e *Executor) truncateReaderItems(items []any, cfg *ReaderConfig, mapInput return items, nil } -// resolveItemsFromReader reads items from S3 using the ItemReader configuration. -// Supports JSON arrays, newline-delimited JSON (JSON Lines), and CSV. +// resolveItemsFromReader reads items from S3 using the ItemReader's Resource +// and ReaderConfig, wrapping any failure as States.ItemReaderFailed -- +// AWS's documented error for a Distributed Map ItemReader that can't read +// its dataset (input-output-itemreader.html). func (e *Executor) resolveItemsFromReader(ctx context.Context, reader *ItemReader) ([]any, error) { + items, err := e.readItemReaderSource(ctx, reader) + if err != nil { + return nil, &FailError{ErrCode: errCodeStatesItemReaderFailed, Cause: err.Error()} + } + + return items, nil +} + +func (e *Executor) readItemReaderSource(ctx context.Context, reader *ItemReader) ([]any, error) { + switch reader.Resource { + case "", itemReaderResourceGetObject: + return e.resolveItemsFromS3GetObject(ctx, reader) + case itemReaderResourceListObjectsV2: + return e.resolveItemsFromS3List(ctx, reader) + default: + return nil, fmt.Errorf("%w %q", ErrItemReaderUnsupportedResource, reader.Resource) + } +} + +// resolveItemsFromS3GetObject implements the s3:getObject Resource: a single +// S3 object decoded as JSON, JSON Lines, CSV, or (via ManifestType/InputType +// MANIFEST) an S3 Inventory manifest fanning out to multiple CSV data files. +func (e *Executor) resolveItemsFromS3GetObject(ctx context.Context, reader *ItemReader) ([]any, error) { if e.s3 == nil { return nil, ErrS3ReaderNotConfigured } @@ -2204,7 +2298,210 @@ func (e *Executor) resolveItemsFromReader(ctx context.Context, reader *ItemReade return nil, fmt.Errorf("ItemReader S3 get error: %w", err) } - return decodeReaderItems(data, reader.ReaderConfig) + cfg := reader.ReaderConfig + if isManifestReaderConfig(cfg) { + return e.resolveManifestItems(ctx, bucket, data, cfg) + } + + return decodeReaderItems(data, cfg) +} + +// resolveItemsFromS3List implements the s3:listObjectsV2 Resource: by +// default, one item per listed object's metadata; with +// ReaderConfig.Transformation LOAD_AND_FLATTEN, each listed object's content +// is read and decoded, fanning out into per-record items. +func (e *Executor) resolveItemsFromS3List(ctx context.Context, reader *ItemReader) ([]any, error) { + if e.s3 == nil { + return nil, ErrS3ReaderNotConfigured + } + + lister, ok := e.s3.(S3ListReader) + if !ok { + return nil, ErrS3ListReaderNotConfigured + } + + bucket, _ := reader.Parameters["Bucket"].(string) + prefix, _ := reader.Parameters["Prefix"].(string) + + objs, err := lister.ListObjectsV2Items(ctx, bucket, prefix) + if err != nil { + return nil, fmt.Errorf("ItemReader S3 list error: %w", err) + } + + cfg := reader.ReaderConfig + if cfg != nil && strings.EqualFold(cfg.Transformation, "LOAD_AND_FLATTEN") { + return e.flattenListedObjects(ctx, bucket, objs, cfg) + } + + items := make([]any, len(objs)) + for i, o := range objs { + items[i] = map[string]any{ + "Etag": o.ETag, + "Key": o.Key, + "LastModified": awstime.Epoch(o.LastModified), + "Size": o.Size, + "StorageClass": o.StorageClass, + } + } + + return items, nil +} + +// flattenListedObjects reads and decodes each listed object's content per +// InputType, fanning out into per-record items (AWS docs: "Processing +// nested data sets"). Zero-byte keys ending in "/" are S3 console folder +// placeholders with no content to decode, so they're skipped. +func (e *Executor) flattenListedObjects( + ctx context.Context, + bucket string, + objs []S3ObjectItem, + cfg *ReaderConfig, +) ([]any, error) { + if cfg.InputType == "" { + return nil, fmt.Errorf( + "%w: InputType is required when Transformation is LOAD_AND_FLATTEN", + ErrItemReaderInvalidData, + ) + } + + var items []any + + for _, o := range objs { + if o.Size == 0 && strings.HasSuffix(o.Key, "/") { + continue + } + + data, err := e.s3.GetObjectBytes(ctx, bucket, o.Key) + if err != nil { + return nil, fmt.Errorf("ItemReader flatten %q: %w", o.Key, err) + } + + objItems, err := decodeReaderItems(data, cfg) + if err != nil { + return nil, fmt.Errorf("ItemReader flatten %q: %w", o.Key, err) + } + + items = append(items, objItems...) + } + + return items, nil +} + +// isManifestReaderConfig reports whether cfg names an S3 Inventory/Athena +// manifest rather than a plain data object -- either the legacy +// InputType=MANIFEST form or the newer ManifestType field. +func isManifestReaderConfig(cfg *ReaderConfig) bool { + if cfg == nil { + return false + } + + return strings.EqualFold(cfg.InputType, "MANIFEST") || cfg.ManifestType != "" +} + +// s3InventoryManifest is the manifest.json shape AWS S3 Inventory writes +// alongside its CSV data files (AWS docs: input-output-itemreader.html). +type s3InventoryManifest struct { + FileSchema string `json:"fileSchema"` + Files []struct { + Key string `json:"key"` + } `json:"files"` +} + +// resolveManifestItems dispatches on ManifestType (S3_INVENTORY, the only +// InputType=MANIFEST target has ever meant, or ATHENA_DATA, unsupported). +func (e *Executor) resolveManifestItems( + ctx context.Context, + bucket string, + manifestData []byte, + cfg *ReaderConfig, +) ([]any, error) { + manifestType := strings.ToUpper(cfg.ManifestType) + if manifestType == "" { + manifestType = "S3_INVENTORY" + } + + switch manifestType { + case "S3_INVENTORY": + return e.resolveS3InventoryManifest(ctx, bucket, manifestData, cfg) + case "ATHENA_DATA": + return nil, ErrAthenaManifestUnsupported + default: + return nil, fmt.Errorf("%w: unsupported ManifestType %q", ErrItemReaderInvalidData, cfg.ManifestType) + } +} + +// resolveS3InventoryManifest reads an S3 Inventory manifest.json, then reads +// and decodes each listed (optionally gzip-compressed) CSV data file, using +// the manifest's fileSchema as the CSV headers. cfg's CSVDelimiter (if any) +// carries through to the data files, matching AWS's "CSVDelimiter ... when +// InputType is CSV or MANIFEST". +func (e *Executor) resolveS3InventoryManifest( + ctx context.Context, + bucket string, + manifestData []byte, + cfg *ReaderConfig, +) ([]any, error) { + var manifest s3InventoryManifest + if err := json.Unmarshal(manifestData, &manifest); err != nil { + return nil, fmt.Errorf("%w: manifest.json: %w", ErrItemReaderInvalidData, err) + } + + headers := splitManifestFileSchema(manifest.FileSchema) + fileCfg := &ReaderConfig{CSVHeaderLocation: "GIVEN", CSVHeaders: headers} + + if cfg != nil { + fileCfg.CSVDelimiter = cfg.CSVDelimiter + } + + var items []any + + for _, f := range manifest.Files { + data, err := e.s3.GetObjectBytes(ctx, bucket, f.Key) + if err != nil { + return nil, fmt.Errorf("ItemReader manifest data file %q: %w", f.Key, err) + } + + if strings.HasSuffix(strings.ToLower(f.Key), ".gz") { + data, err = gunzipBytes(data) + if err != nil { + return nil, fmt.Errorf("%w: gunzip %q: %w", ErrItemReaderInvalidData, f.Key, err) + } + } + + fileItems, err := decodeCSVItems(data, fileCfg) + if err != nil { + return nil, fmt.Errorf("ItemReader manifest data file %q: %w", f.Key, err) + } + + items = append(items, fileItems...) + } + + return items, nil +} + +// splitManifestFileSchema splits an S3 Inventory manifest's fileSchema +// ("Bucket, Key, Size, LastModifiedDate") into CSV headers. +func splitManifestFileSchema(schema string) []string { + parts := strings.Split(schema, ",") + headers := make([]string, len(parts)) + + for i, p := range parts { + headers[i] = strings.TrimSpace(p) + } + + return headers +} + +// gunzipBytes decompresses gzip-compressed S3 object data (AWS docs: ItemReader +// input files support GZIP/ZSTD external compression; only GZIP is implemented). +func gunzipBytes(data []byte) ([]byte, error) { + r, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + return nil, err + } + defer r.Close() + + return io.ReadAll(r) } // decodeReaderItems parses S3 object bytes into Map items based on the @@ -2221,12 +2518,99 @@ func decodeReaderItems(data []byte, cfg *ReaderConfig) ([]any, error) { case "JSONL", "JSON_LINES": return decodeJSONLines(data) case "", "JSON": - return decodeJSONAuto(data) + return decodeJSONItems(data, cfg) + case "PARQUET": + return nil, ErrParquetUnsupported default: return nil, fmt.Errorf("%w: unsupported InputType %q", ErrItemReaderInvalidData, inputType) } } +// decodeJSONItems decodes a JSON InputType object, applying ReaderConfig's +// ItemsPointer (RFC 6901 JSON Pointer) to select a nested array when set -- +// AWS docs: input-output-itemreader.html, "ItemsPointer". Without it, falls +// back to the pre-existing JSON-array-then-JSON-lines auto-detection. +func decodeJSONItems(data []byte, cfg *ReaderConfig) ([]any, error) { + if cfg == nil || cfg.ItemsPointer == "" { + return decodeJSONAuto(data) + } + + var doc any + if err := json.Unmarshal(data, &doc); err != nil { + return nil, fmt.Errorf("%w: %w", ErrItemReaderInvalidData, err) + } + + val, err := resolveJSONPointer(doc, cfg.ItemsPointer) + if err != nil { + return nil, err + } + + arr, ok := val.([]any) + if !ok { + return nil, fmt.Errorf( + "%w: ItemsPointer %q does not reference a JSON array", + ErrItemReaderInvalidData, cfg.ItemsPointer, + ) + } + + return arr, nil +} + +// errJSONPointerNotFound is resolveJSONPointer's internal not-found signal, +// always re-wrapped as ErrItemReaderInvalidData before it leaves this file. +var errJSONPointerNotFound = errors.New("path not found") + +// resolveJSONPointer resolves an RFC 6901 JSON Pointer ("/data/items") +// against a decoded JSON document: forward slashes separate nesting levels, +// array indices are plain decimal integers, and "~1"/"~0" escape "/" and "~" +// in a token (AWS docs: ItemsPointer "JSONPointer syntax"). +func resolveJSONPointer(doc any, pointer string) (any, error) { + if pointer == "" || pointer == "/" { + return doc, nil + } + + if !strings.HasPrefix(pointer, "/") { + return nil, fmt.Errorf(`%w: ItemsPointer %q must start with "/"`, ErrItemReaderInvalidData, pointer) + } + + cur := doc + + for tok := range strings.SplitSeq(pointer[1:], "/") { + tok = strings.ReplaceAll(tok, "~1", "/") + tok = strings.ReplaceAll(tok, "~0", "~") + + next, err := stepJSONPointer(cur, tok) + if err != nil { + return nil, fmt.Errorf("%w: ItemsPointer %q: %w", ErrItemReaderInvalidData, pointer, err) + } + + cur = next + } + + return cur, nil +} + +func stepJSONPointer(cur any, tok string) (any, error) { + switch v := cur.(type) { + case map[string]any: + next, ok := v[tok] + if !ok { + return nil, errJSONPointerNotFound + } + + return next, nil + case []any: + idx, err := strconv.Atoi(tok) + if err != nil || idx < 0 || idx >= len(v) { + return nil, errJSONPointerNotFound + } + + return v[idx], nil + default: + return nil, errJSONPointerNotFound + } +} + func decodeJSONAuto(data []byte) ([]any, error) { var arr []any if jsonErr := json.Unmarshal(data, &arr); jsonErr == nil { @@ -2258,8 +2642,39 @@ func decodeJSONLines(data []byte) ([]any, error) { return items, nil } +// csvDelimiterRune maps ReaderConfig.CSVDelimiter to the field separator +// AWS documents for CSV/MANIFEST InputType: COMMA (default), PIPE, +// SEMICOLON, SPACE, TAB (input-output-itemreader.html). +func csvDelimiterRune(cfg *ReaderConfig) (rune, error) { + delim := "" + if cfg != nil { + delim = strings.ToUpper(cfg.CSVDelimiter) + } + + switch delim { + case "", "COMMA": + return ',', nil + case "PIPE": + return '|', nil + case "SEMICOLON": + return ';', nil + case "SPACE": + return ' ', nil + case "TAB": + return '\t', nil + default: + return 0, fmt.Errorf("%w: unsupported CSVDelimiter %q", ErrItemReaderInvalidData, delim) + } +} + func decodeCSVItems(data []byte, cfg *ReaderConfig) ([]any, error) { + delim, err := csvDelimiterRune(cfg) + if err != nil { + return nil, err + } + reader := csv.NewReader(strings.NewReader(string(data))) + reader.Comma = delim reader.FieldsPerRecord = -1 rows, err := reader.ReadAll() @@ -2673,7 +3088,7 @@ func applyPath(path string, value any, pathCache ...*jsonPathCache) (any, error) // If ResultPath is "$.field", result is written to input[field]. // If ResultPath is "null", result is discarded (input passes through). func applyResultPath(resultPath string, input, result any) (any, error) { - if resultPath == "null" { + if resultPath == aslNullLiteral { return input, nil } diff --git a/services/stepfunctions/asl/intrinsics.go b/services/stepfunctions/asl/intrinsics.go index 17c0ad04dd..8c610314ab 100644 --- a/services/stepfunctions/asl/intrinsics.go +++ b/services/stepfunctions/asl/intrinsics.go @@ -209,7 +209,7 @@ func evalIntrinsicArg(arg string, input any) (any, error) { } // Null literal. - if arg == "null" { + if arg == aslNullLiteral { return nil, nil //nolint:nilnil // null is a valid ASL literal value } diff --git a/services/stepfunctions/asl/intrinsics_extras_test.go b/services/stepfunctions/asl/intrinsics_extras_test.go index cd3deca487..edb50c31fd 100644 --- a/services/stepfunctions/asl/intrinsics_extras_test.go +++ b/services/stepfunctions/asl/intrinsics_extras_test.go @@ -190,6 +190,69 @@ func TestDecodeReaderItems(t *testing.T) { data: []byte("ignored"), wantErr: true, }, + { + name: "csv_pipe_delimiter", + cfg: &asl.ReaderConfig{InputType: "CSV", CSVDelimiter: "PIPE"}, + data: []byte("col1|col2\nx|1\ny|2\n"), + want: []any{ + map[string]any{"col1": "x", "col2": "1"}, + map[string]any{"col1": "y", "col2": "2"}, + }, + }, + { + name: "csv_semicolon_delimiter_lowercase", + cfg: &asl.ReaderConfig{InputType: "CSV", CSVDelimiter: "semicolon"}, + data: []byte("a;b\n1;2\n"), + want: []any{map[string]any{"a": "1", "b": "2"}}, + }, + { + name: "csv_tab_delimiter", + cfg: &asl.ReaderConfig{InputType: "CSV", CSVDelimiter: "TAB"}, + data: []byte("a\tb\n1\t2\n"), + want: []any{map[string]any{"a": "1", "b": "2"}}, + }, + { + name: "csv_space_delimiter", + cfg: &asl.ReaderConfig{InputType: "CSV", CSVDelimiter: "SPACE"}, + data: []byte("a b\n1 2\n"), + want: []any{map[string]any{"a": "1", "b": "2"}}, + }, + { + name: "csv_unsupported_delimiter", + cfg: &asl.ReaderConfig{InputType: "CSV", CSVDelimiter: "COLON"}, + data: []byte("a:b\n1:2\n"), + wantErr: true, + }, + { + name: "items_pointer_selects_nested_array", + cfg: &asl.ReaderConfig{InputType: "JSON", ItemsPointer: "/data/items"}, + data: []byte(`{"data":{"items":[{"id":1.0},{"id":2.0}]}}`), + want: []any{map[string]any{"id": 1.0}, map[string]any{"id": 2.0}}, + }, + { + name: "items_pointer_array_index_segment", + cfg: &asl.ReaderConfig{InputType: "JSON", ItemsPointer: "/data/0/items"}, + data: []byte(`{"data":[{"items":[{"id":1.0}]}]}`), + want: []any{map[string]any{"id": 1.0}}, + }, + { + name: "items_pointer_not_an_array", + cfg: &asl.ReaderConfig{InputType: "JSON", ItemsPointer: "/data"}, + data: []byte(`{"data":{"id":1}}`), + wantErr: true, + }, + { + name: "items_pointer_missing_path", + cfg: &asl.ReaderConfig{InputType: "JSON", ItemsPointer: "/nope"}, + data: []byte(`{"data":[1,2]}`), + wantErr: true, + }, + { + name: "items_pointer_must_start_with_slash", + cfg: &asl.ReaderConfig{InputType: "JSON", ItemsPointer: "data"}, + data: []byte(`{"data":[1,2]}`), + wantErr: true, + }, } for _, tt := range tests { diff --git a/services/stepfunctions/asl/parser.go b/services/stepfunctions/asl/parser.go index 2d6f61f529..90ec23474d 100644 --- a/services/stepfunctions/asl/parser.go +++ b/services/stepfunctions/asl/parser.go @@ -53,26 +53,42 @@ type ItemReader struct { } // ReaderConfig describes how the ItemReader should interpret S3 object data. -// InputType: "JSON" (default), "JSONL", or "CSV". +// InputType: "JSON" (default), "JSONL", "CSV", "MANIFEST", or "PARQUET" +// (PARQUET is parsed but not decoded -- see PARITY.md). // CSVHeaderLocation: "FIRST_ROW" or "GIVEN". // CSVHeaders: explicit headers when CSVHeaderLocation == "GIVEN". +// CSVDelimiter: "COMMA" (default), "PIPE", "SEMICOLON", "SPACE", or "TAB" -- +// only meaningful when InputType is CSV or MANIFEST. +// ItemsPointer: an RFC 6901 JSON Pointer ("/data/items") selecting a nested +// array within a JSON InputType file; only meaningful when InputType is +// JSON (or omitted). // MaxItems: optional cap on number of items returned (0 = unlimited). // MaxItemsPath is MaxItems' reference-path sibling, mutually exclusive with -// it and resolved against the Map state's pre-Parameters input (AWS docs: -// input-output-itemreader.html). +// it and resolved against the Map state's pre-Parameters input. +// Transformation ("NONE" default, or "LOAD_AND_FLATTEN") only applies to the +// s3:listObjectsV2 Resource: LOAD_AND_FLATTEN reads and decodes each listed +// object's content (per InputType) instead of returning object metadata. +// ManifestType ("S3_INVENTORY" or "ATHENA_DATA", only ATHENA_DATA unsupported +// -- see PARITY.md) or InputType "MANIFEST" treats the fetched object as an +// S3 Inventory manifest.json listing CSV data files. +// (AWS docs: input-output-itemreader.html). type ReaderConfig struct { InputType string `json:"InputType,omitempty"` CSVHeaderLocation string `json:"CSVHeaderLocation,omitempty"` + CSVDelimiter string `json:"CSVDelimiter,omitempty"` MaxItemsPath string `json:"MaxItemsPath,omitempty"` + Transformation string `json:"Transformation,omitempty"` + ManifestType string `json:"ManifestType,omitempty"` + ItemsPointer string `json:"ItemsPointer,omitempty"` CSVHeaders []string `json:"CSVHeaders,omitempty"` MaxItems int `json:"MaxItems,omitempty"` } // ResultWriter configures exporting a Distributed Map state's per-item -// results to S3 instead of returning them inline as the state's output -// (AWS docs: input-output-resultwriter.html). Only the Resource+Parameters -// (S3 export) combination is applied; WriterConfig is parsed but not -// honored -- see Executor.exportMapResults. +// results to S3, and/or formatting the state's own output, per AWS docs: +// input-output-resultwriter.html. Resource+Parameters name the S3 +// destination; WriterConfig controls formatting -- see +// Executor.exportMapResults. type ResultWriter struct { Parameters map[string]any `json:"Parameters,omitempty"` WriterConfig *ResultWriterConfig `json:"WriterConfig,omitempty"` @@ -80,8 +96,8 @@ type ResultWriter struct { } // ResultWriterConfig is ResultWriter.WriterConfig: Transformation -// ("NONE"|"COMPACT"|"FLATTEN") and OutputType ("JSON"|"JSONL"). Parsed for -// forward compatibility but not currently applied. +// ("NONE"|"COMPACT"|"FLATTEN") and OutputType ("JSON"|"JSONL") -- AWS docs: +// input-output-resultwriter.html. type ResultWriterConfig struct { Transformation string `json:"Transformation,omitempty"` OutputType string `json:"OutputType,omitempty"` diff --git a/services/stepfunctions/asl/result_writer.go b/services/stepfunctions/asl/result_writer.go index 227d45feb1..6ad7027665 100644 --- a/services/stepfunctions/asl/result_writer.go +++ b/services/stepfunctions/asl/result_writer.go @@ -1,6 +1,7 @@ package asl import ( + "bytes" "context" "encoding/json" "errors" @@ -47,60 +48,114 @@ type resultManifestFile struct { Size int `json:"Size"` } -// mapItemRecord is one entry in a SUCCEEDED_0.json/FAILED_0.json result -// file. Real AWS records also carry a per-item ExecutionArn/Name/StartDate/ -// StopDate, since each Distributed Map iteration is a real child workflow -// execution there. gopherstack runs Map iterations as in-process -// sub-executors with no separate Execution resource to point to, so those -// identity fields are omitted rather than fabricated. +const ( + transformationNone = "NONE" + transformationCompact = "COMPACT" + transformationFlatten = "FLATTEN" + + outputTypeJSON = "JSON" + outputTypeJSONL = "JSONL" + + redriveStatusRedrivable = "REDRIVABLE" + redriveStatusNotRedrivable = "NOT_REDRIVABLE" + redriveReasonSucceeded = "Execution is SUCCEEDED and cannot be redriven." + + statusSucceeded = "SUCCEEDED" + statusFailed = "FAILED" +) + +// detailsIncluded mirrors CloudWatchEventsExecutionDataDetails's one real +// member (sfn@v1.49.0 types.go): whether the data was included, never +// truncated in this emulator. +type detailsIncluded struct { + Included bool `json:"Included"` +} + +// mapItemRecord is one entry in a Distributed Map ResultWriter's +// Transformation: NONE output -- the "workflow metadata" AWS docs +// (input-output-resultwriter.html) describe: the full per-child-execution +// record, Input/Output as JSON-encoded strings (matching a real +// DescribeExecution's Input/Output shape). ExecutionArn/Name/StartDate/ +// StopDate are populated only for DISTRIBUTED Map items, which run as real +// child Executions (see DistributedMapItemResult); an INLINE Map's +// in-process iterations have no such resource to report and leave them +// empty, honestly, rather than fabricating them. type mapItemRecord struct { - Input any `json:"Input"` - Output any `json:"Output,omitempty"` - Error string `json:"Error,omitempty"` - Cause string `json:"Cause,omitempty"` - Status string `json:"Status"` - Index int `json:"Index"` + OutputDetails *detailsIncluded `json:"OutputDetails,omitempty"` + Error string `json:"Error,omitempty"` + RedriveStatus string `json:"RedriveStatus,omitempty"` + Name string `json:"Name,omitempty"` + Output string `json:"Output,omitempty"` + Input string `json:"Input"` + ExecutionArn string `json:"ExecutionArn,omitempty"` + Cause string `json:"Cause,omitempty"` + Status string `json:"Status"` + RedriveStatusReason string `json:"RedriveStatusReason,omitempty"` + StateMachineArn string `json:"StateMachineArn,omitempty"` + RedriveCount int `json:"RedriveCount"` + StartDate float64 `json:"StartDate,omitempty"` + StopDate float64 `json:"StopDate,omitempty"` + InputDetails detailsIncluded `json:"InputDetails"` } const resultWriterFileIndex = 0 -// exportMapResults writes a Distributed Map's per-item results plus a -// manifest to the wired S3Writer and returns ResultWriterDetails in place -// of inline results (AWS docs: input-output-resultwriter.html). It also -// returns the number of successful results actually written, for -// MapRunItemCounts.ResultsWritten. +// exportMapResults applies a Distributed Map's ResultWriter.WriterConfig +// (Transformation/OutputType) and, when Resource+Parameters name an S3 +// destination, writes the per-item results plus a manifest to the wired +// S3Writer -- returning ResultWriterDetails in place of inline results (AWS +// docs: input-output-resultwriter.html). It also returns the number of +// successful results actually written, for MapRunItemCounts.ResultsWritten. // -// Only the Resource+Parameters(Bucket,Prefix) S3-export combination is -// supported; WriterConfig's Transformation/OutputType are parsed but not -// applied. When no S3Writer is wired, or Parameters.Bucket is unset, -// results are returned inline unchanged instead of failing the Map state -- -// the computation already succeeded, only its export is unavailable. Both -// that fallback and an unapplied WriterConfig log a warning naming the -// state, so the degradation is diagnosable instead of silent. +// AWS documents three valid ResultWriter shapes (required field +// combinations): WriterConfig alone previews the formatted output without +// exporting; Resource+Parameters alone exports with NONE/JSON defaults; all +// three format AND export. When no S3Writer is wired, or Parameters.Bucket +// is unset despite Resource being set, results still degrade to the +// formatted inline output instead of failing the Map state -- the +// computation already succeeded, only its export is unavailable -- logging +// a warning naming the state so the degradation is diagnosable. func (e *Executor) exportMapResults( - ctx context.Context, state *State, stateName, mapRunARN string, items, results []any, errs []error, + ctx context.Context, state *State, stateName, mapRunARN string, + items, results []any, errs []error, meta []DistributedMapItemResult, stateMachineArn string, ) (any, int, error) { - if wc := state.ResultWriter.WriterConfig; wc != nil && writerConfigUnsupported(wc) { - logger.Load(ctx).WarnContext(ctx, - "stepfunctions: ResultWriter WriterConfig not applied, writing default JSON array result files", - "state", stateName, "transformation", wc.Transformation, "outputType", wc.OutputType) + rw := state.ResultWriter + bucket, _ := rw.Parameters["Bucket"].(string) + exporting := rw.Resource != "" || bucket != "" + + transformation := resolveTransformation(rw.WriterConfig, exporting) + outputType := resolveOutputType(rw.WriterConfig) + + records := buildMapItemRecords(items, results, errs, meta, stateMachineArn) + + if !exporting { + preview, err := previewValue(records, results, transformation, outputType) + if err != nil { + return nil, 0, fmt.Errorf("ResultWriter: %w", err) + } + + return preview, 0, nil } - bucket, _ := state.ResultWriter.Parameters["Bucket"].(string) - if e.s3w == nil || bucket == "" { + if e.s3w == nil { logger.Load(ctx).WarnContext(ctx, "stepfunctions: ResultWriter configured but export unavailable, returning inline results", "state", stateName, "bucket", bucket) - return results, 0, nil + preview, err := previewValue(records, results, transformation, outputType) + if err != nil { + return nil, 0, fmt.Errorf("ResultWriter: %w", err) + } + + return preview, 0, nil } - prefix, _ := state.ResultWriter.Parameters["Prefix"].(string) - folder := resultFolderKey(prefix, mapRunARN) + succeededCount := countStatus(records, statusSucceeded) - succeeded, failed := partitionMapResults(items, results, errs) + prefix, _ := rw.Parameters["Prefix"].(string) + folder := resultFolderKey(prefix, mapRunARN) - files, err := e.writeMapResultFiles(ctx, bucket, folder, succeeded, failed) + files, err := e.writeMapResultFiles(ctx, bucket, folder, records, results, transformation, outputType) if err != nil { return nil, 0, fmt.Errorf("ResultWriter: %w", err) } @@ -125,17 +180,198 @@ func (e *Executor) exportMapResults( ResultWriterDetails: ResultWriterDetails{Bucket: bucket, Key: manifestKey}, } - return out, len(succeeded), nil + return out, succeededCount, nil +} + +// resolveTransformation applies AWS's documented WriterConfig.Transformation +// defaults: NONE when exporting to S3 and unspecified, COMPACT otherwise +// (input-output-resultwriter.html, "Contents of the ResultWriter field"). +func resolveTransformation(wc *ResultWriterConfig, exporting bool) string { + if wc != nil && wc.Transformation != "" { + return strings.ToUpper(wc.Transformation) + } + + if exporting { + return transformationNone + } + + return transformationCompact +} + +// resolveOutputType applies WriterConfig.OutputType's documented default: JSON. +func resolveOutputType(wc *ResultWriterConfig) string { + if wc != nil && strings.ToUpper(wc.OutputType) == outputTypeJSONL { + return outputTypeJSONL + } + + return outputTypeJSON +} + +// buildMapItemRecords builds one mapItemRecord per Map item, in original +// order, regardless of Transformation -- callers pick which fields of each +// record to surface. +func buildMapItemRecords( + items, results []any, errs []error, meta []DistributedMapItemResult, stateMachineArn string, +) []mapItemRecord { + records := make([]mapItemRecord, len(items)) + + for i, item := range items { + var m DistributedMapItemResult + if i < len(meta) { + m = meta[i] + } + + records[i] = buildMapItemRecord(item, results[i], errs[i], m, stateMachineArn) + } + + return records +} + +func buildMapItemRecord( + item, result any, + err error, + meta DistributedMapItemResult, + stateMachineArn string, +) mapItemRecord { + rec := mapItemRecord{ + Input: stringifyJSON(item), + InputDetails: detailsIncluded{Included: true}, + ExecutionArn: meta.ExecutionArn, + Name: meta.Name, + StartDate: meta.StartDate, + StopDate: meta.StopDate, + StateMachineArn: stateMachineArn, + } + + if err != nil { + rec.Status = statusFailed + rec.Error, rec.Cause = mapResultErrorCodeAndCause(err) + rec.RedriveStatus = redriveStatusRedrivable + + return rec + } + + rec.Status = statusSucceeded + rec.Output = stringifyJSON(result) + rec.OutputDetails = &detailsIncluded{Included: true} + rec.RedriveStatus = redriveStatusNotRedrivable + rec.RedriveStatusReason = redriveReasonSucceeded + + return rec +} + +func stringifyJSON(v any) string { + b, err := json.Marshal(v) + if err != nil { + return aslNullLiteral + } + + return string(b) } -// writerConfigUnsupported reports whether wc requests a Transformation or -// OutputType other than the defaults, neither of which exportMapResults -// applies. -func writerConfigUnsupported(wc *ResultWriterConfig) bool { - transform := wc.Transformation == "COMPACT" || wc.Transformation == "FLATTEN" - outputType := wc.OutputType == "JSONL" +func countStatus(records []mapItemRecord, status string) int { + n := 0 - return transform || outputType + for _, r := range records { + if r.Status == status { + n++ + } + } + + return n +} + +// previewValue renders the WriterConfig-formatted result the Map state +// returns inline: entries in original item order, mixing SUCCEEDED (per +// Transformation) and FAILED (always the full NONE-shaped record -- AWS +// docs: "If a child workflow execution fails, Step Functions returns its +// execution result unchanged"). OutputType JSON returns the native array; +// JSONL returns a newline-delimited string, matching what a JSONL S3 object +// would contain. +func previewValue(records []mapItemRecord, results []any, transformation, outputType string) (any, error) { + entries := make([]any, len(records)) + + for i, rec := range records { + if rec.Status == statusFailed || transformation == transformationNone { + entries[i] = rec + + continue + } + + entries[i] = results[i] + } + + if transformation == transformationFlatten { + entries = flattenValues(entries) + } + + return encodeEntriesInline(entries, outputType) +} + +// flattenValues implements Transformation: FLATTEN -- when an entry is +// itself a JSON array, its elements are spliced into the result in place of +// the array (AWS docs: "If a child workflow execution returns an array, +// this option flattens the array"). Non-array entries (including FAILED +// mapItemRecord entries) pass through unchanged. +func flattenValues(vals []any) []any { + out := make([]any, 0, len(vals)) + + for _, v := range vals { + if arr, ok := v.([]any); ok { + out = append(out, arr...) + + continue + } + + out = append(out, v) + } + + return out +} + +func encodeEntriesInline(entries []any, outputType string) (any, error) { + if outputType != outputTypeJSONL { + return entries, nil + } + + data, err := encodeJSONLines(entries) + if err != nil { + return nil, err + } + + return string(data), nil +} + +// encodeJSONLines renders entries as JSON Lines: one JSON value per line, +// no enclosing array (WriterConfig.OutputType: JSONL). +func encodeJSONLines(entries []any) ([]byte, error) { + var buf bytes.Buffer + + for i, e := range entries { + if i > 0 { + buf.WriteByte('\n') + } + + b, err := json.Marshal(e) + if err != nil { + return nil, fmt.Errorf("marshal JSONL entry: %w", err) + } + + buf.Write(b) + } + + return buf.Bytes(), nil +} + +// mapResultErrorCodeAndCause splits a Map item's error into AWS's separate +// Error/Cause fields; a *FailError already carries them apart, anything +// else has no distinct Cause. +func mapResultErrorCodeAndCause(err error) (string, string) { + if failErr, ok := errors.AsType[*FailError](err); ok { + return failErr.ErrCode, failErr.Cause + } + + return errCodeStatesTaskFailed, err.Error() } // resultFolderKey builds the slash-terminated S3 key prefix holding one Map @@ -160,53 +396,26 @@ func resultFolderKey(prefix, mapRunARN string) string { return strings.TrimSuffix(prefix, "/") + "/" + id + "/" } -// partitionMapResults splits Map iteration results into AWS's SUCCEEDED/ -// FAILED result-file buckets by per-item error. -func partitionMapResults(items, results []any, errs []error) ([]mapItemRecord, []mapItemRecord) { - var succeeded, failed []mapItemRecord - - for i, item := range items { - if errs[i] != nil { - failed = append(failed, mapItemRecord{ - Index: i, - Input: item, - Status: "FAILED", - Error: mapResultErrorCode(errs[i]), - Cause: errs[i].Error(), - }) - - continue - } - - succeeded = append(succeeded, mapItemRecord{ - Index: i, - Input: item, - Output: results[i], - Status: "SUCCEEDED", - }) - } - - return succeeded, failed -} - -func mapResultErrorCode(err error) string { - if failErr, ok := errors.AsType[*FailError](err); ok { - return failErr.ErrCode - } - - return errCodeStatesTaskFailed -} - // writeMapResultFiles writes SUCCEEDED_0.json/FAILED_0.json, each only when // non-empty -- AWS's own manifest only references files that were actually -// created. +// created. Filenames keep the .json extension regardless of OutputType +// (AWS's manifest.json documentation names them unconditionally); only the +// bytes written differ between a JSON array and JSON Lines. func (e *Executor) writeMapResultFiles( - ctx context.Context, bucket, folder string, succeeded, failed []mapItemRecord, + ctx context.Context, + bucket, folder string, + records []mapItemRecord, + results []any, + transformation, outputType string, ) (resultManifestFiles, error) { var files resultManifestFiles + succeeded, succeededResults, failed := partitionRecords(records, results) + if len(succeeded) > 0 { - entry, err := e.writeMapResultFile(ctx, bucket, folder, "SUCCEEDED", succeeded) + entries := succeededEntries(succeeded, succeededResults, transformation) + + entry, err := e.writeMapResultFile(ctx, bucket, folder, "SUCCEEDED", entries, outputType) if err != nil { return files, err } @@ -215,7 +424,12 @@ func (e *Executor) writeMapResultFiles( } if len(failed) > 0 { - entry, err := e.writeMapResultFile(ctx, bucket, folder, "FAILED", failed) + entries := make([]any, len(failed)) + for i, r := range failed { + entries[i] = r + } + + entry, err := e.writeMapResultFile(ctx, bucket, folder, "FAILED", entries, outputType) if err != nil { return files, err } @@ -226,10 +440,64 @@ func (e *Executor) writeMapResultFiles( return files, nil } +// partitionRecords splits records (and their parallel raw results) into +// SUCCEEDED and FAILED groups, preserving relative order within each group. +func partitionRecords( + records []mapItemRecord, + results []any, +) ([]mapItemRecord, []any, []mapItemRecord) { + var succ, failed []mapItemRecord + + var succResults []any + + for i, rec := range records { + if rec.Status == statusFailed { + failed = append(failed, rec) + + continue + } + + succ = append(succ, rec) + succResults = append(succResults, results[i]) + } + + return succ, succResults, failed +} + +// succeededEntries formats only-successful items per Transformation: NONE +// keeps the full record, COMPACT returns each child's raw output, FLATTEN +// additionally splices any array output into the result. +func succeededEntries(records []mapItemRecord, results []any, transformation string) []any { + if transformation == transformationNone { + entries := make([]any, len(records)) + for i, r := range records { + entries[i] = r + } + + return entries + } + + if transformation == transformationFlatten { + return flattenValues(results) + } + + return results +} + func (e *Executor) writeMapResultFile( - ctx context.Context, bucket, folder, status string, recs []mapItemRecord, + ctx context.Context, bucket, folder, status string, entries []any, outputType string, ) (resultManifestFile, error) { - data, err := json.Marshal(recs) + var ( + data []byte + err error + ) + + if outputType == outputTypeJSONL { + data, err = encodeJSONLines(entries) + } else { + data, err = json.Marshal(entries) + } + if err != nil { return resultManifestFile{}, fmt.Errorf("marshal %s results: %w", status, err) } diff --git a/services/stepfunctions/distributed_map.go b/services/stepfunctions/distributed_map.go index 589d24d22c..16684e5ffb 100644 --- a/services/stepfunctions/distributed_map.go +++ b/services/stepfunctions/distributed_map.go @@ -23,7 +23,7 @@ func (d *distributedMapChildRunner) RunDistributedMapItem( iterator *asl.StateMachine, _ int, item any, -) (any, error) { +) (asl.DistributedMapItemResult, error) { return d.backend.runDistributedMapChild(ctx, executionARN, mapRunARN, iterator, item) } @@ -110,14 +110,14 @@ func (b *InMemoryBackend) runDistributedMapChild( parentExecARN, mapRunARN string, iterator *asl.StateMachine, item any, -) (any, error) { +) (asl.DistributedMapItemResult, error) { b.mu.RLock("runDistributedMapChild.context") cc, ok := b.distributedMapChildContextLocked(parentExecARN) integrations := b.snapshotIntegrationsLocked() b.mu.RUnlock() if !ok { - return nil, fmt.Errorf("%w: %s", ErrExecutionDoesNotExist, parentExecARN) + return asl.DistributedMapItemResult{}, fmt.Errorf("%w: %s", ErrExecutionDoesNotExist, parentExecARN) } input := marshalDistributedMapInput(item) @@ -170,15 +170,26 @@ func (b *InMemoryBackend) runDistributedMapChild( } else if childExec.Status == statusRunning { b.finalizeExecutionRecordLocked(childExec, childExecARN, result, execErr) } + + meta := asl.DistributedMapItemResult{ + ExecutionArn: childExecARN, + Name: childName, + StartDate: childExec.StartDate, + } + if childExec.StopDate != nil { + meta.StopDate = *childExec.StopDate + } b.mu.Unlock() if execErr != nil { - return nil, execErr + return meta, execErr } if result.Failed { - return nil, &asl.FailError{ErrCode: result.Error, Cause: result.Cause} + return meta, &asl.FailError{ErrCode: result.Error, Cause: result.Cause} } - return result.Output, nil + meta.Output = result.Output + + return meta, nil } diff --git a/services/stepfunctions/execution_history_test.go b/services/stepfunctions/execution_history_test.go index a96525e147..c910f7c59e 100644 --- a/services/stepfunctions/execution_history_test.go +++ b/services/stepfunctions/execution_history_test.go @@ -5,7 +5,7 @@ import ( "encoding/json" "net/http" "testing" - "time" + "testing/synctest" "github.com/labstack/echo/v5" "github.com/stretchr/testify/assert" @@ -52,34 +52,34 @@ func TestGetExecutionHistory(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - arn := tt.executionArn - if tt.createExec { - sm, err := b.CreateStateMachine(context.Background(), "hist-sm", passDefinition, "arn:role", "STANDARD") - require.NoError(t, err) - exec, err := b.StartExecution(sm.StateMachineArn, "exec-h", "") + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") + + arn := tt.executionArn + if tt.createExec { + sm, err := b.CreateStateMachine( + context.Background(), "hist-sm", passDefinition, "arn:role", "STANDARD", + ) + require.NoError(t, err) + exec, err := b.StartExecution(sm.StateMachineArn, "exec-h", "") + require.NoError(t, err) + arn = exec.ExecutionArn + synctest.Wait() + } + + events, next, err := b.GetExecutionHistory(arn, "", 0, tt.reverse) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + + return + } require.NoError(t, err) - arn = exec.ExecutionArn - // Wait for async execution to complete. - require.Eventually(t, func() bool { - desc, descErr := b.DescribeExecution(arn) - - return descErr == nil && desc.Status != "RUNNING" - }, 5*time.Second, 50*time.Millisecond) - } - - events, next, err := b.GetExecutionHistory(arn, "", 0, tt.reverse) - if tt.wantErr != nil { - require.ErrorIs(t, err, tt.wantErr) - - return - } - require.NoError(t, err) - assert.Empty(t, next) - assert.Len(t, events, tt.wantLen) - assert.Equal(t, tt.wantFirst, events[0].Type) - assert.Equal(t, tt.wantSecond, events[1].Type) + assert.Empty(t, next) + assert.Len(t, events, tt.wantLen) + assert.Equal(t, tt.wantFirst, events[0].Type) + assert.Equal(t, tt.wantSecond, events[1].Type) + }) }) } } @@ -102,100 +102,94 @@ func TestGetExecutionHistory_TaskEventDetails(t *testing.T) { t.Run("succeeded_task_populates_resource_and_output", func(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - b.SetLambdaInvoker(&mockLambdaForBackend{}) - - sm, err := b.CreateStateMachine( - context.Background(), - "hist-task-sm", - taskLambdaDefinition, - "arn:role", - "STANDARD", - ) - require.NoError(t, err) - - exec, err := b.StartExecution(sm.StateMachineArn, "exec-task-ok", `{"in": 1}`) - require.NoError(t, err) - - require.Eventually(t, func() bool { - desc, descErr := b.DescribeExecution(exec.ExecutionArn) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") + b.SetLambdaInvoker(&mockLambdaForBackend{}) - return descErr == nil && desc.Status != "RUNNING" - }, 5*time.Second, 50*time.Millisecond) + sm, err := b.CreateStateMachine( + context.Background(), + "hist-task-sm", + taskLambdaDefinition, + "arn:role", + "STANDARD", + ) + require.NoError(t, err) - events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 0, false) - require.NoError(t, err) + exec, err := b.StartExecution(sm.StateMachineArn, "exec-task-ok", `{"in": 1}`) + require.NoError(t, err) + synctest.Wait() - var sawScheduled, sawSucceeded, sawStateEntered bool + events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 0, false) + require.NoError(t, err) - for _, ev := range events { - switch ev.Type { - case "TaskScheduled": - require.NotNil(t, ev.TaskScheduledEventDetails) - assert.Equal( - t, - "arn:aws:lambda:us-east-1:000000000000:function:fn", - ev.TaskScheduledEventDetails.Resource, - ) - assert.Equal(t, "lambda", ev.TaskScheduledEventDetails.ResourceType) - sawScheduled = true - case "TaskSucceeded": - require.NotNil(t, ev.TaskSucceededEventDetails) - assert.Contains(t, ev.TaskSucceededEventDetails.Output, "ok") - require.NotNil(t, ev.TaskSucceededEventDetails.OutputDetails) - assert.False(t, ev.TaskSucceededEventDetails.OutputDetails.Truncated) - sawSucceeded = true - case "TaskStateEntered": - require.NotNil(t, ev.StateEnteredEventDetails) - assert.Contains(t, ev.StateEnteredEventDetails.Input, `"in":1`) - sawStateEntered = true + var sawScheduled, sawSucceeded, sawStateEntered bool + + for _, ev := range events { + switch ev.Type { + case "TaskScheduled": + require.NotNil(t, ev.TaskScheduledEventDetails) + assert.Equal( + t, + "arn:aws:lambda:us-east-1:000000000000:function:fn", + ev.TaskScheduledEventDetails.Resource, + ) + assert.Equal(t, "lambda", ev.TaskScheduledEventDetails.ResourceType) + sawScheduled = true + case "TaskSucceeded": + require.NotNil(t, ev.TaskSucceededEventDetails) + assert.Contains(t, ev.TaskSucceededEventDetails.Output, "ok") + require.NotNil(t, ev.TaskSucceededEventDetails.OutputDetails) + assert.False(t, ev.TaskSucceededEventDetails.OutputDetails.Truncated) + sawSucceeded = true + case "TaskStateEntered": + require.NotNil(t, ev.StateEnteredEventDetails) + assert.Contains(t, ev.StateEnteredEventDetails.Input, `"in":1`) + sawStateEntered = true + } } - } - assert.True(t, sawScheduled, "expected a TaskScheduled event") - assert.True(t, sawSucceeded, "expected a TaskSucceeded event") - assert.True(t, sawStateEntered, "expected a TaskStateEntered event with populated input") + assert.True(t, sawScheduled, "expected a TaskScheduled event") + assert.True(t, sawSucceeded, "expected a TaskSucceeded event") + assert.True(t, sawStateEntered, "expected a TaskStateEntered event with populated input") + }) }) t.Run("failed_task_populates_error_and_cause", func(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - b.SetLambdaInvoker(&mockLambdaForBackend{returnErr: assert.AnError}) - - sm, err := b.CreateStateMachine( - context.Background(), - "hist-task-fail-sm", - taskLambdaDefinition, - "arn:role", - "STANDARD", - ) - require.NoError(t, err) - - exec, err := b.StartExecution(sm.StateMachineArn, "exec-task-fail", `{}`) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") + b.SetLambdaInvoker(&mockLambdaForBackend{returnErr: assert.AnError}) - require.Eventually(t, func() bool { - desc, descErr := b.DescribeExecution(exec.ExecutionArn) + sm, err := b.CreateStateMachine( + context.Background(), + "hist-task-fail-sm", + taskLambdaDefinition, + "arn:role", + "STANDARD", + ) + require.NoError(t, err) - return descErr == nil && desc.Status != "RUNNING" - }, 5*time.Second, 50*time.Millisecond) + exec, err := b.StartExecution(sm.StateMachineArn, "exec-task-fail", `{}`) + require.NoError(t, err) + synctest.Wait() - events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 0, false) - require.NoError(t, err) + events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 0, false) + require.NoError(t, err) - var sawFailed bool + var sawFailed bool - for _, ev := range events { - if ev.Type == "TaskFailed" { - require.NotNil(t, ev.TaskFailedEventDetails) - assert.NotEmpty(t, ev.TaskFailedEventDetails.Error) - assert.NotEmpty(t, ev.TaskFailedEventDetails.Cause) - sawFailed = true + for _, ev := range events { + if ev.Type == "TaskFailed" { + require.NotNil(t, ev.TaskFailedEventDetails) + assert.NotEmpty(t, ev.TaskFailedEventDetails.Error) + assert.NotEmpty(t, ev.TaskFailedEventDetails.Cause) + sawFailed = true + } } - } - assert.True(t, sawFailed, "expected a TaskFailed event") + assert.True(t, sawFailed, "expected a TaskFailed event") + }) }) } @@ -218,20 +212,9 @@ func TestHandler_GetExecutionHistory(t *testing.T) { smArn := createSM(ctx, t, h, e, "hist-sm") execArn := startExec(ctx, t, h, e, smArn, "hist-exec") - // Wait for the async execution to complete before checking history. - require.Eventually(t, func() bool { - rec := sfnPost(ctx, t, h, e, "DescribeExecution", - `{"executionArn":"`+execArn+`"}`) - if rec.Code != http.StatusOK { - return false - } - var resp map[string]any - if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { - return false - } - - return resp["status"] != "RUNNING" - }, 5*time.Second, 50*time.Millisecond) + // The async execution runs in this goroutine's bubble; Wait + // blocks until it finishes before checking history. + synctest.Wait() return execArn }, @@ -250,27 +233,29 @@ func TestHandler_GetExecutionHistory(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - ctx := t.Context() - h, e := newSFNHandler(t) + synctest.Test(t, func(t *testing.T) { + ctx := t.Context() + h, e := newSFNHandler(t) - var setupResult string - if tt.setup != nil { - setupResult = tt.setup(t, ctx, h, e) - } + var setupResult string + if tt.setup != nil { + setupResult = tt.setup(t, ctx, h, e) + } - body := tt.body - if tt.bodyFn != nil { - body = tt.bodyFn(setupResult) - } + body := tt.body + if tt.bodyFn != nil { + body = tt.bodyFn(setupResult) + } - rec := sfnPost(ctx, t, h, e, "GetExecutionHistory", body) - assert.Equal(t, tt.wantCode, rec.Code) + rec := sfnPost(ctx, t, h, e, "GetExecutionHistory", body) + assert.Equal(t, tt.wantCode, rec.Code) - if tt.wantEvents > 0 { - var resp map[string]any - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) - assert.Len(t, resp["events"].([]any), tt.wantEvents) - } + if tt.wantEvents > 0 { + var resp map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + assert.Len(t, resp["events"].([]any), tt.wantEvents) + } + }) }) } } @@ -278,143 +263,131 @@ func TestHandler_GetExecutionHistory(t *testing.T) { func TestGetExecutionHistory_HasExecutionStarted(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackend() - sm, err := b.CreateStateMachine( - context.Background(), - "hist-sm", - minimalDefinition, - validRoleARN, - "STANDARD", - ) - require.NoError(t, err) - defer b.Destroy() - - exec, err := b.StartExecution(sm.StateMachineArn, "hist-exec", "{}") - require.NoError(t, err) - - // Allow time for history to populate. - require.Eventually(t, func() bool { - events, _, err2 := b.GetExecutionHistory(exec.ExecutionArn, "", 100, false) - - return err2 == nil && len(events) >= 1 - }, 5*time.Second, 20*time.Millisecond) - - events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, false) - require.NoError(t, err) - assert.Equal(t, "ExecutionStarted", events[0].Type) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + sm, err := b.CreateStateMachine( + context.Background(), + "hist-sm", + minimalDefinition, + validRoleARN, + "STANDARD", + ) + require.NoError(t, err) + defer b.Destroy() + + exec, err := b.StartExecution(sm.StateMachineArn, "hist-exec", "{}") + require.NoError(t, err) + synctest.Wait() + + events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, false) + require.NoError(t, err) + require.NotEmpty(t, events) + assert.Equal(t, "ExecutionStarted", events[0].Type) + }) } func TestGetExecutionHistory_ReverseOrder(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackend() - sm, err := b.CreateStateMachine( - context.Background(), - "rev-hist-sm", - minimalDefinition, - validRoleARN, - "STANDARD", - ) - require.NoError(t, err) - defer b.Destroy() - - exec, err := b.StartExecution(sm.StateMachineArn, "rev-hist-exec", "{}") - require.NoError(t, err) - - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) - - return e == nil && d.Status == "SUCCEEDED" - }, 5*time.Second, 20*time.Millisecond) - - events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, true) - require.NoError(t, err) - require.NotEmpty(t, events) - // Last event in forward order should be first in reverse. - assert.Equal(t, "ExecutionSucceeded", events[0].Type) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + sm, err := b.CreateStateMachine( + context.Background(), + "rev-hist-sm", + minimalDefinition, + validRoleARN, + "STANDARD", + ) + require.NoError(t, err) + defer b.Destroy() + + exec, err := b.StartExecution(sm.StateMachineArn, "rev-hist-exec", "{}") + require.NoError(t, err) + synctest.Wait() + + events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, true) + require.NoError(t, err) + require.NotEmpty(t, events) + // Last event in forward order should be first in reverse. + assert.Equal(t, "ExecutionSucceeded", events[0].Type) + }) } func TestGetExecutionHistory_Pagination(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackend() - sm, err := b.CreateStateMachine( - context.Background(), - "page-hist-sm", - minimalDefinition, - validRoleARN, - "STANDARD", - ) - require.NoError(t, err) - defer b.Destroy() - - exec, err := b.StartExecution(sm.StateMachineArn, "page-hist-exec", "{}") - require.NoError(t, err) - - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) - - return e == nil && d.Status == "SUCCEEDED" - }, 5*time.Second, 20*time.Millisecond) - - // Get all events first. - all, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, false) - require.NoError(t, err) - require.NotEmpty(t, all) - - // Paginate with maxResults=1. - var collected []stepfunctions.HistoryEvent - tok := "" - - for { - page, next, err2 := b.GetExecutionHistory(exec.ExecutionArn, tok, 1, false) - require.NoError(t, err2) - collected = append(collected, page...) - - if next == "" { - break - } + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + sm, err := b.CreateStateMachine( + context.Background(), + "page-hist-sm", + minimalDefinition, + validRoleARN, + "STANDARD", + ) + require.NoError(t, err) + defer b.Destroy() - tok = next - } + exec, err := b.StartExecution(sm.StateMachineArn, "page-hist-exec", "{}") + require.NoError(t, err) + synctest.Wait() + + // Get all events first. + all, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, false) + require.NoError(t, err) + require.NotEmpty(t, all) - assert.Len(t, collected, len(all)) + // Paginate with maxResults=1. + var collected []stepfunctions.HistoryEvent + tok := "" + + for { + page, next, err2 := b.GetExecutionHistory(exec.ExecutionArn, tok, 1, false) + require.NoError(t, err2) + collected = append(collected, page...) + + if next == "" { + break + } + + tok = next + } + + assert.Len(t, collected, len(all)) + }) } func TestGetExecutionHistory_EventIDsMonotonicallyIncreasing(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackend() - sm, err := b.CreateStateMachine( - context.Background(), - "mono-sm", - minimalDefinition, - validRoleARN, - "STANDARD", - ) - require.NoError(t, err) - defer b.Destroy() - - exec, err := b.StartExecution(sm.StateMachineArn, "mono-exec", "{}") - require.NoError(t, err) - - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) - - return e == nil && d.Status == "SUCCEEDED" - }, 5*time.Second, 20*time.Millisecond) - - events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, false) - require.NoError(t, err) - - for i := 1; i < len(events); i++ { - assert.Greater( - t, - events[i].ID, - events[i-1].ID, - "event IDs must be monotonically increasing", + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + sm, err := b.CreateStateMachine( + context.Background(), + "mono-sm", + minimalDefinition, + validRoleARN, + "STANDARD", ) - } + require.NoError(t, err) + defer b.Destroy() + + exec, err := b.StartExecution(sm.StateMachineArn, "mono-exec", "{}") + require.NoError(t, err) + synctest.Wait() + + events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, false) + require.NoError(t, err) + + for i := 1; i < len(events); i++ { + assert.Greater( + t, + events[i].ID, + events[i-1].ID, + "event IDs must be monotonically increasing", + ) + } + }) } // ─── ListExecutions ─────────────────────────────────────────────────────────── @@ -447,32 +420,28 @@ func TestHistoryEventCap(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := newSFBackend() - sm, err := b.CreateStateMachine(context.Background(), "cap-sm", exprPassDef, "arn:role", "STANDARD") - require.NoError(t, err) - - exec, err := b.StartExecution(sm.StateMachineArn, "cap-exec", "{}") - require.NoError(t, err) - - execARN := exec.ExecutionArn + synctest.Test(t, func(t *testing.T) { + b := newSFBackend() + sm, err := b.CreateStateMachine(context.Background(), "cap-sm", exprPassDef, "arn:role", "STANDARD") + require.NoError(t, err) - // Wait for execution to reach terminal state so goroutine is done. - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(execARN) + exec, err := b.StartExecution(sm.StateMachineArn, "cap-exec", "{}") + require.NoError(t, err) - return e == nil && d.Status != "RUNNING" - }, 5*time.Second, 50*time.Millisecond) + execARN := exec.ExecutionArn + synctest.Wait() - // Pre-fill history to the desired count using the exported test helper. - b.FillHistoryForTest(execARN, tt.preFill) + // Pre-fill history to the desired count using the exported test helper. + b.FillHistoryForTest(execARN, tt.preFill) - // Try to add more events via the exported recorder helper. - for range tt.addMoreEvents { - b.RecordStateEnteredForTest(execARN, "ExtraState", "Pass") - } + // Try to add more events via the exported recorder helper. + for range tt.addMoreEvents { + b.RecordStateEnteredForTest(execARN, "ExtraState", "Pass") + } - histLen := b.HistoryLenForTest(execARN) - assert.Equal(t, tt.wantLen, histLen) + histLen := b.HistoryLenForTest(execARN) + assert.Equal(t, tt.wantLen, histLen) + }) }) } } @@ -501,24 +470,22 @@ func TestBackend_GetExecutionHistory_ReverseOrder(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackend() - sm, err := b.CreateStateMachine(context.Background(), "hist-sm", sfnPassDefinition, "arn:role", "STANDARD") - require.NoError(t, err) - - exec, err := b.StartExecution(sm.StateMachineArn, "hist-exec", `{}`) - require.NoError(t, err) - - // Wait for execution to complete - require.Eventually(t, func() bool { - desc, descErr := b.DescribeExecution(exec.ExecutionArn) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + sm, err := b.CreateStateMachine( + context.Background(), "hist-sm", sfnPassDefinition, "arn:role", "STANDARD", + ) + require.NoError(t, err) - return descErr == nil && desc.Status == "SUCCEEDED" - }, 5*time.Second, 50*time.Millisecond) + exec, err := b.StartExecution(sm.StateMachineArn, "hist-exec", `{}`) + require.NoError(t, err) + synctest.Wait() - events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 0, tt.reverseOrder) - require.NoError(t, err) - require.NotEmpty(t, events) - assert.Equal(t, tt.wantFirst, events[0].Type) + events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 0, tt.reverseOrder) + require.NoError(t, err) + require.NotEmpty(t, events) + assert.Equal(t, tt.wantFirst, events[0].Type) + }) }) } } @@ -549,43 +516,40 @@ func TestExecutionHistory_Events(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := newSFBackend() - sm, err := b.CreateStateMachine( - context.Background(), - "hist-"+tt.name, - tt.definition, - "arn:role", - "STANDARD", - ) - require.NoError(t, err) - - exec, err := b.StartExecution(sm.StateMachineArn, "hist-exec", "{}") - require.NoError(t, err) - - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) - - return e == nil && d.Status != "RUNNING" - }, 10*time.Second, 25*time.Millisecond) - - events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, false) - require.NoError(t, err) - require.NotEmpty(t, events) - - // Collect event types. - types := make([]string, len(events)) - for i, e := range events { - types[i] = e.Type - } + synctest.Test(t, func(t *testing.T) { + b := newSFBackend() + sm, err := b.CreateStateMachine( + context.Background(), + "hist-"+tt.name, + tt.definition, + "arn:role", + "STANDARD", + ) + require.NoError(t, err) - for _, wantType := range tt.wantEventTypes { - assert.Contains(t, types, wantType, "expected event type %q in history", wantType) - } + exec, err := b.StartExecution(sm.StateMachineArn, "hist-exec", "{}") + require.NoError(t, err) + synctest.Wait() - // Verify IDs are monotonically increasing. - for i := 1; i < len(events); i++ { - assert.Greater(t, events[i].ID, events[i-1].ID, "event IDs should increase") - } + events, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, false) + require.NoError(t, err) + require.NotEmpty(t, events) + + // Collect event types. + types := make([]string, len(events)) + for i, e := range events { + types[i] = e.Type + } + + for _, wantType := range tt.wantEventTypes { + assert.Contains(t, types, wantType, "expected event type %q in history", wantType) + } + + // Verify IDs are monotonically increasing. + for i := 1; i < len(events); i++ { + assert.Greater(t, events[i].ID, events[i-1].ID, "event IDs should increase") + } + }) }) } } @@ -593,31 +557,28 @@ func TestExecutionHistory_Events(t *testing.T) { func TestExecutionHistory_ReverseOrder(t *testing.T) { t.Parallel() - b := newSFBackend() - sm, err := b.CreateStateMachine(context.Background(), "hist-rev", exprPassDef, "arn:role", "STANDARD") - require.NoError(t, err) - - exec, err := b.StartExecution(sm.StateMachineArn, "rev-exec", "{}") - require.NoError(t, err) - - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) + synctest.Test(t, func(t *testing.T) { + b := newSFBackend() + sm, err := b.CreateStateMachine(context.Background(), "hist-rev", exprPassDef, "arn:role", "STANDARD") + require.NoError(t, err) - return e == nil && d.Status != "RUNNING" - }, 10*time.Second, 25*time.Millisecond) + exec, err := b.StartExecution(sm.StateMachineArn, "rev-exec", "{}") + require.NoError(t, err) + synctest.Wait() - forward, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, false) - require.NoError(t, err) + forward, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, false) + require.NoError(t, err) - reverse, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, true) - require.NoError(t, err) + reverse, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, true) + require.NoError(t, err) - require.Len(t, reverse, len(forward)) + require.Len(t, reverse, len(forward)) - // Reverse order means IDs should decrease. - for i := 1; i < len(reverse); i++ { - assert.Less(t, reverse[i].ID, reverse[i-1].ID) - } + // Reverse order means IDs should decrease. + for i := 1; i < len(reverse); i++ { + assert.Less(t, reverse[i].ID, reverse[i-1].ID) + } + }) } // TestResourceTypeFromResource verifies TaskScheduled/TaskSucceeded/TaskFailed's diff --git a/services/stepfunctions/executions.go b/services/stepfunctions/executions.go index efb4db02c8..e3b7973a40 100644 --- a/services/stepfunctions/executions.go +++ b/services/stepfunctions/executions.go @@ -8,6 +8,8 @@ import ( "sort" "time" + "github.com/google/uuid" + "github.com/blackbirdworks/gopherstack/services/stepfunctions/asl" ) @@ -125,7 +127,7 @@ func (b *InMemoryBackend) StartSyncExecution( } if name == "" { - name = fmt.Sprintf("sync-%d", time.Now().UnixNano()) + name = uuid.NewString() } // Execution/MapRun ARNs are always keyed off the base (unqualified) state diff --git a/services/stepfunctions/executions_asl_test.go b/services/stepfunctions/executions_asl_test.go index 06a7d6e946..05fc05db4a 100644 --- a/services/stepfunctions/executions_asl_test.go +++ b/services/stepfunctions/executions_asl_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -44,38 +45,39 @@ func TestStartExecutionASL(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - sm, err := b.CreateStateMachine(context.Background(), "asl-"+tt.name, tt.definition, "arn:role", "STANDARD") - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - exec, err := b.StartExecution(sm.StateMachineArn, "asl-exec", tt.input) - require.NoError(t, err) + sm, err := b.CreateStateMachine( + context.Background(), "asl-"+tt.name, tt.definition, "arn:role", "STANDARD", + ) + require.NoError(t, err) - if tt.checkInitStatus { - // Use DescribeExecution (returns a copy) to safely read status — avoids a data race - // with the goroutine launched inside StartExecution that also writes to the execution struct. - initialDesc, initDescErr := b.DescribeExecution(exec.ExecutionArn) - require.NoError(t, initDescErr) - assert.Contains(t, []string{"RUNNING", "SUCCEEDED"}, initialDesc.Status) - } + exec, err := b.StartExecution(sm.StateMachineArn, "asl-exec", tt.input) + require.NoError(t, err) - require.Eventually(t, func() bool { - desc, descErr := b.DescribeExecution(exec.ExecutionArn) + if tt.checkInitStatus { + // Use DescribeExecution (returns a copy) to safely read status — avoids a data race + // with the goroutine launched inside StartExecution that also writes to the execution struct. + initialDesc, initDescErr := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, initDescErr) + assert.Contains(t, []string{"RUNNING", "SUCCEEDED"}, initialDesc.Status) + } - return descErr == nil && desc.Status == tt.wantStatus - }, 5*time.Second, 50*time.Millisecond, "execution should reach "+tt.wantStatus) + synctest.Wait() - desc, err := b.DescribeExecution(exec.ExecutionArn) - require.NoError(t, err) - assert.Equal(t, tt.wantStatus, desc.Status) + desc, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + assert.Equal(t, tt.wantStatus, desc.Status) - if tt.wantOutputKey != "" { - assert.Contains(t, desc.Output, tt.wantOutputKey) - } - if tt.wantError != "" { - assert.Equal(t, tt.wantError, desc.Error) - } + if tt.wantOutputKey != "" { + assert.Contains(t, desc.Output, tt.wantOutputKey) + } + if tt.wantError != "" { + assert.Equal(t, tt.wantError, desc.Error) + } + }) }) } } @@ -83,61 +85,55 @@ func TestStartExecutionASL(t *testing.T) { func TestExecution_SucceedsAfterPass(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackend() - sm, err := b.CreateStateMachine( - context.Background(), - "succ-sm", - minimalDefinition, - validRoleARN, - "STANDARD", - ) - require.NoError(t, err) - defer b.Destroy() - - exec, err := b.StartExecution(sm.StateMachineArn, "succ-exec", "{}") - require.NoError(t, err) - - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) - - return e == nil && d.Status == "SUCCEEDED" - }, 5*time.Second, 20*time.Millisecond) - - desc, err := b.DescribeExecution(exec.ExecutionArn) - require.NoError(t, err) - assert.Equal(t, "SUCCEEDED", desc.Status) - assert.NotNil(t, desc.StopDate) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + sm, err := b.CreateStateMachine( + context.Background(), + "succ-sm", + minimalDefinition, + validRoleARN, + "STANDARD", + ) + require.NoError(t, err) + defer b.Destroy() + + exec, err := b.StartExecution(sm.StateMachineArn, "succ-exec", "{}") + require.NoError(t, err) + synctest.Wait() + + desc, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + assert.Equal(t, "SUCCEEDED", desc.Status) + assert.NotNil(t, desc.StopDate) + }) } func TestExecution_FailStateProducesFailedStatus(t *testing.T) { t.Parallel() - failDef := `{"StartAt":"F","States":{"F":{"Type":"Fail","Error":"ErrFoo","Cause":"test cause","End":true}}}` - b := stepfunctions.NewInMemoryBackend() - sm, err := b.CreateStateMachine( - context.Background(), - "fail-sm", - failDef, - validRoleARN, - "STANDARD", - ) - require.NoError(t, err) - defer b.Destroy() - - exec, err := b.StartExecution(sm.StateMachineArn, "fail-exec", "{}") - require.NoError(t, err) - - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) - - return e == nil && d.Status == "FAILED" - }, 5*time.Second, 20*time.Millisecond) - - desc, err := b.DescribeExecution(exec.ExecutionArn) - require.NoError(t, err) - assert.Equal(t, "FAILED", desc.Status) - assert.Equal(t, "ErrFoo", desc.Error) - assert.Equal(t, "test cause", desc.Cause) + synctest.Test(t, func(t *testing.T) { + failDef := `{"StartAt":"F","States":{"F":{"Type":"Fail","Error":"ErrFoo","Cause":"test cause","End":true}}}` + b := stepfunctions.NewInMemoryBackend() + sm, err := b.CreateStateMachine( + context.Background(), + "fail-sm", + failDef, + validRoleARN, + "STANDARD", + ) + require.NoError(t, err) + defer b.Destroy() + + exec, err := b.StartExecution(sm.StateMachineArn, "fail-exec", "{}") + require.NoError(t, err) + synctest.Wait() + + desc, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + assert.Equal(t, "FAILED", desc.Status) + assert.Equal(t, "ErrFoo", desc.Error) + assert.Equal(t, "test cause", desc.Cause) + }) } // TestAudit_StartExecution_ExpressMachineSucceeds verifies that @@ -362,53 +358,46 @@ func TestStartExecution_WaitForTaskToken(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := newSFBackend() - sqsMock := &mockStepFunctionsSQS{} - b.SetSQSIntegration(sqsMock) + synctest.Test(t, func(t *testing.T) { + b := newSFBackend() + sqsMock := &mockStepFunctionsSQS{} + b.SetSQSIntegration(sqsMock) - sm, err := b.CreateStateMachine(context.Background(), "wait-token-sm-"+tt.name, def, "arn:role", "STANDARD") - require.NoError(t, err) + sm, err := b.CreateStateMachine( + context.Background(), "wait-token-sm-"+tt.name, def, "arn:role", "STANDARD", + ) + require.NoError(t, err) - exec, err := b.StartExecution(sm.StateMachineArn, "wait-token-exec-"+tt.name, `{}`) - require.NoError(t, err) + exec, err := b.StartExecution(sm.StateMachineArn, "wait-token-exec-"+tt.name, `{}`) + require.NoError(t, err) + + // The executor durably blocks awaiting the callback once it + // registers the task token. + synctest.Wait() - var taskToken string - require.Eventually(t, func() bool { tokens := b.TaskTokensForTest() - if len(tokens) == 0 { - return false - } - taskToken = tokens[0] + require.NotEmpty(t, tokens) - return taskToken != "" - }, 5*time.Second, 25*time.Millisecond) + err = tt.sendResult(b, tokens[0]) + require.NoError(t, err) + synctest.Wait() - err = tt.sendResult(b, taskToken) - require.NoError(t, err) + described, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) - var described *stepfunctions.Execution - require.Eventually(t, func() bool { - execution, describeErr := b.DescribeExecution(exec.ExecutionArn) - if describeErr != nil { - return false + assert.Equal(t, tt.wantStatus, described.Status) + if tt.wantOutput != "" { + assert.JSONEq(t, tt.wantOutput, described.Output) + } + if tt.wantError != "" { + assert.Equal(t, tt.wantError, described.Error) + } + if tt.wantCauseSubstr != "" { + assert.Contains(t, described.Cause, tt.wantCauseSubstr) } - described = execution - - return described.Status != "RUNNING" - }, 5*time.Second, 25*time.Millisecond) - - assert.Equal(t, tt.wantStatus, described.Status) - if tt.wantOutput != "" { - assert.JSONEq(t, tt.wantOutput, described.Output) - } - if tt.wantError != "" { - assert.Equal(t, tt.wantError, described.Error) - } - if tt.wantCauseSubstr != "" { - assert.Contains(t, described.Cause, tt.wantCauseSubstr) - } - assert.Equal(t, 1, sqsMock.callCount) + assert.Equal(t, 1, sqsMock.callCount) + }) }) } } @@ -437,28 +426,25 @@ func TestBackend_RunParsedExecution_FailState(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackend() - sm, err := b.CreateStateMachine( - context.Background(), - "run-sm-"+tt.name, - tt.definition, - "arn:role", - "STANDARD", - ) - require.NoError(t, err) - - exec, err := b.StartExecution(sm.StateMachineArn, "run-exec", `{}`) - require.NoError(t, err) - - require.Eventually(t, func() bool { - desc, descErr := b.DescribeExecution(exec.ExecutionArn) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + sm, err := b.CreateStateMachine( + context.Background(), + "run-sm-"+tt.name, + tt.definition, + "arn:role", + "STANDARD", + ) + require.NoError(t, err) - return descErr == nil && desc.Status == tt.wantStatus - }, 5*time.Second, 50*time.Millisecond) + exec, err := b.StartExecution(sm.StateMachineArn, "run-exec", `{}`) + require.NoError(t, err) + synctest.Wait() - desc, err := b.DescribeExecution(exec.ExecutionArn) - require.NoError(t, err) - assert.Equal(t, tt.wantStatus, desc.Status) + desc, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + assert.Equal(t, tt.wantStatus, desc.Status) + }) }) } } @@ -494,32 +480,31 @@ func TestParallelState_WithCatch(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := newSFBackend() - sm, err := b.CreateStateMachine( - context.Background(), - "parallel-catch-"+tt.name, - tt.definition, - "arn:role", - "STANDARD", - ) - require.NoError(t, err) - - exec, err := b.StartExecution(sm.StateMachineArn, "exec-"+tt.name, tt.input) - require.NoError(t, err) - - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) + synctest.Test(t, func(t *testing.T) { + b := newSFBackend() + sm, err := b.CreateStateMachine( + context.Background(), + "parallel-catch-"+tt.name, + tt.definition, + "arn:role", + "STANDARD", + ) + require.NoError(t, err) - return e == nil && d.Status != "RUNNING" - }, 10*time.Second, 25*time.Millisecond) + exec, err := b.StartExecution(sm.StateMachineArn, "exec-"+tt.name, tt.input) + require.NoError(t, err) + synctest.Wait() - d, err := b.DescribeExecution(exec.ExecutionArn) - require.NoError(t, err) - assert.Equal(t, tt.wantStatus, d.Status, "unexpected execution status") + d, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + assert.Equal(t, tt.wantStatus, d.Status, "unexpected execution status") + }) }) } } +// Not synctest-wrapped: the bubble clock starts at 2000-01-01, so the "past" +// 2020 timestamp would be 20 virtual years ahead. func TestWaitState_TimestampPast(t *testing.T) { t.Parallel() diff --git a/services/stepfunctions/executions_test.go b/services/stepfunctions/executions_test.go index e0a769d281..bdf31d9216 100644 --- a/services/stepfunctions/executions_test.go +++ b/services/stepfunctions/executions_test.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -148,38 +149,38 @@ func TestDescribeExecution(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - arn := tt.executionArn - if tt.createExec { - sm, err := b.CreateStateMachine( - context.Background(), - "desc-exec-sm", - passDefinition, - "arn:role", - "STANDARD", - ) - require.NoError(t, err) - exec, err := b.StartExecution(sm.StateMachineArn, "exec1", tt.input) - require.NoError(t, err) - arn = exec.ExecutionArn - // Wait for the async executor to finish. - require.Eventually(t, func() bool { - desc, descErr := b.DescribeExecution(arn) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") + + arn := tt.executionArn + if tt.createExec { + sm, err := b.CreateStateMachine( + context.Background(), + "desc-exec-sm", + passDefinition, + "arn:role", + "STANDARD", + ) + require.NoError(t, err) + exec, err := b.StartExecution(sm.StateMachineArn, "exec1", tt.input) + require.NoError(t, err) + arn = exec.ExecutionArn + // The async executor runs in this goroutine's bubble; Wait + // blocks until it finishes (or durably blocks). + synctest.Wait() + } - return descErr == nil && desc.Status != "RUNNING" - }, 5*time.Second, 50*time.Millisecond) - } + got, err := b.DescribeExecution(arn) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) - got, err := b.DescribeExecution(arn) - if tt.wantErr != nil { - require.ErrorIs(t, err, tt.wantErr) - - return - } - require.NoError(t, err) - assert.Equal(t, tt.wantStatus, got.Status) - assert.Equal(t, tt.wantInput, got.Input) + return + } + require.NoError(t, err) + assert.Equal(t, tt.wantStatus, got.Status) + assert.Equal(t, tt.wantInput, got.Input) + }) }) } } @@ -217,40 +218,32 @@ func TestListExecutions(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - sm, err := b.CreateStateMachine( - context.Background(), - "list-exec-sm", - passDefinition, - "arn:role", - "STANDARD", - ) - require.NoError(t, err) - for _, name := range tt.execNames { - _, err = b.StartExecution(sm.StateMachineArn, name, "") - require.NoError(t, err) - } + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - // Wait for async executions to complete before checking status filters. - require.Eventually(t, func() bool { - execs, _, listErr := b.ListExecutions(sm.StateMachineArn, "", "", 0) - if listErr != nil { - return false - } - for _, ex := range execs { - if ex.Status == "RUNNING" { - return false - } + sm, err := b.CreateStateMachine( + context.Background(), + "list-exec-sm", + passDefinition, + "arn:role", + "STANDARD", + ) + require.NoError(t, err) + for _, name := range tt.execNames { + _, err = b.StartExecution(sm.StateMachineArn, name, "") + require.NoError(t, err) } - return true - }, 5*time.Second, 50*time.Millisecond) + // Wait for the async executions to complete before checking + // status filters. + synctest.Wait() - execs, next, err := b.ListExecutions(sm.StateMachineArn, tt.statusFilter, "", 0) - require.NoError(t, err) - assert.Empty(t, next) - assert.Len(t, execs, tt.wantCount) + execs, next, err := b.ListExecutions(sm.StateMachineArn, tt.statusFilter, "", 0) + require.NoError(t, err) + assert.Empty(t, next) + assert.Len(t, execs, tt.wantCount) + }) }) } } @@ -288,37 +281,39 @@ func TestStopExecution(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - arn := tt.executionArn - if tt.createExec { - sm, err := b.CreateStateMachine(context.Background(), "stop-sm", waitDefinition, "arn:role", "STANDARD") - require.NoError(t, err) - exec, err := b.StartExecution(sm.StateMachineArn, "exec-stop", "") - require.NoError(t, err) - arn = exec.ExecutionArn - // Wait for execution to enter RUNNING before stopping it. - require.Eventually(t, func() bool { - desc, descErr := b.DescribeExecution(arn) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") + + arn := tt.executionArn + if tt.createExec { + sm, err := b.CreateStateMachine( + context.Background(), "stop-sm", waitDefinition, "arn:role", "STANDARD", + ) + require.NoError(t, err) + exec, err := b.StartExecution(sm.StateMachineArn, "exec-stop", "") + require.NoError(t, err) + arn = exec.ExecutionArn + // The executor blocks on the Wait state's timer once + // RUNNING; Wait returns once it's durably blocked there. + synctest.Wait() + } - return descErr == nil && desc.Status == "RUNNING" - }, 5*time.Second, 10*time.Millisecond) - } + err := b.StopExecution(arn, tt.stopError, tt.stopCause) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) - err := b.StopExecution(arn, tt.stopError, tt.stopCause) - if tt.wantErr != nil { - require.ErrorIs(t, err, tt.wantErr) - - return - } - require.NoError(t, err) + return + } + require.NoError(t, err) - got, err := b.DescribeExecution(arn) - require.NoError(t, err) - assert.Equal(t, tt.wantStatus, got.Status) - assert.Equal(t, tt.wantError, got.Error) - assert.Equal(t, tt.wantCause, got.Cause) - assert.NotNil(t, got.StopDate) + got, err := b.DescribeExecution(arn) + require.NoError(t, err) + assert.Equal(t, tt.wantStatus, got.Status) + assert.Equal(t, tt.wantError, got.Error) + assert.Equal(t, tt.wantCause, got.Cause) + assert.NotNil(t, got.StopDate) + }) }) } } @@ -347,50 +342,33 @@ func TestRedriveExecution_RedriveCount(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - failDef := `{"StartAt":"F","States":{"F":{"Type":"Fail","Error":"Err","Cause":"test"}}}` - sm, err := b.CreateStateMachine( - context.Background(), - "redrive-sm-"+tt.name, - failDef, - "arn:role", - "STANDARD", - ) - require.NoError(t, err) - - exec, err := b.StartExecution(sm.StateMachineArn, "redrive-exec-"+tt.name, `{}`) - require.NoError(t, err) - - // Wait for failure. - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) - - return e == nil && d.Status == "FAILED" - }, 5*time.Second, 50*time.Millisecond) - - // Perform redrives. - for range tt.redrives { - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) - - return e == nil && d.Status == "FAILED" - }, 5*time.Second, 50*time.Millisecond) - - _, redriveErr := b.RedriveExecution(exec.ExecutionArn) - require.NoError(t, redriveErr) - } + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") + failDef := `{"StartAt":"F","States":{"F":{"Type":"Fail","Error":"Err","Cause":"test"}}}` + sm, err := b.CreateStateMachine( + context.Background(), + "redrive-sm-"+tt.name, + failDef, + "arn:role", + "STANDARD", + ) + require.NoError(t, err) - // Wait for final completion. - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) + exec, err := b.StartExecution(sm.StateMachineArn, "redrive-exec-"+tt.name, `{}`) + require.NoError(t, err) + synctest.Wait() - return e == nil && d.Status != "RUNNING" - }, 5*time.Second, 50*time.Millisecond) + for range tt.redrives { + _, redriveErr := b.RedriveExecution(exec.ExecutionArn) + require.NoError(t, redriveErr) + synctest.Wait() + } - described, err := b.DescribeExecution(exec.ExecutionArn) - require.NoError(t, err) - assert.Equal(t, tt.wantRedriveCount, described.RedriveCount) - assert.NotNil(t, described.RedriveDate) + described, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + assert.Equal(t, tt.wantRedriveCount, described.RedriveCount) + assert.NotNil(t, described.RedriveDate) + }) }) } } @@ -541,34 +519,31 @@ func TestDescribeExecution_ParityFields(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - t.Cleanup(b.Destroy) - - sm, err := b.CreateStateMachine(t.Context(), "sm-"+tt.name, tt.def, validRoleARN, "STANDARD") - require.NoError(t, err) - - exec, err := b.StartExecutionWithTrace(sm.StateMachineArn, "exec-"+tt.name, `{"in":1}`, tt.traceHeader) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") + t.Cleanup(b.Destroy) - require.Eventually(t, func() bool { - desc, descErr := b.DescribeExecution(exec.ExecutionArn) + sm, err := b.CreateStateMachine(t.Context(), "sm-"+tt.name, tt.def, validRoleARN, "STANDARD") + require.NoError(t, err) - return descErr == nil && desc.Status != "RUNNING" - }, 5*time.Second, 50*time.Millisecond) + exec, err := b.StartExecutionWithTrace(sm.StateMachineArn, "exec-"+tt.name, `{"in":1}`, tt.traceHeader) + require.NoError(t, err) + synctest.Wait() - desc, err := b.DescribeExecution(exec.ExecutionArn) - require.NoError(t, err) - assert.Equal(t, tt.wantStatus, desc.Status) - assert.Equal(t, tt.wantRedriveStatus, desc.RedriveStatus) - require.NotNil(t, desc.InputDetails) - assert.True(t, desc.InputDetails.Included) - if tt.traceHeader != "" { - assert.Equal(t, tt.traceHeader, desc.TraceHeader) - } - if tt.wantStatus == "SUCCEEDED" { - require.NotNil(t, desc.OutputDetails) - assert.True(t, desc.OutputDetails.Included) - } + desc, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + assert.Equal(t, tt.wantStatus, desc.Status) + assert.Equal(t, tt.wantRedriveStatus, desc.RedriveStatus) + require.NotNil(t, desc.InputDetails) + assert.True(t, desc.InputDetails.Included) + if tt.traceHeader != "" { + assert.Equal(t, tt.traceHeader, desc.TraceHeader) + } + if tt.wantStatus == "SUCCEEDED" { + require.NotNil(t, desc.OutputDetails) + assert.True(t, desc.OutputDetails.Included) + } + }) }) } } @@ -708,35 +683,31 @@ func TestStopExecution_SetsAborted(t *testing.T) { func TestStopExecution_IdempotentOnTerminalState(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackend() - sm, err := b.CreateStateMachine( - context.Background(), - "idm-sm", - minimalDefinition, - validRoleARN, - "STANDARD", - ) - require.NoError(t, err) - defer b.Destroy() - - exec, err := b.StartExecution(sm.StateMachineArn, "idm-exec", "{}") - require.NoError(t, err) - - // Wait for execution to reach terminal state. - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + sm, err := b.CreateStateMachine( + context.Background(), + "idm-sm", + minimalDefinition, + validRoleARN, + "STANDARD", + ) + require.NoError(t, err) + defer b.Destroy() - return e == nil && d.Status != "RUNNING" - }, 5*time.Second, 20*time.Millisecond) + exec, err := b.StartExecution(sm.StateMachineArn, "idm-exec", "{}") + require.NoError(t, err) + synctest.Wait() - // Must not error and must not overwrite terminal status. - err = b.StopExecution(exec.ExecutionArn, "ShouldNotOverwrite", "nope") - require.NoError(t, err) + // Must not error and must not overwrite terminal status. + err = b.StopExecution(exec.ExecutionArn, "ShouldNotOverwrite", "nope") + require.NoError(t, err) - desc, err := b.DescribeExecution(exec.ExecutionArn) - require.NoError(t, err) - assert.Equal(t, "SUCCEEDED", desc.Status, "terminal status must not be overwritten") - assert.NotEqual(t, "ShouldNotOverwrite", desc.Error) + desc, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + assert.Equal(t, "SUCCEEDED", desc.Status, "terminal status must not be overwritten") + assert.NotEqual(t, "ShouldNotOverwrite", desc.Error) + }) } func TestStopExecution_NotFound(t *testing.T) { @@ -753,36 +724,33 @@ func TestStopExecution_NotFound(t *testing.T) { func TestListExecutions_StatusFilter_RUNNING(t *testing.T) { t.Parallel() - waitDef := `{"StartAt":"W","States":{"W":{"Type":"Wait","Seconds":3600,"End":true}}}` - b := stepfunctions.NewInMemoryBackend() - sm, err := b.CreateStateMachine( - context.Background(), - "list-sm", - waitDef, - validRoleARN, - "STANDARD", - ) - require.NoError(t, err) - defer b.Destroy() - - exec, err := b.StartExecution(sm.StateMachineArn, "list-run-e", "{}") - require.NoError(t, err) - - // Give the execution time to start. - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) + synctest.Test(t, func(t *testing.T) { + waitDef := `{"StartAt":"W","States":{"W":{"Type":"Wait","Seconds":3600,"End":true}}}` + b := stepfunctions.NewInMemoryBackend() + sm, err := b.CreateStateMachine( + context.Background(), + "list-sm", + waitDef, + validRoleARN, + "STANDARD", + ) + require.NoError(t, err) + defer b.Destroy() - return e == nil && d.Status == "RUNNING" - }, 5*time.Second, 20*time.Millisecond) + exec, err := b.StartExecution(sm.StateMachineArn, "list-run-e", "{}") + require.NoError(t, err) + // The executor durably blocks on the Wait state's timer once RUNNING. + synctest.Wait() - running, _, err := b.ListExecutions(sm.StateMachineArn, "RUNNING", "", 100) - require.NoError(t, err) - assert.Len(t, running, 1) - assert.Equal(t, exec.ExecutionArn, running[0].ExecutionArn) + running, _, err := b.ListExecutions(sm.StateMachineArn, "RUNNING", "", 100) + require.NoError(t, err) + assert.Len(t, running, 1) + assert.Equal(t, exec.ExecutionArn, running[0].ExecutionArn) - succeeded, _, err := b.ListExecutions(sm.StateMachineArn, "SUCCEEDED", "", 100) - require.NoError(t, err) - assert.Empty(t, succeeded) + succeeded, _, err := b.ListExecutions(sm.StateMachineArn, "SUCCEEDED", "", 100) + require.NoError(t, err) + assert.Empty(t, succeeded) + }) } func TestListExecutions_Pagination(t *testing.T) { @@ -864,30 +832,28 @@ func TestDescribeStateMachineForExecution(t *testing.T) { func TestRedriveExecution_NotRedrivable(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackend() - sm, err := b.CreateStateMachine( - context.Background(), - "redrive-sm", - minimalDefinition, - validRoleARN, - "STANDARD", - ) - require.NoError(t, err) - defer b.Destroy() - - exec, err := b.StartExecution(sm.StateMachineArn, "rd-exec", "{}") - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + sm, err := b.CreateStateMachine( + context.Background(), + "redrive-sm", + minimalDefinition, + validRoleARN, + "STANDARD", + ) + require.NoError(t, err) + defer b.Destroy() - // SUCCEEDED executions cannot be redriven. - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) + exec, err := b.StartExecution(sm.StateMachineArn, "rd-exec", "{}") + require.NoError(t, err) - return e == nil && d.Status == "SUCCEEDED" - }, 5*time.Second, 20*time.Millisecond) + // SUCCEEDED executions cannot be redriven. + synctest.Wait() - _, err = b.RedriveExecution(exec.ExecutionArn) - require.Error(t, err) - assert.ErrorIs(t, err, stepfunctions.ErrExecutionNotRedrivable) + _, err = b.RedriveExecution(exec.ExecutionArn) + require.Error(t, err) + assert.ErrorIs(t, err, stepfunctions.ErrExecutionNotRedrivable) + }) } // ─── roleArn validation ─────────────────────────────────────────────────────── @@ -936,35 +902,37 @@ func TestInput_OverLimit_Fails(t *testing.T) { func TestListExecutions_OrderedByStartDateDesc(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - sm, err := b.CreateStateMachine(context.Background(), "order-sm", minimalDefinition, validRoleARN, "STANDARD") - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") + sm, err := b.CreateStateMachine(context.Background(), "order-sm", minimalDefinition, validRoleARN, "STANDARD") + require.NoError(t, err) - names := []string{"exec-a", "exec-b", "exec-c"} - startDates := make([]float64, len(names)) + names := []string{"exec-a", "exec-b", "exec-c"} + startDates := make([]float64, len(names)) - for i, name := range names { - exec, execErr := b.StartExecution(sm.StateMachineArn, name, "{}") - require.NoError(t, execErr) + for i, name := range names { + exec, execErr := b.StartExecution(sm.StateMachineArn, name, "{}") + require.NoError(t, execErr) - startDates[i] = exec.StartDate + startDates[i] = exec.StartDate - // Small sleep to ensure distinct start timestamps. - time.Sleep(5 * time.Millisecond) - } + // Small sleep to ensure distinct start timestamps. + time.Sleep(5 * time.Millisecond) + } - execs, _, err := b.ListExecutions(sm.StateMachineArn, "", "", 10) - require.NoError(t, err) - require.Len(t, execs, 3) + execs, _, err := b.ListExecutions(sm.StateMachineArn, "", "", 10) + require.NoError(t, err) + require.Len(t, execs, 3) - // Most recent first. - for i := 1; i < len(execs); i++ { - assert.GreaterOrEqual(t, execs[i-1].StartDate, execs[i].StartDate, - "expected descending startDate order at index %d", i) - } + // Most recent first. + for i := 1; i < len(execs); i++ { + assert.GreaterOrEqual(t, execs[i-1].StartDate, execs[i].StartDate, + "expected descending startDate order at index %d", i) + } - // First result should be the last started. - assert.Equal(t, names[2], execs[0].Name) + // First result should be the last started. + assert.Equal(t, names[2], execs[0].Name) + }) } const ( @@ -1001,37 +969,37 @@ func (m *mockStepFunctionsSQS) SFNSendMessage( func TestListExecutionsStatusIndex(t *testing.T) { t.Parallel() - bk := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - ctx := context.Background() + const numExecs = 5 - sm, err := bk.CreateStateMachine( - ctx, "perf-sm", - `{"StartAt":"P","States":{"P":{"Type":"Pass","End":true}}}`, - "arn:aws:iam::123456789012:role/r", "STANDARD", + var ( + bk *stepfunctions.InMemoryBackend + execARNs []string + smARN string ) - require.NoError(t, err) - smARN := sm.StateMachineArn - const numExecs = 5 - execARNs := make([]string, 0, numExecs) + synctest.Test(t, func(t *testing.T) { + bk = stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") + ctx := context.Background() - for i := range numExecs { - exec, startErr := bk.StartExecution(smARN, fmt.Sprintf("exec-%d", i), `{}`) - require.NoError(t, startErr) - execARNs = append(execARNs, exec.ExecutionArn) - } + sm, err := bk.CreateStateMachine( + ctx, "perf-sm", + `{"StartAt":"P","States":{"P":{"Type":"Pass","End":true}}}`, + "arn:aws:iam::123456789012:role/r", "STANDARD", + ) + require.NoError(t, err) + smARN = sm.StateMachineArn - // Wait for all executions to finish. - require.Eventually(t, func() bool { - for _, arn := range execARNs { - exec, descErr := bk.DescribeExecution(arn) - if descErr != nil || exec.Status == "RUNNING" { - return false - } + execARNs = make([]string, 0, numExecs) + + for i := range numExecs { + exec, startErr := bk.StartExecution(smARN, fmt.Sprintf("exec-%d", i), `{}`) + require.NoError(t, startErr) + execARNs = append(execARNs, exec.ExecutionArn) } - return true - }, 5*time.Second, 20*time.Millisecond) + // Wait for all executions to finish. + synctest.Wait() + }) // Count actual statuses. succeededCount := 0 @@ -1176,40 +1144,28 @@ func TestBackend_ListExecutions_Pagination(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackend() - sm, err := b.CreateStateMachine( - context.Background(), - "page-sm", - `{"StartAt":"S","States":{"S":{"Type":"Pass","End":true}}}`, - "arn:role", - "STANDARD", - ) - require.NoError(t, err) - - // Create two executions so we have something to paginate - _, _ = b.StartExecution(sm.StateMachineArn, "exec-a", `{}`) - _, _ = b.StartExecution(sm.StateMachineArn, "exec-b", `{}`) - - // Wait for executions to complete to avoid race condition - require.Eventually(t, func() bool { - execs, _, listErr := b.ListExecutions(sm.StateMachineArn, "", "", 0) - if listErr != nil { - return false - } + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + sm, err := b.CreateStateMachine( + context.Background(), + "page-sm", + `{"StartAt":"S","States":{"S":{"Type":"Pass","End":true}}}`, + "arn:role", + "STANDARD", + ) + require.NoError(t, err) - doneCount := 0 - for _, e := range execs { - if e.Status != "RUNNING" { - doneCount++ - } - } + // Create two executions so we have something to paginate + _, _ = b.StartExecution(sm.StateMachineArn, "exec-a", `{}`) + _, _ = b.StartExecution(sm.StateMachineArn, "exec-b", `{}`) - return doneCount == 2 - }, 5*time.Second, 50*time.Millisecond) + // Wait for both to complete before checking pagination. + synctest.Wait() - execs, _, err := b.ListExecutions(sm.StateMachineArn, "", tt.nextToken, tt.maxResults) - require.NoError(t, err) - assert.Len(t, execs, tt.wantLen) + execs, _, err := b.ListExecutions(sm.StateMachineArn, "", tt.nextToken, tt.maxResults) + require.NoError(t, err) + assert.Len(t, execs, tt.wantLen) + }) }) } } diff --git a/services/stepfunctions/handler_activities_test.go b/services/stepfunctions/handler_activities_test.go index 8f4d3690fb..167ce463b6 100644 --- a/services/stepfunctions/handler_activities_test.go +++ b/services/stepfunctions/handler_activities_test.go @@ -3,12 +3,12 @@ package stepfunctions_test import ( "context" "encoding/json" - "errors" "fmt" "net/http" "strconv" "strings" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -240,65 +240,49 @@ func TestHandler_SendTaskSuccess_WithRealToken(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - ctx := t.Context() - h, e := newSFNHandler(t) - - // Create an activity. - rec := sfnPost(ctx, t, h, e, "CreateActivity", `{"name":"send-act-`+tt.name+`"}`) - require.Equal(t, http.StatusOK, rec.Code) - - var actResp map[string]any - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &actResp)) - actARN := actResp["activityArn"].(string) - - // Enqueue a task by calling InvokeActivity from the backend. - bk, ok := h.Backend.(*stepfunctions.InMemoryBackend) - require.True(t, ok) - - taskCh := make(chan string, 1) - go func() { - out, err := bk.InvokeActivity(t.Context(), actARN, `{"in":1}`, 0) - if err == nil { - taskCh <- out - } else { - taskCh <- "" - } - }() - - // Poll for the task via the handler. - var taskToken string - - require.Eventually(t, func() bool { - pollCtx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) - defer cancel() + synctest.Test(t, func(t *testing.T) { + ctx := t.Context() + h, e := newSFNHandler(t) - // Use the backend directly since GetActivityTask is context-aware. - task, pollErr := bk.GetActivityTask(pollCtx, actARN, "worker") - if pollErr != nil || task == nil || task.TaskToken == "" { - return false - } + // Create an activity. + rec := sfnPost(ctx, t, h, e, "CreateActivity", `{"name":"send-act-`+tt.name+`"}`) + require.Equal(t, http.StatusOK, rec.Code) - taskToken = task.TaskToken + var actResp map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &actResp)) + actARN := actResp["activityArn"].(string) - return true - }, 5*time.Second, 50*time.Millisecond) + // Enqueue a task by calling InvokeActivity from the backend. + bk, ok := h.Backend.(*stepfunctions.InMemoryBackend) + require.True(t, ok) - require.NotEmpty(t, taskToken) + taskCh := make(chan string, 1) + go func() { + out, err := bk.InvokeActivity(t.Context(), actARN, `{"in":1}`, 0) + if err == nil { + taskCh <- out + } else { + taskCh <- "" + } + }() - // Send success via HTTP handler. - body, _ := json.Marshal(map[string]string{ - "taskToken": taskToken, - "output": tt.output, + // GetActivityTask long-polls; it unblocks as soon as the + // goroutine above enqueues the task. + task, pollErr := bk.GetActivityTask(ctx, actARN, "worker") + require.NoError(t, pollErr) + require.NotEmpty(t, task.TaskToken) + + // Send success via HTTP handler. + body, _ := json.Marshal(map[string]string{ + "taskToken": task.TaskToken, + "output": tt.output, + }) + rec = sfnPost(ctx, t, h, e, "SendTaskSuccess", string(body)) + assert.Equal(t, http.StatusOK, rec.Code) + + synctest.Wait() + assert.Equal(t, tt.output, <-taskCh) }) - rec = sfnPost(ctx, t, h, e, "SendTaskSuccess", string(body)) - assert.Equal(t, http.StatusOK, rec.Code) - - select { - case out := <-taskCh: - assert.Equal(t, tt.output, out) - case <-time.After(5 * time.Second): - t.Fatal("timeout waiting for InvokeActivity to complete") - } }) } } @@ -884,102 +868,77 @@ func TestActivity_ListAndPaginate(t *testing.T) { func TestActivity_SendTaskSuccess(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackend() - defer b.Destroy() - - act, err := b.CreateActivity(context.Background(), "send-act") - require.NoError(t, err) - - actDef := fmt.Sprintf(`{"StartAt":"A","States":{"A":{"Type":"Task","Resource":%q,"End":true}}}`, - act.ActivityArn) - sm, err := b.CreateStateMachine( - context.Background(), - "act-sm", - actDef, - validRoleARN, - "STANDARD", - ) - require.NoError(t, err) - - exec, err := b.StartExecution(sm.StateMachineArn, "act-exec", `{"in":1}`) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + defer b.Destroy() - // Poll for the task. - var task *stepfunctions.ActivityTask - - require.Eventually(t, func() bool { - ctx2, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - defer cancel() - - t2, e := b.GetActivityTask(ctx2, act.ActivityArn, "worker1") - - if e == nil && t2 != nil { - task = t2 + act, err := b.CreateActivity(context.Background(), "send-act") + require.NoError(t, err) - return true - } + actDef := fmt.Sprintf(`{"StartAt":"A","States":{"A":{"Type":"Task","Resource":%q,"End":true}}}`, + act.ActivityArn) + sm, err := b.CreateStateMachine( + context.Background(), + "act-sm", + actDef, + validRoleARN, + "STANDARD", + ) + require.NoError(t, err) - return false - }, 5*time.Second, 50*time.Millisecond) + exec, err := b.StartExecution(sm.StateMachineArn, "act-exec", `{"in":1}`) + require.NoError(t, err) - require.NotNil(t, task) - require.NoError(t, b.SendTaskSuccess(task.TaskToken, `{"out":2}`)) + // GetActivityTask long-polls; it unblocks once the executor reaches + // the Task state and enqueues the task. + task, err := b.GetActivityTask(context.Background(), act.ActivityArn, "worker1") + require.NoError(t, err) + require.NotEmpty(t, task.TaskToken) - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, b.SendTaskSuccess(task.TaskToken, `{"out":2}`)) + synctest.Wait() - return e == nil && d.Status == "SUCCEEDED" - }, 5*time.Second, 20*time.Millisecond) + d, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + assert.Equal(t, "SUCCEEDED", d.Status) + }) } func TestActivity_SendTaskFailure(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackend() - defer b.Destroy() - - act, err := b.CreateActivity(context.Background(), "fail-act") - require.NoError(t, err) - - actDef := fmt.Sprintf(`{"StartAt":"A","States":{"A":{"Type":"Task","Resource":%q,"End":true}}}`, - act.ActivityArn) - sm, err := b.CreateStateMachine( - context.Background(), - "act-fail-sm", - actDef, - validRoleARN, - "STANDARD", - ) - require.NoError(t, err) - - exec, err := b.StartExecution(sm.StateMachineArn, "act-fail-exec", "{}") - require.NoError(t, err) - - var task *stepfunctions.ActivityTask - - require.Eventually(t, func() bool { - ctx2, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - defer cancel() - - t2, e := b.GetActivityTask(ctx2, act.ActivityArn, "worker1") + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + defer b.Destroy() - if e == nil && t2 != nil { - task = t2 + act, err := b.CreateActivity(context.Background(), "fail-act") + require.NoError(t, err) - return true - } + actDef := fmt.Sprintf(`{"StartAt":"A","States":{"A":{"Type":"Task","Resource":%q,"End":true}}}`, + act.ActivityArn) + sm, err := b.CreateStateMachine( + context.Background(), + "act-fail-sm", + actDef, + validRoleARN, + "STANDARD", + ) + require.NoError(t, err) - return false - }, 5*time.Second, 50*time.Millisecond) + exec, err := b.StartExecution(sm.StateMachineArn, "act-fail-exec", "{}") + require.NoError(t, err) - require.NotNil(t, task) - require.NoError(t, b.SendTaskFailure(task.TaskToken, "MyErr", "failed on purpose")) + task, err := b.GetActivityTask(context.Background(), act.ActivityArn, "worker1") + require.NoError(t, err) + require.NotEmpty(t, task.TaskToken) - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, b.SendTaskFailure(task.TaskToken, "MyErr", "failed on purpose")) + synctest.Wait() - return e == nil && d.Status == "FAILED" - }, 5*time.Second, 20*time.Millisecond) + d, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + assert.Equal(t, "FAILED", d.Status) + }) } func TestActivity_SendTaskSuccessUnknownToken(t *testing.T) { @@ -994,41 +953,29 @@ func TestActivity_SendTaskSuccessUnknownToken(t *testing.T) { func TestActivity_SendTaskHeartbeat(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackend() - defer b.Destroy() - - act, err := b.CreateActivity(context.Background(), "hb-act") - require.NoError(t, err) - - actDef := fmt.Sprintf( - `{"StartAt":"A","States":{"A":{"Type":"Task","Resource":%q,"HeartbeatSeconds":60,"End":true}}}`, - act.ActivityArn, - ) - sm, err := b.CreateStateMachine(context.Background(), "hb-sm", actDef, validRoleARN, "STANDARD") - require.NoError(t, err) - - _, err = b.StartExecution(sm.StateMachineArn, "hb-exec", "{}") - require.NoError(t, err) - - var task *stepfunctions.ActivityTask - - require.Eventually(t, func() bool { - ctx2, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) - defer cancel() + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + defer b.Destroy() - t2, e := b.GetActivityTask(ctx2, act.ActivityArn, "hb-worker") + act, err := b.CreateActivity(context.Background(), "hb-act") + require.NoError(t, err) - if e == nil && t2 != nil { - task = t2 + actDef := fmt.Sprintf( + `{"StartAt":"A","States":{"A":{"Type":"Task","Resource":%q,"HeartbeatSeconds":60,"End":true}}}`, + act.ActivityArn, + ) + sm, err := b.CreateStateMachine(context.Background(), "hb-sm", actDef, validRoleARN, "STANDARD") + require.NoError(t, err) - return true - } + _, err = b.StartExecution(sm.StateMachineArn, "hb-exec", "{}") + require.NoError(t, err) - return false - }, 5*time.Second, 50*time.Millisecond) + task, err := b.GetActivityTask(context.Background(), act.ActivityArn, "hb-worker") + require.NoError(t, err) + require.NotEmpty(t, task.TaskToken) - require.NotNil(t, task) - require.NoError(t, b.SendTaskHeartbeat(task.TaskToken)) + require.NoError(t, b.SendTaskHeartbeat(task.TaskToken)) + }) } // ─── Versions ───────────────────────────────────────────────────────────────── @@ -1276,40 +1223,38 @@ func TestActivity_InvokeCancellationRemovesTaskToken(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := newSFBackend() - a, err := b.CreateActivity(context.Background(), "cancel-act-"+tt.name) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := newSFBackend() + a, err := b.CreateActivity(context.Background(), "cancel-act-"+tt.name) + require.NoError(t, err) - invokeCtx, cancelInvoke := context.WithCancel(t.Context()) - defer cancelInvoke() + invokeCtx, cancelInvoke := context.WithCancel(t.Context()) + defer cancelInvoke() - invokeErrCh := make(chan error, 1) - go func() { - _, invokeErr := b.InvokeActivity(invokeCtx, a.ActivityArn, `{}`, 0) - invokeErrCh <- invokeErr - }() - - pollCtx, cancelPoll := context.WithTimeout(t.Context(), 5*time.Second) - defer cancelPoll() + invokeErrCh := make(chan error, 1) + go func() { + _, invokeErr := b.InvokeActivity(invokeCtx, a.ActivityArn, `{}`, 0) + invokeErrCh <- invokeErr + }() - task, err := b.GetActivityTask(pollCtx, a.ActivityArn, "worker-1") - require.NoError(t, err) - require.NotNil(t, task) - require.NotEmpty(t, task.TaskToken) + task, err := b.GetActivityTask(t.Context(), a.ActivityArn, "worker-1") + require.NoError(t, err) + require.NotNil(t, task) + require.NotEmpty(t, task.TaskToken) - cancelInvoke() + cancelInvoke() + synctest.Wait() - require.Eventually(t, func() bool { select { case invokeErr := <-invokeErrCh: - return errors.Is(invokeErr, context.Canceled) + require.ErrorIs(t, invokeErr, context.Canceled) default: - return false + t.Fatal("InvokeActivity did not observe cancellation") } - }, 2*time.Second, 25*time.Millisecond) - err = tt.sendResult(b, task.TaskToken) - require.ErrorIs(t, err, stepfunctions.ErrTaskTokenNotFound) + err = tt.sendResult(b, task.TaskToken) + require.ErrorIs(t, err, stepfunctions.ErrTaskTokenNotFound) + }) }) } } @@ -1345,44 +1290,43 @@ func TestActivity_DeleteActivityRemovesOutstandingTaskTokens(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := newSFBackend() - a, err := b.CreateActivity(context.Background(), "delete-act-"+tt.name) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := newSFBackend() + a, err := b.CreateActivity(context.Background(), "delete-act-"+tt.name) + require.NoError(t, err) - invokeCtx, cancelInvoke := context.WithCancel(t.Context()) - defer cancelInvoke() + invokeCtx, cancelInvoke := context.WithCancel(t.Context()) + defer cancelInvoke() - invokeErrCh := make(chan error, 1) - go func() { - _, invokeErr := b.InvokeActivity(invokeCtx, a.ActivityArn, `{}`, 0) - invokeErrCh <- invokeErr - }() + invokeErrCh := make(chan error, 1) + go func() { + _, invokeErr := b.InvokeActivity(invokeCtx, a.ActivityArn, `{}`, 0) + invokeErrCh <- invokeErr + }() - pollCtx, cancelPoll := context.WithTimeout(t.Context(), 5*time.Second) - defer cancelPoll() + task, err := b.GetActivityTask(t.Context(), a.ActivityArn, "worker-1") + require.NoError(t, err) + require.NotNil(t, task) + require.NotEmpty(t, task.TaskToken) - task, err := b.GetActivityTask(pollCtx, a.ActivityArn, "worker-1") - require.NoError(t, err) - require.NotNil(t, task) - require.NotEmpty(t, task.TaskToken) + err = b.DeleteActivity(a.ActivityArn) + require.NoError(t, err) - err = b.DeleteActivity(a.ActivityArn) - require.NoError(t, err) + err = tt.sendResult(b, task.TaskToken) + require.ErrorIs(t, err, stepfunctions.ErrTaskTokenNotFound) - err = tt.sendResult(b, task.TaskToken) - require.ErrorIs(t, err, stepfunctions.ErrTaskTokenNotFound) + // DeleteActivity signals resultCh for in-flight tasks, so InvokeActivity + // must unblock and return an error without requiring context cancellation. + synctest.Wait() - // DeleteActivity signals resultCh for in-flight tasks, so InvokeActivity - // must unblock and return an error without requiring context cancellation. - require.Eventually(t, func() bool { select { case invokeErr := <-invokeErrCh: - return invokeErr != nil + require.Error(t, invokeErr) default: - return false + t.Fatal("InvokeActivity did not unblock after DeleteActivity") } - }, 2*time.Second, 25*time.Millisecond) - cancelInvoke() + cancelInvoke() + }) }) } } @@ -1412,17 +1356,15 @@ func TestSweepTaskTokensRLock(t *testing.T) { act, err := bk.CreateActivity(ctx, "sweep-test-act") require.NoError(t, err) - done := make(chan struct{}) go func() { - defer close(done) // InvokeActivity registers a token; we never complete it. bk.InvokeActivity(ctx, act.ActivityArn, `{}`, 0) }() - // Give the goroutine time to register its token. - require.Eventually(t, func() bool { - return bk.TaskTokenCount() > 0 - }, time.Second, 5*time.Millisecond) + // Wait until the goroutine above registers its token and + // durably blocks awaiting the result. + synctest.Wait() + require.Positive(t, bk.TaskTokenCount()) // Age all tokens well past the TTL. bk.AgeTaskTokensForTest(2 * stepfunctions.DefaultTaskTokenTTLForTest) @@ -1435,11 +1377,13 @@ func TestSweepTaskTokensRLock(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - bk := stepfunctions.NewInMemoryBackend() - tt.setupFn(t, bk) + synctest.Test(t, func(t *testing.T) { + bk := stepfunctions.NewInMemoryBackend() + tt.setupFn(t, bk) - evicted := bk.SweepTaskTokens() - assert.Equal(t, tt.wantEvictions, evicted) + evicted := bk.SweepTaskTokens() + assert.Equal(t, tt.wantEvictions, evicted) + }) }) } } diff --git a/services/stepfunctions/handler_executions_test.go b/services/stepfunctions/handler_executions_test.go index 16de4d9a01..c5b46a5248 100644 --- a/services/stepfunctions/handler_executions_test.go +++ b/services/stepfunctions/handler_executions_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "strings" "testing" + "testing/synctest" "time" "github.com/labstack/echo/v5" @@ -327,39 +328,41 @@ func TestStartExecution_ResponseContainsARNAndStartDate(t *testing.T) { func TestListExecutions_OrderedByStartDateDesc_ViaHandler(t *testing.T) { t.Parallel() - ctx := t.Context() - h, e := newSFNHandler(t) - smARN := createSM(ctx, t, h, e, "order-handler-sm") - - execNames := []string{"exec-z", "exec-a", "exec-m"} - for _, name := range execNames { - body, err := json.Marshal(map[string]string{ - "stateMachineArn": smARN, - "name": name, - "input": "{}", - }) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + ctx := t.Context() + h, e := newSFNHandler(t) + smARN := createSM(ctx, t, h, e, "order-handler-sm") - rec := sfnPost(ctx, t, h, e, "StartExecution", string(body)) - require.Equal(t, http.StatusOK, rec.Code) - time.Sleep(5 * time.Millisecond) - } + execNames := []string{"exec-z", "exec-a", "exec-m"} + for _, name := range execNames { + body, err := json.Marshal(map[string]string{ + "stateMachineArn": smARN, + "name": name, + "input": "{}", + }) + require.NoError(t, err) - listBody, err := json.Marshal(map[string]string{"stateMachineArn": smARN}) - require.NoError(t, err) + rec := sfnPost(ctx, t, h, e, "StartExecution", string(body)) + require.Equal(t, http.StatusOK, rec.Code) + time.Sleep(5 * time.Millisecond) + } - rec := sfnPost(ctx, t, h, e, "ListExecutions", string(listBody)) - require.Equal(t, http.StatusOK, rec.Code) + listBody, err := json.Marshal(map[string]string{"stateMachineArn": smARN}) + require.NoError(t, err) - var resp map[string]any - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + rec := sfnPost(ctx, t, h, e, "ListExecutions", string(listBody)) + require.Equal(t, http.StatusOK, rec.Code) - rawExecs, _ := resp["executions"].([]any) - require.Len(t, rawExecs, 3) + var resp map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) - // exec-m started last, should appear first. - first, _ := rawExecs[0].(map[string]any) - assert.Equal(t, "exec-m", first["name"]) + rawExecs, _ := resp["executions"].([]any) + require.Len(t, rawExecs, 3) + + // exec-m started last, should appear first. + first, _ := rawExecs[0].(map[string]any) + assert.Equal(t, "exec-m", first["name"]) + }) } func TestSFN_DescribeStateMachineForExecution(t *testing.T) { diff --git a/services/stepfunctions/handler_state_machines_test.go b/services/stepfunctions/handler_state_machines_test.go index 69950644df..59b132e987 100644 --- a/services/stepfunctions/handler_state_machines_test.go +++ b/services/stepfunctions/handler_state_machines_test.go @@ -7,7 +7,6 @@ import ( "net/http/httptest" "strings" "testing" - "time" "github.com/labstack/echo/v5" "github.com/stretchr/testify/assert" @@ -160,16 +159,14 @@ func TestHandler_StartExecution_StateMachineDeleting(t *testing.T) { sfnPost(ctx, t, h, e, "StopExecution", `{"executionArn":"`+execArn+`","error":"Test","cause":"cleanup"}`) }) - require.Eventually(t, func() bool { - descRec := sfnPost(ctx, t, h, e, "DescribeExecution", `{"executionArn":"`+execArn+`"}`) - if descRec.Code != http.StatusOK { - return false - } + // StartExecution's response only returns once the execution's status is + // already set to RUNNING, so no wait is needed here. + runningRec := sfnPost(ctx, t, h, e, "DescribeExecution", `{"executionArn":"`+execArn+`"}`) + require.Equal(t, http.StatusOK, runningRec.Code) - var desc map[string]any - - return json.Unmarshal(descRec.Body.Bytes(), &desc) == nil && desc["status"] == "RUNNING" - }, 5*time.Second, 10*time.Millisecond) + var runningDesc map[string]any + require.NoError(t, json.Unmarshal(runningRec.Body.Bytes(), &runningDesc)) + require.Equal(t, "RUNNING", runningDesc["status"]) delRec := sfnPost(ctx, t, h, e, "DeleteStateMachine", `{"stateMachineArn":"`+smArn+`"}`) require.Equal(t, http.StatusOK, delRec.Code) diff --git a/services/stepfunctions/id_generation_test.go b/services/stepfunctions/id_generation_test.go new file mode 100644 index 0000000000..716deffa95 --- /dev/null +++ b/services/stepfunctions/id_generation_test.go @@ -0,0 +1,43 @@ +package stepfunctions_test + +import ( + "context" + "regexp" + "testing" + "testing/synctest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/stepfunctions" +) + +// syncExecutionNamePattern locks in the fix for StartSyncExecution's +// auto-generated name, which used to collide under synctest. +var syncExecutionNamePattern = regexp.MustCompile( + `^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`, +) + +func TestStepFunctionsBackend_SyncExecutionName_Unique(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + + sm, err := b.CreateStateMachine( + context.Background(), "sync-id-sm", minimalDefinition, validRoleARN, "EXPRESS", + ) + require.NoError(t, err) + + res1, err := b.StartSyncExecution(sm.StateMachineArn, "", "{}") + require.NoError(t, err) + + res2, err := b.StartSyncExecution(sm.StateMachineArn, "", "{}") + require.NoError(t, err) + + assert.NotEqual(t, res1.Name, res2.Name, + "two sync executions started back-to-back must get distinct names") + assert.Regexp(t, syncExecutionNamePattern, res1.Name) + assert.Regexp(t, syncExecutionNamePattern, res2.Name) + }) +} diff --git a/services/stepfunctions/integration_test.go b/services/stepfunctions/integration_test.go index 27447b0e9f..087cef70ea 100644 --- a/services/stepfunctions/integration_test.go +++ b/services/stepfunctions/integration_test.go @@ -3,7 +3,7 @@ package stepfunctions_test import ( "context" "testing" - "time" + "testing/synctest" "github.com/aws/aws-sdk-go-v2/aws" awsdynamodb "github.com/aws/aws-sdk-go-v2/service/dynamodb" @@ -464,31 +464,34 @@ func TestRecordTask_SucceededAndFailed(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") - b.SetLambdaInvoker(tt.invoker) - sm, err := b.CreateStateMachine(context.Background(), tt.smName, lambdaTaskDef, "arn:role", "STANDARD") - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackendWithConfig("123456789012", "us-east-1") + b.SetLambdaInvoker(tt.invoker) - exec, err := b.StartExecution(sm.StateMachineArn, tt.execName, `{}`) - require.NoError(t, err) + sm, err := b.CreateStateMachine(context.Background(), tt.smName, lambdaTaskDef, "arn:role", "STANDARD") + require.NoError(t, err) - require.Eventually(t, func() bool { - desc, _ := b.DescribeExecution(exec.ExecutionArn) + exec, err := b.StartExecution(sm.StateMachineArn, tt.execName, `{}`) + require.NoError(t, err) - return desc != nil && desc.Status == tt.wantStatus - }, 5*time.Second, 50*time.Millisecond) + synctest.Wait() - history, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, false) - require.NoError(t, err) + desc, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + require.Equal(t, tt.wantStatus, desc.Status) - eventTypes := make([]string, 0, len(history)) - for _, ev := range history { - eventTypes = append(eventTypes, ev.Type) - } - for _, wantType := range tt.wantEventTypes { - assert.Contains(t, eventTypes, wantType) - } + history, _, err := b.GetExecutionHistory(exec.ExecutionArn, "", 100, false) + require.NoError(t, err) + + eventTypes := make([]string, 0, len(history)) + for _, ev := range history { + eventTypes = append(eventTypes, ev.Type) + } + for _, wantType := range tt.wantEventTypes { + assert.Contains(t, eventTypes, wantType) + } + }) }) } } diff --git a/services/stepfunctions/integrations.go b/services/stepfunctions/integrations.go index ea51c1941f..150fe0c3c6 100644 --- a/services/stepfunctions/integrations.go +++ b/services/stepfunctions/integrations.go @@ -84,8 +84,11 @@ type s3Adapter struct { backend s3pkg.StorageBackend } -// Compile-time assertion: s3Adapter must implement asl.S3Reader. -var _ asl.S3Reader = (*s3Adapter)(nil) +// Compile-time assertion: s3Adapter must implement asl.S3Reader and asl.S3ListReader. +var ( + _ asl.S3Reader = (*s3Adapter)(nil) + _ asl.S3ListReader = (*s3Adapter)(nil) +) // NewS3Integration creates a new S3 integration adapter for Map state ItemReader. func NewS3Integration(backend s3pkg.StorageBackend) asl.S3Reader { @@ -111,6 +114,44 @@ func (a *s3Adapter) GetObjectBytes(ctx context.Context, bucket, key string) ([]b return data, nil } +// ListObjectsV2Items implements asl.S3ListReader, paginating through every +// object under bucket/prefix. +func (a *s3Adapter) ListObjectsV2Items(ctx context.Context, bucket, prefix string) ([]asl.S3ObjectItem, error) { + var ( + items []asl.S3ObjectItem + continuationToken *string + ) + + for { + out, err := a.backend.ListObjectsV2(ctx, &awss3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(prefix), + ContinuationToken: continuationToken, + }) + if err != nil { + return nil, err + } + + for _, obj := range out.Contents { + items = append(items, asl.S3ObjectItem{ + Key: aws.ToString(obj.Key), + ETag: aws.ToString(obj.ETag), + LastModified: aws.ToTime(obj.LastModified), + Size: aws.ToInt64(obj.Size), + StorageClass: string(obj.StorageClass), + }) + } + + if !aws.ToBool(out.IsTruncated) || aws.ToString(out.NextContinuationToken) == "" { + break + } + + continuationToken = out.NextContinuationToken + } + + return items, nil +} + // s3ResultWriterAdapter adapts s3.StorageBackend to asl.S3Writer, used to // export Distributed Map ResultWriter output to S3. type s3ResultWriterAdapter struct { diff --git a/services/stepfunctions/item_reader_s3_resource_test.go b/services/stepfunctions/item_reader_s3_resource_test.go new file mode 100644 index 0000000000..31364c257a --- /dev/null +++ b/services/stepfunctions/item_reader_s3_resource_test.go @@ -0,0 +1,411 @@ +package stepfunctions_test + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + sfnsdk "github.com/aws/aws-sdk-go-v2/service/sfn" + sfntypes "github.com/aws/aws-sdk-go-v2/service/sfn/types" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + s3pkg "github.com/blackbirdworks/gopherstack/services/s3" + "github.com/blackbirdworks/gopherstack/services/stepfunctions" +) + +// itemReaderMapDef wraps an ItemReader clause in a plain (INLINE) Map state +// whose ItemProcessor echoes each item back via a Pass state, so the Map's +// own output is the exact array of items the ItemReader produced. +func itemReaderMapDef(itemReaderJSON string) string { + return `{ + "StartAt": "M", + "States": { + "M": { + "Type": "Map", + "ItemReader": ` + itemReaderJSON + `, + "ItemProcessor": { + "StartAt": "P", + "States": {"P": {"Type": "Pass", "End": true}} + }, + "End": true + } + } + }` +} + +// putS3Object writes data to bucket/key on s3Bk, failing the test on error. +func putS3Object(t *testing.T, s3Bk *s3pkg.InMemoryBackend, bucket, key string, data []byte) { + t.Helper() + + _, err := s3Bk.PutObject(context.Background(), &awss3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + Body: bytes.NewReader(data), + }) + require.NoError(t, err) +} + +func gzipBytes(t *testing.T, data []byte) []byte { + t.Helper() + + var buf bytes.Buffer + + w := gzip.NewWriter(&buf) + _, err := w.Write(data) + require.NoError(t, err) + require.NoError(t, w.Close()) + + return buf.Bytes() +} + +// runItemReaderExecution wires s3Bk into a fresh stepfunctions backend, runs +// itemReaderJSON's Map state through the real SFN SDK client, and returns +// the terminal execution's status/output/error/cause. +func runItemReaderExecution( + t *testing.T, s3Bk *s3pkg.InMemoryBackend, itemReaderJSON string, +) (sfntypes.ExecutionStatus, string, string, string) { + t.Helper() + + backend := stepfunctions.NewInMemoryBackend() + backend.SetS3Reader(stepfunctions.NewS3Integration(s3Bk)) + h := stepfunctions.NewHandler(backend) + client := newSFNSDKClient(t, h) + ctx := t.Context() + + createSM, err := client.CreateStateMachine(ctx, &sfnsdk.CreateStateMachineInput{ + Name: aws.String("item-reader-" + uuid.NewString()[:8]), + Definition: aws.String(itemReaderMapDef(itemReaderJSON)), + RoleArn: aws.String(validRoleARN), + Type: sfntypes.StateMachineTypeStandard, + }) + require.NoError(t, err) + + startOut, err := client.StartExecution(ctx, &sfnsdk.StartExecutionInput{ + StateMachineArn: createSM.StateMachineArn, + Input: aws.String(`{}`), + }) + require.NoError(t, err) + + waitTerminal(ctx, t, client, aws.ToString(startOut.ExecutionArn)) + + desc, err := client.DescribeExecution(ctx, &sfnsdk.DescribeExecutionInput{ + ExecutionArn: startOut.ExecutionArn, + }) + require.NoError(t, err) + + return desc.Status, aws.ToString(desc.Output), aws.ToString(desc.Error), aws.ToString(desc.Cause) +} + +// TestItemReader_S3ListObjectsV2 covers the Resource arn:aws:states:::s3:listObjectsV2, +// both its default object-metadata mode and its LOAD_AND_FLATTEN transformation +// (AWS docs: input-output-itemreader.html). +func TestItemReader_S3ListObjectsV2(t *testing.T) { + t.Parallel() + + t.Run("default lists object metadata", func(t *testing.T) { + t.Parallel() + + const bucket = "list-meta-bucket" + + s3Bk := newBucketBackedS3(t, bucket) + putS3Object(t, s3Bk, bucket, "data/a.txt", []byte("hello")) + putS3Object(t, s3Bk, bucket, "data/b.txt", []byte("world!!")) + + itemReader := `{ + "Resource": "arn:aws:states:::s3:listObjectsV2", + "Parameters": {"Bucket": "` + bucket + `", "Prefix": "data/"} + }` + + status, output, errCode, cause := runItemReaderExecution(t, s3Bk, itemReader) + require.Equal(t, sfntypes.ExecutionStatusSucceeded, status, "error=%s cause=%s", errCode, cause) + + var items []map[string]any + require.NoError(t, json.Unmarshal([]byte(output), &items)) + require.Len(t, items, 2) + + assert.Equal(t, "data/a.txt", items[0]["Key"]) + assert.InDelta(t, 5.0, items[0]["Size"], 0) + assert.Equal(t, "STANDARD", items[0]["StorageClass"]) + assert.NotEmpty(t, items[0]["Etag"]) + assert.Positive(t, items[0]["LastModified"]) + + assert.Equal(t, "data/b.txt", items[1]["Key"]) + assert.InDelta(t, 7.0, items[1]["Size"], 0) + }) + + t.Run("LOAD_AND_FLATTEN JSON reads and flattens object contents", func(t *testing.T) { + t.Parallel() + + const bucket = "list-flatten-json-bucket" + + s3Bk := newBucketBackedS3(t, bucket) + putS3Object(t, s3Bk, bucket, "flat/a.json", []byte(`[{"n":1},{"n":2}]`)) + putS3Object(t, s3Bk, bucket, "flat/b.json", []byte(`[{"n":3}]`)) + + itemReader := `{ + "Resource": "arn:aws:states:::s3:listObjectsV2", + "ReaderConfig": {"InputType": "JSON", "Transformation": "LOAD_AND_FLATTEN"}, + "Parameters": {"Bucket": "` + bucket + `", "Prefix": "flat/"} + }` + + status, output, errCode, cause := runItemReaderExecution(t, s3Bk, itemReader) + require.Equal(t, sfntypes.ExecutionStatusSucceeded, status, "error=%s cause=%s", errCode, cause) + assert.JSONEq(t, `[{"n":1},{"n":2},{"n":3}]`, output) + }) + + t.Run("LOAD_AND_FLATTEN CSV reads and flattens object contents", func(t *testing.T) { + t.Parallel() + + const bucket = "list-flatten-csv-bucket" + + s3Bk := newBucketBackedS3(t, bucket) + putS3Object(t, s3Bk, bucket, "flat/a.csv", []byte("col\nx\ny\n")) + putS3Object(t, s3Bk, bucket, "flat/b.csv", []byte("col\nz\n")) + + itemReader := `{ + "Resource": "arn:aws:states:::s3:listObjectsV2", + "ReaderConfig": {"InputType": "CSV", "Transformation": "LOAD_AND_FLATTEN"}, + "Parameters": {"Bucket": "` + bucket + `", "Prefix": "flat/"} + }` + + status, output, errCode, cause := runItemReaderExecution(t, s3Bk, itemReader) + require.Equal(t, sfntypes.ExecutionStatusSucceeded, status, "error=%s cause=%s", errCode, cause) + assert.JSONEq(t, `[{"col":"x"},{"col":"y"},{"col":"z"}]`, output) + }) + + t.Run("missing bucket fails with States.ItemReaderFailed", func(t *testing.T) { + t.Parallel() + + s3Bk := newBucketBackedS3(t, "unrelated-bucket") + + itemReader := `{ + "Resource": "arn:aws:states:::s3:listObjectsV2", + "Parameters": {"Bucket": "does-not-exist", "Prefix": ""} + }` + + status, _, errCode, cause := runItemReaderExecution(t, s3Bk, itemReader) + assert.Equal(t, sfntypes.ExecutionStatusFailed, status) + assert.Equal(t, "States.ItemReaderFailed", errCode) + assert.Contains(t, cause, "NoSuchBucket") + }) +} + +// TestItemReader_S3Manifest covers ReaderConfig ManifestType S3_INVENTORY and +// the legacy InputType=MANIFEST alias, including a gzip-compressed data file +// (AWS docs: input-output-itemreader.html, "Amazon S3 inventory"). +func TestItemReader_S3Manifest(t *testing.T) { + t.Parallel() + + const inventoryCSV = `"src-bucket","csvDataset/titles.csv","3399671","2022-11-16T00:29:32.000Z"` + "\n" + + `"src-bucket","imageDataset/pic.jpg","27034","2022-11-15T20:02:16.000Z"` + "\n" + + buildManifest := func(t *testing.T, dataKey string) string { + t.Helper() + + manifest := map[string]any{ + "sourceBucket": "src-bucket", + "destinationBucket": "arn:aws:s3:::inv-bucket", + "version": "2016-11-30", + "fileFormat": "CSV", + "fileSchema": "Bucket, Key, Size, LastModifiedDate", + "files": []map[string]any{{"key": dataKey, "size": len(inventoryCSV)}}, + } + + b, err := json.Marshal(manifest) + require.NoError(t, err) + + return string(b) + } + + t.Run("ManifestType S3_INVENTORY, gzip data file", func(t *testing.T) { + t.Parallel() + + const bucket = "inv-bucket" + + s3Bk := newBucketBackedS3(t, bucket) + putS3Object(t, s3Bk, bucket, "inv/data0.csv.gz", gzipBytes(t, []byte(inventoryCSV))) + putS3Object(t, s3Bk, bucket, "inv/manifest.json", []byte(buildManifest(t, "inv/data0.csv.gz"))) + + itemReader := `{ + "Resource": "arn:aws:states:::s3:getObject", + "ReaderConfig": {"ManifestType": "S3_INVENTORY"}, + "Parameters": {"Bucket": "` + bucket + `", "Key": "inv/manifest.json"} + }` + + status, output, errCode, cause := runItemReaderExecution(t, s3Bk, itemReader) + require.Equal(t, sfntypes.ExecutionStatusSucceeded, status, "error=%s cause=%s", errCode, cause) + + var items []map[string]any + require.NoError(t, json.Unmarshal([]byte(output), &items)) + require.Len(t, items, 2) + assert.Equal(t, map[string]any{ + "Bucket": "src-bucket", "Key": "csvDataset/titles.csv", + "Size": "3399671", "LastModifiedDate": "2022-11-16T00:29:32.000Z", + }, items[0]) + }) + + t.Run("legacy InputType MANIFEST, plain data file", func(t *testing.T) { + t.Parallel() + + const bucket = "inv-bucket-legacy" + + s3Bk := newBucketBackedS3(t, bucket) + putS3Object(t, s3Bk, bucket, "inv/data0.csv", []byte(inventoryCSV)) + putS3Object(t, s3Bk, bucket, "inv/manifest.json", []byte(buildManifest(t, "inv/data0.csv"))) + + itemReader := `{ + "Resource": "arn:aws:states:::s3:getObject", + "ReaderConfig": {"InputType": "MANIFEST"}, + "Parameters": {"Bucket": "` + bucket + `", "Key": "inv/manifest.json"} + }` + + status, output, errCode, cause := runItemReaderExecution(t, s3Bk, itemReader) + require.Equal(t, sfntypes.ExecutionStatusSucceeded, status, "error=%s cause=%s", errCode, cause) + + var items []map[string]any + require.NoError(t, json.Unmarshal([]byte(output), &items)) + require.Len(t, items, 2) + assert.Equal(t, "imageDataset/pic.jpg", items[1]["Key"]) + }) + + t.Run("ManifestType S3_INVENTORY with CSVDelimiter carries through to data files", func(t *testing.T) { + t.Parallel() + + const bucket = "inv-bucket-pipe" + + pipeCSV := `"src-bucket"|"csvDataset/titles.csv"|"3399671"|"2022-11-16T00:29:32.000Z"` + "\n" + + s3Bk := newBucketBackedS3(t, bucket) + putS3Object(t, s3Bk, bucket, "inv/data0.csv", []byte(pipeCSV)) + putS3Object(t, s3Bk, bucket, "inv/manifest.json", []byte(buildManifest(t, "inv/data0.csv"))) + + itemReader := `{ + "Resource": "arn:aws:states:::s3:getObject", + "ReaderConfig": {"ManifestType": "S3_INVENTORY", "CSVDelimiter": "PIPE"}, + "Parameters": {"Bucket": "` + bucket + `", "Key": "inv/manifest.json"} + }` + + status, output, errCode, cause := runItemReaderExecution(t, s3Bk, itemReader) + require.Equal(t, sfntypes.ExecutionStatusSucceeded, status, "error=%s cause=%s", errCode, cause) + + var items []map[string]any + require.NoError(t, json.Unmarshal([]byte(output), &items)) + require.Len(t, items, 1) + assert.Equal(t, "csvDataset/titles.csv", items[0]["Key"]) + }) + + t.Run("ManifestType ATHENA_DATA is a recorded gap", func(t *testing.T) { + t.Parallel() + + const bucket = "athena-bucket" + + s3Bk := newBucketBackedS3(t, bucket) + putS3Object(t, s3Bk, bucket, "athena/manifest.csv", []byte("s3://athena-bucket/data/f1.csv\n")) + + itemReader := `{ + "Resource": "arn:aws:states:::s3:getObject", + "ReaderConfig": {"ManifestType": "ATHENA_DATA", "InputType": "CSV"}, + "Parameters": {"Bucket": "` + bucket + `", "Key": "athena/manifest.csv"} + }` + + status, _, errCode, cause := runItemReaderExecution(t, s3Bk, itemReader) + assert.Equal(t, sfntypes.ExecutionStatusFailed, status) + assert.Equal(t, "States.ItemReaderFailed", errCode) + assert.Contains(t, cause, "ATHENA_DATA") + }) +} + +// TestItemReader_S3GetObject_Errors covers ItemReader failure behavior for +// the s3:getObject Resource: a missing key, and the recorded PARQUET gap. +func TestItemReader_S3GetObject_Errors(t *testing.T) { + t.Parallel() + + t.Run("missing key fails with States.ItemReaderFailed", func(t *testing.T) { + t.Parallel() + + const bucket = "getobject-bucket" + s3Bk := newBucketBackedS3(t, bucket) + + itemReader := `{ + "Resource": "arn:aws:states:::s3:getObject", + "Parameters": {"Bucket": "` + bucket + `", "Key": "does-not-exist.json"} + }` + + status, _, errCode, cause := runItemReaderExecution(t, s3Bk, itemReader) + assert.Equal(t, sfntypes.ExecutionStatusFailed, status) + assert.Equal(t, "States.ItemReaderFailed", errCode) + assert.Contains(t, cause, "NoSuchKey") + }) + + t.Run("PARQUET InputType is a recorded gap", func(t *testing.T) { + t.Parallel() + + const bucket = "parquet-bucket" + s3Bk := newBucketBackedS3(t, bucket) + putS3Object(t, s3Bk, bucket, "data.parquet", []byte("not really parquet")) + + itemReader := `{ + "Resource": "arn:aws:states:::s3:getObject", + "ReaderConfig": {"InputType": "PARQUET"}, + "Parameters": {"Bucket": "` + bucket + `", "Key": "data.parquet"} + }` + + status, _, errCode, cause := runItemReaderExecution(t, s3Bk, itemReader) + assert.Equal(t, sfntypes.ExecutionStatusFailed, status) + assert.Equal(t, "States.ItemReaderFailed", errCode) + assert.Contains(t, cause, "PARQUET") + }) +} + +// TestItemReader_ItemsPointer covers ReaderConfig.ItemsPointer, the RFC 6901 +// JSON Pointer selecting a nested array within a JSON InputType file (AWS +// docs: input-output-itemreader.html, "ItemsPointer"). +func TestItemReader_ItemsPointer(t *testing.T) { + t.Parallel() + + t.Run("selects nested array", func(t *testing.T) { + t.Parallel() + + const bucket = "pointer-bucket" + + s3Bk := newBucketBackedS3(t, bucket) + putS3Object(t, s3Bk, bucket, "nested.json", + []byte(`{"inventory":{"products":{"featured":[{"id":1},{"id":2}]}}}`)) + + itemReader := `{ + "Resource": "arn:aws:states:::s3:getObject", + "ReaderConfig": {"InputType": "JSON", "ItemsPointer": "/inventory/products/featured"}, + "Parameters": {"Bucket": "` + bucket + `", "Key": "nested.json"} + }` + + status, output, errCode, cause := runItemReaderExecution(t, s3Bk, itemReader) + require.Equal(t, sfntypes.ExecutionStatusSucceeded, status, "error=%s cause=%s", errCode, cause) + assert.JSONEq(t, `[{"id":1},{"id":2}]`, output) + }) + + t.Run("path not pointing at an array fails with States.ItemReaderFailed", func(t *testing.T) { + t.Parallel() + + const bucket = "pointer-bucket-not-array" + + s3Bk := newBucketBackedS3(t, bucket) + putS3Object(t, s3Bk, bucket, "nested.json", []byte(`{"data":{"id":1}}`)) + + itemReader := `{ + "Resource": "arn:aws:states:::s3:getObject", + "ReaderConfig": {"InputType": "JSON", "ItemsPointer": "/data"}, + "Parameters": {"Bucket": "` + bucket + `", "Key": "nested.json"} + }` + + status, _, errCode, cause := runItemReaderExecution(t, s3Bk, itemReader) + assert.Equal(t, sfntypes.ExecutionStatusFailed, status) + assert.Equal(t, "States.ItemReaderFailed", errCode) + assert.Contains(t, cause, "ItemsPointer") + }) +} diff --git a/services/stepfunctions/leak_test.go b/services/stepfunctions/leak_test.go index 51e13725c0..d3531a3937 100644 --- a/services/stepfunctions/leak_test.go +++ b/services/stepfunctions/leak_test.go @@ -412,11 +412,11 @@ func TestDeletedExecsTombstoneCleanup(t *testing.T) { exec, err := bk.StartExecution(sm.StateMachineArn, "tomb-exec", `{}`) require.NoError(t, err) - require.Eventually(t, func() bool { - e, descErr := bk.DescribeExecution(exec.ExecutionArn) + synctest.Wait() - return descErr == nil && e.Status != "RUNNING" - }, 3*time.Second, 10*time.Millisecond) + e, descErr := bk.DescribeExecution(exec.ExecutionArn) + require.NoError(t, descErr) + require.NotEqual(t, "RUNNING", e.Status) // Return a cutoff far in the future to prune everything. return float64(time.Now().Add(10 * time.Second).Unix()) @@ -428,16 +428,18 @@ func TestDeletedExecsTombstoneCleanup(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - bk := sfn.NewInMemoryBackend() - cutoff := tt.setupFn(t, bk) + synctest.Test(t, func(t *testing.T) { + bk := sfn.NewInMemoryBackend() + cutoff := tt.setupFn(t, bk) - beforeTombstones := bk.DeletedExecsCountForTest() - bk.PruneExecutionsForTest(cutoff) - afterTombstones := bk.DeletedExecsCountForTest() + beforeTombstones := bk.DeletedExecsCountForTest() + bk.PruneExecutionsForTest(cutoff) + afterTombstones := bk.DeletedExecsCountForTest() - // Tombstone count should not increase after pruning. - assert.LessOrEqual(t, afterTombstones, beforeTombstones, - "tombstone count should not increase after prune") + // Tombstone count should not increase after pruning. + assert.LessOrEqual(t, afterTombstones, beforeTombstones, + "tombstone count should not increase after prune") + }) }) } } diff --git a/services/stepfunctions/list_executions_golden_test.go b/services/stepfunctions/list_executions_golden_test.go index 14e219cc58..95c63e645a 100644 --- a/services/stepfunctions/list_executions_golden_test.go +++ b/services/stepfunctions/list_executions_golden_test.go @@ -10,7 +10,7 @@ import ( "strconv" "strings" "testing" - "time" + "testing/synctest" "github.com/labstack/echo/v5" "github.com/stretchr/testify/require" @@ -35,85 +35,82 @@ func normalizeListExecutionsGolden(b []byte) []byte { func TestListExecutions_PageGolden(t *testing.T) { t.Parallel() - bk := stepfunctions.NewInMemoryBackend() - h := stepfunctions.NewHandler(bk) - ctx := context.Background() - - sm, err := bk.CreateStateMachine( - ctx, "golden-list-executions", - `{"StartAt":"S","States":{"S":{"Type":"Pass","End":true}}}`, - "arn:role", "STANDARD", - ) - require.NoError(t, err) - - const total = 12 - for i := range total { - exec, startErr := bk.StartExecution(sm.StateMachineArn, "exec-"+strconv.Itoa(i), `{}`) - require.NoError(t, startErr) - - // Pin a distinct, strictly increasing StartDate per execution so - // ListExecutions' descending sort has no ties -- without this, - // executions started within the same wall-clock second tie on - // StartDate and their relative order is unspecified (it falls back - // to the index's iteration order, which is not stable). - bk.SetExecutionStartDateForTest(exec.ExecutionArn, float64(1700000000+i)) - } - - require.Eventually(t, func() bool { - execs, _, listErr := bk.ListExecutions(sm.StateMachineArn, "", "", total+1) - if listErr != nil || len(execs) != total { - return false + synctest.Test(t, func(t *testing.T) { + bk := stepfunctions.NewInMemoryBackend() + h := stepfunctions.NewHandler(bk) + ctx := context.Background() + + sm, err := bk.CreateStateMachine( + ctx, "golden-list-executions", + `{"StartAt":"S","States":{"S":{"Type":"Pass","End":true}}}`, + "arn:role", "STANDARD", + ) + require.NoError(t, err) + + const total = 12 + for i := range total { + exec, startErr := bk.StartExecution(sm.StateMachineArn, "exec-"+strconv.Itoa(i), `{}`) + require.NoError(t, startErr) + + // Pin a distinct, strictly increasing StartDate per execution so + // ListExecutions' descending sort has no ties -- without this, + // executions started within the same wall-clock second tie on + // StartDate and their relative order is unspecified (it falls back + // to the index's iteration order, which is not stable). + bk.SetExecutionStartDateForTest(exec.ExecutionArn, float64(1700000000+i)) } + synctest.Wait() + + execs, _, listErr := bk.ListExecutions(sm.StateMachineArn, "", "", total+1) + require.NoError(t, listErr) + require.Len(t, execs, total) + for _, exec := range execs { - if exec.Status == "RUNNING" { - return false - } + require.NotEqual(t, "RUNNING", exec.Status) } - return true - }, 30*time.Second, 20*time.Millisecond) + e := echo.New() - e := echo.New() + var got strings.Builder - var got strings.Builder + token := "" + for page := range 3 { + body := `{"stateMachineArn":"` + sm.StateMachineArn + `","maxResults":5` + if token != "" { + body += `,"nextToken":"` + token + `"` + } + body += "}" - token := "" - for page := range 3 { - body := `{"stateMachineArn":"` + sm.StateMachineArn + `","maxResults":5` - if token != "" { - body += `,"nextToken":"` + token + `"` - } - body += "}" + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + req.Header.Set("X-Amz-Target", "AmazonStates.ListExecutions") - req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) - req.Header.Set("X-Amz-Target", "AmazonStates.ListExecutions") + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + require.NoError(t, h.Handler()(c)) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) - rec := httptest.NewRecorder() - c := e.NewContext(req, rec) - require.NoError(t, h.Handler()(c)) - require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + got.WriteString("--- page ") + got.WriteString(strconv.Itoa(page)) + got.WriteString(" ---\n") + got.Write(normalizeListExecutionsGolden(rec.Body.Bytes())) + got.WriteString("\n") - got.WriteString("--- page ") - got.WriteString(strconv.Itoa(page)) - got.WriteString(" ---\n") - got.Write(normalizeListExecutionsGolden(rec.Body.Bytes())) - got.WriteString("\n") + var resp struct { + NextToken string `json:"nextToken"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) - var resp struct { - NextToken string `json:"nextToken"` - } - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + if resp.NextToken == "" { + break + } - if resp.NextToken == "" { - break + token = resp.NextToken } - token = resp.NextToken - } - - want, err := os.ReadFile("testdata/list_executions_golden.txt") - require.NoError(t, err) + want, err := os.ReadFile("testdata/list_executions_golden.txt") + require.NoError(t, err) - require.Equal(t, string(want), got.String()) + require.Equal(t, string(want), got.String()) + }) } diff --git a/services/stepfunctions/persistence_test.go b/services/stepfunctions/persistence_test.go index 89d7b9e24b..e731347d07 100644 --- a/services/stepfunctions/persistence_test.go +++ b/services/stepfunctions/persistence_test.go @@ -3,7 +3,7 @@ package stepfunctions_test import ( "context" "testing" - "time" + "testing/synctest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -88,55 +88,53 @@ func TestInMemoryBackend_RestoreInvalidData(t *testing.T) { func TestRestore_RebuildsStatusIndex(t *testing.T) { t.Parallel() - const def = `{"StartAt":"P","States":{"P":{"Type":"Pass","End":true}}}` - const role = "arn:aws:iam::000000000000:role/test" - - original := stepfunctions.NewInMemoryBackendWithConfig("000000000000", "us-east-1") - ctx := t.Context() - - sm, err := original.CreateStateMachine(ctx, "index-sm", def, role, "STANDARD") - require.NoError(t, err) - smARN := sm.StateMachineArn - - // Manually inject a SUCCEEDED execution via snapshot-level approach: - // start, wait for completion. - exec, err := original.StartExecution(smARN, "exec-a", `{}`) - require.NoError(t, err) - execARN := exec.ExecutionArn - - // Wait for Pass state to complete. - require.Eventually(t, func() bool { - e, _ := original.DescribeExecution(execARN) - - return e != nil && e.Status != "RUNNING" - }, 5*time.Second, 10*time.Millisecond) - - // Verify status before snapshot. - e, err := original.DescribeExecution(execARN) - require.NoError(t, err) - wantStatus := e.Status - - // Snapshot → restore. - snap := original.Snapshot(ctx) - require.NotNil(t, snap) - - fresh := stepfunctions.NewInMemoryBackendWithConfig("000000000000", "us-east-1") - require.NoError(t, fresh.Restore(ctx, snap)) - - // Status bucket should be populated for the terminal status. - count := fresh.SMExecsByStatusCountForTest(smARN, wantStatus) - assert.Equal(t, 1, count, "smExecsByStatus[%s][%s] should have 1 entry after Restore", smARN, wantStatus) - - // ListExecutions with status filter should return the execution. - execs, _, listErr := fresh.ListExecutions(smARN, wantStatus, "", 0) - require.NoError(t, listErr) - require.Len(t, execs, 1) - assert.Equal(t, execARN, execs[0].ExecutionArn) - - // ListExecutions with a non-matching status filter should return nothing. - execs2, _, listErr2 := fresh.ListExecutions(smARN, "FAILED", "", 0) - require.NoError(t, listErr2) - assert.Empty(t, execs2) + synctest.Test(t, func(t *testing.T) { + const def = `{"StartAt":"P","States":{"P":{"Type":"Pass","End":true}}}` + const role = "arn:aws:iam::000000000000:role/test" + + original := stepfunctions.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + ctx := t.Context() + + sm, err := original.CreateStateMachine(ctx, "index-sm", def, role, "STANDARD") + require.NoError(t, err) + smARN := sm.StateMachineArn + + // Manually inject a SUCCEEDED execution via snapshot-level approach: + // start, wait for completion. + exec, err := original.StartExecution(smARN, "exec-a", `{}`) + require.NoError(t, err) + execARN := exec.ExecutionArn + + synctest.Wait() + + // Verify status before snapshot. + e, err := original.DescribeExecution(execARN) + require.NoError(t, err) + require.NotEqual(t, "RUNNING", e.Status) + wantStatus := e.Status + + // Snapshot → restore. + snap := original.Snapshot(ctx) + require.NotNil(t, snap) + + fresh := stepfunctions.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + require.NoError(t, fresh.Restore(ctx, snap)) + + // Status bucket should be populated for the terminal status. + count := fresh.SMExecsByStatusCountForTest(smARN, wantStatus) + assert.Equal(t, 1, count, "smExecsByStatus[%s][%s] should have 1 entry after Restore", smARN, wantStatus) + + // ListExecutions with status filter should return the execution. + execs, _, listErr := fresh.ListExecutions(smARN, wantStatus, "", 0) + require.NoError(t, listErr) + require.Len(t, execs, 1) + assert.Equal(t, execARN, execs[0].ExecutionArn) + + // ListExecutions with a non-matching status filter should return nothing. + execs2, _, listErr2 := fresh.ListExecutions(smARN, "FAILED", "", 0) + require.NoError(t, listErr2) + assert.Empty(t, execs2) + }) } // TestInMemoryBackend_FullStateSnapshotRestoreRoundTrip exercises a full @@ -149,72 +147,74 @@ func TestRestore_RebuildsStatusIndex(t *testing.T) { func TestInMemoryBackend_FullStateSnapshotRestoreRoundTrip(t *testing.T) { t.Parallel() - const def = `{"StartAt":"P","States":{"P":{"Type":"Pass","End":true}}}` - const role = "arn:aws:iam::000000000000:role/test" - - ctx := t.Context() - original := stepfunctions.NewInMemoryBackendWithConfig("000000000000", "us-east-1") - - sm, err := original.CreateStateMachine(ctx, "full-state-sm", def, role, "STANDARD") - require.NoError(t, err) - - act, err := original.CreateActivity(ctx, "full-state-activity") - require.NoError(t, err) - - v, err := original.PublishStateMachineVersion(sm.StateMachineArn, "v1", "") - require.NoError(t, err) - - // Started via the version-qualified ARN so StateMachineVersionArn is - // non-empty on the Execution record, exercising the persistence DTO - // field added alongside qualified-ARN execution resolution. - exec, err := original.StartExecution(v.StateMachineVersionArn, "full-state-exec", `{"k":"v"}`) - require.NoError(t, err) - - require.Eventually(t, func() bool { - e, describeErr := original.DescribeExecution(exec.ExecutionArn) - - return describeErr == nil && e.Status != "RUNNING" - }, 5*time.Second, 10*time.Millisecond) - - wantHistory, _, err := original.GetExecutionHistory(exec.ExecutionArn, "", 0, false) - require.NoError(t, err) - require.NotEmpty(t, wantHistory, "execution should have recorded at least one history event") - - snap := original.Snapshot(ctx) - require.NotNil(t, snap) - - fresh := stepfunctions.NewInMemoryBackendWithConfig("000000000000", "us-east-1") - require.NoError(t, fresh.Restore(ctx, snap)) - - // State machine survives the round trip. - restoredSM, err := fresh.DescribeStateMachine(sm.StateMachineArn) - require.NoError(t, err) - assert.Equal(t, sm.Name, restoredSM.Name) - assert.Equal(t, sm.Definition, restoredSM.Definition) - assert.Equal(t, sm.RoleArn, restoredSM.RoleArn) - - // Activity survives the round trip. - restoredAct, err := fresh.DescribeActivity(act.ActivityArn) - require.NoError(t, err) - assert.Equal(t, act.Name, restoredAct.Name) - - // Execution survives the round trip, including its inline history. - restoredExec, err := fresh.DescribeExecution(exec.ExecutionArn) - require.NoError(t, err) - assert.Equal(t, exec.ExecutionArn, restoredExec.ExecutionArn) - assert.Equal(t, sm.StateMachineArn, restoredExec.StateMachineArn) - assert.Equal(t, v.StateMachineVersionArn, restoredExec.StateMachineVersionArn, - "StateMachineVersionArn must survive the snapshot/restore round trip") - assert.JSONEq(t, `{"k":"v"}`, restoredExec.Input) - - gotHistory, _, err := fresh.GetExecutionHistory(exec.ExecutionArn, "", 0, false) - require.NoError(t, err) - require.Len(t, gotHistory, len(wantHistory)) - - for i, wantEvent := range wantHistory { - assert.Equal(t, wantEvent.Type, gotHistory[i].Type, "history event %d type mismatch after restore", i) - assert.Equal(t, wantEvent.ID, gotHistory[i].ID, "history event %d ID mismatch after restore", i) - } + synctest.Test(t, func(t *testing.T) { + const def = `{"StartAt":"P","States":{"P":{"Type":"Pass","End":true}}}` + const role = "arn:aws:iam::000000000000:role/test" + + ctx := t.Context() + original := stepfunctions.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + + sm, err := original.CreateStateMachine(ctx, "full-state-sm", def, role, "STANDARD") + require.NoError(t, err) + + act, err := original.CreateActivity(ctx, "full-state-activity") + require.NoError(t, err) + + v, err := original.PublishStateMachineVersion(sm.StateMachineArn, "v1", "") + require.NoError(t, err) + + // Started via the version-qualified ARN so StateMachineVersionArn is + // non-empty on the Execution record, exercising the persistence DTO + // field added alongside qualified-ARN execution resolution. + exec, err := original.StartExecution(v.StateMachineVersionArn, "full-state-exec", `{"k":"v"}`) + require.NoError(t, err) + + synctest.Wait() + + terminalDesc, err := original.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + require.NotEqual(t, "RUNNING", terminalDesc.Status) + + wantHistory, _, err := original.GetExecutionHistory(exec.ExecutionArn, "", 0, false) + require.NoError(t, err) + require.NotEmpty(t, wantHistory, "execution should have recorded at least one history event") + + snap := original.Snapshot(ctx) + require.NotNil(t, snap) + + fresh := stepfunctions.NewInMemoryBackendWithConfig("000000000000", "us-east-1") + require.NoError(t, fresh.Restore(ctx, snap)) + + // State machine survives the round trip. + restoredSM, err := fresh.DescribeStateMachine(sm.StateMachineArn) + require.NoError(t, err) + assert.Equal(t, sm.Name, restoredSM.Name) + assert.Equal(t, sm.Definition, restoredSM.Definition) + assert.Equal(t, sm.RoleArn, restoredSM.RoleArn) + + // Activity survives the round trip. + restoredAct, err := fresh.DescribeActivity(act.ActivityArn) + require.NoError(t, err) + assert.Equal(t, act.Name, restoredAct.Name) + + // Execution survives the round trip, including its inline history. + restoredExec, err := fresh.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + assert.Equal(t, exec.ExecutionArn, restoredExec.ExecutionArn) + assert.Equal(t, sm.StateMachineArn, restoredExec.StateMachineArn) + assert.Equal(t, v.StateMachineVersionArn, restoredExec.StateMachineVersionArn, + "StateMachineVersionArn must survive the snapshot/restore round trip") + assert.JSONEq(t, `{"k":"v"}`, restoredExec.Input) + + gotHistory, _, err := fresh.GetExecutionHistory(exec.ExecutionArn, "", 0, false) + require.NoError(t, err) + require.Len(t, gotHistory, len(wantHistory)) + + for i, wantEvent := range wantHistory { + assert.Equal(t, wantEvent.Type, gotHistory[i].Type, "history event %d type mismatch after restore", i) + assert.Equal(t, wantEvent.ID, gotHistory[i].ID, "history event %d ID mismatch after restore", i) + } + }) } func TestSFNHandler_SnapshotRestore_Delegation(t *testing.T) { diff --git a/services/stepfunctions/result_writer_test.go b/services/stepfunctions/result_writer_test.go index 11c4beb46d..900a5d0f21 100644 --- a/services/stepfunctions/result_writer_test.go +++ b/services/stepfunctions/result_writer_test.go @@ -8,9 +8,14 @@ import ( "strings" "sync" "testing" + "testing/synctest" "time" + "github.com/aws/aws-sdk-go-v2/aws" awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + sfnsdk "github.com/aws/aws-sdk-go-v2/service/sfn" + sfntypes "github.com/aws/aws-sdk-go-v2/service/sfn/types" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -124,17 +129,15 @@ func getS3ObjectBytes(t *testing.T, bk *s3pkg.InMemoryBackend, bucket, key strin return data } +// waitForTerminalExecution must run inside a synctest bubble. func waitForTerminalExecution(t *testing.T, b *stepfunctions.InMemoryBackend, execARN string) *stepfunctions.Execution { t.Helper() - require.Eventually(t, func() bool { - d, err := b.DescribeExecution(execARN) - - return err == nil && d.Status != "RUNNING" - }, 5*time.Second, 10*time.Millisecond) + synctest.Wait() d, err := b.DescribeExecution(execARN) require.NoError(t, err) + require.NotEqual(t, "RUNNING", d.Status) return d } @@ -219,8 +222,12 @@ func TestDistributedMapResultWriter(t *testing.T) { require.NoError(t, json.Unmarshal(succeededBytes, &records)) require.Len(t, records, 3) assert.Equal(t, "SUCCEEDED", records[0]["Status"]) - assert.InDelta(t, 1, records[0]["Input"], 0) - assert.InDelta(t, 1, records[0]["Output"], 0) + // Default Transformation (NONE, since ResultWriter exports without a + // WriterConfig) reports Input/Output as JSON-encoded strings, matching + // a real child execution's DescribeExecution shape. + assert.Equal(t, "1", records[0]["Input"]) + assert.Equal(t, "1", records[0]["Output"]) + assert.Equal(t, true, records[0]["InputDetails"].(map[string]any)["Included"]) runs, _, err := b.ListMapRuns(exec.ExecutionArn, "", 0) require.NoError(t, err) @@ -281,9 +288,15 @@ func TestDistributedMapResultWriter(t *testing.T) { "a missing S3 writer must degrade to inline results, not fail the execution: cause=%s error=%s", d.Cause, d.Error) - var arr []float64 - require.NoError(t, json.Unmarshal([]byte(d.Output), &arr)) - assert.Equal(t, []float64{1, 2, 3}, arr) + // Resource+Parameters with no WriterConfig defaults to Transformation + // NONE, same as the writes-to-S3 case -- the missing S3 writer only + // changes whether the formatted result is exported, not its shape. + var records []map[string]any + require.NoError(t, json.Unmarshal([]byte(d.Output), &records)) + require.Len(t, records, 3) + assert.Equal(t, "SUCCEEDED", records[0]["Status"]) + assert.Equal(t, "1", records[0]["Input"]) + assert.Equal(t, "1", records[0]["Output"]) }, }, } @@ -291,7 +304,7 @@ func TestDistributedMapResultWriter(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - tt.fn(t) + synctest.Test(t, tt.fn) }) } } @@ -300,6 +313,406 @@ func TestDistributedMapResultWriter(t *testing.T) { // records exportMapResults emits when a configured ResultWriter degrades // silently in its effect (still SUCCEEDED, inline/default output) — checking // the log is the only way to tell that case apart from a real export. +// writerConfigStateMachineDef builds a 3-item Map+ResultWriter state +// machine whose Iterator always outputs the fixed array [10, 20] via a Pass +// state, regardless of the input item -- enough to exercise +// Transformation's array-handling (COMPACT keeps it nested, FLATTEN +// splices it) without needing per-item computed values. bucket=="" omits +// Resource/Parameters entirely (WriterConfig-only preview, no S3 export); +// transformation/outputType=="" omit that WriterConfig sub-field, letting +// AWS's documented defaults apply. +func writerConfigStateMachineDef(bucket, prefix, transformation, outputType string) string { + var rwFields []string + + if bucket != "" { + rwFields = append(rwFields, + `"Resource":"arn:aws:states:::s3:putObject"`, + `"Parameters":{"Bucket":"`+bucket+`","Prefix":"`+prefix+`"}`, + ) + } + + var wcFields []string + if transformation != "" { + wcFields = append(wcFields, `"Transformation":"`+transformation+`"`) + } + + if outputType != "" { + wcFields = append(wcFields, `"OutputType":"`+outputType+`"`) + } + + if len(wcFields) > 0 { + rwFields = append(rwFields, `"WriterConfig":{`+strings.Join(wcFields, ",")+`}`) + } + + return `{ + "StartAt": "M", + "States": { + "M": { + "Type": "Map", + "End": true, + "ItemsPath": "$", + "MaxConcurrency": 1, + "ResultWriter": {` + strings.Join(rwFields, ",") + `}, + "Iterator": { + "StartAt": "P", + "States": {"P": {"Type": "Pass", "Result": [10, 20], "End": true}} + } + } + } + }` +} + +// startWriterConfigExecution creates and starts def against a 3-item input +// through the real aws-sdk-go-v2 sfn client, waiting for it to leave +// RUNNING, and returns the terminal DescribeExecutionOutput. +func startWriterConfigExecution( + t *testing.T, client *sfnsdk.Client, def, namePrefix string, +) *sfnsdk.DescribeExecutionOutput { + t.Helper() + + ctx := t.Context() + + createSM, err := client.CreateStateMachine(ctx, &sfnsdk.CreateStateMachineInput{ + Name: aws.String(namePrefix + "-" + uuid.NewString()[:8]), + Definition: aws.String(def), + RoleArn: aws.String(validRoleARN), + Type: sfntypes.StateMachineTypeStandard, + }) + require.NoError(t, err) + + startOut, err := client.StartExecution(ctx, &sfnsdk.StartExecutionInput{ + StateMachineArn: createSM.StateMachineArn, + Input: aws.String(`[1,2,3]`), + }) + require.NoError(t, err) + + require.Eventually(t, func() bool { + d, dErr := client.DescribeExecution(ctx, &sfnsdk.DescribeExecutionInput{ExecutionArn: startOut.ExecutionArn}) + + return dErr == nil && d.Status != sfntypes.ExecutionStatusRunning + }, 5*time.Second, 10*time.Millisecond, "execution should leave RUNNING") + + desc, err := client.DescribeExecution(ctx, &sfnsdk.DescribeExecutionInput{ExecutionArn: startOut.ExecutionArn}) + require.NoError(t, err) + + return desc +} + +// assertTransformationEntries checks succeeded-file/preview entries against +// what each Transformation must produce for three items whose child output +// is always [10, 20] (AWS docs: input-output-resultwriter.html). +func assertTransformationEntries(t *testing.T, transformation string, raw []json.RawMessage) { + t.Helper() + + switch transformation { + case "FLATTEN": + require.Len(t, raw, 6, "FLATTEN splices each [10,20] output into the outer array") + + want := []float64{10, 20, 10, 20, 10, 20} + for i, r := range raw { + var v float64 + require.NoError(t, json.Unmarshal(r, &v)) + assert.InDelta(t, want[i], v, 0) + } + case "COMPACT": + require.Len(t, raw, 3) + + for _, r := range raw { + var v []float64 + require.NoError(t, json.Unmarshal(r, &v)) + assert.Equal(t, []float64{10, 20}, v) + } + default: // NONE + require.Len(t, raw, 3) + + for _, r := range raw { + var rec map[string]any + require.NoError(t, json.Unmarshal(r, &rec)) + assert.Equal(t, "SUCCEEDED", rec["Status"]) + assert.Equal(t, "[10,20]", rec["Output"], "NONE stringifies Output like a real DescribeExecution") + assert.Equal(t, true, rec["InputDetails"].(map[string]any)["Included"]) + } + } +} + +// decodeResultEntries parses a SUCCEEDED_0.json/FAILED_0.json file's bytes +// into individual JSON values, per OutputType: JSON is one array, JSONL is +// one value per newline. +func decodeResultEntries(t *testing.T, data []byte, outputType string) []json.RawMessage { + t.Helper() + + if outputType != "JSONL" { + var arr []json.RawMessage + require.NoError(t, json.Unmarshal(data, &arr)) + + return arr + } + + lines := strings.Split(strings.TrimSpace(string(data)), "\n") + raw := make([]json.RawMessage, len(lines)) + + for i, l := range lines { + raw[i] = json.RawMessage(l) + } + + return raw +} + +// TestDistributedMapResultWriter_TransformationOutputType drives a real +// state machine through the SDK client for every Transformation x +// OutputType combination ResultWriter.WriterConfig documents +// (input-output-resultwriter.html), asserting the exact SUCCEEDED_0.json +// bytes written to the wired in-process S3 backend. +func TestDistributedMapResultWriter_TransformationOutputType(t *testing.T) { + t.Parallel() + + transformations := []string{"NONE", "COMPACT", "FLATTEN"} + outputTypes := []string{"JSON", "JSONL"} + + for _, transformation := range transformations { + for _, outputType := range outputTypes { + t.Run(transformation+"_"+outputType, func(t *testing.T) { + t.Parallel() + + bucket := "wc-matrix-" + strings.ToLower(transformation+outputType) + + s3Bk := newBucketBackedS3(t, bucket) + backend := stepfunctions.NewInMemoryBackend() + backend.SetS3ResultWriter(stepfunctions.NewS3ResultWriterIntegration(s3Bk)) + + client := newSFNSDKClient(t, stepfunctions.NewHandler(backend)) + + def := writerConfigStateMachineDef(bucket, "jobs", transformation, outputType) + desc := startWriterConfigExecution(t, client, def, "wc-matrix") + require.Equal(t, sfntypes.ExecutionStatusSucceeded, desc.Status, + "cause=%s error=%s", aws.ToString(desc.Cause), aws.ToString(desc.Error)) + + var out mapExportOutput + require.NoError(t, json.Unmarshal([]byte(aws.ToString(desc.Output)), &out)) + require.Equal(t, bucket, out.ResultWriterDetails.Bucket) + assert.True(t, strings.HasPrefix(out.ResultWriterDetails.Key, "jobs/")) + assert.True(t, strings.HasSuffix(out.ResultWriterDetails.Key, "manifest.json")) + + manifestBytes := getS3ObjectBytes(t, s3Bk, bucket, out.ResultWriterDetails.Key) + + var manifest resultManifest + require.NoError(t, json.Unmarshal(manifestBytes, &manifest)) + require.Len(t, manifest.ResultFiles.Succeeded, 1) + assert.Empty(t, manifest.ResultFiles.Failed) + assert.True(t, strings.HasSuffix(manifest.ResultFiles.Succeeded[0].Key, "SUCCEEDED_0.json")) + + succeededBytes := getS3ObjectBytes(t, s3Bk, bucket, manifest.ResultFiles.Succeeded[0].Key) + entries := decodeResultEntries(t, succeededBytes, outputType) + assertTransformationEntries(t, transformation, entries) + }) + } + } +} + +// TestDistributedMapResultWriter_FailedItemsKeepFullRecord proves that a +// FAILED item's exported record is always the full NONE-shaped record +// regardless of Transformation (AWS docs: "If a child workflow execution +// fails, Step Functions returns its execution result unchanged"), while +// SUCCEEDED items still honor COMPACT. +func TestDistributedMapResultWriter_FailedItemsKeepFullRecord(t *testing.T) { + t.Parallel() + + const bucket = "wc-failed-bucket" + + s3Bk := newBucketBackedS3(t, bucket) + backend := stepfunctions.NewInMemoryBackend() + backend.SetS3ResultWriter(stepfunctions.NewS3ResultWriterIntegration(s3Bk)) + + client := newSFNSDKClient(t, stepfunctions.NewHandler(backend)) + + def := `{ + "StartAt": "M", + "States": { + "M": { + "Type": "Map", + "End": true, + "ItemsPath": "$", + "MaxConcurrency": 1, + "ToleratedFailureCount": 1, + "ResultWriter": { + "Resource": "arn:aws:states:::s3:putObject", + "Parameters": {"Bucket": "` + bucket + `", "Prefix": "jobs"}, + "WriterConfig": {"Transformation": "COMPACT"} + }, + "Iterator": { + "StartAt": "Check", + "States": { + "Check": { + "Type": "Choice", + "Choices": [{"Variable": "$", "NumericEquals": 2, "Next": "Boom"}], + "Default": "OK" + }, + "Boom": {"Type": "Fail", "Error": "States.TaskFailed", "Cause": "item 2 always fails"}, + "OK": {"Type": "Pass", "Result": [10, 20], "End": true} + } + } + } + } + }` + + desc := startWriterConfigExecution(t, client, def, "wc-failed") + require.Equal( + t, + sfntypes.ExecutionStatusSucceeded, + desc.Status, + "1 failure is within ToleratedFailureCount: cause=%s error=%s", + aws.ToString(desc.Cause), + aws.ToString(desc.Error), + ) + + var out mapExportOutput + require.NoError(t, json.Unmarshal([]byte(aws.ToString(desc.Output)), &out)) + + manifestBytes := getS3ObjectBytes(t, s3Bk, bucket, out.ResultWriterDetails.Key) + + var manifest resultManifest + require.NoError(t, json.Unmarshal(manifestBytes, &manifest)) + require.Len(t, manifest.ResultFiles.Succeeded, 1) + require.Len(t, manifest.ResultFiles.Failed, 1) + + succeededBytes := getS3ObjectBytes(t, s3Bk, bucket, manifest.ResultFiles.Succeeded[0].Key) + + var succeeded [][]float64 + require.NoError(t, json.Unmarshal(succeededBytes, &succeeded)) + require.Len(t, succeeded, 2, "2 of 3 items succeed") + assert.Equal(t, []float64{10, 20}, succeeded[0]) + + failedBytes := getS3ObjectBytes(t, s3Bk, bucket, manifest.ResultFiles.Failed[0].Key) + + var failed []map[string]any + require.NoError(t, json.Unmarshal(failedBytes, &failed)) + require.Len(t, failed, 1) + assert.Equal(t, "FAILED", failed[0]["Status"]) + assert.Equal(t, "2", failed[0]["Input"]) + assert.Equal(t, "States.TaskFailed", failed[0]["Error"]) + assert.Equal(t, "item 2 always fails", failed[0]["Cause"]) + assert.Equal(t, "REDRIVABLE", failed[0]["RedriveStatus"]) +} + +// TestDistributedMapResultWriter_DistributedChildIdentity proves that a +// DISTRIBUTED Map's ResultWriter NONE records carry the real child +// execution's ExecutionArn/Name/StartDate, unlike an INLINE Map's (which +// has no such resource -- see TestDistributedMapResultWriter_ +// TransformationOutputType's NONE case, which leaves them empty). +func TestDistributedMapResultWriter_DistributedChildIdentity(t *testing.T) { + t.Parallel() + + const bucket = "wc-distributed-bucket" + + s3Bk := newBucketBackedS3(t, bucket) + backend := stepfunctions.NewInMemoryBackend() + backend.SetS3ResultWriter(stepfunctions.NewS3ResultWriterIntegration(s3Bk)) + + client := newSFNSDKClient(t, stepfunctions.NewHandler(backend)) + + def := `{ + "StartAt": "M", + "States": { + "M": { + "Type": "Map", + "End": true, + "ItemsPath": "$", + "MaxConcurrency": 1, + "ResultWriter": { + "Resource": "arn:aws:states:::s3:putObject", + "Parameters": {"Bucket": "` + bucket + `", "Prefix": "jobs"} + }, + "ItemProcessor": { + "ProcessorConfig": {"Mode": "DISTRIBUTED", "ExecutionType": "STANDARD"}, + "StartAt": "P", + "States": {"P": {"Type": "Pass", "Result": [10, 20], "End": true}} + } + } + } + }` + + desc := startWriterConfigExecution(t, client, def, "wc-distributed") + require.Equal(t, sfntypes.ExecutionStatusSucceeded, desc.Status, + "cause=%s error=%s", aws.ToString(desc.Cause), aws.ToString(desc.Error)) + + var out mapExportOutput + require.NoError(t, json.Unmarshal([]byte(aws.ToString(desc.Output)), &out)) + + manifestBytes := getS3ObjectBytes(t, s3Bk, bucket, out.ResultWriterDetails.Key) + + var manifest resultManifest + require.NoError(t, json.Unmarshal(manifestBytes, &manifest)) + require.Len(t, manifest.ResultFiles.Succeeded, 1) + + succeededBytes := getS3ObjectBytes(t, s3Bk, bucket, manifest.ResultFiles.Succeeded[0].Key) + + var records []map[string]any + require.NoError(t, json.Unmarshal(succeededBytes, &records)) + require.Len(t, records, 3) + + seen := map[string]bool{} + + for _, rec := range records { + execArn, _ := rec["ExecutionArn"].(string) + assert.NotEmpty(t, execArn, "a DISTRIBUTED Map item runs as a real child execution") + assert.False(t, seen[execArn], "each child execution must have a unique ExecutionArn") + seen[execArn] = true + + assert.NotEmpty(t, rec["Name"]) + assert.Positive(t, rec["StartDate"]) + assert.NotEmpty(t, rec["StateMachineArn"]) + } +} + +// TestDistributedMapResultWriter_PreviewWithoutExport covers ResultWriter's +// WriterConfig-only shape (AWS docs' "Required field combinations": no +// Resource/Parameters means no S3 export, only a formatted state output). +func TestDistributedMapResultWriter_PreviewWithoutExport(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + outputType string + }{ + {name: "JSON", outputType: "JSON"}, + {name: "JSONL", outputType: "JSONL"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + backend := stepfunctions.NewInMemoryBackend() + client := newSFNSDKClient(t, stepfunctions.NewHandler(backend)) + + def := writerConfigStateMachineDef("", "", "FLATTEN", tt.outputType) + desc := startWriterConfigExecution(t, client, def, "wc-preview-"+strings.ToLower(tt.outputType)) + require.Equal(t, sfntypes.ExecutionStatusSucceeded, desc.Status, + "cause=%s error=%s", aws.ToString(desc.Cause), aws.ToString(desc.Error)) + + output := aws.ToString(desc.Output) + + if tt.outputType == "JSONL" { + var jsonl string + require.NoError(t, json.Unmarshal([]byte(output), &jsonl)) + + lines := strings.Split(jsonl, "\n") + require.Len(t, lines, 6) + + var v float64 + require.NoError(t, json.Unmarshal([]byte(lines[0]), &v)) + assert.InDelta(t, 10, v, 0) + + return + } + + var arr []float64 + require.NoError(t, json.Unmarshal([]byte(output), &arr)) + assert.Equal(t, []float64{10, 20, 10, 20, 10, 20}, arr) + }) + } +} + func TestDistributedMapResultWriterWarnLogs(t *testing.T) { t.Parallel() @@ -337,47 +750,12 @@ func TestDistributedMapResultWriterWarnLogs(t *testing.T) { assert.Equal(t, "nowhere-bucket", attrs["bucket"]) }, }, - { - name: "unsupported writerconfig warns with state and settings", - fn: func(t *testing.T) { - t.Helper() - - const bucket = "wc-bucket" - - s3Bk := newBucketBackedS3(t, bucket) - b, rh := newLoggingBackend(t) - b.SetS3ResultWriter(stepfunctions.NewS3ResultWriterIntegration(s3Bk)) - - def := resultWriterMapDef( - `"ResultWriter": {"Resource":"arn:aws:states:::s3:putObject",` + - `"Parameters":{"Bucket":"` + bucket + `"},` + - `"WriterConfig":{"Transformation":"COMPACT","OutputType":"JSONL"}},`, - ) - - sm, err := b.CreateStateMachine(context.Background(), "rw-wc-sm", def, validRoleARN, "STANDARD") - require.NoError(t, err) - - exec, err := b.StartExecution(sm.StateMachineArn, "rw-wc-exec", `[1,2,3]`) - require.NoError(t, err) - - d := waitForTerminalExecution(t, b, exec.ExecutionArn) - require.Equal(t, "SUCCEEDED", d.Status, "cause=%s error=%s", d.Cause, d.Error) - - rec := rh.findWarn("ResultWriter WriterConfig not applied") - require.NotNil(t, rec, "expected a warn log for the unapplied WriterConfig") - - attrs := recordAttrs(rec) - assert.Equal(t, "M", attrs["state"]) - assert.Equal(t, "COMPACT", attrs["transformation"]) - assert.Equal(t, "JSONL", attrs["outputType"]) - }, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - tt.fn(t) + synctest.Test(t, tt.fn) }) } } diff --git a/services/stepfunctions/s3_item_reader_test.go b/services/stepfunctions/s3_item_reader_test.go index 0960585d32..cb29db16d2 100644 --- a/services/stepfunctions/s3_item_reader_test.go +++ b/services/stepfunctions/s3_item_reader_test.go @@ -4,7 +4,7 @@ import ( "context" "encoding/json" "testing" - "time" + "testing/synctest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -38,46 +38,44 @@ func (f *fakeS3Reader) GetObjectBytes(_ context.Context, _, _ string) ([]byte, e func TestStartExecution_MapItemReader_S3(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackend() - b.SetS3Reader(&fakeS3Reader{data: []byte(`[{"n":1},{"n":2},{"n":3}]`)}) - - def := `{ - "StartAt": "M", - "States": { - "M": { - "Type": "Map", - "ItemReader": { - "Resource": "arn:aws:states:::s3:getObject", - "Parameters": {"Bucket": "test-bucket", "Key": "items.json"} - }, - "ItemProcessor": { - "StartAt": "P", - "States": {"P": {"Type": "Pass", "End": true}} - }, - "End": true + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + b.SetS3Reader(&fakeS3Reader{data: []byte(`[{"n":1},{"n":2},{"n":3}]`)}) + + def := `{ + "StartAt": "M", + "States": { + "M": { + "Type": "Map", + "ItemReader": { + "Resource": "arn:aws:states:::s3:getObject", + "Parameters": {"Bucket": "test-bucket", "Key": "items.json"} + }, + "ItemProcessor": { + "StartAt": "P", + "States": {"P": {"Type": "Pass", "End": true}} + }, + "End": true + } } - } - }` + }` - sm, err := b.CreateStateMachine(context.Background(), "s3-itemreader-sm", def, validRoleARN, "STANDARD") - require.NoError(t, err) + sm, err := b.CreateStateMachine(context.Background(), "s3-itemreader-sm", def, validRoleARN, "STANDARD") + require.NoError(t, err) - exec, err := b.StartExecution(sm.StateMachineArn, "s3-exec", "{}") - require.NoError(t, err) + exec, err := b.StartExecution(sm.StateMachineArn, "s3-exec", "{}") + require.NoError(t, err) - require.Eventually(t, func() bool { - described, descErr := b.DescribeExecution(exec.ExecutionArn) + synctest.Wait() - return descErr == nil && described.Status != "RUNNING" - }, 5*time.Second, 10*time.Millisecond) + described, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + require.Equal(t, "SUCCEEDED", described.Status, "cause=%s error=%s", described.Cause, described.Error) - described, err := b.DescribeExecution(exec.ExecutionArn) - require.NoError(t, err) - require.Equal(t, "SUCCEEDED", described.Status, "cause=%s error=%s", described.Cause, described.Error) - - var output []map[string]any - require.NoError(t, json.Unmarshal([]byte(described.Output), &output)) - assert.Len(t, output, 3, "expected one output entry per S3-sourced item") + var output []map[string]any + require.NoError(t, json.Unmarshal([]byte(described.Output), &output)) + assert.Len(t, output, 3, "expected one output entry per S3-sourced item") + }) } // TestStartExecution_MapItemReader_NoS3Reader verifies the documented @@ -88,39 +86,37 @@ func TestStartExecution_MapItemReader_S3(t *testing.T) { func TestStartExecution_MapItemReader_NoS3Reader(t *testing.T) { t.Parallel() - b := stepfunctions.NewInMemoryBackend() - - def := `{ - "StartAt": "M", - "States": { - "M": { - "Type": "Map", - "ItemReader": { - "Resource": "arn:aws:states:::s3:getObject", - "Parameters": {"Bucket": "test-bucket", "Key": "items.json"} - }, - "ItemProcessor": { - "StartAt": "P", - "States": {"P": {"Type": "Pass", "End": true}} - }, - "End": true + synctest.Test(t, func(t *testing.T) { + b := stepfunctions.NewInMemoryBackend() + + def := `{ + "StartAt": "M", + "States": { + "M": { + "Type": "Map", + "ItemReader": { + "Resource": "arn:aws:states:::s3:getObject", + "Parameters": {"Bucket": "test-bucket", "Key": "items.json"} + }, + "ItemProcessor": { + "StartAt": "P", + "States": {"P": {"Type": "Pass", "End": true}} + }, + "End": true + } } - } - }` - - sm, err := b.CreateStateMachine(context.Background(), "no-s3-itemreader-sm", def, validRoleARN, "STANDARD") - require.NoError(t, err) + }` - exec, err := b.StartExecution(sm.StateMachineArn, "no-s3-exec", "{}") - require.NoError(t, err) + sm, err := b.CreateStateMachine(context.Background(), "no-s3-itemreader-sm", def, validRoleARN, "STANDARD") + require.NoError(t, err) - require.Eventually(t, func() bool { - described, descErr := b.DescribeExecution(exec.ExecutionArn) + exec, err := b.StartExecution(sm.StateMachineArn, "no-s3-exec", "{}") + require.NoError(t, err) - return descErr == nil && described.Status != "RUNNING" - }, 5*time.Second, 10*time.Millisecond) + synctest.Wait() - described, err := b.DescribeExecution(exec.ExecutionArn) - require.NoError(t, err) - assert.Equal(t, "FAILED", described.Status) + described, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + assert.Equal(t, "FAILED", described.Status) + }) } diff --git a/services/stepfunctions/state_machines_test.go b/services/stepfunctions/state_machines_test.go index 27ec121ffc..aa07990723 100644 --- a/services/stepfunctions/state_machines_test.go +++ b/services/stepfunctions/state_machines_test.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -332,11 +333,10 @@ func TestDeleteStateMachine_DeletingObservableWhileExecutionRunning(t *testing.T _ = b.StopExecution(exec.ExecutionArn, "Test", "cleanup") }) - require.Eventually(t, func() bool { - d, dErr := b.DescribeExecution(exec.ExecutionArn) - - return dErr == nil && d.Status == "RUNNING" - }, 5*time.Second, 10*time.Millisecond) + // StartExecution and StopExecution both set status synchronously; no wait needed. + runningDesc, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + assert.Equal(t, "RUNNING", runningDesc.Status) require.NoError(t, b.DeleteStateMachine(smARN)) @@ -348,11 +348,10 @@ func TestDeleteStateMachine_DeletingObservableWhileExecutionRunning(t *testing.T require.ErrorIs(t, err, stepfunctions.ErrStateMachineDeleting) require.NoError(t, b.StopExecution(exec.ExecutionArn, "Test", "cleanup")) - require.Eventually(t, func() bool { - d, dErr := b.DescribeExecution(exec.ExecutionArn) - return dErr == nil && d.Status != "RUNNING" - }, 5*time.Second, 10*time.Millisecond) + stoppedDesc, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + assert.NotEqual(t, "RUNNING", stoppedDesc.Status) swept := b.SweepDeletingStateMachines(context.Background()) assert.Equal(t, 1, swept) @@ -401,11 +400,11 @@ func TestStateMachineDeleting_BlocksClientCallableOps(t *testing.T) { _ = b.StopExecution(exec.ExecutionArn, "Test", "cleanup") }) - require.Eventually(t, func() bool { - d, dErr := b.DescribeExecution(exec.ExecutionArn) - - return dErr == nil && d.Status == "RUNNING" - }, 5*time.Second, 10*time.Millisecond) + // StartExecution sets the RUNNING status synchronously before the ASL + // interpreter goroutine is launched, so no wait is needed here. + runningDesc, err := b.DescribeExecution(exec.ExecutionArn) + require.NoError(t, err) + assert.Equal(t, "RUNNING", runningDesc.Status) require.NoError(t, b.DeleteStateMachine(smARN)) @@ -936,28 +935,30 @@ func TestDeleteStateMachine_TombstoneOnlyRunning(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - b := newSFBackend() - sm, err := b.CreateStateMachine(context.Background(), "tomb-sm", exprPassDef, "arn:role", "STANDARD") - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := newSFBackend() + sm, err := b.CreateStateMachine(context.Background(), "tomb-sm", exprPassDef, "arn:role", "STANDARD") + require.NoError(t, err) - exec, err := b.StartExecution(sm.StateMachineArn, "tomb-exec", "{}") - require.NoError(t, err) + exec, err := b.StartExecution(sm.StateMachineArn, "tomb-exec", "{}") + require.NoError(t, err) - execARN := exec.ExecutionArn + execARN := exec.ExecutionArn - if tt.waitForCompletion { - require.Eventually(t, func() bool { - d, e := b.DescribeExecution(execARN) + if tt.waitForCompletion { + synctest.Wait() - return e == nil && d.Status != "RUNNING" - }, 5*time.Second, 50*time.Millisecond) - } + d, dErr := b.DescribeExecution(execARN) + require.NoError(t, dErr) + require.NotEqual(t, "RUNNING", d.Status) + } - err = b.DeleteStateMachine(sm.StateMachineArn) - require.NoError(t, err) + err = b.DeleteStateMachine(sm.StateMachineArn) + require.NoError(t, err) - hasTombstone := b.HasTombstoneForTest(execARN) - assert.Equal(t, tt.wantTombstone, hasTombstone) + hasTombstone := b.HasTombstoneForTest(execARN) + assert.Equal(t, tt.wantTombstone, hasTombstone) + }) }) } } diff --git a/services/sts/PARITY.md b/services/sts/PARITY.md index 79b150e9e2..f820262b56 100644 --- a/services/sts/PARITY.md +++ b/services/sts/PARITY.md @@ -35,13 +35,13 @@ ops: GetAccessKeyInfo: {wire: ok, errors: ok, state: ok, persist: ok, note: "session lookup then well-formed-prefix fallback to backend account ID — re-verified correct"} DecodeAuthorizationMessage: {wire: ok, errors: fixed, state: ok, persist: n/a, note: "HMAC-signed self-issued messages verified; foreign base64 blobs decoded permissively for emulator usability — re-verified correct. gopherstack-yatn orphan-code triage FIX: missing-EncodedMessage rejection emitted \"InvalidParameter\", which is not a real STS/AWS-Query code anywhere (absent from sts@v1.45.4 entirely, and absent from the AWS STS Common Errors page, which documents \"MissingParameter\"/\"InvalidParameterValue\" but no bare \"InvalidParameter\"). EncodedMessage is a required member (DecodeAuthorizationMessageInput, api_op_DecodeAuthorizationMessage.go) whose absence is exactly what the doc's \"MissingParameter\" ('A required parameter for the specified action isn't included in the request') describes; reclassified ErrMissingEncodedMessage into the same MissingParameter bucket as every sibling ErrMissingXxx sentinel in mapValidationErrorToCode (control: ErrMissingRoleArn et al.). Malformed-message-content handling (InvalidAuthorizationMessageException, this op's one genuinely declared exception) is untouched. Verified via TestDecodeAuthorizationMessageEmpty (pre-existing test's old \"InvalidParameter\" assertion corrected -- confirmed failing pre-fix)."} families: - trust-policy-evaluation: {status: ok, note: "Principal (AWS/Federated/Service/wildcard), Action (incl. wildcard glob), Effect Allow/Deny, Condition (StringEquals/StringLike/StringEqualsIgnoreCase/StringNotEquals/StringNotLike/Bool/Null/ArnEquals/ArnLike/ArnNotEquals/ArnNotLike + IfExists, case-insensitive keys) implemented in trust_policy.go and verified against the statements in AssumeRole/WithSAML/WithWebIdentity. Bool operator + aws:multifactorauthpresent condition key added (gopherstack-41fl) for AssumeRole only -- AssumeRoleWithSAML/WithWebIdentity have no SerialNumber/TokenCode request members in the real API (federated identities cannot present MFA through those operations), so a Bool MFA condition in a trust policy assumed via those two ops remains unenforced by design, matching AWS's own operation surface, not a gap in this emulator. FIXED (gopherstack-yg95): conditionOperatorHolds's default branch returned true for every operator it did not model, and an unknown condition key also returned true (satisfied) unconditionally -- both meant a restrictive trust policy's condition could be silently ignored. Added Null (tests key presence via conditionValue's known result, not value -- must run before the generic unknown-key fallback or Null:false would always pass through it) and ArnEquals/ArnLike/ArnNotEquals/ArnNotLike (AWS documents ArnEquals/ArnLike as behaving identically, both wildcard-capable; this emulator reuses the same general-purpose glob matcher as StringLike rather than AWS's six-segment-aware ARN matching, since trust-policy ARN conditions in practice wildcard within one segment, e.g. role/prod-*). Confirmed the IfExists suffix stripping in normalizeConditionOp is correct for every operator it runs before: IfExists only changes absent-key handling, which happens uniformly at the single !known check, not per-operator -- Null is the one AWS-documented exception (no IfExists variant), handled by returning before that check. Numeric*/Date*/IpAddress/NotIpAddress/BinaryEquals remain unenforced: STRUCTURAL, not deferred -- this evaluator has no numeric, timestamp, source-IP, or binary-valued request-context anywhere (confirmed by inventory: the only condition keys ever populated are sts:ExternalId, aws:PrincipalArn, aws:MultiFactorAuthPresent, and WebIdentity's per-issuer :aud/:sub claims, all string- or bool-valued) -- there is no value to compare and adding the operator without a real value would be dead plumbing. DECISION: both fallback paths (unmodeled operator, unmodeled/unknown key) remain fail-open (permit) rather than flipping to fail-closed, but now log at WARN (services/sts/trust_policy.go's warnUnmodeledCondition) naming the specific operator/key, closing the 'silent' half of the bug without a behavioral break for existing callers whose trust policies carry a condition on a key this emulator cannot evaluate. This mirrors the file's pre-existing, deliberate 'enforce only what is positively known' design (see evaluateAssumeRoleTrust's and conditionValue's docstrings) rather than reversing it unilaterally; flipping the global default to fail-closed is flagged as a call that would benefit from explicit human sign-off, since the same permissive-mock philosophy is embedded by design throughout this same file (and, per pkgs-catalog.md's shared conventions, plausibly elsewhere in the emulator) rather than being local to this one bug."} + trust-policy-evaluation: {status: ok, note: "Principal (AWS/Federated/Service/wildcard), Action (incl. wildcard glob), Effect Allow/Deny, Condition (StringEquals/StringLike/StringEqualsIgnoreCase/StringNotEquals/StringNotLike/Bool/Null/ArnEquals/ArnLike/ArnNotEquals/ArnNotLike + IfExists, case-insensitive keys) implemented in trust_policy.go and verified against the statements in AssumeRole/WithSAML/WithWebIdentity. Bool operator + aws:multifactorauthpresent condition key added (gopherstack-41fl) for AssumeRole only -- AssumeRoleWithSAML/WithWebIdentity have no SerialNumber/TokenCode request members in the real API (federated identities cannot present MFA through those operations), so a Bool MFA condition in a trust policy assumed via those two ops remains unenforced by design, matching AWS's own operation surface, not a gap in this emulator. FIXED (gopherstack-yg95): conditionOperatorHolds's default branch returned true for every operator it did not model, and an unknown condition key also returned true (satisfied) unconditionally -- both meant a restrictive trust policy's condition could be silently ignored. Added Null (tests key presence via conditionValue's known result, not value -- must run before the generic unknown-key fallback or Null:false would always pass through it) and ArnEquals/ArnLike/ArnNotEquals/ArnNotLike. Confirmed the IfExists suffix stripping in normalizeConditionOp is correct for every operator it runs before: IfExists only changes absent-key handling, which happens uniformly at the single !known check, not per-operator -- Null is the one AWS-documented exception (no IfExists variant), handled by returning before that check. Numeric*/IpAddress/NotIpAddress/BinaryEquals remain unenforced: STRUCTURAL, not deferred -- this evaluator has no numeric, source-IP, or binary-valued request-context anywhere (confirmed by inventory: the only condition keys ever populated are sts:ExternalId, aws:PrincipalArn, aws:MultiFactorAuthPresent, aws:CurrentTime, aws:EpochTime, and WebIdentity's per-issuer :aud/:sub claims) -- there is no value to compare and adding the operator without a real value would be dead plumbing. DECISION: both fallback paths (unmodeled operator, unmodeled/unknown key) remain fail-open (permit) rather than flipping to fail-closed, but now log at WARN (services/sts/trust_policy.go's warnUnmodeledCondition) naming the specific operator/key, closing the 'silent' half of the bug without a behavioral break for existing callers whose trust policies carry a condition on a key this emulator cannot evaluate. This mirrors the file's pre-existing, deliberate 'enforce only what is positively known' design (see evaluateAssumeRoleTrust's and conditionValue's docstrings) rather than reversing it unilaterally; flipping the global default to fail-closed is flagged as a call that would benefit from explicit human sign-off, since the same permissive-mock philosophy is embedded by design throughout this same file (and, per pkgs-catalog.md's shared conventions, plausibly elsewhere in the emulator) rather than being local to this one bug. 2026-09-26 (condition-operator sweep): ArnEquals/ArnLike/ArnNotEquals/ArnNotLike now compare each of the six colon-delimited ARN components separately via the shared condeval.ArnMatch (previously a single glob over the whole string, which let a wildcard incorrectly span a segment boundary) -- extracted to pkgs/condeval alongside services/iam/conditions.go's identical ARN-matching and Date-parsing code, which had begun being copy-pasted verbatim between the two evaluators. Date (DateEquals/DateNotEquals/DateLessThan/DateLessThanEquals/DateGreaterThan/DateGreaterThanEquals), accepting both ISO 8601 and epoch-seconds operands, is now enforced for aws:CurrentTime/aws:EpochTime -- the only two Date-typed keys this evaluator sources honestly (conditionValue defaults them to time.Now() unless a test overrides via conditionCtx); other Date-typed keys such as aws:TokenIssueTime still fall through the unmodeled-key fallback. conditionOperatorHolds refactored from a 12-case switch to a conditionOperatorFuncs dispatch table (cyclop was over budget after the Date case landed)."} session-tag-validation: {status: ok, note: "key/value length, charset, aws: reserved prefix, case-insensitive dup detection, MaxTagCount=50, transitive-tag merge on role chaining — verified correct; AssumeRoleWithSAML's TransitiveTagKeys (assertion-derived, previously never wired to the session at all) is now also propagated, closing a related chaining gap"} locking: {status: ok, note: "InMemoryBackend.mu is *lockmetrics.RWMutex (New(\"sts\")) per pkgs-catalog.md; every new lock path added this pass (GetWebIdentityToken/AssumeRoot CallerSession lookups, and this pass's checkOutboundWebIdentityFederationEnabled) reuses the existing LookupSession/RLock accessors — no new raw sync.Mutex, no lock ordering changes"} gaps: [] items_still_open: - "2026-08-14 (gopherstack-3tpf): independently re-confirmed via a mechanical struct-field diff (cmd/structfielddiff), not by re-reading this file's prior claims -- every Input/Output/nested struct across all 11 ops, expanded through AssumedRoleUser/Credentials/FederatedUser/PolicyDescriptorType/ProvidedContext/Tag, diffed field-by-field against aws-sdk-go-v2/service/sts@v1.45.4. Zero new gaps: every real field this SDK declares has a matching gopherstack field (Go casing differences like AssumedRoleID/AssumedRoleId excluded as known noise), matching this manifest's own A grade. No code changes made to this service this pass." - - "STRUCTURAL (gopherstack-yg95): Numeric*/Date*/IpAddress/NotIpAddress/BinaryEquals trust-policy condition operators are unenforced for every condition key this evaluator carries, because none of those keys are numeric-, timestamp-, IP-, or binary-valued -- see the trust-policy-evaluation family note above for the full key inventory and the fail-open-plus-WARN-log decision. Not a deferred-effort gap: implementing any of these operators today would have nothing real to compare against." + - "STRUCTURAL (gopherstack-yg95, Date* closed 2026-09-26): Numeric*/IpAddress/NotIpAddress/BinaryEquals trust-policy condition operators remain unenforced for every condition key this evaluator carries, because none of those keys are numeric-, IP-, or binary-valued -- see the trust-policy-evaluation family note above for the full key inventory and the fail-open-plus-WARN-log decision. Not a deferred-effort gap: implementing any of these operators today would have nothing real to compare against. Date* (DateEquals/.../DateGreaterThanEquals, ISO 8601 and epoch seconds) is now enforced for aws:CurrentTime/aws:EpochTime -- see the family note's 2026-09-26 entry." - "IMPOSSIBLE (re-confirmed gopherstack-yewt): JWTPayloadSizeExceededException (aws-sdk-go-v2/service/sts/types, dispatched specifically on GetWebIdentityToken's error branch) has no discoverable numeric threshold anywhere searched: (1) the generated SDK doc comment on the type itself says only 'The requested token payload size exceeds the maximum allowed size. Reduce the number of request tags...' -- no byte number; (2) aws-sdk-go-v2/service/sts@v1.44.0's validators.go's validateOpGetWebIdentityTokenInput only checks Audience/SigningAlgorithm required-ness and delegates Tags to validateTagListType (per-tag key/value length limits, not an aggregate payload-size limit) -- no length/size constraint of any kind is client-side-enforced for this op; (3) no botocore/smithy api-2.json model with a `length` trait for this newer STS operation was found in any locally-vendored SDK (aws-sdk-go v1.55.5's models/apis/sts predates GetWebIdentityToken entirely -- confirmed via `ls .../models/apis/sts` finding no api-2.json referencing this op); (4) WebSearch for 'JWTPayloadSizeExceededException STS GetWebIdentityToken maximum size bytes' returned only the same threshold-free doc comment, restated by boto3/re:Post/awsfundamentals.com sources, plus AWS's general (unrelated) guidance that STS credential/token sizes should never be assumed fixed. Implementing a threshold here would mean inventing an arbitrary number with no spec to verify it against -- the opposite of parity. Genuinely unimplementable without an undocumented number AWS does not publish. (bd: gopherstack-p05, follow-up -- OutboundWebIdentityFederationDisabledException, the other half of this original gap entry, WAS closed this pass, see GetWebIdentityToken above)" - "STALE ISSUE PREMISE (gopherstack-yewt re-triage): the follow-up issue's item (2), 'OutboundWebIdentityFederationDisabledException -- needs account-level settings model gopherstack lacks + no API to toggle,' is already fully resolved as of this same PARITY.md's GetWebIdentityToken row above (parity-3 phase 2) -- re-confirmed this pass by reading the actual code, not just this file: web_identity.go's checkOutboundWebIdentityFederationEnabled (called from GetWebIdentityToken, web_identity.go:365) gates on real state via services/iam/account.go's EnableOutboundWebIdentityFederation/DisableOutboundWebIdentityFederation/GetOutboundWebIdentityFederationInfo/OutboundWebIdentityFederationEnabled (all real methods, not stubs -- confirmed by reading their bodies), and both handler_test.go and web_identity_test.go carry OutboundWebIdentityFederationDisabledException regression coverage. No code change needed; the bd issue's premise predates the fix that already landed in this same file." deferred: diff --git a/services/sts/README.md b/services/sts/README.md index 0810de9464..802fe929d7 100644 --- a/services/sts/README.md +++ b/services/sts/README.md @@ -16,7 +16,7 @@ ### Known gaps - 2026-08-14 (gopherstack-3tpf): independently re-confirmed via a mechanical struct-field diff (cmd/structfielddiff), not by re-reading this file's prior claims -- every Input/Output/nested struct across all 11 ops, expanded through AssumedRoleUser/Credentials/FederatedUser/PolicyDescriptorType/ProvidedContext/Tag, diffed field-by-field against aws-sdk-go-v2/service/sts@v1.45.4. Zero new gaps: every real field this SDK declares has a matching gopherstack field (Go casing differences like AssumedRoleID/AssumedRoleId excluded as known noise), matching this manifest's own A grade. No code changes made to this service this pass. -- STRUCTURAL (gopherstack-yg95): Numeric*/Date*/IpAddress/NotIpAddress/BinaryEquals trust-policy condition operators are unenforced for every condition key this evaluator carries, because none of those keys are numeric-, timestamp-, IP-, or binary-valued -- see the trust-policy-evaluation family note above for the full key inventory and the fail-open-plus-WARN-log decision. Not a deferred-effort gap: implementing any of these operators today would have nothing real to compare against. +- STRUCTURAL (gopherstack-yg95, Date* closed 2026-09-26): Numeric*/IpAddress/NotIpAddress/BinaryEquals trust-policy condition operators remain unenforced for every condition key this evaluator carries, because none of those keys are numeric-, IP-, or binary-valued -- see the trust-policy-evaluation family note above for the full key inventory and the fail-open-plus-WARN-log decision. Not a deferred-effort gap: implementing any of these operators today would have nothing real to compare against. Date* (DateEquals/.../DateGreaterThanEquals, ISO 8601 and epoch seconds) is now enforced for aws:CurrentTime/aws:EpochTime -- see the family note's 2026-09-26 entry. - IMPOSSIBLE (re-confirmed gopherstack-yewt): JWTPayloadSizeExceededException (aws-sdk-go-v2/service/sts/types, dispatched specifically on GetWebIdentityToken's error branch) has no discoverable numeric threshold anywhere searched: (1) the generated SDK doc comment on the type itself says only 'The requested token payload size exceeds the maximum allowed size. Reduce the number of request tags...' -- no byte number; (2) aws-sdk-go-v2/service/sts@v1.44.0's validators.go's validateOpGetWebIdentityTokenInput only checks Audience/SigningAlgorithm required-ness and delegates Tags to validateTagListType (per-tag key/value length limits, not an aggregate payload-size limit) -- no length/size constraint of any kind is client-side-enforced for this op; (3) no botocore/smithy api-2.json model with a `length` trait for this newer STS operation was found in any locally-vendored SDK (aws-sdk-go v1.55.5's models/apis/sts predates GetWebIdentityToken entirely -- confirmed via `ls .../models/apis/sts` finding no api-2.json referencing this op); (4) WebSearch for 'JWTPayloadSizeExceededException STS GetWebIdentityToken maximum size bytes' returned only the same threshold-free doc comment, restated by boto3/re:Post/awsfundamentals.com sources, plus AWS's general (unrelated) guidance that STS credential/token sizes should never be assumed fixed. Implementing a threshold here would mean inventing an arbitrary number with no spec to verify it against -- the opposite of parity. Genuinely unimplementable without an undocumented number AWS does not publish. (bd: gopherstack-p05, follow-up -- OutboundWebIdentityFederationDisabledException, the other half of this original gap entry, WAS closed this pass, see GetWebIdentityToken above) - STALE ISSUE PREMISE (gopherstack-yewt re-triage): the follow-up issue's item (2), 'OutboundWebIdentityFederationDisabledException -- needs account-level settings model gopherstack lacks + no API to toggle,' is already fully resolved as of this same PARITY.md's GetWebIdentityToken row above (parity-3 phase 2) -- re-confirmed this pass by reading the actual code, not just this file: web_identity.go's checkOutboundWebIdentityFederationEnabled (called from GetWebIdentityToken, web_identity.go:365) gates on real state via services/iam/account.go's EnableOutboundWebIdentityFederation/DisableOutboundWebIdentityFederation/GetOutboundWebIdentityFederationInfo/OutboundWebIdentityFederationEnabled (all real methods, not stubs -- confirmed by reading their bodies), and both handler_test.go and web_identity_test.go carry OutboundWebIdentityFederationDisabledException regression coverage. No code change needed; the bd issue's premise predates the fix that already landed in this same file. diff --git a/services/sts/trust_policy.go b/services/sts/trust_policy.go index 4b16000f93..889a8ee7fe 100644 --- a/services/sts/trust_policy.go +++ b/services/sts/trust_policy.go @@ -7,7 +7,9 @@ import ( "slices" "strconv" "strings" + "time" + "github.com/blackbirdworks/gopherstack/pkgs/condeval" "github.com/blackbirdworks/gopherstack/pkgs/logger" ) @@ -33,17 +35,25 @@ const ( condKeyExternalID = "sts:externalid" condKeyPrincipalArn = "aws:principalarn" condKeyMFAPresent = "aws:multifactorauthpresent" + condKeyCurrentTime = "aws:currenttime" + condKeyEpochTime = "aws:epochtime" // Normalized (lowercased, IfExists-stripped, see normalizeConditionOp) forms // of the AWS condition operators this evaluator models beyond the String // family. ArnEquals and ArnLike are documented by AWS as behaving // identically (both wildcard-capable), so both map to the same case. - condOperatorBool = "bool" - condOperatorNull = "null" - condOperatorArnEquals = "arnequals" - condOperatorArnLike = "arnlike" - condOperatorArnNotEquals = "arnnotequals" - condOperatorArnNotLike = "arnnotlike" + condOperatorBool = "bool" + condOperatorNull = "null" + condOperatorArnEquals = "arnequals" + condOperatorArnLike = "arnlike" + condOperatorArnNotEquals = "arnnotequals" + condOperatorArnNotLike = "arnnotlike" + condOperatorDateEquals = "dateequals" + condOperatorDateNotEquals = "datenotequals" + condOperatorDateLessThan = "datelessthan" + condOperatorDateLessThanEq = "datelessthanequals" + condOperatorDateGreaterThan = "dategreaterthan" + condOperatorDateGreaterThanEq = "dategreaterthanequals" ) // trustEval carries the caller context evaluated against a role trust policy. @@ -82,17 +92,30 @@ func (e trustEval) principalLabel() string { // unmodelled keys are treated as satisfied so unfamiliar conditions never cause // a spurious denial. func (e trustEval) conditionValue(key string) (string, bool) { - switch strings.ToLower(key) { + lower := strings.ToLower(key) + + switch lower { case condKeyExternalID: return e.externalID, true case condKeyPrincipalArn: return e.callerArn, true } - if v, ok := e.conditionCtx[strings.ToLower(key)]; ok { + if v, ok := e.conditionCtx[lower]; ok { return v, true } + // aws:CurrentTime/aws:EpochTime need no request plumbing (unlike + // aws:SourceIp, which this evaluator has nowhere to source honestly -- + // see services/sts/PARITY.md's gopherstack-yg95 entry): the value is + // always "now" unless a test overrides it via conditionCtx above. + switch lower { + case condKeyCurrentTime: + return time.Now().UTC().Format(time.RFC3339), true + case condKeyEpochTime: + return strconv.FormatInt(time.Now().UTC().Unix(), 10), true + } + return "", false } @@ -462,8 +485,18 @@ func conditionsSatisfied(cond map[string]map[string]json.RawMessage, ev trustEva // paths log loudly at WARN so the gap is discoverable at runtime instead of // silent -- that is the fix for gopherstack-yg95, not a change of default. func conditionOperatorHolds(op, key string, raw json.RawMessage, ev trustEval) bool { - normOp := normalizeConditionOp(op) - isIfExists := isIfExistsConditionOp(op) + // AWS docs: IfExists may suffix any operator except Null (which already + // tests presence), so "NullIfExists" is left as an unrecognized operator. + //nolint:lll // AWS doc URL, cannot be split + // https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html#Conditions_IfExists + lowerOp := strings.ToLower(op) + normOp := lowerOp + isIfExists := false + + if trimmed, ok := strings.CutSuffix(lowerOp, "ifexists"); ok && trimmed != condOperatorNull { + normOp = trimmed + isIfExists = true + } actual, known := ev.conditionValue(key) @@ -488,37 +521,12 @@ func conditionOperatorHolds(op, key string, raw json.RawMessage, ev trustEval) b return true } - want := extractStringValues(raw) - - switch normOp { - case "stringequals": - return anyEquals(want, actual, false) - case "stringequalsignorecase": - return anyEquals(want, actual, true) - case "stringnotequals": - return !anyEquals(want, actual, false) - case "stringlike": - return anyWildcard(want, actual) - case "stringnotlike": - return !anyWildcard(want, actual) - case condOperatorBool: - return anyEquals(want, actual, true) - case condOperatorArnEquals, condOperatorArnLike: - // AWS documents ArnEquals/ArnLike as behaving identically, both - // wildcard-capable. Real AWS matches each of the six colon-delimited - // ARN segments separately and disallows wildcards spanning segments; - // this emulator uses the same general-purpose glob matcher as - // StringLike instead of segment-aware matching, a deliberate - // simplification since trust-policy ARN conditions in practice - // wildcard within a single segment (e.g. role/prod-*). - return anyWildcard(want, actual) - case condOperatorArnNotEquals, condOperatorArnNotLike: - return !anyWildcard(want, actual) - default: - // Numeric*/Date*/IpAddress/NotIpAddress/BinaryEquals are not modeled: - // this evaluator has no numeric, timestamp, source-IP, or binary - // request-context value to compare against for any condition key it - // carries (structural, not deferred -- see PARITY.md). + fn, ok := conditionOperatorFuncs[normOp] + if !ok { + // Numeric*/IpAddress/NotIpAddress/BinaryEquals remain unmodeled: this + // evaluator has no numeric, source-IP, or binary request-context + // value to compare against for any condition key it carries + // (structural, not deferred -- see PARITY.md). if ev.strictConditions { return false } @@ -526,11 +534,71 @@ func conditionOperatorHolds(op, key string, raw json.RawMessage, ev trustEval) b return true } + + return fn(extractStringValues(raw), actual) } -// isIfExistsConditionOp reports whether op ends with the IfExists suffix. -func isIfExistsConditionOp(op string) bool { - return strings.HasSuffix(strings.ToLower(op), "ifexists") +// dateCompare returns a conditionOperatorFuncs entry for one of the six Date +// operators, closing over which comparison dateOperatorHolds should run. +func dateCompare(op string) func(want []string, actual string) bool { + return func(want []string, actual string) bool { + return dateOperatorHolds(op, want, actual) + } +} + +// arnCompare returns a conditionOperatorFuncs entry for one of the four Arn +// operators, negating condeval.AnyArnMatch's result for the NotEquals/NotLike pair. +func arnCompare(negate bool) func(want []string, actual string) bool { + return func(want []string, actual string) bool { + return condeval.AnyArnMatch(want, actual, wildcardMatch) != negate + } +} + +// conditionOperatorFuncs maps a normalized (lower-case, IfExists-stripped) +// condition operator to its want/actual matcher. Null is handled separately +// by conditionOperatorHolds (it tests key presence, not value); an operator +// absent from this table is unrecognized and fails open there too. AWS +// documents ArnEquals/ArnLike (and their NotEquals/NotLike negations) as +// behaving identically -- each of the six colon-delimited ARN components is +// wildcard-matched separately, not one glob over the whole string. +// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html#Conditions_ARN +// +//nolint:gochecknoglobals // read-only dispatch table +var conditionOperatorFuncs = map[string]func(want []string, actual string) bool{ + "stringequals": func(want []string, actual string) bool { return anyEquals(want, actual, false) }, + "stringequalsignorecase": func(want []string, actual string) bool { return anyEquals(want, actual, true) }, + "stringnotequals": func(want []string, actual string) bool { return !anyEquals(want, actual, false) }, + "stringlike": anyWildcard, + "stringnotlike": func(want []string, actual string) bool { return !anyWildcard(want, actual) }, + condOperatorBool: func(want []string, actual string) bool { return anyEquals(want, actual, true) }, + condOperatorArnEquals: arnCompare(false), + condOperatorArnLike: arnCompare(false), + condOperatorArnNotEquals: arnCompare(true), + condOperatorArnNotLike: arnCompare(true), + condOperatorDateEquals: dateCompare(condOperatorDateEquals), + condOperatorDateNotEquals: dateCompare(condOperatorDateNotEquals), + condOperatorDateLessThan: dateCompare(condOperatorDateLessThan), + condOperatorDateLessThanEq: dateCompare(condOperatorDateLessThanEq), + condOperatorDateGreaterThan: dateCompare(condOperatorDateGreaterThan), + condOperatorDateGreaterThanEq: dateCompare(condOperatorDateGreaterThanEq), +} + +// dateOperatorHolds evaluates a Date condition operator: actual matches if it +// satisfies the comparison against any value in want (OR-within-key). +func dateOperatorHolds(normOp string, want []string, actual string) bool { + actualTime, ok := condeval.ParseDate(actual) + if !ok { + return false + } + + for _, v := range want { + condTime, okParse := condeval.ParseDate(v) + if okParse && condeval.CompareDate(normOp, actualTime, condTime) { + return true + } + } + + return false } // nullConditionHolds evaluates AWS's Null condition operator: "true" requires diff --git a/services/sts/trust_policy_test.go b/services/sts/trust_policy_test.go index 766589d763..b4aea2595d 100644 --- a/services/sts/trust_policy_test.go +++ b/services/sts/trust_policy_test.go @@ -473,6 +473,122 @@ func TestEvaluateAssumeRoleTrust_ArnOperators(t *testing.T) { } } +// TestEvaluateAssumeRoleTrust_DateOperators exercises the Date condition +// family against aws:CurrentTime, one of the two Date-typed keys this +// evaluator can source honestly without new request plumbing (the other is +// aws:EpochTime); see conditionValue and services/sts/PARITY.md's +// gopherstack-yg95 entry. ConditionCtx overrides the key to a fixed instant +// so the test is deterministic (no time.Sleep, no wall-clock dependency). +func TestEvaluateAssumeRoleTrust_DateOperators(t *testing.T) { + t.Parallel() + + const caller = "arn:aws:iam::123456789012:user/alice" + + policy := func(op, value string) string { + return `{"Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},` + + `"Action":"sts:AssumeRole","Condition":{"` + op + `":{"aws:CurrentTime":"` + value + `"}}}]}` + } + + tests := []struct { + name string + policy string + now string + wantErr bool + }{ + { + name: "date_less_than_true", policy: policy("DateLessThan", "2025-01-01T00:00:00Z"), + now: "2024-01-01T00:00:00Z", wantErr: false, + }, + { + name: "date_less_than_false", policy: policy("DateLessThan", "2025-01-01T00:00:00Z"), + now: "2026-01-01T00:00:00Z", wantErr: true, + }, + { + name: "date_greater_than_epoch_seconds", policy: policy("DateGreaterThan", "1704067199"), + now: "2025-01-01T00:00:00Z", wantErr: false, + }, + { + name: "date_greater_than_epoch_seconds_false", policy: policy("DateGreaterThan", "1704067199"), + now: "2023-01-01T00:00:00Z", wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ev := sts.TrustEvalForTest{ + Action: sts.ActionAssumeRole, CallerArn: caller, + ConditionCtx: map[string]string{"aws:currenttime": tt.now}, + } + + err := sts.EvaluateAssumeRoleTrust(tt.policy, ev) + if !tt.wantErr { + require.NoError(t, err) + + return + } + + require.Error(t, err) + assert.ErrorIs(t, err, sts.ErrAccessDenied) + }) + } +} + +// TestEvaluateAssumeRoleTrust_NullIfExistsUnrecognized proves "NullIfExists" +// is not silently accepted as a stripped-suffix alias for Null: AWS documents +// IfExists as valid on any operator except Null. +// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html#Conditions_IfExists +// +//nolint:lll // AWS doc URL, cannot be split +func TestEvaluateAssumeRoleTrust_NullIfExistsUnrecognized(t *testing.T) { + t.Parallel() + + const caller = "arn:aws:iam::123456789012:user/alice" + + policy := `{"Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},` + + `"Action":"sts:AssumeRole","Condition":{"NullIfExists":{"custom:ticket":"true"}}}]}` + + // Permissive mode (default): unrecognized operators fail open, same as + // TestEvaluateAssumeRoleTrust_UnmodeledOperatorPermitsByDesign. + err := sts.EvaluateAssumeRoleTrust(policy, sts.TrustEvalForTest{ + Action: sts.ActionAssumeRole, CallerArn: caller, + }) + require.NoError(t, err) + + // Strict mode: an unrecognized operator now denies -- proving + // "NullIfExists" took the unrecognized-operator path (which strict mode + // distinguishes) rather than nullConditionHolds's permissive Null path + // (which "custom:ticket":"true", key absent, would have satisfied). + err = sts.EvaluateAssumeRoleTrust(policy, sts.TrustEvalForTest{ + Action: sts.ActionAssumeRole, CallerArn: caller, StrictConditions: true, + }) + require.Error(t, err) + assert.ErrorIs(t, err, sts.ErrAccessDenied) +} + +// TestEvaluateAssumeRoleTrust_ArnSegmentWise proves ArnLike compares each of +// the six colon-delimited ARN components separately instead of one glob over +// the whole string, so a wildcard cannot span a segment boundary. +// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html#Conditions_ARN +func TestEvaluateAssumeRoleTrust_ArnSegmentWise(t *testing.T) { + t.Parallel() + + const caller = "arn:aws:sts::123456789012:assumed-role/AppRole/session" + + // A malformed pattern missing a colon must not match by letting '*' span + // the region+account segment boundary of aws:PrincipalArn. + policy := `{"Statement":[{"Effect":"Allow","Principal":{"AWS":"*"},` + + `"Action":"sts:AssumeRole","Condition":{"ArnLike":{"aws:PrincipalArn":` + + `"arn:aws:sts:*:assumed-role/AppRole/session"}}}]}` + + err := sts.EvaluateAssumeRoleTrust(policy, sts.TrustEvalForTest{ + Action: sts.ActionAssumeRole, CallerArn: caller, + }) + require.Error(t, err) + assert.ErrorIs(t, err, sts.ErrAccessDenied) +} + // TestEvaluateAssumeRoleTrust_UnmodeledOperatorPermitsByDesign documents the // deliberate fail-open decision (gopherstack-yg95) for condition operators // this evaluator has no request-context value to check against (Numeric*, diff --git a/services/swf/PARITY.md b/services/swf/PARITY.md index 1435d13509..6f462032bc 100644 --- a/services/swf/PARITY.md +++ b/services/swf/PARITY.md @@ -71,6 +71,16 @@ leaks: {status: clean, note: "no goroutines/timers spawned by this service, incl ## Notes +### 2026-09-26: closed-execution retention pruning (unbounded-growth audit) + +Closed workflow executions were only ever removed by the unrelated +maxWorkflowExecutions=10_000 FIFO cap, never by the domain's own +workflowExecutionRetentionPeriodInDays (AWS RegisterDomain doc). Added +sweepExpiredClosedExecutionsLocked (timeout_sweep.go), wired into the same +lazy per-op sweep as timeout enforcement -- no new goroutine. See +TestSweepExpiredClosedExecutionsLocked_Evaluation / +TestListClosedWorkflowExecutions_SweepsRetentionOnRead. + ### 2026-09-19 over-wide-response sweep cmd/overwidecandidates flagged all 5 List ops. All 5 already emitted exactly diff --git a/services/swf/timeout_sweep.go b/services/swf/timeout_sweep.go index b029be7bbc..171d43323d 100644 --- a/services/swf/timeout_sweep.go +++ b/services/swf/timeout_sweep.go @@ -6,6 +6,47 @@ import ( "time" ) +// closedExecutionRetentionCutoffLocked returns the epoch cutoff for closed executions +// and false when retention is NONE/unset. Caller holds the read lock. +// https://docs.aws.amazon.com/amazonswf/latest/apireference/API_RegisterDomain.html +func (b *InMemoryBackend) closedExecutionRetentionCutoffLocked(domain string, now time.Time) (float64, bool) { + d, ok := b.domains.Get(domain) + if !ok || d.WorkflowExecutionRetentionPeriodInDays == "" || + d.WorkflowExecutionRetentionPeriodInDays == retentionNone { + return 0, false + } + + days, err := strconv.Atoi(d.WorkflowExecutionRetentionPeriodInDays) + if err != nil || days < 0 { + return 0, false + } + + return float64(now.AddDate(0, 0, -days).Unix()), true +} + +// sweepExpiredClosedExecutionsLocked evicts closed executions past their domain's +// retention. Caller holds the write lock. +func (b *InMemoryBackend) sweepExpiredClosedExecutionsLocked(now time.Time) { + var toEvict []string + + for _, exec := range b.executions.All() { + if exec.Status == statusRunning || exec.CloseTimestamp == 0 { + continue + } + + cutoff, finite := b.closedExecutionRetentionCutoffLocked(exec.Domain, now) + if !finite || exec.CloseTimestamp >= cutoff { + continue + } + + toEvict = append(toEvict, executionKey(exec.Domain, exec.WorkflowID, exec.RunID)) + } + + for _, key := range toEvict { + b.evictExecutionLocked(key) + } +} + // executionDeadline returns exec's ExecutionStartToCloseTimeout deadline as // epoch seconds, and whether one is configured at all -- an empty or "NONE" // timeout (validateDuration's accepted sentinel for "no timeout") never @@ -34,8 +75,8 @@ func executionDeadline(exec *WorkflowExecution) (float64, bool) { // the next such call rather than at the real wall-clock instant it expired. // now is a parameter rather than an internal time.Now() call so the sweep's // evaluation instant is directly controllable in tests, without sleeping or -// a background goroutine. Caller must hold the write lock. Returns the -// number of executions closed (timer fires are not counted). +// a background goroutine, and evicts closed executions past retention. Caller must +// hold the write lock. Returns executions closed (evictions not counted). func (b *InMemoryBackend) sweepTimedOutExecutionsLocked(now time.Time) int { nowEpoch := float64(now.UnixMilli()) / milliDivisor @@ -56,6 +97,8 @@ func (b *InMemoryBackend) sweepTimedOutExecutionsLocked(now time.Time) int { swept++ } + b.sweepExpiredClosedExecutionsLocked(now) + return swept } diff --git a/services/swf/timeout_sweep_whitebox_test.go b/services/swf/timeout_sweep_whitebox_test.go index 5481b3f8e4..9db83f906b 100644 --- a/services/swf/timeout_sweep_whitebox_test.go +++ b/services/swf/timeout_sweep_whitebox_test.go @@ -183,6 +183,100 @@ func TestTimeoutExecutionLocked_CascadesChildPolicy(t *testing.T) { } } +// TestSweepExpiredClosedExecutionsLocked_Evaluation proves closed workflow +// executions are evicted once they cross their domain's +// workflowExecutionRetentionPeriodInDays (see timeout_sweep.go's citation) -- +// previously only the unrelated maxWorkflowExecutions FIFO cap ever removed a +// closed execution, regardless of the domain's configured retention. +func TestSweepExpiredClosedExecutionsLocked_Evaluation(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + retention string + closedAge time.Duration + leaveOpen bool + wantEvicted bool + }{ + { + name: "past a 1-day retention is evicted", retention: "1", + closedAge: 25 * time.Hour, wantEvicted: true, + }, + { + name: "within a 1-day retention is kept", retention: "1", + closedAge: 23 * time.Hour, wantEvicted: false, + }, + { + name: "NONE retention is never evicted", retention: "NONE", + closedAge: 365 * 24 * time.Hour, wantEvicted: false, + }, + { + name: "0-day retention evicts immediately", retention: "0", + closedAge: time.Second, wantEvicted: true, + }, + { + name: "still-open execution is never evicted", retention: "0", + leaveOpen: true, closedAge: 365 * 24 * time.Hour, wantEvicted: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := NewInMemoryBackend() + require.NoError(t, b.RegisterDomain("dom", "", tt.retention)) + + started, err := b.StartWorkflowExecution(StartWorkflowExecutionInput{ + Domain: "dom", WorkflowID: "wf-1", TaskList: "tasks", + }) + require.NoError(t, err) + + evalAt := time.Now() + + if !tt.leaveOpen { + require.NoError(t, b.TerminateWorkflowExecution("dom", "wf-1", started.RunID, "", "", "")) + live := mustLiveExecution(t, b, "wf-1", started.RunID) + live.CloseTimestamp = float64(evalAt.Add(-tt.closedAge).Unix()) + } + + b.mu.Lock("test") + b.sweepExpiredClosedExecutionsLocked(evalAt) + b.mu.Unlock() + + _, err = b.DescribeWorkflowExecution("dom", "wf-1", started.RunID) + if tt.wantEvicted { + require.ErrorIs(t, err, ErrNotFound) + } else { + require.NoError(t, err) + } + }) + } +} + +// TestListClosedWorkflowExecutions_SweepsRetentionOnRead verifies the +// retention sweep is wired into a public entry point, not just callable in +// isolation: a closed execution backdated past its domain's retention +// disappears from ListClosedWorkflowExecutions on the next call. +func TestListClosedWorkflowExecutions_SweepsRetentionOnRead(t *testing.T) { + t.Parallel() + + b := NewInMemoryBackend() + require.NoError(t, b.RegisterDomain("dom", "", "1")) + + started, err := b.StartWorkflowExecution(StartWorkflowExecutionInput{ + Domain: "dom", WorkflowID: "wf-1", TaskList: "tasks", + }) + require.NoError(t, err) + require.NoError(t, b.TerminateWorkflowExecution("dom", "wf-1", started.RunID, "", "", "")) + + live := mustLiveExecution(t, b, "wf-1", started.RunID) + live.CloseTimestamp -= float64((25 * time.Hour) / time.Second) + + out := b.ListClosedWorkflowExecutions("dom", ExecutionFilter{}) + assert.Empty(t, out) +} + // TestDescribeWorkflowExecution_SweepsOnRead verifies the sweep is actually // wired into a public read entry point, not just callable in isolation: // backdating StartTimestamp into the real past (no sleep, no fabricated diff --git a/services/textract/handler_adapter_versions_test.go b/services/textract/handler_adapter_versions_test.go index 651ba506ea..ac9c2983a1 100644 --- a/services/textract/handler_adapter_versions_test.go +++ b/services/textract/handler_adapter_versions_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -273,62 +274,65 @@ func TestHandler_CreateAdapterVersion_DatasetConfig(t *testing.T) { func TestHandler_AdapterVersion_InProgressThenActive(t *testing.T) { t.Parallel() - // Use a backend with a short async delay. - b := textract.NewInMemoryBackendSync("123456789012", "us-east-1") - textract.SetBackendAsyncDelay(b, 50*time.Millisecond) - h := textract.NewHandler(b) + synctest.Test(t, func(t *testing.T) { + // Use a backend with a short async delay. + b := textract.NewInMemoryBackendSync("123456789012", "us-east-1") + textract.SetBackendAsyncDelay(b, 50*time.Millisecond) + h := textract.NewHandler(b) - createAdapterRec := doTextractRequest(t, h, "CreateAdapter", map[string]any{ - "AdapterName": "lifecycle-adapter", - "FeatureTypes": []string{"FORMS"}, - }) - require.Equal(t, http.StatusOK, createAdapterRec.Code) + createAdapterRec := doTextractRequest(t, h, "CreateAdapter", map[string]any{ + "AdapterName": "lifecycle-adapter", + "FeatureTypes": []string{"FORMS"}, + }) + require.Equal(t, http.StatusOK, createAdapterRec.Code) - var createAdapterResp map[string]string - require.NoError(t, json.Unmarshal(createAdapterRec.Body.Bytes(), &createAdapterResp)) - adapterID := createAdapterResp["AdapterId"] + var createAdapterResp map[string]string + require.NoError(t, json.Unmarshal(createAdapterRec.Body.Bytes(), &createAdapterResp)) + adapterID := createAdapterResp["AdapterId"] - createVersionRec := doTextractRequest(t, h, "CreateAdapterVersion", map[string]any{ - "AdapterId": adapterID, - "DatasetConfig": map[string]any{ - "ManifestS3Object": map[string]any{ - "Bucket": "test-dataset-bucket", - "Name": "manifest.json", + createVersionRec := doTextractRequest(t, h, "CreateAdapterVersion", map[string]any{ + "AdapterId": adapterID, + "DatasetConfig": map[string]any{ + "ManifestS3Object": map[string]any{ + "Bucket": "test-dataset-bucket", + "Name": "manifest.json", + }, }, - }, - "OutputConfig": map[string]any{ - "S3Bucket": "test-output-bucket", - }, - }) - require.Equal(t, http.StatusOK, createVersionRec.Code) + "OutputConfig": map[string]any{ + "S3Bucket": "test-output-bucket", + }, + }) + require.Equal(t, http.StatusOK, createVersionRec.Code) - var createVersionResp map[string]string - require.NoError(t, json.Unmarshal(createVersionRec.Body.Bytes(), &createVersionResp)) - adapterVersion := createVersionResp["AdapterVersion"] + var createVersionResp map[string]string + require.NoError(t, json.Unmarshal(createVersionRec.Body.Bytes(), &createVersionResp)) + adapterVersion := createVersionResp["AdapterVersion"] - // Immediately check: should be CREATION_IN_PROGRESS. - getRec1 := doTextractRequest(t, h, "GetAdapterVersion", map[string]any{ - "AdapterId": adapterID, - "AdapterVersion": adapterVersion, - }) - require.Equal(t, http.StatusOK, getRec1.Code) + // Immediately check: should be CREATION_IN_PROGRESS. + getRec1 := doTextractRequest(t, h, "GetAdapterVersion", map[string]any{ + "AdapterId": adapterID, + "AdapterVersion": adapterVersion, + }) + require.Equal(t, http.StatusOK, getRec1.Code) - var getResp1 map[string]any - require.NoError(t, json.Unmarshal(getRec1.Body.Bytes(), &getResp1)) - assert.Equal(t, "CREATION_IN_PROGRESS", getResp1["Status"]) + var getResp1 map[string]any + require.NoError(t, json.Unmarshal(getRec1.Body.Bytes(), &getResp1)) + assert.Equal(t, "CREATION_IN_PROGRESS", getResp1["Status"]) - // After delay, should be ACTIVE. - time.Sleep(200 * time.Millisecond) + // After delay, should be ACTIVE. + time.Sleep(200 * time.Millisecond) + synctest.Wait() - getRec2 := doTextractRequest(t, h, "GetAdapterVersion", map[string]any{ - "AdapterId": adapterID, - "AdapterVersion": adapterVersion, - }) - require.Equal(t, http.StatusOK, getRec2.Code) + getRec2 := doTextractRequest(t, h, "GetAdapterVersion", map[string]any{ + "AdapterId": adapterID, + "AdapterVersion": adapterVersion, + }) + require.Equal(t, http.StatusOK, getRec2.Code) - var getResp2 map[string]any - require.NoError(t, json.Unmarshal(getRec2.Body.Bytes(), &getResp2)) - assert.Equal(t, "ACTIVE", getResp2["Status"]) + var getResp2 map[string]any + require.NoError(t, json.Unmarshal(getRec2.Body.Bytes(), &getResp2)) + assert.Equal(t, "ACTIVE", getResp2["Status"]) + }) } // TestHandler_AdapterVersion_EvaluationMetrics verifies GetAdapterVersion diff --git a/services/textract/handler_document_analysis_test.go b/services/textract/handler_document_analysis_test.go index fbc52d6bde..5904aa9747 100644 --- a/services/textract/handler_document_analysis_test.go +++ b/services/textract/handler_document_analysis_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "net/http" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -662,40 +663,43 @@ func TestHandler_GetDocumentAnalysis_JobStatusSucceeded(t *testing.T) { func TestHandler_StartDocumentAnalysis_AsyncInProgressThenSucceeded(t *testing.T) { t.Parallel() - // Use a backend with a short async delay (not zero, so we can observe IN_PROGRESS). - b := textract.NewInMemoryBackendSync("123456789012", "us-east-1") - textract.SetBackendAsyncDelay(b, 50*time.Millisecond) - h := textract.NewHandler(b) + synctest.Test(t, func(t *testing.T) { + // Use a backend with a short async delay (not zero, so we can observe IN_PROGRESS). + b := textract.NewInMemoryBackendSync("123456789012", "us-east-1") + textract.SetBackendAsyncDelay(b, 50*time.Millisecond) + h := textract.NewHandler(b) - startRec := doTextractRequest(t, h, "StartDocumentAnalysis", map[string]any{ - "DocumentLocation": map[string]any{ - "S3Object": map[string]any{"Bucket": "b", "Name": "doc.pdf"}, - }, - "FeatureTypes": []string{"FORMS"}, - }) - require.Equal(t, http.StatusOK, startRec.Code) + startRec := doTextractRequest(t, h, "StartDocumentAnalysis", map[string]any{ + "DocumentLocation": map[string]any{ + "S3Object": map[string]any{"Bucket": "b", "Name": "doc.pdf"}, + }, + "FeatureTypes": []string{"FORMS"}, + }) + require.Equal(t, http.StatusOK, startRec.Code) - var startResp map[string]string - require.NoError(t, json.Unmarshal(startRec.Body.Bytes(), &startResp)) - jobID := startResp["JobId"] + var startResp map[string]string + require.NoError(t, json.Unmarshal(startRec.Body.Bytes(), &startResp)) + jobID := startResp["JobId"] - // Immediately after start, job should be IN_PROGRESS. - getRec1 := doTextractRequest(t, h, "GetDocumentAnalysis", map[string]any{"JobId": jobID}) - require.Equal(t, http.StatusOK, getRec1.Code) + // Immediately after start, job should be IN_PROGRESS. + getRec1 := doTextractRequest(t, h, "GetDocumentAnalysis", map[string]any{"JobId": jobID}) + require.Equal(t, http.StatusOK, getRec1.Code) - var getResp1 map[string]any - require.NoError(t, json.Unmarshal(getRec1.Body.Bytes(), &getResp1)) - assert.Equal(t, "IN_PROGRESS", getResp1["JobStatus"]) + var getResp1 map[string]any + require.NoError(t, json.Unmarshal(getRec1.Body.Bytes(), &getResp1)) + assert.Equal(t, "IN_PROGRESS", getResp1["JobStatus"]) - // After delay, job should be SUCCEEDED. - time.Sleep(200 * time.Millisecond) + // After delay, job should be SUCCEEDED. + time.Sleep(200 * time.Millisecond) + synctest.Wait() - getRec2 := doTextractRequest(t, h, "GetDocumentAnalysis", map[string]any{"JobId": jobID}) - require.Equal(t, http.StatusOK, getRec2.Code) + getRec2 := doTextractRequest(t, h, "GetDocumentAnalysis", map[string]any{"JobId": jobID}) + require.Equal(t, http.StatusOK, getRec2.Code) - var getResp2 map[string]any - require.NoError(t, json.Unmarshal(getRec2.Body.Bytes(), &getResp2)) - assert.Equal(t, "SUCCEEDED", getResp2["JobStatus"]) + var getResp2 map[string]any + require.NoError(t, json.Unmarshal(getRec2.Body.Bytes(), &getResp2)) + assert.Equal(t, "SUCCEEDED", getResp2["JobStatus"]) + }) } // TestHandler_StartDocumentAnalysis_AsyncInitialStatusInProgress verifies that diff --git a/services/translate/concurrency_race_test.go b/services/translate/concurrency_race_test.go new file mode 100644 index 0000000000..a61e6c48c1 --- /dev/null +++ b/services/translate/concurrency_race_test.go @@ -0,0 +1,121 @@ +package translate_test + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/translate" +) + +const raceIterations = 200 + +// TestListFieldsRaceWithInPlaceAdvance proves that fields read off a List +// result can't race a concurrent op advancing the same value in place. +func TestListFieldsRaceWithInPlaceAdvance(t *testing.T) { + t.Parallel() + + tests := []struct { + setup func(t *testing.T, b *translate.InMemoryBackend) string + reader func(b *translate.InMemoryBackend) + mutator func(b *translate.InMemoryBackend, id string) + name string + }{ + { + name: "jobs", + setup: func(t *testing.T, b *translate.InMemoryBackend) string { + t.Helper() + + return startJob(t, b, "race-job").JobID + }, + reader: func(b *translate.InMemoryBackend) { + list, _ := b.ListTextTranslationJobs(translate.TextTranslationJobFilter{}, 10, "") + for _, j := range list { + _ = j.JobStatus + _ = j.EndAt + _ = j.Message + } + }, + mutator: func(b *translate.InMemoryBackend, id string) { + _, _ = b.DescribeTextTranslationJob(id) + _, _ = b.StopTextTranslationJob(id) + }, + }, + { + name: "parallel_data", + setup: func(t *testing.T, b *translate.InMemoryBackend) string { + t.Helper() + + _, err := b.CreateParallelData("race-pd", "d", nil, nil, nil) + require.NoError(t, err) + + return "race-pd" + }, + reader: func(b *translate.InMemoryBackend) { + list, _ := b.ListParallelData(10, "") + for _, pd := range list { + _ = pd.Status + _ = pd.LastUpdatedAt + } + }, + mutator: func(b *translate.InMemoryBackend, name string) { + _, _ = b.GetParallelData(name) + _, _ = b.UpdateParallelData(name, "d2", nil) + }, + }, + { + name: "terminology", + setup: func(t *testing.T, b *translate.InMemoryBackend) string { + t.Helper() + + data := &translate.TerminologyData{File: []byte("en,es\nhello,hola\n"), Format: "CSV"} + _, err := b.ImportTerminology("race-term", "d", data, nil, nil) + require.NoError(t, err) + + return "race-term" + }, + reader: func(b *translate.InMemoryBackend) { + list, _ := b.ListTerminologies(10, "") + for _, term := range list { + _ = term.Description + _ = term.LastUpdatedAt + } + }, + mutator: func(b *translate.InMemoryBackend, name string) { + data := &translate.TerminologyData{File: []byte("en,es\nhello,hola\n"), Format: "CSV"} + _, _ = b.ImportTerminology(name, "d2", data, nil, nil) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := newTestBackend(t) + id := tt.setup(t, b) + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + for range raceIterations { + tt.reader(b) + } + }() + + go func() { + defer wg.Done() + + for range raceIterations { + tt.mutator(b, id) + } + }() + + wg.Wait() + }) + } +} diff --git a/services/translate/parallel_data.go b/services/translate/parallel_data.go index b54518ccf2..3e52e077e6 100644 --- a/services/translate/parallel_data.go +++ b/services/translate/parallel_data.go @@ -11,6 +11,14 @@ func (b *InMemoryBackend) parallelDataARN(name string) string { return arn.Build("translate", b.region, b.accountID, "parallel-data/"+name) } +// cloneParallelData copies pd so callers reading its fields after the lock +// releases don't race a concurrent in-place mutator like advanceParallelData. +func cloneParallelData(pd *ParallelData) *ParallelData { + cp := *pd + + return &cp +} + // validateParallelDataConfig rejects a ParallelDataConfig.Format outside the // modeled CSV|TMX|TSV enum. Format is not itself a required member of the // ParallelDataConfig shape (api-2.json), so an absent Format is left to @@ -87,7 +95,7 @@ func (b *InMemoryBackend) CreateParallelData( b.tags[resourceARN] = copyMap(tags) } - return pd, nil + return cloneParallelData(pd), nil } // advanceParallelData moves pd one step through its async lifecycle, called @@ -122,7 +130,7 @@ func (b *InMemoryBackend) GetParallelData(name string) (*ParallelData, error) { advanceParallelData(pd) - return pd, nil + return cloneParallelData(pd), nil } // UpdateParallelData updates an existing parallel data resource. Real AWS @@ -170,7 +178,7 @@ func (b *InMemoryBackend) UpdateParallelData( pd.Status = parallelDataStatusUpdating pd.LatestUpdateAttemptStatus = parallelDataStatusUpdating - return pd, nil + return cloneParallelData(pd), nil } // DeleteParallelData removes a parallel data resource by name. @@ -187,7 +195,7 @@ func (b *InMemoryBackend) DeleteParallelData(name string) (*ParallelData, error) b.parallelData.Delete(name) delete(b.tags, resourceARN) - return pd, nil + return cloneParallelData(pd), nil } // ListParallelData returns a paginated list of parallel data resources. @@ -197,5 +205,12 @@ func (b *InMemoryBackend) ListParallelData(maxResults int, nextToken string) ([] names := sortedNames(b.parallelData.All(), func(pd *ParallelData) string { return pd.Name }) - return paginate(names, func(n string) *ParallelData { return tableGet(b.parallelData, n) }, maxResults, nextToken) + return paginate(names, func(n string) *ParallelData { + pd := tableGet(b.parallelData, n) + if pd == nil { + return nil + } + + return cloneParallelData(pd) + }, maxResults, nextToken) } diff --git a/services/translate/terminologies.go b/services/translate/terminologies.go index 25ee7de024..81d3becdc1 100644 --- a/services/translate/terminologies.go +++ b/services/translate/terminologies.go @@ -12,6 +12,14 @@ func (b *InMemoryBackend) terminologyARN(name string) string { return arn.Build("translate", b.region, b.accountID, "terminology/"+name) } +// cloneTerminology copies t so callers reading its fields after the lock +// releases don't race a concurrent re-import mutating the same value. +func cloneTerminology(t *Terminology) *Terminology { + cp := *t + + return &cp +} + // parseCSVLanguages extracts source/target language codes and term count from CSV bytes. // CSV header row is: sourceLang,targetLang1[,targetLang2,...]; subsequent rows are terms. func parseCSVLanguages(csvBytes []byte) (string, []string, int) { @@ -124,7 +132,7 @@ func (b *InMemoryBackend) ImportTerminology( b.tags[resourceARN] = copyMap(tags) } - return existing, nil + return cloneTerminology(existing), nil } term := &Terminology{ @@ -149,7 +157,7 @@ func (b *InMemoryBackend) ImportTerminology( b.tags[resourceARN] = copyMap(tags) } - return term, nil + return cloneTerminology(term), nil } // GetTerminology retrieves a terminology by name. @@ -162,7 +170,7 @@ func (b *InMemoryBackend) GetTerminology(name string) (*Terminology, error) { return nil, fmt.Errorf("%w: terminology %q not found", ErrNotFound, name) } - return t, nil + return cloneTerminology(t), nil } // LookupTerminologies returns terminology entries for the given names. A @@ -184,7 +192,7 @@ func (b *InMemoryBackend) LookupTerminologies(names []string) ([]*Terminology, e return nil, fmt.Errorf("%w: terminology %q not found", ErrNotFound, name) } - out = append(out, t) + out = append(out, cloneTerminology(t)) } return out, nil @@ -213,5 +221,12 @@ func (b *InMemoryBackend) ListTerminologies(maxResults int, nextToken string) ([ names := sortedNames(b.terminologies.All(), func(t *Terminology) string { return t.Name }) - return paginate(names, func(n string) *Terminology { return tableGet(b.terminologies, n) }, maxResults, nextToken) + return paginate(names, func(n string) *Terminology { + t := tableGet(b.terminologies, n) + if t == nil { + return nil + } + + return cloneTerminology(t) + }, maxResults, nextToken) } diff --git a/services/translate/text_translation_jobs.go b/services/translate/text_translation_jobs.go index 223d03b9c7..6d02cc1b0d 100644 --- a/services/translate/text_translation_jobs.go +++ b/services/translate/text_translation_jobs.go @@ -9,6 +9,14 @@ import ( "github.com/google/uuid" ) +// cloneJob copies job so callers reading its fields after the lock releases +// don't race a concurrent in-place mutator like advanceJob. +func cloneJob(job *TranslationJob) *TranslationJob { + cp := *job + + return &cp +} + // StartTextTranslationJob creates a new async translation job. func (b *InMemoryBackend) StartTextTranslationJob( jobName, dataAccessRoleARN, sourceLang string, @@ -56,7 +64,7 @@ func (b *InMemoryBackend) StartTextTranslationJob( } b.jobs.Put(job) - return job, nil + return cloneJob(job), nil } // StopTextTranslationJob requests stop of a translation job. @@ -81,7 +89,7 @@ func (b *InMemoryBackend) StopTextTranslationJob(jobID string) (*TranslationJob, job.EndAt = time.Now().UTC() } - return job, nil + return cloneJob(job), nil } // DescribeTextTranslationJob retrieves a translation job and advances it one @@ -103,7 +111,7 @@ func (b *InMemoryBackend) DescribeTextTranslationJob(jobID string) (*Translation advanceJob(job) - return job, nil + return cloneJob(job), nil } // advanceJob moves job one step through its lifecycle. Called from @@ -206,5 +214,12 @@ func (b *InMemoryBackend) ListTextTranslationJobs( ids[i] = job.JobID } - return paginate(ids, func(id string) *TranslationJob { return tableGet(b.jobs, id) }, maxResults, nextToken) + return paginate(ids, func(id string) *TranslationJob { + j := tableGet(b.jobs, id) + if j == nil { + return nil + } + + return cloneJob(j) + }, maxResults, nextToken) } diff --git a/services/waf/ip_sets.go b/services/waf/ip_sets.go index 5e2c8e2844..71ea0e8986 100644 --- a/services/waf/ip_sets.go +++ b/services/waf/ip_sets.go @@ -35,7 +35,7 @@ func (b *InMemoryBackend) CreateIPSet(name, changeToken string, tags map[string] b.tags[b.ipSetARN(id)] = maps.Clone(tags) } - return ipSet, nil + return cloneIPSet(ipSet), nil } // GetIPSet retrieves an IPSet by ID. @@ -48,7 +48,17 @@ func (b *InMemoryBackend) GetIPSet(id string) (*IPSet, error) { return nil, ErrNotFound } - return ipSet, nil + return cloneIPSet(ipSet), nil +} + +// cloneIPSet stops a caller from racing UpdateIPSet. A shallow copy is not +// enough: applyEntryUpdate's delete path reuses the descriptors backing array. +func cloneIPSet(ipSet *IPSet) *IPSet { + cp := *ipSet + cp.IPSetDescriptors = make([]IPSetDescriptor, len(ipSet.IPSetDescriptors)) + copy(cp.IPSetDescriptors, ipSet.IPSetDescriptors) + + return &cp } // UpdateIPSet updates an IPSet's descriptors. diff --git a/services/waf/match_sets.go b/services/waf/match_sets.go index b4bec14a84..3e126bd944 100644 --- a/services/waf/match_sets.go +++ b/services/waf/match_sets.go @@ -41,7 +41,17 @@ func (b *InMemoryBackend) CreateByteMatchSet(name, changeToken string) (*ByteMat } b.byteMatchSets.Put(bms) - return bms, nil + return cloneByteMatchSet(bms), nil +} + +// cloneByteMatchSet stops a caller from racing UpdateByteMatchSet. A shallow +// copy is not enough: applyEntryUpdate reuses the tuples backing array. +func cloneByteMatchSet(bms *ByteMatchSet) *ByteMatchSet { + cp := *bms + cp.ByteMatchTuples = make([]ByteMatchTuple, len(bms.ByteMatchTuples)) + copy(cp.ByteMatchTuples, bms.ByteMatchTuples) + + return &cp } // GetByteMatchSet retrieves a ByteMatchSet by ID. @@ -54,7 +64,7 @@ func (b *InMemoryBackend) GetByteMatchSet(id string) (*ByteMatchSet, error) { return nil, ErrNotFound } - return bms, nil + return cloneByteMatchSet(bms), nil } // UpdateByteMatchSet updates a ByteMatchSet's tuples. @@ -158,7 +168,17 @@ func (b *InMemoryBackend) CreateSizeConstraintSet(name, changeToken string) (*Si } b.sizeConstraintSets.Put(scs) - return scs, nil + return cloneSizeConstraintSet(scs), nil +} + +// cloneSizeConstraintSet stops a caller from racing UpdateSizeConstraintSet. +// A shallow copy is not enough: applyEntryUpdate reuses the constraints backing array. +func cloneSizeConstraintSet(scs *SizeConstraintSet) *SizeConstraintSet { + cp := *scs + cp.SizeConstraints = make([]SizeConstraint, len(scs.SizeConstraints)) + copy(cp.SizeConstraints, scs.SizeConstraints) + + return &cp } // GetSizeConstraintSet retrieves a SizeConstraintSet by ID. @@ -171,7 +191,7 @@ func (b *InMemoryBackend) GetSizeConstraintSet(id string) (*SizeConstraintSet, e return nil, ErrNotFound } - return scs, nil + return cloneSizeConstraintSet(scs), nil } // UpdateSizeConstraintSet updates a SizeConstraintSet's constraints. @@ -284,7 +304,17 @@ func (b *InMemoryBackend) CreateSqlInjectionMatchSet( } b.sqlInjectionMatchSets.Put(sims) - return sims, nil + return cloneSQLInjectionMatchSet(sims), nil +} + +// cloneSQLInjectionMatchSet stops a caller from racing +// UpdateSqlInjectionMatchSet, whose delete path reuses the tuples backing array. +func cloneSQLInjectionMatchSet(sims *SqlInjectionMatchSet) *SqlInjectionMatchSet { + cp := *sims + cp.SqlInjectionMatchTuples = make([]SqlInjectionMatchTuple, len(sims.SqlInjectionMatchTuples)) + copy(cp.SqlInjectionMatchTuples, sims.SqlInjectionMatchTuples) + + return &cp } // GetSqlInjectionMatchSet retrieves a SqlInjectionMatchSet by ID. @@ -299,7 +329,7 @@ func (b *InMemoryBackend) GetSqlInjectionMatchSet(id string) (*SqlInjectionMatch return nil, ErrNotFound } - return sims, nil + return cloneSQLInjectionMatchSet(sims), nil } // UpdateSqlInjectionMatchSet updates a SqlInjectionMatchSet's tuples. @@ -417,7 +447,17 @@ func (b *InMemoryBackend) CreateXssMatchSet(name, changeToken string) (*XssMatch } b.xssMatchSets.Put(xms) - return xms, nil + return cloneXSSMatchSet(xms), nil +} + +// cloneXSSMatchSet stops a caller from racing UpdateXssMatchSet. A shallow +// copy is not enough: applyEntryUpdate reuses the tuples backing array. +func cloneXSSMatchSet(xms *XssMatchSet) *XssMatchSet { + cp := *xms + cp.XssMatchTuples = make([]XssMatchTuple, len(xms.XssMatchTuples)) + copy(cp.XssMatchTuples, xms.XssMatchTuples) + + return &cp } // GetXssMatchSet retrieves an XssMatchSet by ID. @@ -432,7 +472,7 @@ func (b *InMemoryBackend) GetXssMatchSet(id string) (*XssMatchSet, error) { return nil, ErrNotFound } - return xms, nil + return cloneXSSMatchSet(xms), nil } // UpdateXssMatchSet updates an XssMatchSet's tuples. @@ -542,7 +582,17 @@ func (b *InMemoryBackend) CreateGeoMatchSet(name, changeToken string) (*GeoMatch } b.geoMatchSets.Put(gms) - return gms, nil + return cloneGeoMatchSet(gms), nil +} + +// cloneGeoMatchSet stops a caller from racing UpdateGeoMatchSet. A shallow +// copy is not enough: applyEntryUpdate reuses the constraints backing array. +func cloneGeoMatchSet(gms *GeoMatchSet) *GeoMatchSet { + cp := *gms + cp.GeoMatchConstraints = make([]GeoMatchConstraint, len(gms.GeoMatchConstraints)) + copy(cp.GeoMatchConstraints, gms.GeoMatchConstraints) + + return &cp } // GetGeoMatchSet retrieves a GeoMatchSet by ID. @@ -555,7 +605,7 @@ func (b *InMemoryBackend) GetGeoMatchSet(id string) (*GeoMatchSet, error) { return nil, ErrNotFound } - return gms, nil + return cloneGeoMatchSet(gms), nil } // UpdateGeoMatchSet updates a GeoMatchSet's constraints. @@ -774,7 +824,17 @@ func (b *InMemoryBackend) CreateRegexMatchSet(name, changeToken string) (*RegexM } b.regexMatchSets.Put(rms) - return rms, nil + return cloneRegexMatchSet(rms), nil +} + +// cloneRegexMatchSet stops a caller from racing UpdateRegexMatchSet. A +// shallow copy is not enough: applyEntryUpdate reuses the tuples backing array. +func cloneRegexMatchSet(rms *RegexMatchSet) *RegexMatchSet { + cp := *rms + cp.RegexMatchTuples = make([]RegexMatchTuple, len(rms.RegexMatchTuples)) + copy(cp.RegexMatchTuples, rms.RegexMatchTuples) + + return &cp } // GetRegexMatchSet retrieves a RegexMatchSet by ID. @@ -787,7 +847,7 @@ func (b *InMemoryBackend) GetRegexMatchSet(id string) (*RegexMatchSet, error) { return nil, ErrNotFound } - return rms, nil + return cloneRegexMatchSet(rms), nil } // UpdateRegexMatchSet updates a RegexMatchSet's tuples. diff --git a/services/waf/rate_based_rules.go b/services/waf/rate_based_rules.go index 5914558d29..9ff9f7a7dc 100644 --- a/services/waf/rate_based_rules.go +++ b/services/waf/rate_based_rules.go @@ -43,7 +43,17 @@ func (b *InMemoryBackend) CreateRateBasedRule( b.tags[b.rateBasedRuleARN(id)] = maps.Clone(tags) } - return rule, nil + return cloneRateBasedRule(rule), nil +} + +// cloneRateBasedRule stops a caller from racing UpdateRateBasedRule. A +// shallow copy is not enough: applyEntryUpdate reuses the predicates backing array. +func cloneRateBasedRule(rule *RateBasedRule) *RateBasedRule { + cp := *rule + cp.MatchPredicates = make([]Predicate, len(rule.MatchPredicates)) + copy(cp.MatchPredicates, rule.MatchPredicates) + + return &cp } // GetRateBasedRule retrieves a RateBasedRule by ID. @@ -56,7 +66,7 @@ func (b *InMemoryBackend) GetRateBasedRule(id string) (*RateBasedRule, error) { return nil, ErrNotFound } - return rule, nil + return cloneRateBasedRule(rule), nil } // UpdateRateBasedRule updates a RateBasedRule's predicates and rate limit. diff --git a/services/waf/rules.go b/services/waf/rules.go index a1c3f265ad..54ea57d10d 100644 --- a/services/waf/rules.go +++ b/services/waf/rules.go @@ -39,7 +39,17 @@ func (b *InMemoryBackend) CreateRule( b.tags[b.ruleARN(id)] = maps.Clone(tags) } - return rule, nil + return cloneRule(rule), nil +} + +// cloneRule stops a caller from racing UpdateRule. A shallow copy is not +// enough: applyEntryUpdate's delete path reuses the predicates backing array. +func cloneRule(rule *Rule) *Rule { + cp := *rule + cp.Predicates = make([]Predicate, len(rule.Predicates)) + copy(cp.Predicates, rule.Predicates) + + return &cp } // GetRule retrieves a Rule by ID. @@ -52,7 +62,7 @@ func (b *InMemoryBackend) GetRule(id string) (*Rule, error) { return nil, ErrNotFound } - return rule, nil + return cloneRule(rule), nil } // UpdateRule updates a Rule's predicates. diff --git a/services/waf/web_acls.go b/services/waf/web_acls.go index feb2653862..9c28acf23b 100644 --- a/services/waf/web_acls.go +++ b/services/waf/web_acls.go @@ -43,7 +43,17 @@ func (b *InMemoryBackend) CreateWebACL( b.tags[acl.WebACLArn] = maps.Clone(tags) } - return acl, nil + return cloneWebACL(acl), nil +} + +// cloneWebACL stops a caller from racing UpdateWebACL. A shallow copy is not +// enough: its delete path reuses Rules's backing array via "acl.Rules[:0]". +func cloneWebACL(acl *WebACL) *WebACL { + cp := *acl + cp.Rules = make([]ActivatedRule, len(acl.Rules)) + copy(cp.Rules, acl.Rules) + + return &cp } // GetWebACL retrieves a WebACL by ID. @@ -56,7 +66,7 @@ func (b *InMemoryBackend) GetWebACL(id string) (*WebACL, error) { return nil, ErrNotFound } - return acl, nil + return cloneWebACL(acl), nil } // UpdateWebACL updates a WebACL's default action and rules. diff --git a/services/waf/web_acls_race_test.go b/services/waf/web_acls_race_test.go new file mode 100644 index 0000000000..1e884a841b --- /dev/null +++ b/services/waf/web_acls_race_test.go @@ -0,0 +1,115 @@ +package waf_test + +import ( + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/blackbirdworks/gopherstack/services/waf" +) + +// TestWAFClassicResourceConcurrentWithUpdate proves Get/Create must not hand +// back the live pointer whose entry slice the shared Update path mutates. +func TestWAFClassicResourceConcurrentWithUpdate(t *testing.T) { + t.Parallel() + + tests := []struct { + setup func(t *testing.T, b *waf.InMemoryBackend, token string) string + reader func(b *waf.InMemoryBackend, id string) + mutator func(b *waf.InMemoryBackend, id, token string, i int) + name string + }{ + { + name: "WebACL races UpdateWebACL", + setup: func(t *testing.T, b *waf.InMemoryBackend, token string) string { + t.Helper() + + acl, err := b.CreateWebACL("race-acl", "raceMetric", waf.WafAction{Type: "ALLOW"}, token, nil) + require.NoError(t, err) + + return acl.WebACLId + }, + reader: func(b *waf.InMemoryBackend, id string) { + got, err := b.GetWebACL(id) + if err != nil { + return + } + + for _, r := range got.Rules { + _ = r.RuleId + } + }, + mutator: func(b *waf.InMemoryBackend, id, token string, i int) { + update := waf.WebACLUpdate{ + Action: "INSERT", + ActivatedRule: waf.ActivatedRule{RuleId: "rule-race", Priority: int32(i)}, + } + _ = b.UpdateWebACL(id, token, nil, []waf.WebACLUpdate{update}) + + update.Action = "DELETE" + _ = b.UpdateWebACL(id, token, nil, []waf.WebACLUpdate{update}) + }, + }, + { + name: "IPSet races UpdateIPSet (shared applyEntryUpdate helper)", + setup: func(t *testing.T, b *waf.InMemoryBackend, token string) string { + t.Helper() + + ipSet, err := b.CreateIPSet("race-ipset", token, nil) + require.NoError(t, err) + + return ipSet.IPSetId + }, + reader: func(b *waf.InMemoryBackend, id string) { + got, err := b.GetIPSet(id) + if err != nil { + return + } + + for _, d := range got.IPSetDescriptors { + _ = d.Value + } + }, + mutator: func(b *waf.InMemoryBackend, id, token string, _ int) { + descriptor := waf.IPSetDescriptor{Type: "IPV4", Value: "10.0.0.0/8"} + + _ = b.UpdateIPSet(id, token, []waf.IPSetUpdate{{Action: "INSERT", IPSetDescriptor: descriptor}}) + _ = b.UpdateIPSet(id, token, []waf.IPSetUpdate{{Action: "DELETE", IPSetDescriptor: descriptor}}) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + b := waf.NewInMemoryBackend("000000000000", "us-east-1") + token := b.GetChangeToken() + id := tt.setup(t, b, token) + + const iterations = 300 + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + for range iterations { + tt.reader(b, id) + } + }() + + go func() { + defer wg.Done() + + for i := range iterations { + tt.mutator(b, id, token, i) + } + }() + + wg.Wait() + }) + } +} diff --git a/services/xray/handler_sampling_rules_test.go b/services/xray/handler_sampling_rules_test.go index 7d5db2037e..6eed4ea626 100644 --- a/services/xray/handler_sampling_rules_test.go +++ b/services/xray/handler_sampling_rules_test.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -293,42 +294,44 @@ func TestSamplingRuleAttributes(t *testing.T) { func TestSamplingRuleModifiedAtInRecord(t *testing.T) { t.Parallel() - b := xray.NewInMemoryBackend("000000000000", "us-east-1") - _, err := b.CreateSamplingRule(xray.SamplingRule{RuleName: "time-rule", FixedRate: 0.1, Priority: 1}) - require.NoError(t, err) + synctest.Test(t, func(t *testing.T) { + b := xray.NewInMemoryBackend("000000000000", "us-east-1") + _, err := b.CreateSamplingRule(xray.SamplingRule{RuleName: "time-rule", FixedRate: 0.1, Priority: 1}) + require.NoError(t, err) - // Small sleep so Modified and Created timestamps will differ after update. - time.Sleep(time.Millisecond * 2) + // Small sleep so Modified and Created timestamps will differ after update. + time.Sleep(time.Millisecond * 2) - _, err = b.UpdateSamplingRule("time-rule", xray.SamplingRule{ServiceName: "updated"}) - require.NoError(t, err) + _, err = b.UpdateSamplingRule("time-rule", xray.SamplingRule{ServiceName: "updated"}) + require.NoError(t, err) - h := xray.NewHandler(b) - rec := doXrayRequest(t, h, "/GetSamplingRules", nil) - require.Equal(t, http.StatusOK, rec.Code) + h := xray.NewHandler(b) + rec := doXrayRequest(t, h, "/GetSamplingRules", nil) + require.Equal(t, http.StatusOK, rec.Code) - var resp map[string]any - require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + var resp map[string]any + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) - records, ok := resp["SamplingRuleRecords"].([]any) - require.True(t, ok) - // 2 rules: time-rule + Default. - require.Len(t, records, 2) + records, ok := resp["SamplingRuleRecords"].([]any) + require.True(t, ok) + // 2 rules: time-rule + Default. + require.Len(t, records, 2) - // The first record (sorted by priority=1) should be time-rule. - record, ok := records[0].(map[string]any) - require.True(t, ok) + // The first record (sorted by priority=1) should be time-rule. + record, ok := records[0].(map[string]any) + require.True(t, ok) - samplingRule, ok := record["SamplingRule"].(map[string]any) - require.True(t, ok) - require.Equal(t, "time-rule", samplingRule["RuleName"]) + samplingRule, ok := record["SamplingRule"].(map[string]any) + require.True(t, ok) + require.Equal(t, "time-rule", samplingRule["RuleName"]) - createdAt, ok1 := record["CreatedAt"].(float64) - modifiedAt, ok2 := record["ModifiedAt"].(float64) + createdAt, ok1 := record["CreatedAt"].(float64) + modifiedAt, ok2 := record["ModifiedAt"].(float64) - require.True(t, ok1) - require.True(t, ok2) - assert.GreaterOrEqual(t, modifiedAt, createdAt) + require.True(t, ok1) + require.True(t, ok2) + assert.GreaterOrEqual(t, modifiedAt, createdAt) + }) } // TestDefaultSamplingRuleUndeletable verifies the Default rule cannot be deleted via handler. diff --git a/services/xray/sampling_rules_test.go b/services/xray/sampling_rules_test.go index 37ccfae9df..574ed6b2bc 100644 --- a/services/xray/sampling_rules_test.go +++ b/services/xray/sampling_rules_test.go @@ -2,6 +2,7 @@ package xray_test import ( "testing" + "testing/synctest" "time" "github.com/stretchr/testify/assert" @@ -247,22 +248,24 @@ func TestDeleteSamplingRule_ClearsResourceTagsOnRecreate(t *testing.T) { func TestModifiedAtTracking(t *testing.T) { t.Parallel() - b := xray.NewInMemoryBackend("000000000000", "us-east-1") + synctest.Test(t, func(t *testing.T) { + b := xray.NewInMemoryBackend("000000000000", "us-east-1") - r, err := b.CreateSamplingRule(xray.SamplingRule{RuleName: "track-rule", FixedRate: 0.1, Priority: 1}) - require.NoError(t, err) + r, err := b.CreateSamplingRule(xray.SamplingRule{RuleName: "track-rule", FixedRate: 0.1, Priority: 1}) + require.NoError(t, err) - createdAt := r.CreatedAt - modifiedAt := r.ModifiedAt + createdAt := r.CreatedAt + modifiedAt := r.ModifiedAt - // Small sleep to ensure timestamps differ. - time.Sleep(time.Millisecond) + // Small sleep to ensure timestamps differ. + time.Sleep(time.Millisecond) - updated, err := b.UpdateSamplingRule("track-rule", xray.SamplingRule{ServiceName: "svc"}) - require.NoError(t, err) + updated, err := b.UpdateSamplingRule("track-rule", xray.SamplingRule{ServiceName: "svc"}) + require.NoError(t, err) - assert.Equal(t, createdAt, updated.CreatedAt) - assert.True(t, updated.ModifiedAt.After(modifiedAt)) + assert.Equal(t, createdAt, updated.CreatedAt) + assert.True(t, updated.ModifiedAt.After(modifiedAt)) + }) } // TestAddSamplingRuleInternal verifies the seed helper. diff --git a/shutdown_leak_test.go b/shutdown_leak_test.go new file mode 100644 index 0000000000..a9e93ffae2 --- /dev/null +++ b/shutdown_leak_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "context" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/goleak" + + "github.com/blackbirdworks/gopherstack/pkgs/testleak" +) + +// TestServerShutdown_NoGoroutineLeaks boots the full service composition, +// runs every background worker, shuts down, then asserts no goroutine +// started during the run outlived it. +// +// Can't run under synctest: run() opens a real net.Listener and services +// like the IoT MQTT broker do real blocking network I/O. +// +//nolint:paralleltest // uses t.Setenv via parseCLI, which is incompatible with t.Parallel. +func TestServerShutdown_NoGoroutineLeaks(t *testing.T) { + baseline := goleak.IgnoreCurrent() + + port := freeTCPPort(t) + cli := parseCLI(t, map[string]string{ + "PORT": strconv.Itoa(port), + }) + + ctx, cancel := context.WithCancel(t.Context()) + + errCh := make(chan error, 1) + go func() { + errCh <- run(ctx, cli) + }() + + waitForServerReady(t, port) + + cancel() + + select { + case err := <-errCh: + require.NoError(t, err, "server should shut down cleanly") + case <-time.After(shutdownWaitTimeout): + require.FailNow(t, "server did not shut down within timeout") + } + + goleak.VerifyNone(t, append(testleak.DefaultIgnores(), baseline)...) +} diff --git a/test/integration/cloudwatchlogs_test.go b/test/integration/cloudwatchlogs_test.go index 45e54c082a..a955255ca4 100644 --- a/test/integration/cloudwatchlogs_test.go +++ b/test/integration/cloudwatchlogs_test.go @@ -263,6 +263,7 @@ func TestIntegration_CloudWatchLogs_SubscriptionFilter_KinesisDelivery(t *testin _, _ = kinesisClient.DeleteStream(cleanupCtx, &kinesissdk.DeleteStreamInput{StreamName: aws.String(streamName)}) }) + waitKinesisStreamActive(ctx, t, kinesisClient, streamName) // Get Kinesis stream ARN. descKinesis, err := kinesisClient.DescribeStream(ctx, &kinesissdk.DescribeStreamInput{ diff --git a/test/integration/fis_test.go b/test/integration/fis_test.go index 3c814b5fa0..428c31e4e4 100644 --- a/test/integration/fis_test.go +++ b/test/integration/fis_test.go @@ -505,6 +505,7 @@ func TestIntegration_FIS_KinesisThroughputException(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + waitKinesisStreamActive(ctx, t, kinesisClient, streamName) // Obtain the stream ARN so we can reference it in the FIS target. descOut, err := kinesisClient.DescribeStream(ctx, &kinesissdk.DescribeStreamInput{ diff --git a/test/integration/kinesis_lambda_test.go b/test/integration/kinesis_lambda_test.go index c0248e61f0..83064928b0 100644 --- a/test/integration/kinesis_lambda_test.go +++ b/test/integration/kinesis_lambda_test.go @@ -30,6 +30,7 @@ func TestIntegration_Kinesis_EventSourceMapping(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + waitKinesisStreamActive(ctx, t, kinesisClient, streamName) // Get stream ARN descOut, err := kinesisClient.DescribeStreamSummary(ctx, &kinesis.DescribeStreamSummaryInput{ @@ -112,6 +113,7 @@ func TestIntegration_Kinesis_EventSourceMapping_WithRecords(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + waitKinesisStreamActive(ctx, t, kinesisClient, streamName) // Get stream ARN descOut, err := kinesisClient.DescribeStreamSummary(ctx, &kinesis.DescribeStreamSummaryInput{ diff --git a/test/integration/kinesis_test.go b/test/integration/kinesis_test.go index a85c0d5aa7..0ea6c52d2e 100644 --- a/test/integration/kinesis_test.go +++ b/test/integration/kinesis_test.go @@ -31,6 +31,7 @@ func TestIntegration_Kinesis_StreamLifecycle(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + waitKinesisStreamActive(ctx, t, client, streamName) // ListStreams listOut, err := client.ListStreams(ctx, &kinesis.ListStreamsInput{}) @@ -59,6 +60,7 @@ func TestIntegration_Kinesis_StreamLifecycle(t *testing.T) { StreamName: aws.String(streamName), }) require.NoError(t, err) + waitKinesisStreamGone(ctx, t, client, streamName) // Verify gone listOut2, err := client.ListStreams(ctx, &kinesis.ListStreamsInput{}) @@ -81,6 +83,7 @@ func TestIntegration_Kinesis_PutAndGetRecords(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + waitKinesisStreamActive(ctx, t, client, streamName) // Get shard ID from DescribeStream descOut, err := client.DescribeStream(ctx, &kinesis.DescribeStreamInput{ @@ -201,6 +204,7 @@ func TestIntegration_Kinesis_ListShards(t *testing.T) { ShardCount: aws.Int32(3), }) require.NoError(t, err) + waitKinesisStreamActive(ctx, t, client, streamName) listShardsOut, err := client.ListShards(ctx, &kinesis.ListShardsInput{ StreamName: aws.String(streamName), @@ -232,6 +236,7 @@ func TestIntegration_Kinesis_DataIntegrity(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + waitKinesisStreamActive(ctx, t, client, streamName) descOut, err := client.DescribeStream(ctx, &kinesis.DescribeStreamInput{ StreamName: aws.String(streamName), @@ -293,6 +298,7 @@ func TestIntegration_Kinesis_EnhancedFanOut(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + waitKinesisStreamActive(ctx, t, client, streamName) // Get stream ARN descOut, err := client.DescribeStream(ctx, &kinesis.DescribeStreamInput{ @@ -407,6 +413,7 @@ func TestIntegration_Kinesis_UpdateShardCount(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + waitKinesisStreamActive(ctx, t, client, streamName) reshardOut, err := client.UpdateShardCount(ctx, &kinesis.UpdateShardCountInput{ StreamName: aws.String(streamName), @@ -424,6 +431,9 @@ func TestIntegration_Kinesis_UpdateShardCount(t *testing.T) { require.NoError(t, err) assert.Len(t, listShardsOut.Shards, 2) + // UpdateShardCount leaves the stream UPDATING; DeleteStream requires ACTIVE. + waitKinesisStreamActive(ctx, t, client, streamName) + _, err = client.DeleteStream(ctx, &kinesis.DeleteStreamInput{ StreamName: aws.String(streamName), }) @@ -444,6 +454,7 @@ func TestIntegration_Kinesis_EnhancedMonitoring(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + waitKinesisStreamActive(ctx, t, client, streamName) // Enable monitoring enableOut, err := client.EnableEnhancedMonitoring(ctx, &kinesis.EnableEnhancedMonitoringInput{ @@ -487,6 +498,7 @@ func TestIntegration_Kinesis_GetShardIteratorAtTimestamp(t *testing.T) { ShardCount: aws.Int32(1), }) require.NoError(t, err) + waitKinesisStreamActive(ctx, t, client, streamName) descOut, err := client.DescribeStream(ctx, &kinesis.DescribeStreamInput{ StreamName: aws.String(streamName), @@ -549,6 +561,7 @@ func TestIntegration_Kinesis_SplitShard_RoundTrip(t *testing.T) { _, _ = client.DeleteStream(cleanupCtx, &kinesis.DeleteStreamInput{StreamName: aws.String(streamName)}) }) + waitKinesisStreamActive(ctx, t, client, streamName) descOut, err := client.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: aws.String(streamName)}) require.NoError(t, err) @@ -620,6 +633,7 @@ func TestIntegration_Kinesis_MergeShards_RoundTrip(t *testing.T) { _, _ = client.DeleteStream(cleanupCtx, &kinesis.DeleteStreamInput{StreamName: aws.String(streamName)}) }) + waitKinesisStreamActive(ctx, t, client, streamName) descOut, err := client.DescribeStream(ctx, &kinesis.DescribeStreamInput{StreamName: aws.String(streamName)}) require.NoError(t, err) diff --git a/test/integration/kinesis_wait_helpers_test.go b/test/integration/kinesis_wait_helpers_test.go new file mode 100644 index 0000000000..62462e653f --- /dev/null +++ b/test/integration/kinesis_wait_helpers_test.go @@ -0,0 +1,38 @@ +package integration_test + +import ( + "context" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/kinesis" + "github.com/stretchr/testify/require" +) + +const kinesisWaiterMaxWait = 30 * time.Second + +// waitKinesisStreamActive waits for a just-created stream to leave CREATING, +// the way a real SDK caller does before using a new stream. +func waitKinesisStreamActive(ctx context.Context, t *testing.T, client *kinesis.Client, streamName string) { + t.Helper() + + waiter := kinesis.NewStreamExistsWaiter(client, func(o *kinesis.StreamExistsWaiterOptions) { + o.MinDelay = 50 * time.Millisecond + o.MaxDelay = 500 * time.Millisecond + }) + err := waiter.Wait(ctx, &kinesis.DescribeStreamInput{StreamName: aws.String(streamName)}, kinesisWaiterMaxWait) + require.NoError(t, err, "stream %q did not become ACTIVE", streamName) +} + +// waitKinesisStreamGone waits for a deleted stream to actually disappear. +func waitKinesisStreamGone(ctx context.Context, t *testing.T, client *kinesis.Client, streamName string) { + t.Helper() + + waiter := kinesis.NewStreamNotExistsWaiter(client, func(o *kinesis.StreamNotExistsWaiterOptions) { + o.MinDelay = 50 * time.Millisecond + o.MaxDelay = 500 * time.Millisecond + }) + err := waiter.Wait(ctx, &kinesis.DescribeStreamInput{StreamName: aws.String(streamName)}, kinesisWaiterMaxWait) + require.NoError(t, err, "stream %q was not removed", streamName) +} diff --git a/test/terraform/apigatewayv2_apprunner_and_macie_test.go b/test/terraform/apigatewayv2_apprunner_and_macie_test.go index fc32a54fbc..12956848ab 100644 --- a/test/terraform/apigatewayv2_apprunner_and_macie_test.go +++ b/test/terraform/apigatewayv2_apprunner_and_macie_test.go @@ -53,6 +53,7 @@ func TestTerraform_Apigatewayv2ApprunnerAndMacie(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { t.Parallel() + lockMacie2(t) runTFTest(t, tc) }) } diff --git a/test/terraform/aurora_dsql_test.go b/test/terraform/aurora_dsql_test.go new file mode 100644 index 0000000000..9f7020e287 --- /dev/null +++ b/test/terraform/aurora_dsql_test.go @@ -0,0 +1,79 @@ +package terraform_test + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + dsqlsdk "github.com/aws/aws-sdk-go-v2/service/dsql" + "github.com/aws/aws-sdk-go-v2/service/dsql/types" + "github.com/aws/smithy-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// createDSQLClient returns an Aurora DSQL client pointed at the shared test container. +func createDSQLClient(t *testing.T) *dsqlsdk.Client { + t.Helper() + + return createClientWithEndpoint(t, dsqlsdk.NewFromConfig, endpoint) +} + +// TestTerraform_AuroraDsql provisions an aws_dsql_cluster via Terraform, then +// verifies it is visible via the Aurora DSQL SDK with the expected +// deletion-protection setting, tags, and a wire-shaped VPC endpoint service +// name. The pinned aws provider (~> 5.0, resolving to hashicorp/aws +// v5.100.0) does not yet expose an aws_dsql_cluster_policy resource, so +// cluster-policy coverage lives in services/dsql's own unit tests instead. +func TestTerraform_AuroraDsql(t *testing.T) { + t.Parallel() + + tests := []tfTestCase{ + { + name: "success", + fixture: "aurora-dsql", + setup: func(t *testing.T, _ string) map[string]any { + t.Helper() + + return map[string]any{} + }, + verify: func(t *testing.T, ctx context.Context, _ map[string]any) { + t.Helper() + + client := createDSQLClient(t) + + listOut, err := client.ListClusters(ctx, &dsqlsdk.ListClustersInput{}) + require.NoError(t, err, "ListClusters should succeed after terraform apply") + require.Len(t, listOut.Clusters, 1, "exactly one cluster should exist after terraform apply") + + identifier := listOut.Clusters[0].Identifier + + out, err := client.GetCluster(ctx, &dsqlsdk.GetClusterInput{Identifier: identifier}) + require.NoError(t, err, "GetCluster should succeed after terraform apply") + assert.False(t, aws.ToBool(out.DeletionProtectionEnabled)) + assert.Equal(t, map[string]string{"Environment": "test", "Owner": "terraform"}, out.Tags) + + vpcOut, err := client.GetVpcEndpointServiceName(ctx, &dsqlsdk.GetVpcEndpointServiceNameInput{ + Identifier: identifier, + }) + require.NoError(t, err, "GetVpcEndpointServiceName should succeed after terraform apply") + assert.NotEmpty(t, aws.ToString(vpcOut.ServiceName)) + + var apiErr smithy.APIError + + _, err = client.GetClusterPolicy(ctx, &dsqlsdk.GetClusterPolicyInput{Identifier: identifier}) + require.Error(t, err, "no cluster policy was set by the terraform fixture") + require.ErrorAs(t, err, &apiErr) + assert.Equal(t, "ResourceNotFoundException", apiErr.ErrorCode()) + assert.Equal(t, types.ClusterStatusActive, out.Status) + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + runTFTest(t, tc) + }) + } +} diff --git a/test/terraform/ecr_public_repositories_test.go b/test/terraform/ecr_public_repositories_test.go new file mode 100644 index 0000000000..deedd5a54e --- /dev/null +++ b/test/terraform/ecr_public_repositories_test.go @@ -0,0 +1,88 @@ +package terraform_test + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go-v2/aws" + ecrpublicsdk "github.com/aws/aws-sdk-go-v2/service/ecrpublic" + "github.com/google/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// createEcrPublicClient returns an ECR Public client pointed at the shared test container. +func createEcrPublicClient(t *testing.T) *ecrpublicsdk.Client { + t.Helper() + + return createClientWithEndpoint(t, ecrpublicsdk.NewFromConfig, endpoint) +} + +// TestTerraform_EcrPublicRepositories provisions an aws_ecrpublic_repository +// and an aws_ecrpublic_repository_policy via Terraform, then verifies both +// are visible via the Amazon ECR Public SDK. +func TestTerraform_EcrPublicRepositories(t *testing.T) { + t.Parallel() + + tests := []tfTestCase{ + { + name: "success", + fixture: "ecr-public-repositories", + setup: func(t *testing.T, _ string) map[string]any { + t.Helper() + + return map[string]any{ + "RepositoryName": "tf-ecrpublic-" + uuid.NewString()[:8], + } + }, + verify: func(t *testing.T, ctx context.Context, vars map[string]any) { + t.Helper() + + client := createEcrPublicClient(t) + repoName := vars["RepositoryName"].(string) + + out, err := client.DescribeRepositories(ctx, &ecrpublicsdk.DescribeRepositoriesInput{ + RepositoryNames: []string{repoName}, + }) + require.NoError(t, err, "DescribeRepositories should succeed after terraform apply") + require.Len(t, out.Repositories, 1) + + repo := out.Repositories[0] + assert.Equal(t, repoName, aws.ToString(repo.RepositoryName)) + assert.Regexp(t, `^arn:aws:ecr-public::\d+:repository/`+repoName+`$`, aws.ToString(repo.RepositoryArn)) + assert.Regexp(t, `^public\.ecr\.aws/`, aws.ToString(repo.RepositoryUri)) + + catalog, err := client.GetRepositoryCatalogData(ctx, &ecrpublicsdk.GetRepositoryCatalogDataInput{ + RepositoryName: aws.String(repoName), + }) + require.NoError(t, err, "GetRepositoryCatalogData should succeed after terraform apply") + assert.Equal(t, "About "+repoName, aws.ToString(catalog.CatalogData.AboutText)) + assert.Equal(t, []string{"ARM"}, catalog.CatalogData.Architectures) + assert.Equal(t, []string{"Linux"}, catalog.CatalogData.OperatingSystems) + + tagsOut, err := client.ListTagsForResource(ctx, &ecrpublicsdk.ListTagsForResourceInput{ + ResourceArn: repo.RepositoryArn, + }) + require.NoError(t, err, "ListTagsForResource should succeed after terraform apply") + gotTags := make(map[string]string, len(tagsOut.Tags)) + for _, tag := range tagsOut.Tags { + gotTags[aws.ToString(tag.Key)] = aws.ToString(tag.Value) + } + assert.Equal(t, map[string]string{"Environment": "test", "Owner": "terraform"}, gotTags) + + policyOut, err := client.GetRepositoryPolicy(ctx, &ecrpublicsdk.GetRepositoryPolicyInput{ + RepositoryName: aws.String(repoName), + }) + require.NoError(t, err, "GetRepositoryPolicy should succeed after terraform apply") + assert.Contains(t, aws.ToString(policyOut.PolicyText), "AllowPull") + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + runTFTest(t, tc) + }) + } +} diff --git a/test/terraform/fixtures/aurora-dsql.tf b/test/terraform/fixtures/aurora-dsql.tf new file mode 100644 index 0000000000..13058ffdcb --- /dev/null +++ b/test/terraform/fixtures/aurora-dsql.tf @@ -0,0 +1,8 @@ +resource "aws_dsql_cluster" "this" { + deletion_protection_enabled = false + + tags = { + Environment = "test" + Owner = "terraform" + } +} diff --git a/test/terraform/fixtures/ecr-public-repositories.tf b/test/terraform/fixtures/ecr-public-repositories.tf new file mode 100644 index 0000000000..a4609075f5 --- /dev/null +++ b/test/terraform/fixtures/ecr-public-repositories.tf @@ -0,0 +1,34 @@ +resource "aws_ecrpublic_repository" "this" { + repository_name = "{{.RepositoryName}}" + + catalog_data { + about_text = "About {{.RepositoryName}}" + architectures = ["ARM"] + description = "Test repository for {{.RepositoryName}}" + operating_systems = ["Linux"] + usage_text = "Usage instructions for {{.RepositoryName}}" + } + + tags = { + Environment = "test" + Owner = "terraform" + } +} + +resource "aws_ecrpublic_repository_policy" "this" { + repository_name = aws_ecrpublic_repository.this.repository_name + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "AllowPull" + Effect = "Allow" + Principal = "*" + Action = [ + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + ] + } + ] + }) +} diff --git a/test/terraform/fixtures/kinesis-video-streams.tf b/test/terraform/fixtures/kinesis-video-streams.tf new file mode 100644 index 0000000000..375bb2199d --- /dev/null +++ b/test/terraform/fixtures/kinesis-video-streams.tf @@ -0,0 +1,11 @@ +resource "aws_kinesis_video_stream" "this" { + name = "{{.StreamName}}" + data_retention_in_hours = 48 + device_name = "tf-kvs-device" + media_type = "video/h264" + + tags = { + Environment = "test" + Owner = "terraform" + } +} diff --git a/test/terraform/fixtures/msk-connect.tf b/test/terraform/fixtures/msk-connect.tf new file mode 100644 index 0000000000..f65d669c33 --- /dev/null +++ b/test/terraform/fixtures/msk-connect.tf @@ -0,0 +1,118 @@ +resource "aws_vpc" "mskc" { + cidr_block = "10.201.0.0/16" +} + +resource "aws_subnet" "mskc_a" { + vpc_id = aws_vpc.mskc.id + cidr_block = "10.201.1.0/24" +} + +resource "aws_subnet" "mskc_b" { + vpc_id = aws_vpc.mskc.id + cidr_block = "10.201.2.0/24" +} + +resource "aws_security_group" "mskc" { + name = "mskc-sg" + vpc_id = aws_vpc.mskc.id +} + +resource "aws_iam_role" "mskc_connect" { + name = "mskc-connect-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "kafkaconnect.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) +} + +resource "aws_s3_bucket" "mskc" { + bucket = "{{.BucketName}}" + force_destroy = true +} + +resource "aws_s3_object" "mskc_plugin" { + bucket = aws_s3_bucket.mskc.id + key = "plugins/mskc-plugin.zip" + content = "mskc fake plugin bytes" +} + +resource "aws_mskconnect_custom_plugin" "mskc" { + name = "{{.PluginName}}" + content_type = "ZIP" + + location { + s3 { + bucket_arn = aws_s3_bucket.mskc.arn + file_key = aws_s3_object.mskc_plugin.key + } + } +} + +resource "aws_mskconnect_worker_configuration" "mskc" { + name = "{{.WorkerConfigName}}" + + properties_file_content = <