diff --git a/contrib/realengine/blobstore/blobstore.go b/contrib/realengine/blobstore/blobstore.go index 2c4a063e9..bfda4897f 100644 --- a/contrib/realengine/blobstore/blobstore.go +++ b/contrib/realengine/blobstore/blobstore.go @@ -1,9 +1,9 @@ // Package blobstore provides an opt-in real object-storage engine that persists // object bytes to a real local filesystem (no Docker), backing CloudEmu's -// object stores (AWS S3, Azure Blob, GCP GCS). Bytes are written to real files -// under a root directory, so they survive in the store for the process's -// lifetime and can be inspected with ordinary tools. Wire it in with -// config.WithStorageEngine(blobstore.New("")). +// object stores (AWS S3, Azure Blob, GCP GCS, OCI Object Storage). Bytes are +// written to real files under a root directory, so they survive in the store +// for the process's lifetime and can be inspected with ordinary tools. Wire it +// in with config.WithStorageEngine(blobstore.New("")). // // It lives in a separate module on purpose: the storage-backing dependency // stays out of CloudEmu's core. The in-memory provider keeps each object's diff --git a/contrib/realengine/blobstore/oci_test.go b/contrib/realengine/blobstore/oci_test.go new file mode 100644 index 000000000..efbfb707c --- /dev/null +++ b/contrib/realengine/blobstore/oci_test.go @@ -0,0 +1,148 @@ +package blobstore_test + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + cloudemu "github.com/stackshy/cloudemu/v2" + "github.com/stackshy/cloudemu/v2/config" + "github.com/stackshy/cloudemu/v2/contrib/realengine/blobstore" + ociserver "github.com/stackshy/cloudemu/v2/server/oci" +) + +const ociCompartment = "ocid1.compartment.oc1..aaaaaaaablobstore" + +// ociCall issues one Object Storage request against the emulator and fails the +// test on any non-2xx. +func ociCall(t *testing.T, ts *httptest.Server, method, path string, body []byte) []byte { + t.Helper() + + var reader io.Reader + if body != nil { + reader = bytes.NewReader(body) + } + + req, err := http.NewRequestWithContext(t.Context(), method, ts.URL+path, reader) + if err != nil { + t.Fatalf("build %s %s: %v", method, path, err) + } + + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, path, err) + } + defer func() { _ = resp.Body.Close() }() + + out, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read %s %s: %v", method, path, err) + } + + if resp.StatusCode/100 != 2 { + t.Fatalf("%s %s: status %d: %s", method, path, resp.StatusCode, out) + } + + return out +} + +// TestOCIObjectStorageBlobstoreE2E runs the real-user flow against OCI Object +// Storage backed by a real filesystem engine (no Docker, no cloud account): +// read the namespace, create a bucket, put an object, get it, head it, copy it +// with the rename action, delete the original and confirm it is gone — then +// read the surviving bytes straight off disk under the engine root, proving +// they flowed through the engine rather than living only in memory. +// +// The requests are hand-built rather than driven by github.com/oracle/oci-go-sdk +// because that client mandates a signed request with an RSA keypair and a +// ConfigurationProvider, which the emulator does not verify; the wire shape is +// what this test is about. +func TestOCIObjectStorageBlobstoreE2E(t *testing.T) { + eng := blobstore.New("") + t.Cleanup(func() { _ = eng.Close() }) + + cloud := cloudemu.NewOCI( + config.WithStorageEngine(eng), + config.WithCompartmentID(ociCompartment), + ) + ts := httptest.NewServer(ociserver.New(ociserver.Drivers{ + ObjectStorage: cloud.ObjectStorage, + CompartmentID: cloud.CompartmentID, + TenancyOCID: cloud.TenancyOCID, + Region: cloud.Region, + })) + t.Cleanup(ts.Close) + + var namespace string + if err := json.Unmarshal(ociCall(t, ts, http.MethodGet, "/n", nil), &namespace); err != nil { + t.Fatalf("decode namespace: %v", err) + } + + const ( + bucket = "blob-bucket" + object = "docs/greeting.txt" + moved = "docs/greeting-moved.txt" + ) + + body := []byte("hello from the real blobstore engine") + root := "/n/" + namespace + "/b" + + spec, err := json.Marshal(map[string]string{"name": bucket, "compartmentId": ociCompartment}) + if err != nil { + t.Fatalf("marshal bucket spec: %v", err) + } + + ociCall(t, ts, http.MethodPost, root, spec) + ociCall(t, ts, http.MethodPut, root+"/"+bucket+"/o/"+object, body) + + if got := ociCall(t, ts, http.MethodGet, root+"/"+bucket+"/o/"+object, nil); !bytes.Equal(got, body) { + t.Fatalf("object round-trip mismatch: got %q want %q", got, body) + } + + var listed struct { + Objects []struct { + Name string `json:"name"` + Size int64 `json:"size"` + } `json:"objects"` + } + + if err := json.Unmarshal(ociCall(t, ts, http.MethodGet, root+"/"+bucket+"/o", nil), &listed); err != nil { + t.Fatalf("decode list: %v", err) + } + + if len(listed.Objects) != 1 || listed.Objects[0].Size != int64(len(body)) { + t.Fatalf("list must report the real size after the offload: %+v", listed.Objects) + } + + rename, err := json.Marshal(map[string]string{"sourceName": object, "newName": moved}) + if err != nil { + t.Fatalf("marshal rename: %v", err) + } + + ociCall(t, ts, http.MethodPost, root+"/"+bucket+"/actions/renameObject", rename) + + if got := ociCall(t, ts, http.MethodGet, root+"/"+bucket+"/o/"+moved, nil); !bytes.Equal(got, body) { + t.Fatalf("renamed object mismatch: got %q want %q", got, body) + } + + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, ts.URL+root+"/"+bucket+"/o/"+object, nil) + if err != nil { + t.Fatalf("build get: %v", err) + } + + resp, err := ts.Client().Do(req) + if err != nil { + t.Fatalf("get deleted source: %v", err) + } + + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("expected 404 for the renamed-away source, got %d", resp.StatusCode) + } + + assertEngineFileMatches(t, eng, bucket, moved, body) +} diff --git a/docs/coverage/README.md b/docs/coverage/README.md index 3a049805b..2f50a0571 100644 --- a/docs/coverage/README.md +++ b/docs/coverage/README.md @@ -169,7 +169,7 @@ code does not implement. Machine-readable: [`coverage.json`](./coverage.json). | `sql` | - | [SQL](./azure/sql.md) | - | - | 21 | | `sqlvirtualmachine` | - | [SQLVirtualMachine](./azure/sqlvirtualmachine.md) | - | - | 9 | | `sshpublickeys` | - | [Sshpublickeys](./azure/sshpublickeys.md) | - | - | 7 | -| `storage` | [S3](./aws/s3.md) | [BlobStorage](./azure/blobstorage.md) | [GCS](./gcp/gcs.md) | - | 35 | +| `storage` | [S3](./aws/s3.md) | [BlobStorage](./azure/blobstorage.md) | [GCS](./gcp/gcs.md) | [ObjectStorage](./oci/objectstorage.md) | 35 | | `storageaccount` | - | [Storageaccount](./azure/storageaccount.md) | - | - | 10 | | `streamanalytics` | - | [StreamAnalytics](./azure/streamanalytics.md) | - | - | 19 | | `sts` | [STS](./aws/sts.md) | - | - | - | 8 | diff --git a/docs/coverage/coverage.json b/docs/coverage/coverage.json index 9ff0dda5d..3e0c75d5a 100644 --- a/docs/coverage/coverage.json +++ b/docs/coverage/coverage.json @@ -18084,7 +18084,8 @@ "providers": { "aws": "S3", "azure": "BlobStorage", - "gcp": "GCS" + "gcp": "GCS", + "oci": "ObjectStorage" } }, { diff --git a/docs/coverage/oci/README.md b/docs/coverage/oci/README.md index 8a022e2ab..d5f21998b 100644 --- a/docs/coverage/oci/README.md +++ b/docs/coverage/oci/README.md @@ -7,5 +7,6 @@ Services cloudemu emulates for OCI, by native name. Back to the [cross-provider | --- | --- | --- | | [Identity](./identity.md) | `iam` | 40 | | [Monitoring](./monitoring.md) | `monitoring` | 12 | +| [ObjectStorage](./objectstorage.md) | `storage` | 35 | | [VCN](./vcn.md) | `networking` | 57 | | [Workrequest](./workrequest.md) | (provider-native) | 4 | diff --git a/docs/coverage/oci/objectstorage.md b/docs/coverage/oci/objectstorage.md new file mode 100644 index 000000000..da0df0e45 --- /dev/null +++ b/docs/coverage/oci/objectstorage.md @@ -0,0 +1,101 @@ + +# ObjectStorage + +OCI's `storage` service · portable interface `driver.Bucket` · [OCI index](./README.md) + +## Operations (35) + +| Operation | Description | +| --- | --- | +| `AbortMultipartUpload` | | +| `CompleteMultipartUpload` | | +| `CopyObject` | | +| `CreateBucket` | | +| `CreateMultipartUpload` | Multipart uploads | +| `DeleteBucket` | | +| `DeleteBucketPolicy` | | +| `DeleteBucketTagging` | | +| `DeleteCORSConfig` | | +| `DeleteObject` | | +| `DeleteObjectTagging` | | +| `EvaluateLifecycle` | | +| `GeneratePresignedURL` | Presigned URLs | +| `GetBucketPolicy` | | +| `GetBucketTagging` | | +| `GetBucketVersioning` | | +| `GetCORSConfig` | | +| `GetEncryptionConfig` | | +| `GetLifecycleConfig` | | +| `GetObject` | | +| `GetObjectTagging` | | +| `HeadObject` | | +| `ListBuckets` | | +| `ListMultipartUploads` | | +| `ListObjects` | | +| `ListParts` | ListParts returns the parts buffered so far for an in-progress upload, | +| `PutBucketPolicy` | Bucket Policy | +| `PutBucketTagging` | Bucket Tagging | +| `PutCORSConfig` | CORS | +| `PutEncryptionConfig` | Encryption | +| `PutLifecycleConfig` | Lifecycle policies | +| `PutObject` | | +| `PutObjectTagging` | Object Tagging | +| `SetBucketVersioning` | Versioning | +| `UploadPart` | | + +## Optional capabilities + +Discovered by type assertion; only some providers implement these. + +### VersionedBucket + +VersionedBucket is an optional extension a storage provider implements when + +| Operation | Description | +| --- | --- | +| `AbortMultipartUpload` | | +| `CompleteMultipartUpload` | | +| `CopyObject` | | +| `CreateBucket` | | +| `CreateMultipartUpload` | Multipart uploads | +| `DeleteBucket` | | +| `DeleteBucketPolicy` | | +| `DeleteBucketTagging` | | +| `DeleteCORSConfig` | | +| `DeleteObject` | | +| `DeleteObjectTagging` | | +| `DeleteObjectVersion` | DeleteObjectVersion removes a specific version when versionID != "". | +| `EvaluateLifecycle` | | +| `GeneratePresignedURL` | Presigned URLs | +| `GetBucketPolicy` | | +| `GetBucketTagging` | | +| `GetBucketVersioning` | | +| `GetCORSConfig` | | +| `GetEncryptionConfig` | | +| `GetLifecycleConfig` | | +| `GetObject` | | +| `GetObjectTagging` | | +| `GetObjectVersion` | GetObjectVersion / HeadObjectVersion fetch a specific version by ID. A | +| `HeadObject` | | +| `HeadObjectVersion` | | +| `ListBuckets` | | +| `ListMultipartUploads` | | +| `ListObjectVersions` | ListObjectVersions returns the full version history matching opts. | +| `ListObjects` | | +| `ListParts` | ListParts returns the parts buffered so far for an in-progress upload, | +| `PutBucketPolicy` | Bucket Policy | +| `PutBucketTagging` | Bucket Tagging | +| `PutCORSConfig` | CORS | +| `PutEncryptionConfig` | Encryption | +| `PutLifecycleConfig` | Lifecycle policies | +| `PutObject` | | +| `PutObjectTagging` | Object Tagging | +| `SetBucketVersioning` | Versioning | +| `SetVersioningStatus` | SetVersioningStatus sets the bucket's versioning status: "Enabled" or | +| `UploadPart` | | +| `VersioningStatus` | | + +## Not in scope + +- Azure: on the bare emulator host (such as `https://127.0.0.1:4568/`), Blob, Queue and Table share one endpoint, and the account-root calls (`GET /?comp=list`, `?restype=service`, `?restype=account`) look the same for each service. They go to Blob unless the User-Agent carries an Azure SDK Queue or Table product token (`azsdk-go-azqueue`, `azsdk-python-storage-queue`, `azsdk-java-azure-storage-queue`, `azsdk-js-storage-queue`, `azsdk-net-Storage.Queues`, and the matching `data-tables` clients). Other Queue and Table clients should use the `{account}.queue.core.windows.net` or `{account}.table.core.windows.net` host, or the path-style `/{account}/` form. +- Azure: Queue and Table Set Service Properties validate the request and return 202, but do not store logging, metrics or CORS settings. Get Service Stats and Get User Delegation Key are not served. diff --git a/docs/services.md b/docs/services.md index de9a8f545..4143eba6e 100644 --- a/docs/services.md +++ b/docs/services.md @@ -66,7 +66,7 @@ This is a hand-maintained reference for the core service categories on all three ## 1. Storage **Driver interface:** `services/storage/driver/driver.go` -**AWS:** S3 | **Azure:** Blob Storage | **GCP:** GCS +**AWS:** S3 | **Azure:** Blob Storage | **GCP:** GCS | **OCI:** Object Storage (buckets live in a compartment under the tenancy namespace; pre-authenticated requests map to presigned URLs; retention rules and storage tiers have no portable equivalent — bucket policies, CORS and object tags are not OCI concepts and answer `Unimplemented`) ### Bucket Operations @@ -159,6 +159,105 @@ This is a hand-maintained reference for the core service categories on all three **Total: 33 operations** +### OCI Object Storage + +**Optional capability:** `server/oci/objectstorage.Extras` — OCI roots every +path at the tenancy namespace, scopes buckets to a compartment, and carries +bucket settings, object rename, storage tiers, retention rules and +pre-authenticated requests that the portable interface does not express. Its +value types live in `providers/oci/objectstorage`. +**Provider:** `providers/oci/objectstorage` | **Wire:** `server/oci/objectstorage` + +Object Storage carries no API-version prefix; `{ns}` is the tenancy namespace, +which `GET /n` returns. + +| Operation | Route | +|-----------|-------| +| `GetNamespace` | `GET /n` | +| `GetNamespaceMetadata` | `GET /n/{ns}` | +| `CreateBucket` | `POST /n/{ns}/b` | +| `ListBuckets` | `GET /n/{ns}/b` | +| `GetBucket` | `GET /n/{ns}/b/{bucket}` | +| `HeadBucket` | `HEAD /n/{ns}/b/{bucket}` | +| `UpdateBucket` | `POST /n/{ns}/b/{bucket}` | +| `DeleteBucket` | `DELETE /n/{ns}/b/{bucket}` | +| `ListObjects` | `GET /n/{ns}/b/{bucket}/o` | +| `PutObject` | `PUT /n/{ns}/b/{bucket}/o/{object}` | +| `GetObject` | `GET /n/{ns}/b/{bucket}/o/{object}` | +| `HeadObject` | `HEAD /n/{ns}/b/{bucket}/o/{object}` | +| `DeleteObject` | `DELETE /n/{ns}/b/{bucket}/o/{object}` | +| `ListObjectVersions` | `GET /n/{ns}/b/{bucket}/objectversions` | +| `RenameObject` | `POST /n/{ns}/b/{bucket}/actions/renameObject` | +| `CopyObject` | `POST /n/{ns}/b/{bucket}/actions/copyObject` | +| `UpdateObjectStorageTier` | `POST /n/{ns}/b/{bucket}/actions/updateObjectStorageTier` | +| `CreateMultipartUpload` | `POST /n/{ns}/b/{bucket}/u` | +| `ListMultipartUploads` | `GET /n/{ns}/b/{bucket}/u` | +| `UploadPart` | `PUT /n/{ns}/b/{bucket}/u/{object}` | +| `CommitMultipartUpload` | `POST /n/{ns}/b/{bucket}/u/{object}` | +| `ListMultipartUploadParts` | `GET /n/{ns}/b/{bucket}/u/{object}` | +| `AbortMultipartUpload` | `DELETE /n/{ns}/b/{bucket}/u/{object}` | +| `CreatePreauthenticatedRequest` | `POST /n/{ns}/b/{bucket}/p` | +| `ListPreauthenticatedRequests` | `GET /n/{ns}/b/{bucket}/p` | +| `GetPreauthenticatedRequest` | `GET /n/{ns}/b/{bucket}/p/{parId}` | +| `DeletePreauthenticatedRequest` | `DELETE /n/{ns}/b/{bucket}/p/{parId}` | +| PAR redemption | `GET`/`PUT /p/{par}/n/{ns}/b/{bucket}/o/{object}` | +| `CreateRetentionRule` | `POST /n/{ns}/b/{bucket}/retentionRules` | +| `ListRetentionRules` | `GET /n/{ns}/b/{bucket}/retentionRules` | +| `GetRetentionRule` | `GET /n/{ns}/b/{bucket}/retentionRules/{ruleId}` | +| `UpdateRetentionRule` | `PUT /n/{ns}/b/{bucket}/retentionRules/{ruleId}` | +| `DeleteRetentionRule` | `DELETE /n/{ns}/b/{bucket}/retentionRules/{ruleId}` | +| `PutObjectLifecyclePolicy` | `PUT /n/{ns}/b/{bucket}/l` | +| `GetObjectLifecyclePolicy` | `GET /n/{ns}/b/{bucket}/l` | +| `DeleteObjectLifecyclePolicy` | `DELETE /n/{ns}/b/{bucket}/l` | + +`ListBuckets` is the one collection OCI scopes by compartment, so it is the +only route here that requires `compartmentId`; every other list is scoped by +its bucket. Every list pages by `limit` and `page` (`start` for `ListObjects`) +and returns the cursor as `opc-next-page`; an unspecified `limit` yields OCI's +Object Storage page size of 1000, not the 100 the other OCI services default +to. `copyObject` is asynchronous in real OCI, so it returns `202` with an +`opc-work-request-id` the shared work-request poller answers at +`/workRequests/{id}`; every other mutation here is synchronous. A create into, +or a move to, a compartment Identity does not know is `404 +NotAuthorizedOrNotFound`, as for VCN. + +ETags are opaque and minted per write, so re-uploading identical bytes changes +them. `if-match` and `if-none-match` are honored on Put, Get, Head and Delete +object, on UpdateBucket and DeleteBucket, and on both sides of `copyObject`: a +failed write precondition is `412 IfMatchFailed` / `IfNoneMatchFailed`, a +matching `if-none-match` on a read is `304`. GetObject serves a single byte +`Range` with `206` and `Content-Range`; an unsatisfiable range is `416`, a +multi-range request is refused. `copyObject` requires `destinationRegion` and +refuses another region or namespace; a source version, replacement metadata +and a destination storage tier all apply. A multipart commit checks each +part's `etag`. UpdateBucket's `name` renames the bucket, its objects and PARs +moving with it. Errors carry Object Storage's own codes where it has one: +`BucketAlreadyExists`, `BucketNotEmpty`. + +Buckets refuse deletion while they hold objects or uncommitted multipart +uploads. Versioning is the OCI tri-state — `Disabled`, `Enabled`, `Suspended` — +and never returns to `Disabled` once enabled; enabling it seeds the objects +already present as versions, and a `Suspended` bucket reuses the `null` version +rather than appending. Retention rules with an elapsed lock block overwrites +and deletes, and a locked rule cannot be weakened. Lifecycle rules are stored +exactly as sent — unit, `target` and all three `objectNameFilter` lists — and +the policy reports its `timeCreated`; `ABORT` requires the `multipart-uploads` +target, and only rules aimed at `objects` age out live objects. + +A pre-authenticated request requires `timeExpires`, with no S3-style cap, and +on the `AnyObject` types `objectName` is a prefix. Its redemption token is drawn +from `crypto/rand`, so tokens issued after a persist restore never collide with +restored ones. + +Object bytes flow through `config.WithStorageEngine` when one is wired, the +same seam AWS S3, Azure Blob and GCP GCS use, keyed by object version so each +version's bytes are addressed separately. + +Not emulated: `/actions/reencrypt` and `/actions/restoreObjects`, which need +per-object key material and an archive-retrieval lifecycle the storage driver +has no shape for. Both are claimed by the handler and answer `501` with the +reason rather than a bare `404`. + --- ## 2. Compute diff --git a/docs/standalone-server.md b/docs/standalone-server.md index 96445798c..24006e5ac 100644 --- a/docs/standalone-server.md +++ b/docs/standalone-server.md @@ -416,6 +416,7 @@ generated cert's SANs with `--tls-host ` and trust that cert in your clien | `--gcp-grpc-port` | (empty) | port for the GCP gRPC transport (health + reflection); empty disables it. Point `*_EMULATOR_HOST` clients here | | `--account-id` | `000000000000` | AWS account ID (also used for GCP/OCI) | | `--azure-subscription` | `00000000-0000-0000-0000-000000000000` | Azure subscription id (a GUID). Resource ids and Resource Graph scoping use it; discovery is subscription-transparent, so a query scoped to any subscription returns the estate rendered under it | +| `--oci-tenancy` | `ocid1.tenancy.oc1..aaaaaaaacloudemulocaltenancy` | OCI tenancy OCID. It is also the root compartment, and the Object Storage namespace is derived from it | | `--region` | `us-east-1` | default region | | `--project-id` | `cloudemu-local` | GCP project ID | | `--latency` | `0` | artificial per-call latency (e.g. `20ms`) | diff --git a/providers/oci/monitoring/metrics.go b/providers/oci/monitoring/metrics.go index 7cbb33ac4..955a0674e 100644 --- a/providers/oci/monitoring/metrics.go +++ b/providers/oci/monitoring/metrics.go @@ -80,6 +80,21 @@ type metricSeries struct { // PostMetricData records metric data points against a compartment. func (m *Mock) PostMetricData(_ context.Context, compartmentID, resourceGroup string, data []driver.MetricDatum) error { + return m.postMetricData(compartmentID, resourceGroup, data, false) +} + +// PostServiceMetricData records a sibling OCI service's own metrics against a +// compartment. Unlike PostMetricData it admits the oci_ namespaces Oracle +// reserves for service metrics, which only the service itself produces. +func (m *Mock) PostServiceMetricData(_ context.Context, compartmentID string, data []driver.MetricDatum) error { + return m.postMetricData(compartmentID, "", data, true) +} + +// postMetricData records metric data points. allowReserved admits the oci_ +// namespaces; only PostServiceMetricData sets it. +func (m *Mock) postMetricData( + compartmentID, resourceGroup string, data []driver.MetricDatum, allowReserved bool, +) error { if compartmentID == "" { return cerrors.New(cerrors.InvalidArgument, "compartmentId is required") } @@ -89,7 +104,7 @@ func (m *Mock) PostMetricData(_ context.Context, compartmentID, resourceGroup st } for i := range data { - if err := validateDatum(&data[i]); err != nil { + if err := validateDatum(&data[i], allowReserved); err != nil { return err } } @@ -320,7 +335,7 @@ func resolutionOf(interval time.Duration, resolution string) (time.Duration, err // validateDatum rejects a data point real OCI would reject. Namespace and // dimension shapes are checked; the metadata, per-request datapoint cap and // ingestion time window are not. -func validateDatum(d *driver.MetricDatum) error { +func validateDatum(d *driver.MetricDatum, allowReserved bool) error { switch { case d.Namespace == "": return cerrors.New(cerrors.InvalidArgument, "namespace is required") @@ -329,7 +344,7 @@ func validateDatum(d *driver.MetricDatum) error { case !validNamespace(d.Namespace): return cerrors.Newf(cerrors.InvalidArgument, "namespace %q must start with a letter and hold only letters, digits and underscores", d.Namespace) - case reservedNamespace(d.Namespace): + case !allowReserved && reservedNamespace(d.Namespace): return cerrors.Newf(cerrors.InvalidArgument, "namespace %q uses a prefix Oracle reserves", d.Namespace) case d.MetricName == "": return cerrors.New(cerrors.InvalidArgument, "metric name is required") diff --git a/providers/oci/objectstorage/bucket.go b/providers/oci/objectstorage/bucket.go new file mode 100644 index 000000000..42731c65c --- /dev/null +++ b/providers/oci/objectstorage/bucket.go @@ -0,0 +1,587 @@ +package objectstorage + +import ( + "context" + "maps" + "sort" + + "github.com/stackshy/cloudemu/v2/config" + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/idgen" + "github.com/stackshy/cloudemu/v2/internal/memstore" + "github.com/stackshy/cloudemu/v2/services/storage/driver" + "github.com/stackshy/cloudemu/v2/services/storage/storageengine" +) + +// Encryption algorithms the bucket reports: the Oracle-managed default, or a +// customer key held in Vault. +const ( + algorithmAES256 = "AES256" + algorithmKMS = "oci:kms" +) + +// BucketSpec is a bucket to create. Only Name is required; the compartment +// falls back to the provider's default. +type BucketSpec struct { + Name string + CompartmentID string + PublicAccessType string + StorageTier string + Versioning string + KMSKeyID string + AutoTiering string + ObjectEventsEnabled bool + Metadata map[string]string + FreeformTags map[string]string + DefinedTags map[string]map[string]string +} + +// BucketUpdate is a partial bucket update. A nil pointer leaves the field +// alone; OCI's UpdateBucket replaces only what the caller sends. +type BucketUpdate struct { + Name *string + IfMatch string + CompartmentID *string + PublicAccessType *string + Versioning *string + KMSKeyID *string + AutoTiering *string + ObjectEventsEnabled *bool + Metadata map[string]string + FreeformTags map[string]string + DefinedTags map[string]map[string]string +} + +// Bucket is a bucket as OCI reports it. +type Bucket struct { + ID string + Name string + Namespace string + CompartmentID string + CreatedBy string + TimeCreated string + ETag string + PublicAccessType string + StorageTier string + Versioning string + KMSKeyID string + AutoTiering string + ObjectEventsEnabled bool + ReplicationEnabled bool + IsReadOnly bool + Metadata map[string]string + FreeformTags map[string]string + DefinedTags map[string]map[string]string + ApproximateCount int64 + ApproximateSize int64 +} + +// validPublicAccess and friends reject a value OCI would not accept, rather +// than storing something the emulator would report back unchanged. +func validPublicAccess(v string) bool { + switch v { + case AccessNone, AccessObjectRead, AccessObjectReadWithoutList: + return true + } + + return false +} + +func validStorageTier(v string) bool { + switch v { + case TierStandard, TierArchive, TierInfrequentAccess: + return true + } + + return false +} + +func validVersioning(v string) bool { + switch v { + case VersioningDisabled, VersioningEnabled, VersioningSuspended: + return true + } + + return false +} + +func validAutoTiering(v string) bool { + return v == AutoTieringDisabled || v == AutoTieringInfreq +} + +// maxBucketNameLen is OCI's bucket name limit. +const maxBucketNameLen = 256 + +// validateBucketName enforces OCI's bucket name rule: letters, digits, +// hyphens, underscores and periods, up to 256 characters. +func validateBucketName(name string) error { + if name == "" { + return cerrors.New(cerrors.InvalidArgument, "bucket name cannot be empty") + } + + if len(name) > maxBucketNameLen { + return cerrors.Newf(cerrors.InvalidArgument, "bucket name exceeds %d characters", maxBucketNameLen) + } + + for _, c := range name { + switch { + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9', c == '-', c == '_', c == '.': + default: + return cerrors.Newf(cerrors.InvalidArgument, + "bucket name %q may hold only letters, digits, '-', '_' and '.'", name) + } + } + + return nil +} + +// renameBucketLocked moves a bucket and everything under it to a new name. Its +// PARs follow it, and with a storage engine wired the object bytes move to the +// new name's references. Callers hold mu for writing. +func (m *Mock) renameBucketLocked(ctx context.Context, bkt *bucketData, newName string) error { + oldName := bkt.Name + + if m.engineWired() { + for _, ref := range engineRefsLocked(bkt) { + if err := storageengine.Copy(ctx, m.opts.StorageEngine, + engineRef(newName, ref.Key, ref.Version), engineRef(oldName, ref.Key, ref.Version)); err != nil { + return err + } + } + + for _, ref := range engineRefsLocked(bkt) { + _ = storageengine.Delete(ctx, m.opts.StorageEngine, engineRef(oldName, ref.Key, ref.Version)) + } + } + + for _, id := range bkt.pars.Keys() { + if par, ok := bkt.pars.Get(id); ok { + par.Bucket = newName + } + } + + bkt.Name = newName + + m.buckets.Delete(oldName) + m.buckets.Set(newName, bkt) + + return nil +} + +// engineRefsLocked lists every engine reference a bucket's current objects and +// stored versions occupy, each once. Callers hold mu. +func engineRefsLocked(bkt *bucketData) []config.StorageRef { + seen := make(map[config.StorageRef]bool) + + var refs []config.StorageRef + + add := func(key, version string) { + ref := config.StorageRef{Key: key, Version: version} + if !seen[ref] { + seen[ref] = true + + refs = append(refs, ref) + } + } + + for _, name := range bkt.objects.Keys() { + if obj, ok := bkt.objects.Get(name); ok { + add(name, obj.VersionID) + } + } + + for name, chain := range bkt.versions { + for _, v := range chain { + if !v.deleteMarker { + add(name, v.versionID) + } + } + } + + return refs +} + +// CreateBucketWith creates a bucket with OCI's bucket settings, recording the +// compartment it lands in. +// +//nolint:gocritic // BucketSpec is a request shape, passed by value like the driver's own config structs. +func (m *Mock) CreateBucketWith(_ context.Context, spec BucketSpec) (*Bucket, error) { + if err := validateBucketName(spec.Name); err != nil { + return nil, err + } + + if err := validateSpec(spec); err != nil { + return nil, err + } + + m.mu.Lock() + defer m.mu.Unlock() + + if m.buckets.Has(spec.Name) { + return nil, serviceErrorf(CodeBucketAlreadyExists, cerrors.AlreadyExists, "bucket %q already exists", spec.Name) + } + + bkt := &bucketData{ + ID: idgen.OCID(typeBucket, m.opts.Realm, m.opts.OCIRegion()), + Name: spec.Name, + Namespace: m.namespace, + CompartmentID: orDefault(spec.CompartmentID, m.opts.CompartmentID), + CreatedBy: m.opts.TenancyOCID, + TimeCreated: m.now(), + ETag: newETag(), + PublicAccessType: orDefault(spec.PublicAccessType, AccessNone), + StorageTier: orDefault(spec.StorageTier, TierStandard), + Versioning: orDefault(spec.Versioning, VersioningDisabled), + KMSKeyID: spec.KMSKeyID, + AutoTiering: orDefault(spec.AutoTiering, AutoTieringDisabled), + ObjectEventsEnabled: spec.ObjectEventsEnabled, + Metadata: maps.Clone(spec.Metadata), + FreeformTags: maps.Clone(spec.FreeformTags), + DefinedTags: cloneDefinedTags(spec.DefinedTags), + objects: memstore.New[*objectData](), + multiparts: memstore.New[*multipartUpload](), + pars: memstore.New[*parData](), + retention: memstore.New[*retentionRuleData](), + } + + if bkt.Versioning == VersioningEnabled { + bkt.versions = make(map[string][]*objectVersion) + } + + m.buckets.Set(bkt.Name, bkt) + + return projectBucket(bkt), nil +} + +//nolint:gocritic // mirrors CreateBucketWith's by-value spec. +func validateSpec(spec BucketSpec) error { + if spec.PublicAccessType != "" && !validPublicAccess(spec.PublicAccessType) { + return cerrors.Newf(cerrors.InvalidArgument, "unsupported publicAccessType %q", spec.PublicAccessType) + } + + if spec.StorageTier != "" && !validStorageTier(spec.StorageTier) { + return cerrors.Newf(cerrors.InvalidArgument, "unsupported storageTier %q", spec.StorageTier) + } + + if spec.Versioning != "" && !validVersioning(spec.Versioning) { + return cerrors.Newf(cerrors.InvalidArgument, "unsupported versioning %q", spec.Versioning) + } + + if spec.AutoTiering != "" && !validAutoTiering(spec.AutoTiering) { + return cerrors.Newf(cerrors.InvalidArgument, "unsupported autoTiering %q", spec.AutoTiering) + } + + return nil +} + +// BucketDetails returns one bucket as OCI reports it. +func (m *Mock) BucketDetails(_ context.Context, name string) (*Bucket, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, err := m.bucketLocked(name) + if err != nil { + return nil, err + } + + return projectBucket(bkt), nil +} + +// UpdateBucket applies a partial update, replacing only the fields set. +// +//nolint:gocritic // BucketUpdate is a request shape, passed by value like BucketSpec. +func (m *Mock) UpdateBucket(ctx context.Context, name string, upd BucketUpdate) (*Bucket, error) { + if err := validateUpdate(upd); err != nil { + return nil, err + } + + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(name) + if err != nil { + return nil, err + } + + if err := m.checkUpdateLocked(bkt, &upd); err != nil { + return nil, err + } + + if upd.Versioning != nil { + if err := m.setVersioningLocked(ctx, name, bkt, *upd.Versioning); err != nil { + return nil, err + } + } + + if renamed(name, &upd) { + if err := m.renameBucketLocked(ctx, bkt, *upd.Name); err != nil { + return nil, err + } + } + + applyUpdate(bkt, &upd) + bkt.ETag = newETag() + + return projectBucket(bkt), nil +} + +// checkUpdateLocked applies the update's if-match and refuses a rename onto an +// existing bucket. Callers hold mu. +func (m *Mock) checkUpdateLocked(bkt *bucketData, upd *BucketUpdate) error { + if err := checkETag(bkt.ETag, upd.IfMatch, ""); err != nil { + return err + } + + if renamed(bkt.Name, upd) && m.buckets.Has(*upd.Name) { + return serviceErrorf(CodeBucketAlreadyExists, cerrors.AlreadyExists, "bucket %q already exists", *upd.Name) + } + + return nil +} + +func renamed(name string, upd *BucketUpdate) bool { return upd.Name != nil && *upd.Name != name } + +//nolint:gocritic // BucketUpdate is a request shape, passed by value like BucketSpec. +func validateUpdate(upd BucketUpdate) error { + if upd.Name != nil { + if err := validateBucketName(*upd.Name); err != nil { + return err + } + } + + if upd.Versioning != nil && *upd.Versioning == VersioningDisabled { + return cerrors.New(cerrors.InvalidArgument, + "versioning cannot be set back to Disabled once enabled; use Suspended") + } + + return validateUpdateValues(&upd) +} + +// validateUpdateValues rejects an enumerated setting OCI would not accept. +func validateUpdateValues(upd *BucketUpdate) error { + if upd.PublicAccessType != nil && !validPublicAccess(*upd.PublicAccessType) { + return cerrors.Newf(cerrors.InvalidArgument, "unsupported publicAccessType %q", *upd.PublicAccessType) + } + + if upd.Versioning != nil && !validVersioning(*upd.Versioning) { + return cerrors.Newf(cerrors.InvalidArgument, "unsupported versioning %q", *upd.Versioning) + } + + if upd.AutoTiering != nil && !validAutoTiering(*upd.AutoTiering) { + return cerrors.Newf(cerrors.InvalidArgument, "unsupported autoTiering %q", *upd.AutoTiering) + } + + return nil +} + +func applyUpdate(bkt *bucketData, upd *BucketUpdate) { + if upd.CompartmentID != nil { + bkt.CompartmentID = *upd.CompartmentID + } + + if upd.PublicAccessType != nil { + bkt.PublicAccessType = *upd.PublicAccessType + } + + if upd.KMSKeyID != nil { + bkt.KMSKeyID = *upd.KMSKeyID + } + + if upd.AutoTiering != nil { + bkt.AutoTiering = *upd.AutoTiering + } + + if upd.ObjectEventsEnabled != nil { + bkt.ObjectEventsEnabled = *upd.ObjectEventsEnabled + } + + if upd.Metadata != nil { + bkt.Metadata = maps.Clone(upd.Metadata) + } + + if upd.FreeformTags != nil { + bkt.FreeformTags = maps.Clone(upd.FreeformTags) + } + + if upd.DefinedTags != nil { + bkt.DefinedTags = cloneDefinedTags(upd.DefinedTags) + } +} + +// ListBucketsIn returns the buckets in a compartment, ordered by name. +func (m *Mock) ListBucketsIn(_ context.Context, compartmentID string) ([]Bucket, error) { + if compartmentID == "" { + return nil, cerrors.New(cerrors.InvalidArgument, "compartmentId is required") + } + + m.mu.RLock() + defer m.mu.RUnlock() + + names := m.buckets.Keys() + sort.Strings(names) + + out := make([]Bucket, 0, len(names)) + + for _, n := range names { + bkt, ok := m.buckets.Get(n) + if !ok || bkt.CompartmentID != compartmentID { + continue + } + + out = append(out, *projectBucket(bkt)) + } + + return out, nil +} + +// projectBucket copies a bucket out from under mu, summing the objects it +// holds for the approximate counts OCI reports on request. +func projectBucket(bkt *bucketData) *Bucket { + var count, size int64 + + for _, name := range bkt.objects.Keys() { + if obj, ok := bkt.objects.Get(name); ok { + count++ + size += obj.Size + } + } + + return &Bucket{ + ID: bkt.ID, + Name: bkt.Name, + Namespace: bkt.Namespace, + CompartmentID: bkt.CompartmentID, + CreatedBy: bkt.CreatedBy, + TimeCreated: bkt.TimeCreated, + ETag: bkt.ETag, + PublicAccessType: bkt.PublicAccessType, + StorageTier: bkt.StorageTier, + Versioning: bkt.Versioning, + KMSKeyID: bkt.KMSKeyID, + AutoTiering: bkt.AutoTiering, + ObjectEventsEnabled: bkt.ObjectEventsEnabled, + Metadata: cloneMeta(bkt.Metadata), + FreeformTags: cloneMeta(bkt.FreeformTags), + DefinedTags: cloneDefinedTags(bkt.DefinedTags), + ApproximateCount: count, + ApproximateSize: size, + } +} + +func cloneDefinedTags(in map[string]map[string]string) map[string]map[string]string { + if in == nil { + return nil + } + + out := make(map[string]map[string]string, len(in)) + for ns, kv := range in { + out[ns] = maps.Clone(kv) + } + + return out +} + +func orDefault(v, fallback string) string { + if v == "" { + return fallback + } + + return v +} + +// PutBucketTagging sets a bucket's freeform tags. +func (m *Mock) PutBucketTagging(_ context.Context, bucket string, tags map[string]string) error { + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return err + } + + bkt.FreeformTags = maps.Clone(tags) + + return nil +} + +// GetBucketTagging returns a bucket's freeform tags. +func (m *Mock) GetBucketTagging(_ context.Context, bucket string) (map[string]string, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + if bkt.FreeformTags == nil { + return map[string]string{}, nil + } + + return maps.Clone(bkt.FreeformTags), nil +} + +// DeleteBucketTagging clears a bucket's freeform tags. +func (m *Mock) DeleteBucketTagging(_ context.Context, bucket string) error { + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return err + } + + bkt.FreeformTags = nil + + return nil +} + +// PutEncryptionConfig points a bucket at a KMS key. OCI encrypts every bucket +// with an Oracle-managed key by default, so the only thing configurable is +// which customer key replaces it — encryption itself cannot be turned off. +func (m *Mock) PutEncryptionConfig(_ context.Context, bucket string, cfg driver.EncryptionConfig) error { + if !cfg.Enabled { + return cerrors.New(cerrors.InvalidArgument, + "OCI Object Storage encryption cannot be disabled; every bucket is encrypted at rest") + } + + if cfg.Algorithm != "" && cfg.Algorithm != algorithmAES256 && cfg.Algorithm != algorithmKMS { + return cerrors.Newf(cerrors.InvalidArgument, "unsupported encryption algorithm %q", cfg.Algorithm) + } + + if cfg.Algorithm == algorithmKMS && cfg.KeyID == "" { + return cerrors.New(cerrors.InvalidArgument, "kmsKeyId is required for customer-managed encryption") + } + + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return err + } + + bkt.KMSKeyID = cfg.KeyID + bkt.ETag = newETag() + + return nil +} + +// GetEncryptionConfig reports the bucket's encryption. It is always enabled; +// the algorithm reflects whether a customer KMS key is assigned. +func (m *Mock) GetEncryptionConfig(_ context.Context, bucket string) (*driver.EncryptionConfig, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + cfg := driver.EncryptionConfig{Enabled: true, Algorithm: algorithmAES256} + if bkt.KMSKeyID != "" { + cfg.Algorithm, cfg.KeyID = algorithmKMS, bkt.KMSKeyID + } + + return &cfg, nil +} diff --git a/providers/oci/objectstorage/engine.go b/providers/oci/objectstorage/engine.go new file mode 100644 index 000000000..bb8b5a123 --- /dev/null +++ b/providers/oci/objectstorage/engine.go @@ -0,0 +1,97 @@ +package objectstorage + +import ( + "context" + + "github.com/stackshy/cloudemu/v2/config" + "github.com/stackshy/cloudemu/v2/services/storage/storageengine" +) + +// engineWired reports whether object bytes live in a real storage engine +// rather than on the heap. +func (m *Mock) engineWired() bool { return m.opts.StorageEngine != nil } + +// engineRef addresses one object version in the engine. OCI keeps full version +// chains, so the version id is part of the address the way S3's is. +func engineRef(bucket, key, version string) config.StorageRef { + return config.StorageRef{Bucket: bucket, Key: key, Version: version} +} + +// engineStore writes an object's bytes to the engine. Callers must not hold mu: +// the engine is real I/O. +func (m *Mock) engineStore(ctx context.Context, bucket string, obj *objectData) error { + return storageengine.Put(ctx, m.opts.StorageEngine, config.StorageObject{ + Bucket: bucket, + Key: obj.Name, + Version: obj.VersionID, + Data: obj.Data, + ContentType: obj.ContentType, + Metadata: obj.Metadata, + }) +} + +// offloadLocked hands a freshly stored object's bytes to the engine and drops +// the heap copy. It runs under the write lock rather than after it, so no +// reader ever observes an object whose bytes are in neither place. Callers hold +// mu for writing. +func (m *Mock) offloadLocked(ctx context.Context, bkt *bucketData, bucket string, obj *objectData) error { + if !m.engineWired() { + return nil + } + + if err := m.engineStore(ctx, bucket, obj); err != nil { + return err + } + + dropBytesLocked(bkt, obj) + + return nil +} + +// dropBytesLocked releases the in-memory copy of an object and of the version +// record that shares its bytes. Callers hold mu for writing. +func dropBytesLocked(bkt *bucketData, obj *objectData) { + obj.Data = nil + + for _, v := range bkt.versions[obj.Name] { + if v.versionID == obj.VersionID { + v.data = nil + } + } +} + +// purgeLocked removes a deleted object's engine bytes. A delete marker on a +// versioned bucket keeps the prior versions readable, so only an outright +// removal or a suspended-bucket null overwrite purges. Failures are ignored: +// an idempotent delete must not fail because the bytes were already gone. +// Callers hold mu for writing. +func (m *Mock) purgeLocked(ctx context.Context, bucket, key, versionID string, marker bool) { + if !m.engineWired() { + return + } + + if marker && versionID != nullVersionID { + return + } + + _ = storageengine.Delete(ctx, m.opts.StorageEngine, engineRef(bucket, key, versionID)) +} + +// engineLoad returns the bytes for a version, preferring the in-memory copy so +// an unwired emulator behaves exactly as before. +func (m *Mock) engineLoad(ctx context.Context, ref config.StorageRef, inMemory []byte) ([]byte, error) { + if !m.engineWired() || inMemory != nil { + return inMemory, nil + } + + data, ok, err := storageengine.Get(ctx, m.opts.StorageEngine, ref) + if err != nil { + return nil, err + } + + if !ok { + return nil, nil + } + + return data, nil +} diff --git a/providers/oci/objectstorage/engine_test.go b/providers/oci/objectstorage/engine_test.go new file mode 100644 index 000000000..a185eea3c --- /dev/null +++ b/providers/oci/objectstorage/engine_test.go @@ -0,0 +1,263 @@ +package objectstorage_test + +import ( + "context" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stackshy/cloudemu/v2/config" + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/providers/oci/objectstorage" + "github.com/stackshy/cloudemu/v2/services/storage/driver" +) + +// fakeStorageEngine is a version-aware in-memory config.StorageEngine, so a +// test can assert which bytes reached the seam and under which version. +type fakeStorageEngine struct { + mu sync.Mutex + data map[string][]byte + err error +} + +func newFakeStorageEngine() *fakeStorageEngine { + return &fakeStorageEngine{data: make(map[string][]byte)} +} + +func refKey(ref config.StorageRef) string { + return ref.Bucket + "\x00" + ref.Key + "\x00" + ref.Version +} + +//nolint:gocritic // obj is the by-value DTO defined by the StorageEngine contract +func (f *fakeStorageEngine) Put(_ context.Context, obj config.StorageObject) error { + f.mu.Lock() + defer f.mu.Unlock() + + if f.err != nil { + return f.err + } + + f.data[refKey(config.StorageRef{Bucket: obj.Bucket, Key: obj.Key, Version: obj.Version})] = + append([]byte(nil), obj.Data...) + + return nil +} + +func (f *fakeStorageEngine) Get(_ context.Context, ref config.StorageRef) (config.StorageObject, error) { + f.mu.Lock() + defer f.mu.Unlock() + + b, ok := f.data[refKey(ref)] + if !ok { + return config.StorageObject{}, assert.AnError + } + + return config.StorageObject{ + Bucket: ref.Bucket, Key: ref.Key, Version: ref.Version, Data: append([]byte(nil), b...), + }, nil +} + +func (f *fakeStorageEngine) Delete(_ context.Context, ref config.StorageRef) error { + f.mu.Lock() + defer f.mu.Unlock() + + delete(f.data, refKey(ref)) + + return nil +} + +func (f *fakeStorageEngine) Copy(ctx context.Context, dst, src config.StorageRef) error { + obj, err := f.Get(ctx, src) + if err != nil { + return err + } + + return f.Put(ctx, config.StorageObject{Bucket: dst.Bucket, Key: dst.Key, Version: dst.Version, Data: obj.Data}) +} + +func (f *fakeStorageEngine) has(bucket, key, version string) bool { + f.mu.Lock() + defer f.mu.Unlock() + + _, ok := f.data[refKey(config.StorageRef{Bucket: bucket, Key: key, Version: version})] + + return ok +} + +func (f *fakeStorageEngine) len() int { + f.mu.Lock() + defer f.mu.Unlock() + + return len(f.data) +} + +func newEngineMock(t *testing.T, eng config.StorageEngine) *objectstorage.Mock { + t.Helper() + + return objectstorage.New(config.NewOptions( + config.WithRegion("us-ashburn-1"), + config.WithCompartmentID(testCompartment), + config.WithStorageEngine(eng), + )) +} + +func TestStorageEnginePutGetRoundTrip(t *testing.T) { + eng := newFakeStorageEngine() + m := newEngineMock(t, eng) + ctx := context.Background() + newBucket(t, m, testBucket) + + require.NoError(t, m.PutObject(ctx, testBucket, "a.txt", []byte("hello engine"), "text/plain", nil)) + assert.True(t, eng.has(testBucket, "a.txt", ""), "the bytes reached the engine") + + got, err := m.GetObject(ctx, testBucket, "a.txt") + require.NoError(t, err) + assert.Equal(t, []byte("hello engine"), got.Data) + + // Metadata survives the offload: Head and List read Size, not len(Data). + head, err := m.HeadObject(ctx, testBucket, "a.txt") + require.NoError(t, err) + assert.Equal(t, int64(12), head.Size) + + list, err := m.ListObjects(ctx, testBucket, driver.ListOptions{}) + require.NoError(t, err) + require.Len(t, list.Objects, 1) + assert.Equal(t, int64(12), list.Objects[0].Size) + + details, err := m.BucketDetails(ctx, testBucket) + require.NoError(t, err) + assert.Equal(t, int64(12), details.ApproximateSize) + + require.NoError(t, m.DeleteObject(ctx, testBucket, "a.txt")) + assert.Zero(t, eng.len(), "delete purges the engine bytes") +} + +func TestStorageEngineCopyAndRename(t *testing.T) { + eng := newFakeStorageEngine() + m := newEngineMock(t, eng) + ctx := context.Background() + newBucket(t, m, testBucket) + newBucket(t, m, "bucket-b") + + require.NoError(t, m.PutObject(ctx, testBucket, "a.txt", []byte("payload"), "text/plain", nil)) + + require.NoError(t, m.CopyObject(ctx, "bucket-b", "copied.txt", + driver.CopySource{Bucket: testBucket, Key: "a.txt"})) + assert.True(t, eng.has("bucket-b", "copied.txt", "")) + + copied, err := m.GetObject(ctx, "bucket-b", "copied.txt") + require.NoError(t, err) + assert.Equal(t, []byte("payload"), copied.Data) + + moved, err := m.RenameObject(ctx, testBucket, "a.txt", "b.txt") + require.NoError(t, err) + assert.Equal(t, int64(7), moved.Size) + assert.True(t, eng.has(testBucket, "b.txt", "")) + assert.False(t, eng.has(testBucket, "a.txt", ""), "the source bytes are purged") + + renamed, err := m.GetObject(ctx, testBucket, "b.txt") + require.NoError(t, err) + assert.Equal(t, []byte("payload"), renamed.Data) +} + +func TestStorageEngineVersionedRoundTrip(t *testing.T) { + eng := newFakeStorageEngine() + m := newEngineMock(t, eng) + ctx := context.Background() + newBucket(t, m, testBucket) + + require.NoError(t, m.SetVersioningStatus(ctx, testBucket, objectstorage.VersioningEnabled)) + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v1"), "text/plain", nil)) + + first, err := m.HeadObject(ctx, testBucket, "k") + require.NoError(t, err) + + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v2"), "text/plain", nil)) + + second, err := m.HeadObject(ctx, testBucket, "k") + require.NoError(t, err) + assert.Equal(t, 2, eng.len(), "each version's bytes live at their own reference") + + old, err := m.GetObjectVersion(ctx, testBucket, "k", first.VersionID) + require.NoError(t, err) + assert.Equal(t, []byte("v1"), old.Data) + + oldHead, err := m.HeadObjectVersion(ctx, testBucket, "k", first.VersionID) + require.NoError(t, err) + assert.Equal(t, int64(2), oldHead.Size) + + // A top-level delete on a versioned bucket appends a delete marker; the + // prior versions' bytes must survive it. + _, marker, err := m.DeleteObjectVersion(ctx, testBucket, "k", "") + require.NoError(t, err) + require.True(t, marker) + assert.Equal(t, 2, eng.len(), "a delete marker purges nothing") + + still, err := m.GetObjectVersion(ctx, testBucket, "k", first.VersionID) + require.NoError(t, err) + assert.Equal(t, []byte("v1"), still.Data) + + // Removing a version by id does purge that version's bytes. + _, _, err = m.DeleteObjectVersion(ctx, testBucket, "k", second.VersionID) + require.NoError(t, err) + assert.False(t, eng.has(testBucket, "k", second.VersionID)) + assert.True(t, eng.has(testBucket, "k", first.VersionID)) +} + +func TestStorageEngineSuspendedDeletePurgesNullBytes(t *testing.T) { + eng := newFakeStorageEngine() + m := newEngineMock(t, eng) + ctx := context.Background() + newBucket(t, m, testBucket) + + require.NoError(t, m.SetVersioningStatus(ctx, testBucket, objectstorage.VersioningSuspended)) + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v1"), "text/plain", nil)) + assert.True(t, eng.has(testBucket, "k", "null")) + + _, marker, err := m.DeleteObjectVersion(ctx, testBucket, "k", "") + require.NoError(t, err) + require.True(t, marker) + assert.False(t, eng.has(testBucket, "k", "null"), "the null version is overwritten, so its bytes go") +} + +func TestStorageEngineMultipartRoutesAssembledBytes(t *testing.T) { + eng := newFakeStorageEngine() + m := newEngineMock(t, eng) + ctx := context.Background() + newBucket(t, m, testBucket) + + up, err := m.CreateMultipartUpload(ctx, testBucket, "big", "application/octet-stream") + require.NoError(t, err) + + _, err = m.UploadPart(ctx, testBucket, "big", up.UploadID, 1, []byte("aaa")) + require.NoError(t, err) + _, err = m.UploadPart(ctx, testBucket, "big", up.UploadID, 2, []byte("bbb")) + require.NoError(t, err) + + assert.Zero(t, eng.len(), "parts stay on the heap until the upload commits") + + require.NoError(t, m.CompleteMultipartUpload(ctx, testBucket, "big", up.UploadID, + []driver.UploadPart{{PartNumber: 1}, {PartNumber: 2}})) + assert.True(t, eng.has(testBucket, "big", "")) + + got, err := m.GetObject(ctx, testBucket, "big") + require.NoError(t, err) + assert.Equal(t, []byte("aaabbb"), got.Data) +} + +// An engine failure fails the write rather than leaving the emulator reporting +// an object whose bytes are in neither the engine nor memory. +func TestStorageEngineFailureFailsThePut(t *testing.T) { + eng := newFakeStorageEngine() + eng.err = assert.AnError + m := newEngineMock(t, eng) + ctx := context.Background() + newBucket(t, m, testBucket) + + err := m.PutObject(ctx, testBucket, "a.txt", []byte("x"), "text/plain", nil) + require.Error(t, err) + assert.Equal(t, cerrors.Internal, cerrors.GetCode(err)) + assert.Contains(t, err.Error(), "storage engine") +} diff --git a/providers/oci/objectstorage/errors.go b/providers/oci/objectstorage/errors.go new file mode 100644 index 000000000..58b64cfff --- /dev/null +++ b/providers/oci/objectstorage/errors.go @@ -0,0 +1,51 @@ +package objectstorage + +import ( + cerrors "github.com/stackshy/cloudemu/v2/errors" +) + +// Object Storage error codes the portable error codes cannot name. +const ( + CodeBucketAlreadyExists = "BucketAlreadyExists" + CodeBucketNotEmpty = "BucketNotEmpty" + CodeIfMatchFailed = "IfMatchFailed" + CodeIfNoneMatchFailed = "IfNoneMatchFailed" +) + +// ServiceError tags a portable error with the code OCI Object Storage reports +// for it. It unwraps to the portable error, so cerrors.GetCode still applies. +type ServiceError struct { + Code string + Err *cerrors.Error +} + +func (e *ServiceError) Error() string { return e.Err.Error() } + +func (e *ServiceError) Unwrap() error { return e.Err } + +func serviceErrorf(code string, portable cerrors.Code, format string, args ...any) error { + return &ServiceError{Code: code, Err: cerrors.Newf(portable, format, args...)} +} + +// checkETag applies if-match and if-none-match to a write against the current +// ETag, empty when nothing exists yet. if-none-match accepts only "*", as OCI's +// writes do. +func checkETag(current, ifMatch, ifNoneMatch string) error { + if ifMatch != "" && (current == "" || (ifMatch != "*" && ifMatch != current)) { + return serviceErrorf(CodeIfMatchFailed, cerrors.FailedPrecondition, + "the if-match ETag %q does not match the current ETag", ifMatch) + } + + switch ifNoneMatch { + case "": + case "*": + if current != "" { + return serviceErrorf(CodeIfNoneMatchFailed, cerrors.FailedPrecondition, + "if-none-match is * but the resource already exists") + } + default: + return cerrors.Newf(cerrors.InvalidArgument, "if-none-match supports only *, got %q", ifNoneMatch) + } + + return nil +} diff --git a/providers/oci/objectstorage/lifecycle.go b/providers/oci/objectstorage/lifecycle.go new file mode 100644 index 000000000..be5309492 --- /dev/null +++ b/providers/oci/objectstorage/lifecycle.go @@ -0,0 +1,449 @@ +package objectstorage + +import ( + "context" + "regexp" + "sort" + "strings" + "time" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/services/storage/driver" +) + +// Lifecycle rule actions OCI accepts. +const ( + LifecycleDelete = "DELETE" + LifecycleArchive = "ARCHIVE" + LifecycleInfrequent = "INFREQUENT_ACCESS" + LifecycleAbort = "ABORT" +) + +// Lifecycle rule targets. An empty target is OCI's default, objects. +const ( + TargetObjects = "objects" + TargetPreviousVersions = "previous-object-versions" + TargetMultipartUploads = "multipart-uploads" +) + +// Lifecycle time units. +const ( + UnitDays = "DAYS" + UnitYears = "YEARS" +) + +// LifecycleRule is one OCI object lifecycle rule, kept exactly as the caller +// sent it so a policy reads back as it was written. +type LifecycleRule struct { + Name string + Action string + TimeAmount int64 + TimeUnit string + Target string + IsEnabled bool + InclusionPrefixes []string + InclusionPatterns []string + ExclusionPatterns []string +} + +// LifecyclePolicy is a bucket's object lifecycle policy. +type LifecyclePolicy struct { + Rules []LifecycleRule + TimeCreated string +} + +// PutLifecyclePolicy replaces a bucket's lifecycle policy. Every rule is +// validated whole: an action, target, unit or pattern CloudEmu cannot honor +// is refused rather than stored and ignored. +func (m *Mock) PutLifecyclePolicy(_ context.Context, bucket string, rules []LifecycleRule) (*LifecyclePolicy, error) { + for i := range rules { + if err := validateLifecycleRule(&rules[i]); err != nil { + return nil, err + } + } + + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + bkt.lifecycle = &LifecyclePolicy{Rules: cloneLifecycleRules(rules), TimeCreated: m.now()} + + return cloneLifecyclePolicy(bkt.lifecycle), nil +} + +// GetLifecyclePolicy returns a bucket's lifecycle policy. +func (m *Mock) GetLifecyclePolicy(_ context.Context, bucket string) (*LifecyclePolicy, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + if bkt.lifecycle == nil { + return nil, cerrors.Newf(cerrors.NotFound, "no lifecycle policy for bucket %q", bucket) + } + + return cloneLifecyclePolicy(bkt.lifecycle), nil +} + +// DeleteLifecyclePolicy removes a bucket's lifecycle policy. +func (m *Mock) DeleteLifecyclePolicy(_ context.Context, bucket string) error { + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return err + } + + if bkt.lifecycle == nil { + return cerrors.Newf(cerrors.NotFound, "no lifecycle policy for bucket %q", bucket) + } + + bkt.lifecycle = nil + + return nil +} + +func validateLifecycleRule(r *LifecycleRule) error { + if r.Name == "" { + return cerrors.New(cerrors.InvalidArgument, "lifecycle rule name is required") + } + + if r.TimeAmount <= 0 { + return cerrors.Newf(cerrors.InvalidArgument, "rule %s: timeAmount must be positive", r.Name) + } + + if r.TimeUnit != UnitDays && r.TimeUnit != UnitYears { + return cerrors.Newf(cerrors.InvalidArgument, "rule %s: unsupported timeUnit %q, want DAYS or YEARS", r.Name, r.TimeUnit) + } + + if err := validateLifecycleAction(r); err != nil { + return err + } + + for _, p := range append(append([]string{}, r.InclusionPatterns...), r.ExclusionPatterns...) { + if _, err := globRegexp(p); err != nil { + return cerrors.Newf(cerrors.InvalidArgument, "rule %s: invalid pattern %q", r.Name, p) + } + } + + return nil +} + +// validateLifecycleAction checks the target and the action against it. +func validateLifecycleAction(r *LifecycleRule) error { + switch r.Target { + case "", TargetObjects, TargetPreviousVersions, TargetMultipartUploads: + default: + return cerrors.Newf(cerrors.InvalidArgument, "rule %s: unsupported target %q", r.Name, r.Target) + } + + switch r.Action { + case LifecycleDelete, LifecycleArchive, LifecycleInfrequent: + if r.Target == TargetMultipartUploads { + return cerrors.Newf(cerrors.InvalidArgument, + "rule %s: the multipart-uploads target only takes the ABORT action", r.Name) + } + case LifecycleAbort: + if r.Target != TargetMultipartUploads { + return cerrors.Newf(cerrors.InvalidArgument, + "rule %s: ABORT requires the multipart-uploads target", r.Name) + } + default: + return cerrors.Newf(cerrors.InvalidArgument, "rule %s: unsupported lifecycle action %q", r.Name, r.Action) + } + + return nil +} + +// lifecycleSpan converts a rule's time amount to a duration. A year is +// 365 days, the same reckoning retention rules use. +func lifecycleSpan(amount int64, unit string) (time.Duration, error) { + switch unit { + case UnitDays: + return time.Duration(amount) * hoursPerDay * time.Hour, nil + case UnitYears: + return time.Duration(amount) * daysPerYear * hoursPerDay * time.Hour, nil + default: + return 0, cerrors.Newf(cerrors.InvalidArgument, "unsupported timeUnit %q, want DAYS or YEARS", unit) + } +} + +// EvaluateLifecycle reports the current object names an enabled DELETE rule +// targeting live objects has aged out. It reports rather than deletes, as the +// other providers' mocks do; a rule aimed at previous versions or multipart +// uploads never selects a live object. +func (m *Mock) EvaluateLifecycle(_ context.Context, bucket string) ([]string, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + if bkt.lifecycle == nil { + return nil, nil + } + + now := m.opts.Clock.Now().UTC() + + var expired []string + + for _, name := range bkt.objects.Keys() { + obj, ok := bkt.objects.Get(name) + if !ok { + continue + } + + if objectExpired(obj, bkt.lifecycle.Rules, now) { + expired = append(expired, name) + } + } + + sort.Strings(expired) + + return expired, nil +} + +func objectExpired(obj *objectData, rules []LifecycleRule, now time.Time) bool { + modified, err := time.Parse(timeFormat, obj.TimeModified) + if err != nil { + return false + } + + age := now.Sub(modified) + + for i := range rules { + r := &rules[i] + if !r.IsEnabled || r.Action != LifecycleDelete || (r.Target != "" && r.Target != TargetObjects) { + continue + } + + if !ruleSelects(r, obj.Name) { + continue + } + + if span, spanErr := lifecycleSpan(r.TimeAmount, r.TimeUnit); spanErr == nil && age >= span { + return true + } + } + + return false +} + +// ruleSelects applies a rule's object-name filter: the name must carry one of +// the inclusion prefixes and match one of the inclusion patterns, when either +// list is given, and must match none of the exclusion patterns. +func ruleSelects(r *LifecycleRule, name string) bool { + if len(r.InclusionPrefixes) > 0 && !anyPrefix(name, r.InclusionPrefixes) { + return false + } + + if len(r.InclusionPatterns) > 0 && !anyPattern(name, r.InclusionPatterns) { + return false + } + + return !anyPattern(name, r.ExclusionPatterns) +} + +func anyPrefix(name string, prefixes []string) bool { + for _, p := range prefixes { + if strings.HasPrefix(name, p) { + return true + } + } + + return false +} + +func anyPattern(name string, patterns []string) bool { + for _, p := range patterns { + if re, err := globRegexp(p); err == nil && re.MatchString(name) { + return true + } + } + + return false +} + +// globRegexp compiles an OCI object-name pattern: * matches any run of +// characters, slashes included, ? matches one, and [...] a character class. +func globRegexp(pattern string) (*regexp.Regexp, error) { + var b strings.Builder + + b.WriteString("^") + + for i := 0; i < len(pattern); i++ { + switch c := pattern[i]; c { + case '*': + b.WriteString(".*") + case '?': + b.WriteString(".") + case '[': + end := strings.IndexByte(pattern[i:], ']') + if end < 0 { + return nil, cerrors.New(cerrors.InvalidArgument, "unterminated character class") + } + + b.WriteString(pattern[i : i+end+1]) + i += end + default: + b.WriteString(regexp.QuoteMeta(string(c))) + } + } + + b.WriteString("$") + + return regexp.Compile(b.String()) +} + +// PutLifecycleConfig is the portable form: each portable rule becomes the OCI +// rules that express its expiration, transition and multipart-abort settings. +func (m *Mock) PutLifecycleConfig(ctx context.Context, bucket string, cfg driver.LifecycleConfig) error { + var rules []LifecycleRule + + for _, r := range cfg.Rules { + converted := fromPortableRule(r) + if len(converted) == 0 { + return cerrors.Newf(cerrors.InvalidArgument, + "lifecycle rule %s sets no expiration, transition or multipart abort", r.ID) + } + + rules = append(rules, converted...) + } + + _, err := m.PutLifecyclePolicy(ctx, bucket, rules) + + return err +} + +//nolint:gocritic // driver.LifecycleRule is the portable value type, iterated by value. +func fromPortableRule(r driver.LifecycleRule) []LifecycleRule { + base := LifecycleRule{Name: r.ID, IsEnabled: r.Enabled, TimeUnit: UnitDays} + if r.Prefix != "" { + base.InclusionPrefixes = []string{r.Prefix} + } + + var out []LifecycleRule + + add := func(suffix, action, target string, days int) { + rule := base + rule.Name += suffix + rule.Action, rule.Target, rule.TimeAmount = action, target, int64(days) + + rule.InclusionPrefixes = cloneStrings(base.InclusionPrefixes) + out = append(out, rule) + } + + if r.ExpirationDays > 0 { + add("", LifecycleDelete, TargetObjects, r.ExpirationDays) + } + + if r.TransitionDays > 0 { + action := LifecycleArchive + if r.TransitionStorageClass == LifecycleInfrequent || r.TransitionStorageClass == TierInfrequentAccess { + action = LifecycleInfrequent + } + + add("-transition", action, TargetObjects, r.TransitionDays) + } + + if r.AbortMultipartDays > 0 { + add("-abort", LifecycleAbort, TargetMultipartUploads, r.AbortMultipartDays) + } + + return out +} + +// GetLifecycleConfig is the portable read. A rule the portable shape cannot +// carry — a non-object target, patterns, several prefixes or a YEARS unit — is +// reported rather than flattened into one that would mean something else. +func (m *Mock) GetLifecycleConfig(ctx context.Context, bucket string) (*driver.LifecycleConfig, error) { + policy, err := m.GetLifecyclePolicy(ctx, bucket) + if err != nil { + return nil, err + } + + out := &driver.LifecycleConfig{Rules: make([]driver.LifecycleRule, 0, len(policy.Rules))} + + for i := range policy.Rules { + r := &policy.Rules[i] + + rule, ok := toPortableRule(r) + if !ok { + return nil, cerrors.Newf(cerrors.Unimplemented, + "lifecycle rule %s uses a target, filter or unit the portable lifecycle shape cannot express; "+ + "read it with GetLifecyclePolicy", r.Name) + } + + out.Rules = append(out.Rules, rule) + } + + return out, nil +} + +// portable reports whether a rule's unit and filter fit the portable shape: +// days, at most one prefix, no patterns. +func portable(r *LifecycleRule) bool { + return r.TimeUnit == UnitDays && len(r.InclusionPatterns) == 0 && len(r.ExclusionPatterns) == 0 && + len(r.InclusionPrefixes) <= 1 +} + +func toPortableRule(r *LifecycleRule) (driver.LifecycleRule, bool) { + if !portable(r) { + return driver.LifecycleRule{}, false + } + + out := driver.LifecycleRule{ID: r.Name, Enabled: r.IsEnabled} + if len(r.InclusionPrefixes) == 1 { + out.Prefix = r.InclusionPrefixes[0] + } + + days := int(r.TimeAmount) + + switch { + case r.Action == LifecycleAbort && r.Target == TargetMultipartUploads: + out.AbortMultipartDays = days + case r.Target != "" && r.Target != TargetObjects: + return driver.LifecycleRule{}, false + case r.Action == LifecycleDelete: + out.ExpirationDays = days + default: + out.TransitionDays, out.TransitionStorageClass = days, r.Action + } + + return out, true +} + +func cloneLifecycleRules(in []LifecycleRule) []LifecycleRule { + out := make([]LifecycleRule, len(in)) + + for i := range in { + out[i] = in[i] + out[i].InclusionPrefixes = cloneStrings(in[i].InclusionPrefixes) + out[i].InclusionPatterns = cloneStrings(in[i].InclusionPatterns) + out[i].ExclusionPatterns = cloneStrings(in[i].ExclusionPatterns) + } + + return out +} + +func cloneLifecyclePolicy(p *LifecyclePolicy) *LifecyclePolicy { + return &LifecyclePolicy{Rules: cloneLifecycleRules(p.Rules), TimeCreated: p.TimeCreated} +} + +func cloneStrings(in []string) []string { + if in == nil { + return nil + } + + return append([]string(nil), in...) +} diff --git a/providers/oci/objectstorage/multipart.go b/providers/oci/objectstorage/multipart.go new file mode 100644 index 000000000..4edaa9eab --- /dev/null +++ b/providers/oci/objectstorage/multipart.go @@ -0,0 +1,283 @@ +package objectstorage + +import ( + "context" + "sort" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/idgen" + "github.com/stackshy/cloudemu/v2/services/storage/driver" +) + +// Part numbers OCI accepts. +const ( + minPartNumber = 1 + maxPartNumber = 10000 +) + +// MultipartUploadSpec is a multipart upload to create, with the OCI-only +// fields the portable CreateMultipartUpload has no room for. +type MultipartUploadSpec struct { + Object string + ContentType string + StorageTier string + Metadata map[string]string +} + +func (m *Mock) CreateMultipartUpload( + ctx context.Context, bucket, key, contentType string, +) (*driver.MultipartUpload, error) { + return m.CreateMultipartUploadWith(ctx, bucket, MultipartUploadSpec{ + Object: key, ContentType: contentType, + }) +} + +// CreateMultipartUploadWith starts a multipart upload carrying OCI's storage +// tier and user metadata. +func (m *Mock) CreateMultipartUploadWith( + _ context.Context, bucket string, spec MultipartUploadSpec, +) (*driver.MultipartUpload, error) { + if spec.Object == "" { + return nil, cerrors.New(cerrors.InvalidArgument, "object name cannot be empty") + } + + if spec.StorageTier != "" && !validStorageTier(spec.StorageTier) { + return nil, cerrors.Newf(cerrors.InvalidArgument, "unsupported storageTier %q", spec.StorageTier) + } + + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + uploadID := idgen.GenerateID("") + now := m.now() + + bkt.multiparts.Set(uploadID, &multipartUpload{ + id: uploadID, + object: spec.Object, + contentType: orDefault(spec.ContentType, "application/octet-stream"), + metadata: cloneMeta(spec.Metadata), + storageTier: orDefault(spec.StorageTier, bkt.StorageTier), + parts: make(map[int][]byte), + timeCreated: now, + }) + + return &driver.MultipartUpload{ + UploadID: uploadID, Bucket: bucket, Key: spec.Object, CreatedAt: now, + }, nil +} + +func (m *Mock) UploadPart( + _ context.Context, bucket, key, uploadID string, partNumber int, data []byte, +) (*driver.UploadPart, error) { + if partNumber < minPartNumber || partNumber > maxPartNumber { + return nil, cerrors.Newf(cerrors.InvalidArgument, + "uploadPartNum must be between %d and %d, got %d", minPartNumber, maxPartNumber, partNumber) + } + + m.mu.Lock() + defer m.mu.Unlock() + + mp, err := m.uploadLocked(bucket, key, uploadID) + if err != nil { + return nil, err + } + + mp.parts[partNumber] = cloneBytes(data) + + return &driver.UploadPart{ + PartNumber: partNumber, ETag: objectETag(data), Size: int64(len(data)), + }, nil +} + +// ListParts returns the parts buffered so far, ordered by part number. +func (m *Mock) ListParts(_ context.Context, bucket, key, uploadID string) ([]driver.UploadPart, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + mp, err := m.uploadLocked(bucket, key, uploadID) + if err != nil { + return nil, err + } + + nums := make([]int, 0, len(mp.parts)) + for n := range mp.parts { + nums = append(nums, n) + } + + sort.Ints(nums) + + out := make([]driver.UploadPart, 0, len(nums)) + + for _, n := range nums { + out = append(out, driver.UploadPart{ + PartNumber: n, ETag: objectETag(mp.parts[n]), Size: int64(len(mp.parts[n])), + }) + } + + return out, nil +} + +// CompleteMultipartUpload assembles the named parts, in ascending part-number +// order, into the upload's object. +func (m *Mock) CompleteMultipartUpload( + ctx context.Context, bucket, key, uploadID string, parts []driver.UploadPart, +) error { + if len(parts) == 0 { + return cerrors.New(cerrors.InvalidArgument, "partsToCommit cannot be empty") + } + + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return err + } + + mp, ok := bkt.multiparts.Get(uploadID) + if !ok { + return cerrors.Newf(cerrors.NotFound, "upload %q not found in bucket %q", uploadID, bucket) + } + + if key != "" && key != mp.object { + return cerrors.Newf(cerrors.InvalidArgument, + "upload %q is for object %q, not %q", uploadID, mp.object, key) + } + + data, err := assembleParts(mp, parts) + if err != nil { + return err + } + + if err := retentionBlocksLocked(bkt, mp.object, m.opts.Clock.Now()); err != nil { + return err + } + + now := m.now() + obj := &objectData{ + Name: mp.object, + Data: data, + Size: int64(len(data)), + ContentType: mp.contentType, + ContentMD5: contentMD5(data), + ETag: newETag(), + TimeCreated: now, + TimeModified: now, + Metadata: cloneMeta(mp.metadata), + StorageTier: mp.storageTier, + } + storeObjectLocked(bkt, obj) + + // Parts stay on the heap until the upload commits; only the assembled + // object reaches the engine. + if err := m.offloadLocked(ctx, bkt, bucket, obj); err != nil { + return err + } + + bkt.multiparts.Delete(uploadID) + + return nil +} + +// assembleParts concatenates the named parts in ascending part-number order, +// refusing a part never uploaded or one whose etag does not match. +func assembleParts(mp *multipartUpload, parts []driver.UploadPart) ([]byte, error) { + ordered := append([]driver.UploadPart(nil), parts...) + sort.Slice(ordered, func(i, j int) bool { return ordered[i].PartNumber < ordered[j].PartNumber }) + + var data []byte + + for _, p := range ordered { + buf, exists := mp.parts[p.PartNumber] + if !exists { + return nil, cerrors.Newf(cerrors.InvalidArgument, "part %d was never uploaded to %q", p.PartNumber, mp.id) + } + + if p.ETag != "" && p.ETag != objectETag(buf) { + return nil, cerrors.Newf(cerrors.InvalidArgument, + "part %d's etag %q does not match the uploaded part", p.PartNumber, p.ETag) + } + + data = append(data, buf...) + } + + return data, nil +} + +func (m *Mock) AbortMultipartUpload(_ context.Context, bucket, key, uploadID string) error { + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return err + } + + mp, ok := bkt.multiparts.Get(uploadID) + if !ok { + return cerrors.Newf(cerrors.NotFound, "upload %q not found in bucket %q", uploadID, bucket) + } + + if key != "" && key != mp.object { + return cerrors.Newf(cerrors.InvalidArgument, + "upload %q is for object %q, not %q", uploadID, mp.object, key) + } + + bkt.multiparts.Delete(uploadID) + + return nil +} + +func (m *Mock) ListMultipartUploads(_ context.Context, bucket string) ([]driver.MultipartUpload, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + ids := bkt.multiparts.Keys() + sort.Strings(ids) + + out := make([]driver.MultipartUpload, 0, len(ids)) + + for _, id := range ids { + mp, ok := bkt.multiparts.Get(id) + if !ok { + continue + } + + out = append(out, driver.MultipartUpload{ + UploadID: mp.id, Bucket: bucket, Key: mp.object, CreatedAt: mp.timeCreated, + }) + } + + return out, nil +} + +// uploadLocked resolves an upload, checking the object name when the caller +// supplied one. Callers hold mu. +func (m *Mock) uploadLocked(bucket, key, uploadID string) (*multipartUpload, error) { + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + mp, ok := bkt.multiparts.Get(uploadID) + if !ok { + return nil, cerrors.Newf(cerrors.NotFound, "upload %q not found in bucket %q", uploadID, bucket) + } + + if key != "" && key != mp.object { + return nil, cerrors.Newf(cerrors.InvalidArgument, + "upload %q is for object %q, not %q", uploadID, mp.object, key) + } + + return mp, nil +} diff --git a/providers/oci/objectstorage/object.go b/providers/oci/objectstorage/object.go new file mode 100644 index 000000000..b9362e207 --- /dev/null +++ b/providers/oci/objectstorage/object.go @@ -0,0 +1,606 @@ +package objectstorage + +import ( + "context" + "maps" + "sort" + "strings" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/pagination" + "github.com/stackshy/cloudemu/v2/services/storage/driver" + "github.com/stackshy/cloudemu/v2/services/storage/storageengine" +) + +// defaultListLimit is the page size OCI applies when the caller names none. +const defaultListLimit = 1000 + +// ObjectDetails is an object as OCI reports it in list and rename responses. +type ObjectDetails struct { + Name string + Size int64 + MD5 string + ETag string + ContentType string + TimeCreated string + TimeModified string + StorageTier string + Metadata map[string]string + VersionID string + DeleteMarker bool +} + +// PutOptions carries the OCI-only fields that ride on a PutObject: the +// storage tier the object lands in and its opc-meta- user metadata. +type PutOptions struct { + ContentType string + StorageTier string + Metadata map[string]string + IfMatch string + IfNoneMatch string +} + +func (m *Mock) PutObject( + ctx context.Context, bucket, key string, data []byte, contentType string, metadata map[string]string, +) error { + _, err := m.PutObjectWith(ctx, bucket, key, data, PutOptions{ + ContentType: contentType, + Metadata: metadata, + }) + + return err +} + +// PutObjectWith stores an object with OCI's per-object settings and returns +// what OCI stamps on the response. +func (m *Mock) PutObjectWith( + ctx context.Context, bucket, key string, data []byte, opts PutOptions, +) (*ObjectDetails, error) { + if key == "" { + return nil, cerrors.New(cerrors.InvalidArgument, "object name cannot be empty") + } + + if opts.StorageTier != "" && !validStorageTier(opts.StorageTier) { + return nil, cerrors.Newf(cerrors.InvalidArgument, "unsupported storageTier %q", opts.StorageTier) + } + + m.mu.Lock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + m.mu.Unlock() + return nil, err + } + + existing, exists := bkt.objects.Get(key) + + currentETag := "" + if exists { + currentETag = existing.ETag + } + + if err := checkETag(currentETag, opts.IfMatch, opts.IfNoneMatch); err != nil { + m.mu.Unlock() + return nil, err + } + + if err := retentionBlocksLocked(bkt, key, m.opts.Clock.Now()); err != nil { + m.mu.Unlock() + return nil, err + } + + now := m.now() + created := now + + if exists { + created = existing.TimeCreated + } + + obj := &objectData{ + Name: key, + Data: cloneBytes(data), + Size: int64(len(data)), + ContentType: orDefault(opts.ContentType, "application/octet-stream"), + ContentMD5: contentMD5(data), + ETag: newETag(), + TimeCreated: created, + TimeModified: now, + Metadata: cloneMeta(opts.Metadata), + StorageTier: orDefault(opts.StorageTier, bkt.StorageTier), + } + storeObjectLocked(bkt, obj) + + if err := m.offloadLocked(ctx, bkt, bucket, obj); err != nil { + m.mu.Unlock() + return nil, err + } + + details := detailsOf(obj) + m.mu.Unlock() + + m.emitMetric("PutRequests", 1, "Count", bucket) + m.emitMetric("StoredBytes", float64(len(data)), "Bytes", bucket) + + return &details, nil +} + +func (m *Mock) GetObject(ctx context.Context, bucket, key string) (*driver.Object, error) { + m.mu.RLock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + m.mu.RUnlock() + return nil, err + } + + obj, err := objectLocked(bkt, key) + if err != nil { + m.mu.RUnlock() + return nil, err + } + + data, err := m.engineLoad(ctx, engineRef(bucket, key, obj.VersionID), obj.Data) + if err != nil { + m.mu.RUnlock() + return nil, err + } + + out := &driver.Object{Info: infoOf(obj), Data: cloneBytes(data)} + m.mu.RUnlock() + + m.emitMetric("GetRequests", 1, "Count", bucket) + + return out, nil +} + +func (m *Mock) HeadObject(_ context.Context, bucket, key string) (*driver.ObjectInfo, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + obj, err := objectLocked(bkt, key) + if err != nil { + return nil, err + } + + info := infoOf(obj) + + return &info, nil +} + +// ObjectDetailsOf returns the OCI projection of an object's current version, +// carrying the fields the portable ObjectInfo has no room for. +func (m *Mock) ObjectDetailsOf(_ context.Context, bucket, key string) (*ObjectDetails, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + obj, err := objectLocked(bkt, key) + if err != nil { + return nil, err + } + + d := detailsOf(obj) + + return &d, nil +} + +func (m *Mock) DeleteObject(ctx context.Context, bucket, key string) error { + m.mu.Lock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + m.mu.Unlock() + return err + } + + if err := retentionBlocksLocked(bkt, key, m.opts.Clock.Now()); err != nil { + m.mu.Unlock() + return err + } + + vid, marker, existed := m.deleteCurrentLocked(bkt, key) + m.purgeLocked(ctx, bucket, key, vid, marker) + m.mu.Unlock() + + if !existed { + return cerrors.Newf(cerrors.NotFound, "object %q not found in bucket %q", key, bucket) + } + + m.emitMetric("DeleteRequests", 1, "Count", bucket) + + return nil +} + +// RenameObject moves an object to a new name within the same bucket, OCI's +// atomic rename action. newName must not already exist. +func (m *Mock) RenameObject(ctx context.Context, bucket, sourceName, newName string) (*ObjectDetails, error) { + if sourceName == "" || newName == "" { + return nil, cerrors.New(cerrors.InvalidArgument, "sourceName and newName are required") + } + + if sourceName == newName { + return nil, cerrors.New(cerrors.InvalidArgument, "newName must differ from sourceName") + } + + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + src, err := objectLocked(bkt, sourceName) + if err != nil { + return nil, err + } + + if bkt.objects.Has(newName) { + return nil, cerrors.Newf(cerrors.AlreadyExists, "object %q already exists in bucket %q", newName, bucket) + } + + if err := retentionBlocksLocked(bkt, sourceName, m.opts.Clock.Now()); err != nil { + return nil, err + } + + srcVersion := src.VersionID + + moved := *src + moved.Name = newName + moved.TimeModified = m.now() + moved.Data = cloneBytes(src.Data) + moved.Metadata = cloneMeta(src.Metadata) + + storeObjectLocked(bkt, &moved) + + if m.engineWired() { + if err := storageengine.Copy(ctx, m.opts.StorageEngine, + engineRef(bucket, newName, moved.VersionID), engineRef(bucket, sourceName, srcVersion)); err != nil { + return nil, err + } + + dropBytesLocked(bkt, &moved) + } + + vid, marker, _ := m.deleteCurrentLocked(bkt, sourceName) + m.purgeLocked(ctx, bucket, sourceName, vid, marker) + + details := detailsOf(&moved) + + return &details, nil +} + +// CopyObject copies an object between buckets in this namespace. OCI runs the +// copy asynchronously; the wire layer records the work request. +func (m *Mock) CopyObject(ctx context.Context, dstBucket, dstKey string, src driver.CopySource) error { + return m.CopyObjectWith(ctx, CopySpec{ + SourceBucket: src.Bucket, SourceObject: src.Key, + DestinationBucket: dstBucket, DestinationObject: dstKey, + }) +} + +// CopySpec is OCI's CopyObjectDetails within one namespace and region. Nil +// Metadata keeps the source's user metadata; an empty StorageTier keeps its +// tier. +type CopySpec struct { + SourceBucket string + SourceObject string + SourceVersionID string + SourceIfMatch string + DestinationBucket string + DestinationObject string + DestinationIfMatch string + // DestinationIfNoneMatch accepts only "*", as OCI's does. + DestinationIfNoneMatch string + Metadata map[string]string + StorageTier string +} + +// CopyObjectWith copies an object, honoring the source version, both sides' +// ETag preconditions, and a replacement metadata set and storage tier. +// +//nolint:gocritic // CopySpec is a request shape, passed by value like BucketSpec. +func (m *Mock) CopyObjectWith(ctx context.Context, spec CopySpec) error { + if spec.StorageTier != "" && !validStorageTier(spec.StorageTier) { + return cerrors.Newf(cerrors.InvalidArgument, "unsupported storageTier %q", spec.StorageTier) + } + + m.mu.Lock() + defer m.mu.Unlock() + + srcBkt, err := m.bucketLocked(spec.SourceBucket) + if err != nil { + return cerrors.Newf(cerrors.NotFound, "source bucket %q not found", spec.SourceBucket) + } + + src, err := copySourceLocked(srcBkt, spec) + if err != nil { + return err + } + + dstBkt, err := m.copyDestinationLocked(spec, src.etag) + if err != nil { + return err + } + + metadata := cloneMeta(src.metadata) + if spec.Metadata != nil { + metadata = cloneMeta(spec.Metadata) + } + + now := m.now() + dstObj := &objectData{ + Name: spec.DestinationObject, + Data: cloneBytes(src.data), + Size: src.size, + ContentType: src.contentType, + ContentMD5: src.contentMD5, + ETag: newETag(), + TimeCreated: now, + TimeModified: now, + Metadata: metadata, + StorageTier: orDefault(spec.StorageTier, src.storageTier), + } + storeObjectLocked(dstBkt, dstObj) + + if m.engineWired() { + if err := storageengine.Copy(ctx, m.opts.StorageEngine, + engineRef(spec.DestinationBucket, spec.DestinationObject, dstObj.VersionID), + engineRef(spec.SourceBucket, spec.SourceObject, src.versionID)); err != nil { + return err + } + + dropBytesLocked(dstBkt, dstObj) + } + + return nil +} + +// copyDestinationLocked resolves a copy's destination bucket once both sides' +// preconditions and the destination's retention hold. Callers hold mu. +// +//nolint:gocritic // mirrors CopyObjectWith's by-value spec. +func (m *Mock) copyDestinationLocked(spec CopySpec, sourceETag string) (*bucketData, error) { + if err := checkETag(sourceETag, spec.SourceIfMatch, ""); err != nil { + return nil, err + } + + dstBkt, err := m.bucketLocked(spec.DestinationBucket) + if err != nil { + return nil, cerrors.Newf(cerrors.NotFound, "destination bucket %q not found", spec.DestinationBucket) + } + + current := targetETagLocked(dstBkt, spec.DestinationObject, "") + if err := checkETag(current, spec.DestinationIfMatch, spec.DestinationIfNoneMatch); err != nil { + return nil, err + } + + if err := retentionBlocksLocked(dstBkt, spec.DestinationObject, m.opts.Clock.Now()); err != nil { + return nil, err + } + + return dstBkt, nil +} + +// copySourceLocked resolves a copy's source: the current object, or the named +// version. Callers hold mu. +// +//nolint:gocritic // mirrors CopyObjectWith's by-value spec. +func copySourceLocked(bkt *bucketData, spec CopySpec) (*objectVersion, error) { + if spec.SourceVersionID == "" { + obj, err := objectLocked(bkt, spec.SourceObject) + if err != nil { + return nil, cerrors.Newf(cerrors.NotFound, "source object %q not found in bucket %q", + spec.SourceObject, spec.SourceBucket) + } + + return versionOf(obj), nil + } + + for _, v := range bkt.versions[spec.SourceObject] { + if v.versionID == spec.SourceVersionID && !v.deleteMarker { + return v, nil + } + } + + return nil, cerrors.Newf(cerrors.NotFound, "version %q of %q not found", spec.SourceVersionID, spec.SourceObject) +} + +func (m *Mock) ListObjects(_ context.Context, bucket string, opts driver.ListOptions) (*driver.ListResult, error) { + m.mu.RLock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + m.mu.RUnlock() + return nil, err + } + + matched, prefixes := matchObjectsLocked(bkt, opts) + m.mu.RUnlock() + + limit := opts.MaxKeys + if limit <= 0 { + limit = defaultListLimit + } + + page, err := pagination.Paginate(matched, opts.PageToken, limit) + if err != nil { + return nil, cerrors.Newf(cerrors.InvalidArgument, "invalid page token: %v", err) + } + + m.emitMetric("ListRequests", 1, "Count", bucket) + + return &driver.ListResult{ + Objects: page.Items, + CommonPrefixes: prefixes, + NextPageToken: page.NextPageToken, + IsTruncated: page.HasMore, + }, nil +} + +// matchObjectsLocked applies prefix and delimiter to the bucket's current +// objects, returning the matches in name order and the rolled-up prefixes. +func matchObjectsLocked(bkt *bucketData, opts driver.ListOptions) (matched []driver.ObjectInfo, prefixes []string) { + for _, obj := range walkObjectsLocked(bkt, opts, &prefixes) { + matched = append(matched, infoOf(obj)) + } + + return matched, prefixes +} + +// walkObjectsLocked returns the objects matching opts in name order, rolling +// the delimiter-collapsed names into prefixes. Callers hold mu. +func walkObjectsLocked(bkt *bucketData, opts driver.ListOptions, prefixes *[]string) []*objectData { + names := bkt.objects.Keys() + sort.Strings(names) + + var matched []*objectData + + prefixSet := make(map[string]struct{}) + + for _, n := range names { + if opts.Prefix != "" && !strings.HasPrefix(n, opts.Prefix) { + continue + } + + if opts.Delimiter != "" { + rest := n[len(opts.Prefix):] + if idx := strings.Index(rest, opts.Delimiter); idx >= 0 { + prefixSet[opts.Prefix+rest[:idx+len(opts.Delimiter)]] = struct{}{} + continue + } + } + + if obj, ok := bkt.objects.Get(n); ok { + matched = append(matched, obj) + } + } + + out := make([]string, 0, len(prefixSet)) + for p := range prefixSet { + out = append(out, p) + } + + sort.Strings(out) + + *prefixes = out + + return matched +} + +// ListObjectDetails is ListObjects in OCI's shape, carrying the storage tier +// and MD5 the portable ObjectInfo has no field for. +func (m *Mock) ListObjectDetails( + _ context.Context, bucket string, opts driver.ListOptions, +) (items []ObjectDetails, prefixes []string, nextPageToken string, err error) { + m.mu.RLock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + m.mu.RUnlock() + return nil, nil, "", err + } + + objects := walkObjectsLocked(bkt, opts, &prefixes) + + matched := make([]ObjectDetails, 0, len(objects)) + for _, obj := range objects { + matched = append(matched, detailsOf(obj)) + } + + m.mu.RUnlock() + + limit := opts.MaxKeys + if limit <= 0 { + limit = defaultListLimit + } + + page, err := pagination.Paginate(matched, opts.PageToken, limit) + if err != nil { + return nil, nil, "", cerrors.Newf(cerrors.InvalidArgument, "invalid page token: %v", err) + } + + return page.Items, prefixes, page.NextPageToken, nil +} + +func infoOf(obj *objectData) driver.ObjectInfo { + return driver.ObjectInfo{ + Key: obj.Name, + Size: obj.Size, + ContentType: obj.ContentType, + ETag: obj.ETag, + LastModified: obj.TimeModified, + Metadata: cloneMeta(obj.Metadata), + VersionID: obj.VersionID, + } +} + +func detailsOf(obj *objectData) ObjectDetails { + return ObjectDetails{ + Name: obj.Name, + Size: obj.Size, + MD5: obj.ContentMD5, + ETag: obj.ETag, + ContentType: obj.ContentType, + TimeCreated: obj.TimeCreated, + TimeModified: obj.TimeModified, + StorageTier: obj.StorageTier, + Metadata: cloneMeta(obj.Metadata), + VersionID: obj.VersionID, + } +} + +// UpdateObjectMetadata replaces an object's opc-meta- user metadata, OCI's +// updateObjectStorageTier sibling for metadata-only changes. +func (m *Mock) UpdateObjectMetadata(_ context.Context, bucket, key string, metadata map[string]string) error { + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return err + } + + obj, err := objectLocked(bkt, key) + if err != nil { + return err + } + + obj.Metadata = maps.Clone(metadata) + obj.TimeModified = m.now() + + return nil +} + +// UpdateObjectStorageTier moves an object between storage tiers. +func (m *Mock) UpdateObjectStorageTier(_ context.Context, bucket, key, tier string) error { + if !validStorageTier(tier) { + return cerrors.Newf(cerrors.InvalidArgument, "unsupported storageTier %q", tier) + } + + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return err + } + + obj, err := objectLocked(bkt, key) + if err != nil { + return err + } + + obj.StorageTier = tier + + return nil +} diff --git a/providers/oci/objectstorage/objectstorage.go b/providers/oci/objectstorage/objectstorage.go new file mode 100644 index 000000000..18995ed9b --- /dev/null +++ b/providers/oci/objectstorage/objectstorage.go @@ -0,0 +1,408 @@ +// Package objectstorage provides an in-memory mock implementation of OCI +// Object Storage. It implements the portable storage driver: an OCI bucket is +// the bucket, an OCI object is the object. Buckets live under a tenancy +// namespace, which the driver derives from the tenancy OCID and exposes to the +// wire layer; the driver itself keys buckets by name, as the portable +// interface does. +package objectstorage + +import ( + "context" + "crypto/md5" //nolint:gosec // OCI reports object content MD5; not a security primitive + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "maps" + "sort" + "sync" + "time" + + "github.com/stackshy/cloudemu/v2/config" + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/idgen" + "github.com/stackshy/cloudemu/v2/internal/memstore" + mondriver "github.com/stackshy/cloudemu/v2/services/monitoring/driver" + "github.com/stackshy/cloudemu/v2/services/scope" + "github.com/stackshy/cloudemu/v2/services/storage/driver" +) + +const timeFormat = time.RFC3339 + +// OCID resource type segments. +const ( + typeBucket = "bucket" + typePAR = "preauthenticatedrequest" + typeRetentionRule = "retentionrule" +) + +// namespaceLen is the length of the generated tenancy namespace. Real +// namespaces are short opaque lowercase strings, not the tenancy OCID. +const namespaceLen = 12 + +// Public access types a bucket may carry. +const ( + AccessNone = "NoPublicAccess" + AccessObjectRead = "ObjectRead" + AccessObjectReadWithoutList = "ObjectReadWithoutList" +) + +// Bucket storage tiers. +const ( + TierStandard = "Standard" + TierArchive = "Archive" + TierInfrequentAccess = "InfrequentAccess" +) + +// Object versioning states. OCI has three, unlike S3's two. +const ( + VersioningDisabled = "Disabled" + VersioningEnabled = "Enabled" + VersioningSuspended = "Suspended" +) + +// nullVersionID is the version reported for objects written while versioning +// was suspended or never enabled. +const nullVersionID = "null" + +// Auto-tiering settings. +const ( + AutoTieringDisabled = "Disabled" + AutoTieringInfreq = "InfrequentAccess" +) + +// Retention rule time units. +const ( + RetentionDays = "DAYS" + RetentionYears = "YEARS" +) + +// PAR access types. +const ( + PARObjectRead = "ObjectRead" + PARObjectWrite = "ObjectWrite" + PARObjectReadWrite = "ObjectReadWrite" + PARAnyObjectRead = "AnyObjectRead" + PARAnyObjectWrite = "AnyObjectWrite" + PARAnyObjectReadWrite = "AnyObjectReadWrite" +) + +// metricNamespace is the OCI Monitoring namespace Object Storage publishes to. +const metricNamespace = "oci_objectstorage" + +// Compile-time check that Mock implements driver.Bucket. +var _ driver.Bucket = (*Mock)(nil) + +// Optional driver capabilities, discovered by type assertion. +var _ driver.VersionedBucket = (*Mock)(nil) + +// objectData is one stored object at its current version. Data is nil once a +// StorageEngine holds the bytes instead, so Size is tracked independently and +// Head/List stay correct after the offload. +type objectData struct { + Name string + Data []byte + Size int64 + ContentType string + ContentMD5 string + ETag string + TimeCreated string + TimeModified string + Metadata map[string]string + StorageTier string + VersionID string +} + +// objectVersion is one entry in a name's version chain, oldest first. A +// delete marker carries no data. +type objectVersion struct { + versionID string + data []byte + size int64 + contentType string + contentMD5 string + etag string + timeModified string + metadata map[string]string + storageTier string + deleteMarker bool +} + +// multipartUpload is an in-progress multipart upload. Parts are buffered until +// the upload is committed. +type multipartUpload struct { + id string + object string + contentType string + metadata map[string]string + storageTier string + parts map[int][]byte + timeCreated string +} + +// bucketData is a bucket and everything hanging off it. +type bucketData struct { + ID string + Name string + Namespace string + CompartmentID string + CreatedBy string + TimeCreated string + ETag string + PublicAccessType string + StorageTier string + Versioning string + KMSKeyID string + AutoTiering string + ObjectEventsEnabled bool + Metadata map[string]string + FreeformTags map[string]string + DefinedTags map[string]map[string]string + + objects *memstore.Store[*objectData] + multiparts *memstore.Store[*multipartUpload] + pars *memstore.Store[*parData] + retention *memstore.Store[*retentionRuleData] + lifecycle *LifecyclePolicy + // versions maps an object name to its chain, oldest first. Only populated + // once versioning has been enabled on the bucket. + versions map[string][]*objectVersion +} + +// Mock is an in-memory mock implementation of OCI Object Storage. +type Mock struct { + // mu guards the fields of stored values and spans the reads and writes a + // single operation makes across a bucket's stores. Each store locks its + // own map, but the pointers it hands back are mutated in place while list + // calls walk them, and a version chain is read before the current object + // is rewritten. + mu sync.RWMutex + + buckets *memstore.Store[*bucketData] + namespace string + opts *config.Options + monitoring mondriver.Monitoring +} + +// New creates a new OCI Object Storage mock. +func New(opts *config.Options) *Mock { + return &Mock{ + buckets: memstore.New[*bucketData](), + namespace: namespaceFor(opts.TenancyOCID), + opts: opts, + } +} + +// SetMonitoring sets the monitoring backend for auto-metric generation. +func (m *Mock) SetMonitoring(mon mondriver.Monitoring) { + m.mu.Lock() + defer m.mu.Unlock() + + m.monitoring = mon +} + +// namespaceFor derives the tenancy's Object Storage namespace. Real OCI mints +// an opaque short string per tenancy; deriving it keeps it stable across runs. +func namespaceFor(tenancyOCID string) string { + sum := sha256.Sum256([]byte(tenancyOCID)) + + return hex.EncodeToString(sum[:])[:namespaceLen] +} + +// Namespace returns the tenancy's Object Storage namespace. +func (m *Mock) Namespace() string { return m.namespace } + +// Region returns the OCI region the emulated Object Storage endpoint serves. +func (m *Mock) Region() string { return m.opts.OCIRegion() } + +// Scope returns the compartment a bucket was created in. It is an OPTIONAL +// capability, discovered by type assertion: the portable Bucket driver has no +// compartment parameter, so OCI scoping is exposed alongside it. +func (m *Mock) Scope(bucket string) scope.Scope { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, ok := m.buckets.Get(bucket) + if !ok { + return scope.Scope{} + } + + return scope.Scope{Compartment: bkt.CompartmentID} +} + +// CreateBucket creates a bucket in the provider's default compartment. The +// OCI wire layer uses CreateBucketWith, which carries the caller's compartment +// and bucket settings. +func (m *Mock) CreateBucket(ctx context.Context, name string) error { + _, err := m.CreateBucketWith(ctx, BucketSpec{Name: name}) + + return err +} + +// DeleteBucket removes an empty bucket. OCI refuses to delete a bucket that +// still holds objects or uncommitted multipart uploads. +func (m *Mock) DeleteBucket(ctx context.Context, name string) error { + return m.DeleteBucketIf(ctx, name, "") +} + +// DeleteBucketIf is DeleteBucket guarded by an if-match ETag. +func (m *Mock) DeleteBucketIf(_ context.Context, name, ifMatch string) error { + m.mu.Lock() + defer m.mu.Unlock() + + bkt, ok := m.buckets.Get(name) + if !ok { + return cerrors.Newf(cerrors.NotFound, "bucket %q not found", name) + } + + if err := checkETag(bkt.ETag, ifMatch, ""); err != nil { + return err + } + + if bkt.objects.Len() > 0 || len(bkt.versions) > 0 { + return serviceErrorf(CodeBucketNotEmpty, cerrors.FailedPrecondition, "bucket %q is not empty", name) + } + + if bkt.multiparts.Len() > 0 { + return serviceErrorf(CodeBucketNotEmpty, cerrors.FailedPrecondition, + "bucket %q has uncommitted multipart uploads", name) + } + + m.buckets.Delete(name) + + return nil +} + +// ListBuckets returns every bucket, unfiltered. The OCI wire layer uses +// ListBucketsIn, which OCI requires a compartment for. +func (m *Mock) ListBuckets(_ context.Context) ([]driver.BucketInfo, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + return m.bucketInfosLocked(""), nil +} + +// bucketInfosLocked projects the buckets in a compartment (all of them when +// compartmentID is empty) onto the portable shape, ordered by name. +func (m *Mock) bucketInfosLocked(compartmentID string) []driver.BucketInfo { + names := m.buckets.Keys() + sort.Strings(names) + + out := make([]driver.BucketInfo, 0, len(names)) + + for _, n := range names { + bkt, ok := m.buckets.Get(n) + if !ok || (compartmentID != "" && bkt.CompartmentID != compartmentID) { + continue + } + + out = append(out, driver.BucketInfo{ + Name: bkt.Name, + Region: m.opts.OCIRegion(), + CreatedAt: bkt.TimeCreated, + }) + } + + return out +} + +// bucketLocked fetches a bucket or the NotFound error naming it. +func (m *Mock) bucketLocked(name string) (*bucketData, error) { + bkt, ok := m.buckets.Get(name) + if !ok { + return nil, cerrors.Newf(cerrors.NotFound, "bucket %q not found", name) + } + + return bkt, nil +} + +// objectLocked fetches an object's current version or the NotFound error. +func objectLocked(bkt *bucketData, name string) (*objectData, error) { + obj, ok := bkt.objects.Get(name) + if !ok { + return nil, cerrors.Newf(cerrors.NotFound, "object %q not found in bucket %q", name, bkt.Name) + } + + return obj, nil +} + +func (m *Mock) now() string { return m.opts.Clock.Now().UTC().Format(timeFormat) } + +// newETag mints the opaque entity tag OCI stamps on buckets and objects. +// newETag mints an opaque ETag per write. It is random rather than a content +// hash, so re-uploading identical bytes still changes it, and rather than the +// process counter, which restarts after a persist restore. +func newETag() string { return idgen.UUID() } + +func contentMD5(data []byte) string { + sum := md5.Sum(data) //nolint:gosec // OCI reports content MD5; not a security primitive + + return base64.StdEncoding.EncodeToString(sum[:]) +} + +func objectETag(data []byte) string { + return fmt.Sprintf("%x", sha256.Sum256(data)) +} + +// serviceMetricSink records a service's own metrics into a compartment, under +// the oci_ namespaces Oracle reserves for them. providers/oci/monitoring.Mock +// implements it; a portable monitoring driver cannot take a reserved namespace. +type serviceMetricSink interface { + PostServiceMetricData(ctx context.Context, compartmentID string, data []mondriver.MetricDatum) error +} + +// emitMetric publishes one Object Storage metric into the bucket's compartment, +// if monitoring is wired. Callers must not hold mu. +func (m *Mock) emitMetric(name string, value float64, unit, bucket string) { + m.mu.RLock() + sink, ok := m.monitoring.(serviceMetricSink) + + compartmentID := m.opts.CompartmentID + if bkt, found := m.buckets.Get(bucket); found { + compartmentID = bkt.CompartmentID + } + m.mu.RUnlock() + + if !ok { + return + } + + _ = sink.PostServiceMetricData(context.Background(), compartmentID, []mondriver.MetricDatum{{ + Namespace: metricNamespace, MetricName: name, Value: value, Unit: unit, + Dimensions: map[string]string{"bucketName": bucket, "resourceId": bucket}, + Timestamp: m.opts.Clock.Now(), + }}) +} + +func cloneMeta(in map[string]string) map[string]string { + if in == nil { + return nil + } + + return maps.Clone(in) +} + +func cloneBytes(in []byte) []byte { + out := make([]byte, len(in)) + copy(out, in) + + return out +} + +// NamespaceMetadata is the tenancy namespace's Object Storage metadata: the +// compartments the S3 and Swift compatibility endpoints create buckets in. +type NamespaceMetadata struct { + Namespace string + DefaultS3CompartmentID string + DefaultSwiftCompartmentID string +} + +// Metadata returns the namespace metadata. Both compatibility endpoints +// default to the provider's compartment. +func (m *Mock) Metadata(_ context.Context) NamespaceMetadata { + return NamespaceMetadata{ + Namespace: m.namespace, + DefaultS3CompartmentID: m.opts.CompartmentID, + DefaultSwiftCompartmentID: m.opts.CompartmentID, + } +} diff --git a/providers/oci/objectstorage/objectstorage_test.go b/providers/oci/objectstorage/objectstorage_test.go new file mode 100644 index 000000000..f7d07e674 --- /dev/null +++ b/providers/oci/objectstorage/objectstorage_test.go @@ -0,0 +1,1542 @@ +package objectstorage_test + +import ( + "context" + "net/http" + "regexp" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stackshy/cloudemu/v2/config" + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/providers/oci/monitoring" + "github.com/stackshy/cloudemu/v2/providers/oci/objectstorage" + "github.com/stackshy/cloudemu/v2/services/storage/driver" +) + +const ( + testCompartment = "ocid1.compartment.oc1..aaaaaaaatest" + otherCompartment = "ocid1.compartment.oc1..aaaaaaaaother" + testBucket = "bucket-a" +) + +func newMock(t *testing.T) *objectstorage.Mock { + t.Helper() + + return objectstorage.New(config.NewOptions( + config.WithRegion("us-ashburn-1"), + config.WithCompartmentID(testCompartment), + )) +} + +// newBucket creates a bucket in testCompartment and fails if the driver refuses. +func newBucket(t *testing.T, m *objectstorage.Mock, name string) *objectstorage.Bucket { + t.Helper() + + b, err := m.CreateBucketWith(context.Background(), objectstorage.BucketSpec{ + Name: name, CompartmentID: testCompartment, + }) + require.NoError(t, err) + + return b +} + +func TestCreateBucket(t *testing.T) { + tests := []struct { + name string + spec objectstorage.BucketSpec + existing string + expectErr cerrors.Code + }{ + { + name: "creates bucket", + spec: objectstorage.BucketSpec{Name: testBucket, CompartmentID: testCompartment}, + }, + { + name: "empty name rejected", + spec: objectstorage.BucketSpec{CompartmentID: testCompartment}, + expectErr: cerrors.InvalidArgument, + }, + { + name: "duplicate name rejected", + spec: objectstorage.BucketSpec{Name: testBucket, CompartmentID: testCompartment}, + existing: testBucket, + expectErr: cerrors.AlreadyExists, + }, + { + name: "unknown public access type rejected", + spec: objectstorage.BucketSpec{ + Name: testBucket, CompartmentID: testCompartment, PublicAccessType: "Everyone", + }, + expectErr: cerrors.InvalidArgument, + }, + { + name: "unknown storage tier rejected", + spec: objectstorage.BucketSpec{ + Name: testBucket, CompartmentID: testCompartment, StorageTier: "Glacier", + }, + expectErr: cerrors.InvalidArgument, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m := newMock(t) + if tc.existing != "" { + newBucket(t, m, tc.existing) + } + + b, err := m.CreateBucketWith(context.Background(), tc.spec) + if tc.expectErr != cerrors.OK { + require.Error(t, err) + assert.Equal(t, tc.expectErr, cerrors.GetCode(err)) + + return + } + + require.NoError(t, err) + assert.Equal(t, tc.spec.Name, b.Name) + assert.Equal(t, testCompartment, b.CompartmentID) + assert.Equal(t, objectstorage.AccessNone, b.PublicAccessType) + assert.Equal(t, objectstorage.TierStandard, b.StorageTier) + assert.Equal(t, objectstorage.VersioningDisabled, b.Versioning) + assert.Equal(t, m.Namespace(), b.Namespace) + }) + } +} + +func TestBucketOCIDShape(t *testing.T) { + m := newMock(t) + b := newBucket(t, m, testBucket) + + assert.Regexp(t, regexp.MustCompile(`^ocid1\.bucket\.oc1\.iad\.[a-z0-9]+$`), b.ID) + + par, err := m.CreatePAR(context.Background(), testBucket, objectstorage.PARSpec{ + Name: "par", ObjectName: "k", AccessType: objectstorage.PARObjectRead, + TimeExpires: time.Now().Add(time.Hour), + }) + require.NoError(t, err) + assert.Regexp(t, regexp.MustCompile(`^ocid1\.preauthenticatedrequest\.oc1\.iad\.[a-z0-9]+$`), par.ID) + + rule, err := m.CreateRetentionRule(context.Background(), testBucket, objectstorage.RetentionRuleSpec{ + DisplayName: "hold", + }) + require.NoError(t, err) + assert.Regexp(t, regexp.MustCompile(`^ocid1\.retentionrule\.oc1\.iad\.[a-z0-9]+$`), rule.ID) +} + +func TestNamespaceIsStablePerTenancy(t *testing.T) { + a := objectstorage.New(config.NewOptions(config.WithTenancyOCID("ocid1.tenancy.oc1..a"))) + b := objectstorage.New(config.NewOptions(config.WithTenancyOCID("ocid1.tenancy.oc1..a"))) + c := objectstorage.New(config.NewOptions(config.WithTenancyOCID("ocid1.tenancy.oc1..b"))) + + assert.Equal(t, a.Namespace(), b.Namespace()) + assert.NotEqual(t, a.Namespace(), c.Namespace()) + assert.Len(t, a.Namespace(), 12) +} + +func TestListBucketsInFiltersByCompartment(t *testing.T) { + m := newMock(t) + newBucket(t, m, "mine") + + _, err := m.CreateBucketWith(context.Background(), objectstorage.BucketSpec{ + Name: "theirs", CompartmentID: otherCompartment, + }) + require.NoError(t, err) + + mine, err := m.ListBucketsIn(context.Background(), testCompartment) + require.NoError(t, err) + require.Len(t, mine, 1) + assert.Equal(t, "mine", mine[0].Name) + + theirs, err := m.ListBucketsIn(context.Background(), otherCompartment) + require.NoError(t, err) + require.Len(t, theirs, 1) + assert.Equal(t, "theirs", theirs[0].Name) + + _, err = m.ListBucketsIn(context.Background(), "") + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) + + assert.Equal(t, testCompartment, m.Scope("mine").Compartment) + assert.Equal(t, otherCompartment, m.Scope("theirs").Compartment) +} + +func TestBucketLifecycleCRUD(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + got, err := m.BucketDetails(ctx, testBucket) + require.NoError(t, err) + assert.Equal(t, testBucket, got.Name) + + _, err = m.BucketDetails(ctx, "missing") + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) + + access := objectstorage.AccessObjectRead + updated, err := m.UpdateBucket(ctx, testBucket, objectstorage.BucketUpdate{PublicAccessType: &access}) + require.NoError(t, err) + assert.Equal(t, objectstorage.AccessObjectRead, updated.PublicAccessType) + assert.NotEqual(t, got.ETag, updated.ETag) + + bogus := "Everyone" + _, err = m.UpdateBucket(ctx, testBucket, objectstorage.BucketUpdate{PublicAccessType: &bogus}) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) + + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v"), "text/plain", nil)) + err = m.DeleteBucket(ctx, testBucket) + require.Error(t, err) + assert.Equal(t, cerrors.FailedPrecondition, cerrors.GetCode(err)) + + require.NoError(t, m.DeleteObject(ctx, testBucket, "k")) + require.NoError(t, m.DeleteBucket(ctx, testBucket)) + + err = m.DeleteBucket(ctx, testBucket) + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) +} + +func TestObjectCRUD(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + details, err := m.PutObjectWith(ctx, testBucket, "dir/a.txt", []byte("hello"), objectstorage.PutOptions{ + ContentType: "text/plain", + Metadata: map[string]string{"owner": "ada"}, + }) + require.NoError(t, err) + assert.Equal(t, int64(5), details.Size) + assert.NotEmpty(t, details.MD5) + assert.Equal(t, objectstorage.TierStandard, details.StorageTier) + + obj, err := m.GetObject(ctx, testBucket, "dir/a.txt") + require.NoError(t, err) + assert.Equal(t, []byte("hello"), obj.Data) + assert.Equal(t, "ada", obj.Info.Metadata["owner"]) + + _, err = m.GetObject(ctx, testBucket, "missing") + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) + + _, err = m.GetObject(ctx, "no-bucket", "dir/a.txt") + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) + + info, err := m.HeadObject(ctx, testBucket, "dir/a.txt") + require.NoError(t, err) + assert.Equal(t, int64(5), info.Size) + + require.NoError(t, m.DeleteObject(ctx, testBucket, "dir/a.txt")) + + err = m.DeleteObject(ctx, testBucket, "dir/a.txt") + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) +} + +func TestListObjectsPrefixAndDelimiter(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + for _, k := range []string{"a/1", "a/2", "a/sub/3", "b/1", "top"} { + require.NoError(t, m.PutObject(ctx, testBucket, k, []byte("x"), "text/plain", nil)) + } + + res, err := m.ListObjects(ctx, testBucket, driver.ListOptions{Prefix: "a/", Delimiter: "/"}) + require.NoError(t, err) + require.Len(t, res.Objects, 2) + assert.Equal(t, "a/1", res.Objects[0].Key) + assert.Equal(t, []string{"a/sub/"}, res.CommonPrefixes) + + all, err := m.ListObjects(ctx, testBucket, driver.ListOptions{}) + require.NoError(t, err) + assert.Len(t, all.Objects, 5) + + _, err = m.ListObjects(ctx, "missing", driver.ListOptions{}) + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) +} + +func TestRenameAndCopyObject(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + newBucket(t, m, "bucket-b") + require.NoError(t, m.PutObject(ctx, testBucket, "old", []byte("v"), "text/plain", nil)) + + renamed, err := m.RenameObject(ctx, testBucket, "old", "new") + require.NoError(t, err) + assert.Equal(t, "new", renamed.Name) + + _, err = m.GetObject(ctx, testBucket, "old") + require.Error(t, err) + + _, err = m.RenameObject(ctx, testBucket, "old", "other") + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) + + require.NoError(t, m.CopyObject(ctx, "bucket-b", "copied", driver.CopySource{Bucket: testBucket, Key: "new"})) + + got, err := m.GetObject(ctx, "bucket-b", "copied") + require.NoError(t, err) + assert.Equal(t, []byte("v"), got.Data) + + err = m.CopyObject(ctx, "bucket-b", "x", driver.CopySource{Bucket: testBucket, Key: "absent"}) + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) +} + +func TestMultipartUpload(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + up, err := m.CreateMultipartUploadWith(ctx, testBucket, objectstorage.MultipartUploadSpec{ + Object: "big", ContentType: "application/octet-stream", + }) + require.NoError(t, err) + + p1, err := m.UploadPart(ctx, testBucket, "big", up.UploadID, 1, []byte("aaa")) + require.NoError(t, err) + p2, err := m.UploadPart(ctx, testBucket, "big", up.UploadID, 2, []byte("bbb")) + require.NoError(t, err) + + _, err = m.UploadPart(ctx, testBucket, "big", up.UploadID, 0, []byte("x")) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) + + parts, err := m.ListParts(ctx, testBucket, "big", up.UploadID) + require.NoError(t, err) + require.Len(t, parts, 2) + + uploads, err := m.ListMultipartUploads(ctx, testBucket) + require.NoError(t, err) + require.Len(t, uploads, 1) + + require.NoError(t, m.CompleteMultipartUpload(ctx, testBucket, "big", up.UploadID, + []driver.UploadPart{{PartNumber: p2.PartNumber}, {PartNumber: p1.PartNumber}})) + + obj, err := m.GetObject(ctx, testBucket, "big") + require.NoError(t, err) + assert.Equal(t, []byte("aaabbb"), obj.Data) + + err = m.AbortMultipartUpload(ctx, testBucket, "big", up.UploadID) + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) +} + +func TestVersioning(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + require.NoError(t, m.SetVersioningStatus(ctx, testBucket, objectstorage.VersioningEnabled)) + + status, err := m.VersioningStatus(ctx, testBucket) + require.NoError(t, err) + assert.Equal(t, objectstorage.VersioningEnabled, status) + + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v1"), "text/plain", nil)) + first, err := m.HeadObject(ctx, testBucket, "k") + require.NoError(t, err) + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v2"), "text/plain", nil)) + + old, err := m.GetObjectVersion(ctx, testBucket, "k", first.VersionID) + require.NoError(t, err) + assert.Equal(t, []byte("v1"), old.Data) + + _, err = m.GetObjectVersion(ctx, testBucket, "k", "nope") + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) + + vid, marker, err := m.DeleteObjectVersion(ctx, testBucket, "k", "") + require.NoError(t, err) + assert.True(t, marker) + assert.NotEmpty(t, vid) + + list, err := m.ListObjectVersions(ctx, testBucket, driver.ListOptions{}) + require.NoError(t, err) + assert.Len(t, list.Versions, 3) + assert.True(t, list.Versions[0].DeleteMarker) + + err = m.SetVersioningStatus(ctx, testBucket, objectstorage.VersioningDisabled) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) +} + +func TestRetentionRules(t *testing.T) { + clock := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + m := objectstorage.New(config.NewOptions( + config.WithRegion("us-ashburn-1"), + config.WithCompartmentID(testCompartment), + config.WithClock(clock), + )) + ctx := context.Background() + newBucket(t, m, testBucket) + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v"), "text/plain", nil)) + + rule, err := m.CreateRetentionRule(ctx, testBucket, objectstorage.RetentionRuleSpec{ + DisplayName: "thirty-days", + Duration: &objectstorage.RetentionDuration{TimeAmount: 30, TimeUnit: objectstorage.RetentionDays}, + }) + require.NoError(t, err) + + err = m.DeleteObject(ctx, testBucket, "k") + require.Error(t, err) + assert.Equal(t, cerrors.FailedPrecondition, cerrors.GetCode(err)) + + rules, err := m.ListRetentionRules(ctx, testBucket) + require.NoError(t, err) + require.Len(t, rules, 1) + + _, err = m.GetRetentionRule(ctx, testBucket, "missing") + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) + + _, err = m.CreateRetentionRule(ctx, testBucket, objectstorage.RetentionRuleSpec{ + Duration: &objectstorage.RetentionDuration{TimeAmount: 1, TimeUnit: "FORTNIGHTS"}, + }) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) + + require.NoError(t, m.DeleteRetentionRule(ctx, testBucket, rule.ID)) + + clock.Advance(31 * 24 * time.Hour) + require.NoError(t, m.DeleteObject(ctx, testBucket, "k")) +} + +func TestRetentionRuleLockCannotBeWeakened(t *testing.T) { + clock := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + m := objectstorage.New(config.NewOptions( + config.WithRegion("us-ashburn-1"), + config.WithCompartmentID(testCompartment), + config.WithClock(clock), + )) + ctx := context.Background() + newBucket(t, m, testBucket) + + lockAt := clock.Now().Add(-time.Hour) + rule, err := m.CreateRetentionRule(ctx, testBucket, objectstorage.RetentionRuleSpec{ + DisplayName: "locked", + Duration: &objectstorage.RetentionDuration{TimeAmount: 10, TimeUnit: objectstorage.RetentionDays}, + TimeRuleLocked: &lockAt, + }) + require.NoError(t, err) + + _, err = m.UpdateRetentionRule(ctx, testBucket, rule.ID, objectstorage.RetentionRuleSpec{ + Duration: &objectstorage.RetentionDuration{TimeAmount: 5, TimeUnit: objectstorage.RetentionDays}, + }) + require.Error(t, err) + assert.Equal(t, cerrors.FailedPrecondition, cerrors.GetCode(err)) + + _, err = m.UpdateRetentionRule(ctx, testBucket, rule.ID, objectstorage.RetentionRuleSpec{ + Duration: &objectstorage.RetentionDuration{TimeAmount: 20, TimeUnit: objectstorage.RetentionDays}, + }) + require.NoError(t, err) + + err = m.DeleteRetentionRule(ctx, testBucket, rule.ID) + require.Error(t, err) + assert.Equal(t, cerrors.FailedPrecondition, cerrors.GetCode(err)) +} + +func TestPreauthenticatedRequests(t *testing.T) { + clock := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + m := objectstorage.New(config.NewOptions( + config.WithRegion("us-ashburn-1"), + config.WithCompartmentID(testCompartment), + config.WithClock(clock), + )) + ctx := context.Background() + newBucket(t, m, testBucket) + + par, err := m.CreatePAR(ctx, testBucket, objectstorage.PARSpec{ + Name: "read-k", ObjectName: "k", AccessType: objectstorage.PARObjectRead, + TimeExpires: clock.Now().Add(time.Hour), + }) + require.NoError(t, err) + assert.Contains(t, par.AccessURI, "/n/"+m.Namespace()+"/b/"+testBucket+"/o/k") + + got, err := m.GetPAR(ctx, testBucket, par.ID) + require.NoError(t, err) + assert.Empty(t, got.AccessURI, "OCI returns the access URI only from create") + + pars, err := m.ListPARs(ctx, testBucket, "") + require.NoError(t, err) + require.Len(t, pars, 1) + + _, err = m.CreatePAR(ctx, testBucket, objectstorage.PARSpec{ + Name: "bad", AccessType: objectstorage.PARObjectRead, + }) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) + + _, err = m.CreatePAR(ctx, testBucket, objectstorage.PARSpec{ + Name: "bad", ObjectName: "k", AccessType: "Whatever", + }) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) + + require.NoError(t, m.DeletePAR(ctx, testBucket, par.ID)) + + err = m.DeletePAR(ctx, testBucket, par.ID) + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) +} + +func TestPARExpiryIsEnforced(t *testing.T) { + clock := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + m := objectstorage.New(config.NewOptions( + config.WithRegion("us-ashburn-1"), + config.WithCompartmentID(testCompartment), + config.WithClock(clock), + )) + ctx := context.Background() + newBucket(t, m, testBucket) + + url, err := m.GeneratePresignedURL(ctx, driver.PresignedURLRequest{ + Bucket: testBucket, Key: "k", Method: "GET", ExpiresIn: time.Hour, + }) + require.NoError(t, err) + require.Contains(t, url.URL, "/p/") + + token := tokenFrom(t, url.URL) + + resolved, err := m.ResolvePAR(ctx, token) + require.NoError(t, err) + assert.Equal(t, objectstorage.PARObjectRead, resolved.AccessType) + assert.True(t, objectstorage.PARAllows(resolved, "GET", "k")) + assert.False(t, objectstorage.PARAllows(resolved, "PUT", "k")) + assert.False(t, objectstorage.PARAllows(resolved, "GET", "other")) + + clock.Advance(2 * time.Hour) + + _, err = m.ResolvePAR(ctx, token) + require.Error(t, err) + assert.Equal(t, cerrors.PermissionDenied, cerrors.GetCode(err)) +} + +// tokenFrom extracts the redemption token from a PAR access URL. +func tokenFrom(t *testing.T, url string) string { + t.Helper() + + re := regexp.MustCompile(`/p/([^/]+)/n/`) + match := re.FindStringSubmatch(url) + require.Len(t, match, 2) + + return match[1] +} + +func TestUnsupportedOperationsAreNamed(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + tests := []struct { + name string + call func() error + }{ + {"PutBucketPolicy", func() error { return m.PutBucketPolicy(ctx, testBucket, driver.BucketPolicy{}) }}, + {"GetBucketPolicy", func() error { _, err := m.GetBucketPolicy(ctx, testBucket); return err }}, + {"PutCORSConfig", func() error { return m.PutCORSConfig(ctx, testBucket, driver.CORSConfig{}) }}, + {"GetObjectTagging", func() error { _, err := m.GetObjectTagging(ctx, testBucket, "k"); return err }}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := tc.call() + require.Error(t, err) + assert.Equal(t, cerrors.Unimplemented, cerrors.GetCode(err)) + assert.Contains(t, err.Error(), tc.name) + }) + } +} + +func TestEncryptionAndTagging(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + cfg, err := m.GetEncryptionConfig(ctx, testBucket) + require.NoError(t, err) + assert.True(t, cfg.Enabled) + assert.Equal(t, "AES256", cfg.Algorithm) + + require.NoError(t, m.PutEncryptionConfig(ctx, testBucket, driver.EncryptionConfig{ + Enabled: true, Algorithm: "oci:kms", KeyID: "ocid1.key.oc1..aaaa", + })) + + cfg, err = m.GetEncryptionConfig(ctx, testBucket) + require.NoError(t, err) + assert.Equal(t, "ocid1.key.oc1..aaaa", cfg.KeyID) + + err = m.PutEncryptionConfig(ctx, testBucket, driver.EncryptionConfig{Enabled: false}) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) + + require.NoError(t, m.PutBucketTagging(ctx, testBucket, map[string]string{"env": "dev"})) + + tags, err := m.GetBucketTagging(ctx, testBucket) + require.NoError(t, err) + assert.Equal(t, map[string]string{"env": "dev"}, tags) + + require.NoError(t, m.DeleteBucketTagging(ctx, testBucket)) + + tags, err = m.GetBucketTagging(ctx, testBucket) + require.NoError(t, err) + assert.Empty(t, tags) +} + +func TestLifecyclePolicyExpiry(t *testing.T) { + clock := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + m := objectstorage.New(config.NewOptions( + config.WithRegion("us-ashburn-1"), + config.WithCompartmentID(testCompartment), + config.WithClock(clock), + )) + ctx := context.Background() + newBucket(t, m, testBucket) + + _, err := m.GetLifecycleConfig(ctx, testBucket) + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) + + expired, err := m.EvaluateLifecycle(ctx, testBucket) + require.NoError(t, err) + assert.Empty(t, expired, "no policy ages nothing out") + + require.NoError(t, m.PutObject(ctx, testBucket, "logs/old.txt", []byte("a"), "text/plain", nil)) + require.NoError(t, m.PutObject(ctx, testBucket, "keep/old.txt", []byte("b"), "text/plain", nil)) + require.NoError(t, m.PutObject(ctx, testBucket, "logs/disabled.txt", []byte("c"), "text/plain", nil)) + + require.NoError(t, m.PutLifecycleConfig(ctx, testBucket, driver.LifecycleConfig{Rules: []driver.LifecycleRule{ + {ID: "expire-logs", Prefix: "logs/", ExpirationDays: 30, Enabled: true}, + {ID: "off", Prefix: "logs/disabled", ExpirationDays: 1, Enabled: false}, + }})) + + stored, err := m.GetLifecycleConfig(ctx, testBucket) + require.NoError(t, err) + require.Len(t, stored.Rules, 2) + + err = m.PutLifecycleConfig(ctx, testBucket, driver.LifecycleConfig{Rules: []driver.LifecycleRule{ + {ID: "keep", Prefix: "keep/", Enabled: true}, + }}) + require.Error(t, err, "a portable rule with no action is refused, not stored as a no-op") + + expired, err = m.EvaluateLifecycle(ctx, testBucket) + require.NoError(t, err) + assert.Empty(t, expired, "nothing has aged out yet") + + // One hour short of the window, then over it. + clock.Advance(30*hoursPerDay*time.Hour - time.Hour) + + expired, err = m.EvaluateLifecycle(ctx, testBucket) + require.NoError(t, err) + assert.Empty(t, expired, "the window is inclusive of ExpirationDays, not shorter") + + clock.Advance(time.Hour) + + expired, err = m.EvaluateLifecycle(ctx, testBucket) + require.NoError(t, err) + assert.Equal(t, []string{"logs/disabled.txt", "logs/old.txt"}, expired, + "only the enabled logs/ rule ages objects out") + + _, err = m.EvaluateLifecycle(ctx, "missing") + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) + + require.NoError(t, m.DeleteLifecyclePolicy(ctx, testBucket)) + + err = m.DeleteLifecyclePolicy(ctx, testBucket) + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) + + require.Error(t, m.PutLifecycleConfig(ctx, "missing", driver.LifecycleConfig{})) +} + +// hoursPerDay mirrors the provider's own day length. +const hoursPerDay = 24 + +func TestPortableVersioningWrappers(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + on, err := m.GetBucketVersioning(ctx, testBucket) + require.NoError(t, err) + assert.False(t, on) + + require.NoError(t, m.SetBucketVersioning(ctx, testBucket, true)) + + on, err = m.GetBucketVersioning(ctx, testBucket) + require.NoError(t, err) + assert.True(t, on) + + status, err := m.VersioningStatus(ctx, testBucket) + require.NoError(t, err) + assert.Equal(t, objectstorage.VersioningEnabled, status) + + // Disabling suspends: OCI never returns a bucket to Disabled. + require.NoError(t, m.SetBucketVersioning(ctx, testBucket, false)) + + status, err = m.VersioningStatus(ctx, testBucket) + require.NoError(t, err) + assert.Equal(t, objectstorage.VersioningSuspended, status) + + on, err = m.GetBucketVersioning(ctx, testBucket) + require.NoError(t, err) + assert.False(t, on) + + require.Error(t, m.SetBucketVersioning(ctx, "missing", true)) + + _, err = m.GetBucketVersioning(ctx, "missing") + require.Error(t, err) +} + +func TestHeadObjectVersion(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + require.NoError(t, m.SetVersioningStatus(ctx, testBucket, objectstorage.VersioningEnabled)) + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v1"), "text/plain", nil)) + + first, err := m.HeadObject(ctx, testBucket, "k") + require.NoError(t, err) + + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v2-longer"), "text/plain", nil)) + + old, err := m.HeadObjectVersion(ctx, testBucket, "k", first.VersionID) + require.NoError(t, err) + assert.Equal(t, first.VersionID, old.VersionID) + assert.Equal(t, int64(2), old.Size) + + // An empty versionID heads the current object. + current, err := m.HeadObjectVersion(ctx, testBucket, "k", "") + require.NoError(t, err) + assert.Equal(t, int64(9), current.Size) + assert.NotEqual(t, first.VersionID, current.VersionID) + + _, err = m.HeadObjectVersion(ctx, testBucket, "k", "nope") + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) + + _, err = m.HeadObjectVersion(ctx, "missing", "k", "some-version") + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) +} + +// The portable driver.Bucket methods are thin wrappers over the OCI-shaped +// ones; a consumer wired through services/storage/driver only sees these. +func TestPortableDriverWrappers(t *testing.T) { + m := newMock(t) + ctx := context.Background() + + require.NoError(t, m.CreateBucket(ctx, testBucket)) + + buckets, err := m.ListBuckets(ctx) + require.NoError(t, err) + require.Len(t, buckets, 1) + assert.Equal(t, testBucket, buckets[0].Name) + assert.Equal(t, "us-ashburn-1", buckets[0].Region) + + details, err := m.BucketDetails(ctx, testBucket) + require.NoError(t, err) + assert.Equal(t, testCompartment, details.CompartmentID, "CreateBucket lands in the default compartment") + + up, err := m.CreateMultipartUpload(ctx, testBucket, "big", "application/octet-stream") + require.NoError(t, err) + assert.Equal(t, "big", up.Key) + require.NoError(t, m.AbortMultipartUpload(ctx, testBucket, "big", up.UploadID)) + + require.Error(t, m.CreateBucket(ctx, testBucket), "duplicate name") +} + +func TestObjectDetailsAndTiering(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + _, err := m.PutObjectWith(ctx, testBucket, "a.txt", []byte("hello"), objectstorage.PutOptions{ + ContentType: "text/plain", Metadata: map[string]string{"owner": "ana"}, + }) + require.NoError(t, err) + + d, err := m.ObjectDetailsOf(ctx, testBucket, "a.txt") + require.NoError(t, err) + assert.Equal(t, int64(5), d.Size) + assert.Equal(t, objectstorage.TierStandard, d.StorageTier) + assert.NotEmpty(t, d.MD5) + assert.Equal(t, map[string]string{"owner": "ana"}, d.Metadata) + + _, err = m.ObjectDetailsOf(ctx, testBucket, "missing") + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) + + _, err = m.ObjectDetailsOf(ctx, "missing", "a.txt") + require.Error(t, err) + + require.NoError(t, m.PutObject(ctx, testBucket, "logs/b.txt", []byte("x"), "text/plain", nil)) + + items, prefixes, next, err := m.ListObjectDetails(ctx, testBucket, driver.ListOptions{Delimiter: "/"}) + require.NoError(t, err) + assert.Empty(t, next) + assert.Equal(t, []string{"logs/"}, prefixes) + require.Len(t, items, 1) + assert.Equal(t, "a.txt", items[0].Name) + + items, _, _, err = m.ListObjectDetails(ctx, testBucket, driver.ListOptions{Prefix: "logs/"}) + require.NoError(t, err) + require.Len(t, items, 1) + assert.Equal(t, "logs/b.txt", items[0].Name) + + _, _, _, err = m.ListObjectDetails(ctx, "missing", driver.ListOptions{}) + require.Error(t, err) + + require.NoError(t, m.UpdateObjectStorageTier(ctx, testBucket, "a.txt", objectstorage.TierArchive)) + + d, err = m.ObjectDetailsOf(ctx, testBucket, "a.txt") + require.NoError(t, err) + assert.Equal(t, objectstorage.TierArchive, d.StorageTier) + + err = m.UpdateObjectStorageTier(ctx, testBucket, "a.txt", "Glacier") + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err), "an unmodelled tier is named, not stored") + + require.Error(t, m.UpdateObjectStorageTier(ctx, testBucket, "missing", objectstorage.TierArchive)) + require.Error(t, m.UpdateObjectStorageTier(ctx, "missing", "a.txt", objectstorage.TierArchive)) + + require.NoError(t, m.UpdateObjectMetadata(ctx, testBucket, "a.txt", map[string]string{"owner": "bo"})) + + d, err = m.ObjectDetailsOf(ctx, testBucket, "a.txt") + require.NoError(t, err) + assert.Equal(t, map[string]string{"owner": "bo"}, d.Metadata) + + require.Error(t, m.UpdateObjectMetadata(ctx, testBucket, "missing", nil)) + require.Error(t, m.UpdateObjectMetadata(ctx, "missing", "a.txt", nil)) +} + +func TestNamespaceMetadata(t *testing.T) { + m := newMock(t) + + meta := m.Metadata(context.Background()) + assert.Equal(t, m.Namespace(), meta.Namespace) + assert.Equal(t, testCompartment, meta.DefaultS3CompartmentID) + assert.Equal(t, testCompartment, meta.DefaultSwiftCompartmentID) +} + +func TestSuspendedVersioningReusesTheNullVersion(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + require.NoError(t, m.SetVersioningStatus(ctx, testBucket, objectstorage.VersioningEnabled)) + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v1"), "text/plain", nil)) + + kept, err := m.HeadObject(ctx, testBucket, "k") + require.NoError(t, err) + + require.NoError(t, m.SetVersioningStatus(ctx, testBucket, objectstorage.VersioningSuspended)) + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v2"), "text/plain", nil)) + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v3"), "text/plain", nil)) + + list, err := m.ListObjectVersions(ctx, testBucket, driver.ListOptions{}) + require.NoError(t, err) + require.Len(t, list.Versions, 2, "the null version is overwritten, not appended") + + null, err := m.GetObjectVersion(ctx, testBucket, "k", "null") + require.NoError(t, err) + assert.Equal(t, []byte("v3"), null.Data) + + // Deleting the current object under Suspended replaces the null version + // with a delete marker, leaving the enabled-era version reachable. + vid, marker, err := m.DeleteObjectVersion(ctx, testBucket, "k", "") + require.NoError(t, err) + assert.Equal(t, "null", vid) + assert.True(t, marker) + + _, err = m.GetObject(ctx, testBucket, "k") + require.Error(t, err) + + old, err := m.GetObjectVersion(ctx, testBucket, "k", kept.VersionID) + require.NoError(t, err) + assert.Equal(t, []byte("v1"), old.Data) + + // Removing the delete marker by id restores the newest remaining version. + _, _, err = m.DeleteObjectVersion(ctx, testBucket, "k", "null") + require.NoError(t, err) + + current, err := m.GetObject(ctx, testBucket, "k") + require.NoError(t, err) + assert.Equal(t, []byte("v1"), current.Data) + + _, _, err = m.DeleteObjectVersion(ctx, testBucket, "k", kept.VersionID) + require.NoError(t, err) + + _, err = m.GetObject(ctx, testBucket, "k") + require.Error(t, err, "the last version leaves no current object") + + _, _, err = m.DeleteObjectVersion(ctx, testBucket, "k", "nope") + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) + + _, _, err = m.DeleteObjectVersion(ctx, "missing", "k", "") + require.Error(t, err) +} + +func TestBucketUpdateRejectsUnmodelledValues(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + tiering := objectstorage.AutoTieringInfreq + moved := otherCompartment + kms := "ocid1.key.oc1..aaaa" + events := true + + updated, err := m.UpdateBucket(ctx, testBucket, objectstorage.BucketUpdate{ + AutoTiering: &tiering, + CompartmentID: &moved, + KMSKeyID: &kms, + ObjectEventsEnabled: &events, + Metadata: map[string]string{"team": "infra"}, + FreeformTags: map[string]string{"env": "dev"}, + DefinedTags: map[string]map[string]string{"ops": {"tier": "gold"}}, + }) + require.NoError(t, err) + assert.Equal(t, objectstorage.AutoTieringInfreq, updated.AutoTiering) + assert.Equal(t, otherCompartment, updated.CompartmentID) + assert.Equal(t, kms, updated.KMSKeyID) + assert.True(t, updated.ObjectEventsEnabled) + assert.Equal(t, map[string]string{"team": "infra"}, updated.Metadata) + assert.Equal(t, map[string]map[string]string{"ops": {"tier": "gold"}}, updated.DefinedTags) + + // The projection is a copy: mutating it must not reach the stored bucket. + updated.DefinedTags["ops"]["tier"] = "bronze" + + again, err := m.BucketDetails(ctx, testBucket) + require.NoError(t, err) + assert.Equal(t, "gold", again.DefinedTags["ops"]["tier"]) + + bogus := "Aggressive" + _, err = m.UpdateBucket(ctx, testBucket, objectstorage.BucketUpdate{AutoTiering: &bogus}) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) + + suspended := objectstorage.VersioningSuspended + _, err = m.UpdateBucket(ctx, testBucket, objectstorage.BucketUpdate{Versioning: &suspended}) + require.NoError(t, err) + + disabled := objectstorage.VersioningDisabled + _, err = m.UpdateBucket(ctx, testBucket, objectstorage.BucketUpdate{Versioning: &disabled}) + require.Error(t, err) + + _, err = m.UpdateBucket(ctx, "missing", objectstorage.BucketUpdate{}) + require.Error(t, err) +} + +func TestCreateBucketRejectsUnmodelledSettings(t *testing.T) { + m := newMock(t) + ctx := context.Background() + + tests := []struct { + name string + spec objectstorage.BucketSpec + }{ + {"storage tier", objectstorage.BucketSpec{Name: "b", CompartmentID: testCompartment, StorageTier: "Glacier"}}, + {"versioning", objectstorage.BucketSpec{Name: "b", CompartmentID: testCompartment, Versioning: "On"}}, + {"auto tiering", objectstorage.BucketSpec{Name: "b", CompartmentID: testCompartment, AutoTiering: "Auto"}}, + {"public access", objectstorage.BucketSpec{Name: "b", CompartmentID: testCompartment, PublicAccessType: "All"}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := m.CreateBucketWith(ctx, tc.spec) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) + }) + } +} + +func TestListPARsFiltersByObjectPrefix(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + expiry := time.Now().Add(time.Hour).UTC() + + for _, name := range []string{"logs/a", "logs/b", "photos/c"} { + _, err := m.CreatePAR(ctx, testBucket, objectstorage.PARSpec{ + Name: name, ObjectName: name, AccessType: objectstorage.PARObjectRead, TimeExpires: expiry, + }) + require.NoError(t, err) + } + + all, err := m.ListPARs(ctx, testBucket, "") + require.NoError(t, err) + assert.Len(t, all, 3) + + logs, err := m.ListPARs(ctx, testBucket, "logs/") + require.NoError(t, err) + assert.Len(t, logs, 2) + + // A prefix longer than the object name matches nothing. + none, err := m.ListPARs(ctx, testBucket, "logs/aaaaaaaaaa") + require.NoError(t, err) + assert.Empty(t, none) +} + +// Every portable operation OCI has no equivalent for must name itself and say +// what OCI does instead, rather than silently succeeding. +func TestEveryUnsupportedOperationIsNamed(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + tests := []struct { + name string + call func() error + }{ + {"DeleteBucketPolicy", func() error { return m.DeleteBucketPolicy(ctx, testBucket) }}, + {"GetCORSConfig", func() error { _, err := m.GetCORSConfig(ctx, testBucket); return err }}, + {"DeleteCORSConfig", func() error { return m.DeleteCORSConfig(ctx, testBucket) }}, + {"PutObjectTagging", func() error { return m.PutObjectTagging(ctx, testBucket, "k", nil) }}, + {"DeleteObjectTagging", func() error { return m.DeleteObjectTagging(ctx, testBucket, "k") }}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := tc.call() + require.Error(t, err) + assert.Equal(t, cerrors.Unimplemented, cerrors.GetCode(err)) + assert.Contains(t, err.Error(), tc.name) + }) + } +} + +func TestPARAccessTypesGrantTheRightVerbs(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + tests := []struct { + accessType string + object string + getOK bool + putOK bool + }{ + {objectstorage.PARObjectRead, "a.txt", true, false}, + {objectstorage.PARObjectWrite, "a.txt", false, true}, + {objectstorage.PARObjectReadWrite, "a.txt", true, true}, + {objectstorage.PARAnyObjectRead, "", true, false}, + {objectstorage.PARAnyObjectWrite, "", false, true}, + {objectstorage.PARAnyObjectReadWrite, "", true, true}, + } + + for _, tc := range tests { + t.Run(tc.accessType, func(t *testing.T) { + par, err := m.CreatePAR(ctx, testBucket, objectstorage.PARSpec{ + Name: tc.accessType, ObjectName: tc.object, AccessType: tc.accessType, + TimeExpires: time.Now().Add(time.Hour), + }) + require.NoError(t, err) + + assert.Equal(t, tc.getOK, objectstorage.PARAllows(par, http.MethodGet, "a.txt")) + assert.Equal(t, tc.getOK, objectstorage.PARAllows(par, http.MethodHead, "a.txt")) + assert.Equal(t, tc.putOK, objectstorage.PARAllows(par, http.MethodPut, "a.txt")) + assert.False(t, objectstorage.PARAllows(par, http.MethodDelete, "a.txt"), + "a PAR never authorizes a delete") + + if tc.object != "" { + assert.False(t, objectstorage.PARAllows(par, http.MethodGet, "other.txt"), + "an object-scoped PAR is bound to its object") + } + }) + } + + _, err := m.CreatePAR(ctx, testBucket, objectstorage.PARSpec{ + Name: "bad", ObjectName: "a.txt", AccessType: "ObjectAppend", + }) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) + + // OCI PARs commonly live for months or years; there is no S3-style week cap. + _, err = m.CreatePAR(ctx, testBucket, objectstorage.PARSpec{ + Name: "a-year-out", ObjectName: "a.txt", AccessType: objectstorage.PARObjectRead, + TimeExpires: time.Now().Add(365 * 24 * time.Hour), + }) + require.NoError(t, err) + + _, err = m.CreatePAR(ctx, testBucket, objectstorage.PARSpec{ + Name: "no-expiry", ObjectName: "a.txt", AccessType: objectstorage.PARObjectRead, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "timeExpires is required") + + // On the AnyObject types objectName is a prefix. + scoped, err := m.CreatePAR(ctx, testBucket, objectstorage.PARSpec{ + Name: "logs-only", ObjectName: "logs/", AccessType: objectstorage.PARAnyObjectRead, + TimeExpires: time.Now().Add(time.Hour), + }) + require.NoError(t, err) + assert.True(t, objectstorage.PARAllows(scoped, http.MethodGet, "logs/app.log")) + assert.False(t, objectstorage.PARAllows(scoped, http.MethodGet, "secret/payroll.csv")) + + _, err = m.CreatePAR(ctx, testBucket, objectstorage.PARSpec{ + Name: "past", ObjectName: "a.txt", AccessType: objectstorage.PARObjectRead, + TimeExpires: time.Now().Add(-time.Hour), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "must be in the future") + + _, err = m.GetPAR(ctx, testBucket, "ocid1.preauthenticatedrequest.oc1..missing") + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) + + require.Error(t, m.DeletePAR(ctx, testBucket, "ocid1.preauthenticatedrequest.oc1..missing")) + + _, err = m.ResolvePAR(ctx, "not-a-token") + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) +} + +func TestRetentionRuleDurationsAndLocking(t *testing.T) { + clock := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + m := objectstorage.New(config.NewOptions( + config.WithRegion("us-ashburn-1"), + config.WithCompartmentID(testCompartment), + config.WithClock(clock), + )) + ctx := context.Background() + newBucket(t, m, testBucket) + + // An unmodelled time unit is named rather than stored. + _, err := m.CreateRetentionRule(ctx, testBucket, objectstorage.RetentionRuleSpec{ + DisplayName: "bad-unit", + Duration: &objectstorage.RetentionDuration{TimeAmount: 1, TimeUnit: "MONTHS"}, + }) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) + assert.Contains(t, err.Error(), "DAYS or YEARS") + + _, err = m.CreateRetentionRule(ctx, testBucket, objectstorage.RetentionRuleSpec{ + DisplayName: "zero", + Duration: &objectstorage.RetentionDuration{TimeAmount: 0, TimeUnit: objectstorage.RetentionDays}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "timeAmount must be positive") + + // YEARS is accepted and is longer than the same amount in DAYS. + years, err := m.CreateRetentionRule(ctx, testBucket, objectstorage.RetentionRuleSpec{ + DisplayName: "one-year", + Duration: &objectstorage.RetentionDuration{TimeAmount: 1, TimeUnit: objectstorage.RetentionYears}, + }) + require.NoError(t, err) + + got, err := m.GetRetentionRule(ctx, testBucket, years.ID) + require.NoError(t, err) + assert.Equal(t, objectstorage.RetentionYears, got.Duration.TimeUnit) + + _, err = m.GetRetentionRule(ctx, testBucket, "ocid1.retentionrule.oc1..missing") + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) + + _, err = m.GetRetentionRule(ctx, "missing", years.ID) + require.Error(t, err) + + // A lock that has not yet engaged leaves the rule fully mutable. + lockAt := clock.Now().Add(time.Hour) + pending, err := m.UpdateRetentionRule(ctx, testBucket, years.ID, objectstorage.RetentionRuleSpec{ + DisplayName: "one-year-locked-soon", + Duration: &objectstorage.RetentionDuration{TimeAmount: 1, TimeUnit: objectstorage.RetentionDays}, + TimeRuleLocked: &lockAt, + }) + require.NoError(t, err) + assert.Equal(t, "one-year-locked-soon", pending.DisplayName) + require.NoError(t, m.DeleteRetentionRule(ctx, testBucket, pending.ID)) + + rule, err := m.CreateRetentionRule(ctx, testBucket, objectstorage.RetentionRuleSpec{ + DisplayName: "locked", + Duration: &objectstorage.RetentionDuration{TimeAmount: 10, TimeUnit: objectstorage.RetentionDays}, + TimeRuleLocked: &lockAt, + }) + require.NoError(t, err) + + clock.Advance(2 * time.Hour) + + // Once locked, the duration cannot be removed, shortened, or the rule deleted. + _, err = m.UpdateRetentionRule(ctx, testBucket, rule.ID, objectstorage.RetentionRuleSpec{DisplayName: "no-duration"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot be removed") + + _, err = m.UpdateRetentionRule(ctx, testBucket, rule.ID, objectstorage.RetentionRuleSpec{ + DisplayName: "shorter", + Duration: &objectstorage.RetentionDuration{TimeAmount: 5, TimeUnit: objectstorage.RetentionDays}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "only be extended") + + extended, err := m.UpdateRetentionRule(ctx, testBucket, rule.ID, objectstorage.RetentionRuleSpec{ + DisplayName: "longer", + Duration: &objectstorage.RetentionDuration{TimeAmount: 20, TimeUnit: objectstorage.RetentionDays}, + }) + require.NoError(t, err) + assert.Equal(t, int64(20), extended.Duration.TimeAmount) + + err = m.DeleteRetentionRule(ctx, testBucket, rule.ID) + require.Error(t, err) + assert.Equal(t, cerrors.FailedPrecondition, cerrors.GetCode(err)) + + _, err = m.UpdateRetentionRule(ctx, testBucket, "ocid1.retentionrule.oc1..missing", + objectstorage.RetentionRuleSpec{}) + require.Error(t, err) + + require.Error(t, m.DeleteRetentionRule(ctx, testBucket, "ocid1.retentionrule.oc1..missing")) + require.Error(t, m.DeleteRetentionRule(ctx, "missing", rule.ID)) +} + +func TestRetentionHoldsObjectsUntilTheyAgeOut(t *testing.T) { + clock := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + m := objectstorage.New(config.NewOptions( + config.WithRegion("us-ashburn-1"), + config.WithCompartmentID(testCompartment), + config.WithClock(clock), + )) + ctx := context.Background() + newBucket(t, m, testBucket) + + require.NoError(t, m.PutObject(ctx, testBucket, "held", []byte("v1"), "text/plain", nil)) + + _, err := m.CreateRetentionRule(ctx, testBucket, objectstorage.RetentionRuleSpec{ + DisplayName: "hold-10-days", + Duration: &objectstorage.RetentionDuration{TimeAmount: 10, TimeUnit: objectstorage.RetentionDays}, + }) + require.NoError(t, err) + + err = m.DeleteObject(ctx, testBucket, "held") + require.Error(t, err) + assert.Equal(t, cerrors.FailedPrecondition, cerrors.GetCode(err)) + assert.Contains(t, err.Error(), "is retained until") + + // An object the rule has not seen yet is not held. + require.NoError(t, m.PutObject(ctx, testBucket, "fresh", []byte("v"), "text/plain", nil)) + + clock.Advance(11 * hoursPerDay * time.Hour) + require.NoError(t, m.DeleteObject(ctx, testBucket, "held")) + + // A rule with no duration holds the bucket indefinitely. + _, err = m.CreateRetentionRule(ctx, testBucket, objectstorage.RetentionRuleSpec{DisplayName: "indefinite"}) + require.NoError(t, err) + + err = m.DeleteObject(ctx, testBucket, "fresh") + require.Error(t, err) + assert.Contains(t, err.Error(), "held indefinitely") + + _, err = m.RenameObject(ctx, testBucket, "fresh", "moved") + require.Error(t, err) + + _, err = m.PutObjectWith(ctx, testBucket, "fresh", []byte("v2"), objectstorage.PutOptions{}) + require.Error(t, err) +} + +func TestListObjectVersionsPrefixAndDelimiter(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + require.NoError(t, m.SetVersioningStatus(ctx, testBucket, objectstorage.VersioningEnabled)) + + for _, k := range []string{"logs/a", "logs/b", "photos/c", "root"} { + require.NoError(t, m.PutObject(ctx, testBucket, k, []byte("v"), "text/plain", nil)) + } + + all, err := m.ListObjectVersions(ctx, testBucket, driver.ListOptions{}) + require.NoError(t, err) + assert.Len(t, all.Versions, 4) + + logs, err := m.ListObjectVersions(ctx, testBucket, driver.ListOptions{Prefix: "logs/"}) + require.NoError(t, err) + assert.Len(t, logs.Versions, 2) + + rolled, err := m.ListObjectVersions(ctx, testBucket, driver.ListOptions{Delimiter: "/"}) + require.NoError(t, err) + assert.Equal(t, []string{"logs/", "photos/"}, rolled.CommonPrefixes) + require.Len(t, rolled.Versions, 1) + assert.Equal(t, "root", rolled.Versions[0].Key) + + _, err = m.ListObjectVersions(ctx, "missing", driver.ListOptions{}) + require.Error(t, err) +} + +// A bucket that never had versioning still reports its current objects as the +// reusable "null" version, which is what OCI does. +func TestListObjectVersionsOnUnversionedBucket(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v"), "text/plain", nil)) + + list, err := m.ListObjectVersions(ctx, testBucket, driver.ListOptions{}) + require.NoError(t, err) + require.Len(t, list.Versions, 1) + assert.Equal(t, "null", list.Versions[0].VersionID) + assert.True(t, list.Versions[0].IsLatest) + assert.Equal(t, int64(1), list.Versions[0].Size) +} + +func TestMultipartRejectsUnmodelledInput(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + _, err := m.CreateMultipartUploadWith(ctx, testBucket, objectstorage.MultipartUploadSpec{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "object name cannot be empty") + + _, err = m.CreateMultipartUploadWith(ctx, testBucket, objectstorage.MultipartUploadSpec{ + Object: "big", StorageTier: "Glacier", + }) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) + + _, err = m.CreateMultipartUploadWith(ctx, "missing", objectstorage.MultipartUploadSpec{Object: "big"}) + require.Error(t, err) + + up, err := m.CreateMultipartUploadWith(ctx, testBucket, objectstorage.MultipartUploadSpec{ + Object: "big", StorageTier: objectstorage.TierInfrequentAccess, + }) + require.NoError(t, err) + + _, err = m.UploadPart(ctx, testBucket, "big", "no-such-upload", 1, []byte("a")) + require.Error(t, err) + assert.Equal(t, cerrors.NotFound, cerrors.GetCode(err)) + + _, err = m.UploadPart(ctx, testBucket, "other-object", up.UploadID, 1, []byte("a")) + require.Error(t, err) + + err = m.CompleteMultipartUpload(ctx, testBucket, "big", up.UploadID, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "partsToCommit cannot be empty") + + err = m.CompleteMultipartUpload(ctx, testBucket, "big", up.UploadID, + []driver.UploadPart{{PartNumber: 9}}) + require.Error(t, err) + assert.Contains(t, err.Error(), "never uploaded") + + err = m.CompleteMultipartUpload(ctx, "missing", "big", up.UploadID, []driver.UploadPart{{PartNumber: 1}}) + require.Error(t, err) + + require.Error(t, m.AbortMultipartUpload(ctx, testBucket, "big", "no-such-upload")) + require.Error(t, m.AbortMultipartUpload(ctx, "missing", "big", up.UploadID)) + require.NoError(t, m.AbortMultipartUpload(ctx, testBucket, "big", up.UploadID)) +} + +func TestPutObjectRejectsUnmodelledInput(t *testing.T) { + m := newMock(t) + ctx := context.Background() + newBucket(t, m, testBucket) + + _, err := m.PutObjectWith(ctx, testBucket, "", []byte("v"), objectstorage.PutOptions{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "object name cannot be empty") + + _, err = m.PutObjectWith(ctx, testBucket, "k", []byte("v"), objectstorage.PutOptions{StorageTier: "Glacier"}) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) + + _, err = m.PutObjectWith(ctx, "missing", "k", []byte("v"), objectstorage.PutOptions{}) + require.Error(t, err) + + _, err = m.HeadObject(ctx, "missing", "k") + require.Error(t, err) + + _, err = m.GetObject(ctx, "missing", "k") + require.Error(t, err) + + require.Error(t, m.DeleteObject(ctx, "missing", "k")) + + err = m.PutEncryptionConfig(ctx, "missing", driver.EncryptionConfig{Enabled: true, Algorithm: "AES256"}) + require.Error(t, err) + + _, err = m.GetEncryptionConfig(ctx, "missing") + require.Error(t, err) + + err = m.PutEncryptionConfig(ctx, testBucket, driver.EncryptionConfig{Enabled: true, Algorithm: "RC4"}) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) + + require.Error(t, m.PutBucketTagging(ctx, "missing", nil)) + + _, err = m.GetBucketTagging(ctx, "missing") + require.Error(t, err) + + require.Error(t, m.DeleteBucketTagging(ctx, "missing")) +} + +// Object bytes moving through the driver publish to OCI Monitoring under the +// oci_objectstorage namespace once a backend is wired. +func TestMetricsEmission(t *testing.T) { + opts := config.NewOptions( + config.WithRegion("us-ashburn-1"), + config.WithCompartmentID(testCompartment), + config.WithClock(config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC))), + ) + m := objectstorage.New(opts) + mon := monitoring.New(opts) + ctx := context.Background() + + // Before a backend is wired the emit is a no-op, not a panic. + require.NoError(t, m.CreateBucket(ctx, testBucket)) + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v"), "text/plain", nil)) + + m.SetMonitoring(mon) + + require.NoError(t, m.PutObject(ctx, testBucket, "k2", []byte("hello"), "text/plain", nil)) + _, err := m.GetObject(ctx, testBucket, "k2") + require.NoError(t, err) + require.NoError(t, m.DeleteObject(ctx, testBucket, "k2")) + + names, err := mon.ListMetrics(ctx, "oci_objectstorage") + require.NoError(t, err) + assert.Subset(t, names, []string{"PutRequests", "StoredBytes", "GetRequests", "DeleteRequests"}) +} + +// A bucket's metrics land in the bucket's own compartment, keyed by +// resourceId, as real oci_objectstorage metrics are. +func TestMetricsLandInTheBucketCompartment(t *testing.T) { + const otherCompartment = "ocid1.compartment.oc1..aaaaaaaaother" + + opts := config.NewOptions(config.WithRegion("us-ashburn-1"), config.WithCompartmentID(testCompartment)) + m := objectstorage.New(opts) + mon := monitoring.New(opts) + ctx := context.Background() + + m.SetMonitoring(mon) + + _, err := m.CreateBucketWith(ctx, objectstorage.BucketSpec{Name: "elsewhere", CompartmentID: otherCompartment}) + require.NoError(t, err) + require.NoError(t, m.PutObject(ctx, "elsewhere", "k", []byte("hello"), "text/plain", nil)) + + filter := monitoring.OCIMetricFilter{Namespace: "oci_objectstorage"} + + there, err := mon.ListOCIMetrics(ctx, otherCompartment, filter) + require.NoError(t, err) + require.NotEmpty(t, there, "metrics must land in the bucket's compartment") + + for i := range there { + assert.Equal(t, "elsewhere", there[i].Dimensions["resourceId"]) + } + + here, err := mon.ListOCIMetrics(ctx, testCompartment, filter) + require.NoError(t, err) + assert.Empty(t, here, "nothing may land in the default compartment") +} + +// Enabling versioning on a bucket that already holds objects must keep the +// original as a prior version once it is overwritten. +func TestEnablingVersioningSeedsExistingObjects(t *testing.T) { + ctx := context.Background() + + for _, tc := range []struct { + name string + enable func(m *objectstorage.Mock) error + }{ + {"SetVersioningStatus", func(m *objectstorage.Mock) error { + return m.SetVersioningStatus(ctx, testBucket, objectstorage.VersioningEnabled) + }}, + {"UpdateBucket", func(m *objectstorage.Mock) error { + on := objectstorage.VersioningEnabled + _, err := m.UpdateBucket(ctx, testBucket, objectstorage.BucketUpdate{Versioning: &on}) + + return err + }}, + {"portable SetBucketVersioning", func(m *objectstorage.Mock) error { + return m.SetBucketVersioning(ctx, testBucket, true) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + for _, eng := range []bool{false, true} { + m := newMock(t) + if eng { + m = newEngineMock(t, newFakeStorageEngine()) + } + + newBucket(t, m, testBucket) + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("original"), "text/plain", nil)) + require.NoError(t, tc.enable(m)) + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("second"), "text/plain", nil)) + + list, err := m.ListObjectVersions(ctx, testBucket, driver.ListOptions{}) + require.NoError(t, err) + require.Len(t, list.Versions, 2, "engine=%v", eng) + + oldest := list.Versions[len(list.Versions)-1] + assert.NotEmpty(t, oldest.VersionID) + + got, err := m.GetObjectVersion(ctx, testBucket, "k", oldest.VersionID) + require.NoError(t, err) + assert.Equal(t, []byte("original"), got.Data, "engine=%v", eng) + } + }) + } +} + +// A rule's target and object-name filter govern what it selects: a rule for +// previous versions never ages out a live object, and patterns apply. +func TestLifecycleHonoursTargetAndPatterns(t *testing.T) { + clock := config.NewFakeClock(time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)) + m := objectstorage.New(config.NewOptions( + config.WithRegion("us-ashburn-1"), + config.WithCompartmentID(testCompartment), + config.WithClock(clock), + )) + ctx := context.Background() + newBucket(t, m, testBucket) + + for _, k := range []string{"logs/app.log", "logs/keep-me.log", "logs/data.csv", "other/x.log"} { + require.NoError(t, m.PutObject(ctx, testBucket, k, []byte("v"), "text/plain", nil)) + } + + policy, err := m.PutLifecyclePolicy(ctx, testBucket, []objectstorage.LifecycleRule{ + { + Name: "versions-only", Action: objectstorage.LifecycleDelete, TimeAmount: 1, + TimeUnit: objectstorage.UnitDays, Target: objectstorage.TargetPreviousVersions, IsEnabled: true, + }, + { + Name: "logs", Action: objectstorage.LifecycleDelete, TimeAmount: 1, TimeUnit: objectstorage.UnitYears, + IsEnabled: true, InclusionPrefixes: []string{"logs/"}, + InclusionPatterns: []string{"*.log"}, ExclusionPatterns: []string{"*keep*"}, + }, + }) + require.NoError(t, err) + assert.NotEmpty(t, policy.TimeCreated) + + clock.Advance(2 * hoursPerDay * time.Hour) + + expired, err := m.EvaluateLifecycle(ctx, testBucket) + require.NoError(t, err) + assert.Empty(t, expired, "the previous-versions rule must not touch live objects") + + clock.Advance(365 * hoursPerDay * time.Hour) + + expired, err = m.EvaluateLifecycle(ctx, testBucket) + require.NoError(t, err) + assert.Equal(t, []string{"logs/app.log"}, expired, "prefix, inclusion and exclusion patterns all apply") + + got, err := m.GetLifecyclePolicy(ctx, testBucket) + require.NoError(t, err) + assert.Equal(t, objectstorage.UnitYears, got.Rules[1].TimeUnit, "the unit reads back as sent") + assert.Equal(t, objectstorage.TargetPreviousVersions, got.Rules[0].Target) + + // The portable read refuses rather than flattening a rule it cannot carry. + _, err = m.GetLifecycleConfig(ctx, testBucket) + require.Error(t, err) + assert.Equal(t, cerrors.Unimplemented, cerrors.GetCode(err)) +} diff --git a/providers/oci/objectstorage/par.go b/providers/oci/objectstorage/par.go new file mode 100644 index 000000000..44d5a46df --- /dev/null +++ b/providers/oci/objectstorage/par.go @@ -0,0 +1,396 @@ +package objectstorage + +import ( + "context" + "crypto/rand" + "encoding/base64" + "net/http" + "sort" + "strings" + "time" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/idgen" + "github.com/stackshy/cloudemu/v2/services/storage/driver" +) + +// parTokenBytes is the entropy behind a redemption token. The token is the +// only secret a PAR carries, so it comes from crypto/rand rather than the +// process counter, which restarts after a persist restore. +const parTokenBytes = 48 + +// defaultPresignLifetime is the lifetime GeneratePresignedURL gives a PAR when +// the portable caller names none; OCI itself requires timeExpires. +const defaultPresignLifetime = 7 * hoursPerDay * time.Hour + +// PreauthenticatedRequest is OCI's presigned-URL equivalent: a first-class +// resource with its own OCID and lifetime, listable and revocable, rather than +// a signature baked into a URL. +type PreauthenticatedRequest struct { + ID string + Name string + Bucket string + ObjectName string + AccessType string + BucketListingAction string + TimeCreated string + TimeExpires string + // AccessURI is the path the request is redeemed at. Real OCI returns it + // only from CreatePreauthenticatedRequest, never from a later Get. + AccessURI string +} + +// PARSpec is a pre-authenticated request to create. +type PARSpec struct { + Name string + ObjectName string + AccessType string + BucketListingAction string + TimeExpires time.Time +} + +type parData struct { + ID string + Name string + Bucket string + ObjectName string + AccessType string + BucketListingAction string + TimeCreated string + TimeExpires time.Time + token string +} + +func validPARAccess(v string) bool { + switch v { + case PARObjectRead, PARObjectWrite, PARObjectReadWrite, + PARAnyObjectRead, PARAnyObjectWrite, PARAnyObjectReadWrite: + return true + } + + return false +} + +// parScopedToObject reports whether an access type binds the request to a +// single named object rather than the whole bucket. +func parScopedToObject(accessType string) bool { + switch accessType { + case PARObjectRead, PARObjectWrite, PARObjectReadWrite: + return true + } + + return false +} + +// CreatePAR creates a pre-authenticated request against a bucket or one of its +// objects. +// +//nolint:gocritic // PARSpec is a request shape, passed by value like the driver's own config structs. +func (m *Mock) CreatePAR(_ context.Context, bucket string, spec PARSpec) (*PreauthenticatedRequest, error) { + if spec.Name == "" { + return nil, cerrors.New(cerrors.InvalidArgument, "name is required") + } + + if !validPARAccess(spec.AccessType) { + return nil, cerrors.Newf(cerrors.InvalidArgument, "unsupported accessType %q", spec.AccessType) + } + + if parScopedToObject(spec.AccessType) && spec.ObjectName == "" { + return nil, cerrors.Newf(cerrors.InvalidArgument, "objectName is required for accessType %q", spec.AccessType) + } + + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + now := m.opts.Clock.Now().UTC() + + expires, err := parExpiry(spec.TimeExpires, now) + if err != nil { + return nil, err + } + + token, err := m.newPARTokenLocked() + if err != nil { + return nil, err + } + + par := &parData{ + ID: m.newPARIDLocked(), + Name: spec.Name, + Bucket: bucket, + ObjectName: spec.ObjectName, + AccessType: spec.AccessType, + BucketListingAction: spec.BucketListingAction, + TimeCreated: now.Format(timeFormat), + TimeExpires: expires, + token: token, + } + + bkt.pars.Set(par.ID, par) + + out := projectPAR(par) + out.AccessURI = m.accessURI(par) + + return out, nil +} + +// parExpiry validates timeExpires. OCI requires one and caps it only at the +// far future, so a PAR a year or more out is accepted. +func parExpiry(requested, now time.Time) (time.Time, error) { + if requested.IsZero() { + return time.Time{}, cerrors.New(cerrors.InvalidArgument, "timeExpires is required") + } + + expires := requested.UTC() + if !expires.After(now) { + return time.Time{}, cerrors.New(cerrors.InvalidArgument, "timeExpires must be in the future") + } + + return expires, nil +} + +// newPARTokenLocked mints a redemption token no live PAR holds. Callers hold mu. +func (m *Mock) newPARTokenLocked() (string, error) { + buf := make([]byte, parTokenBytes) + + for { + if _, err := rand.Read(buf); err != nil { + return "", cerrors.Newf(cerrors.Internal, "generate pre-authenticated request token: %v", err) + } + + token := base64.RawURLEncoding.EncodeToString(buf) + if m.findPARLocked(func(p *parData) bool { return p.token == token }) == nil { + return token, nil + } + } +} + +// newPARIDLocked mints a PAR OCID no live PAR holds. The OCID counter restarts +// after a persist restore while restored PARs keep theirs. Callers hold mu. +func (m *Mock) newPARIDLocked() string { + for { + id := idgen.OCID(typePAR, m.opts.Realm, m.opts.OCIRegion()) + if m.findPARLocked(func(p *parData) bool { return p.ID == id }) == nil { + return id + } + } +} + +// findPARLocked returns the first PAR in any bucket matching pred. Callers +// hold mu. +func (m *Mock) findPARLocked(pred func(*parData) bool) *parData { + for _, name := range m.buckets.Keys() { + bkt, ok := m.buckets.Get(name) + if !ok { + continue + } + + for _, id := range bkt.pars.Keys() { + if par, exists := bkt.pars.Get(id); exists && pred(par) { + return par + } + } + } + + return nil +} + +// accessURI is the path a PAR is redeemed at, matching the shape real OCI +// returns: /p/{token}/n/{namespace}/b/{bucket}/o/{object}. +func (m *Mock) accessURI(par *parData) string { + uri := "/p/" + par.token + "/n/" + m.namespace + "/b/" + par.Bucket + "/o/" + if parScopedToObject(par.AccessType) { + uri += par.ObjectName + } + + return uri +} + +// GetPAR returns one pre-authenticated request. Its access URI is not +// repeated, as real OCI does not repeat it either. +func (m *Mock) GetPAR(_ context.Context, bucket, parID string) (*PreauthenticatedRequest, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + par, ok := bkt.pars.Get(parID) + if !ok { + return nil, cerrors.Newf(cerrors.NotFound, "pre-authenticated request %q not found in bucket %q", parID, bucket) + } + + return projectPAR(par), nil +} + +// ListPARs returns a bucket's pre-authenticated requests, ordered by id and +// optionally filtered to those whose name starts with prefix. +func (m *Mock) ListPARs(_ context.Context, bucket, objectNamePrefix string) ([]PreauthenticatedRequest, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + ids := bkt.pars.Keys() + sort.Strings(ids) + + out := make([]PreauthenticatedRequest, 0, len(ids)) + + for _, id := range ids { + par, ok := bkt.pars.Get(id) + if !ok { + continue + } + + if objectNamePrefix != "" && !hasPrefix(par.ObjectName, objectNamePrefix) { + continue + } + + out = append(out, *projectPAR(par)) + } + + return out, nil +} + +func hasPrefix(s, prefix string) bool { + return len(s) >= len(prefix) && s[:len(prefix)] == prefix +} + +// DeletePAR revokes a pre-authenticated request. +func (m *Mock) DeletePAR(_ context.Context, bucket, parID string) error { + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return err + } + + if !bkt.pars.Delete(parID) { + return cerrors.Newf(cerrors.NotFound, "pre-authenticated request %q not found in bucket %q", parID, bucket) + } + + return nil +} + +// ResolvePAR resolves a redemption token to the request it authorizes, +// refusing one that has expired. It is what makes the access URI usable rather +// than decorative. +func (m *Mock) ResolvePAR(_ context.Context, token string) (*PreauthenticatedRequest, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + par := m.findPARLocked(func(p *parData) bool { return p.token == token }) + if par == nil { + return nil, cerrors.New(cerrors.NotFound, "pre-authenticated request not found") + } + + if !m.opts.Clock.Now().UTC().Before(par.TimeExpires) { + return nil, cerrors.Newf(cerrors.PermissionDenied, "pre-authenticated request %q has expired", par.ID) + } + + return projectPAR(par), nil +} + +// parGrantsRead and parGrantsWrite report which verb an access type grants. +func parGrantsRead(accessType string) bool { + switch accessType { + case PARObjectRead, PARObjectReadWrite, PARAnyObjectRead, PARAnyObjectReadWrite: + return true + } + + return false +} + +func parGrantsWrite(accessType string) bool { + switch accessType { + case PARObjectWrite, PARObjectReadWrite, PARAnyObjectWrite, PARAnyObjectReadWrite: + return true + } + + return false +} + +// PARAllows reports whether a resolved request authorizes method on object. +// An object-scoped PAR names one object; on the AnyObject types objectName is a +// prefix, as OCI treats it. +func PARAllows(par *PreauthenticatedRequest, method, object string) bool { + if parScopedToObject(par.AccessType) { + if par.ObjectName != object { + return false + } + } else if !strings.HasPrefix(object, par.ObjectName) { + return false + } + + switch method { + case http.MethodGet, http.MethodHead: + return parGrantsRead(par.AccessType) + case http.MethodPut: + return parGrantsWrite(par.AccessType) + default: + return false + } +} + +func projectPAR(par *parData) *PreauthenticatedRequest { + return &PreauthenticatedRequest{ + ID: par.ID, + Name: par.Name, + Bucket: par.Bucket, + ObjectName: par.ObjectName, + AccessType: par.AccessType, + BucketListingAction: par.BucketListingAction, + TimeCreated: par.TimeCreated, + TimeExpires: par.TimeExpires.UTC().Format(timeFormat), + } +} + +// GeneratePresignedURL is OCI's pre-authenticated request behind the portable +// name: it creates a real PAR resource and returns its access URI. A caller +// that wants to list or revoke it later uses ListPARs and DeletePAR. +func (m *Mock) GeneratePresignedURL(ctx context.Context, req driver.PresignedURLRequest) (*driver.PresignedURL, error) { + var accessType string + + switch req.Method { + case http.MethodGet: + accessType = PARObjectRead + case http.MethodPut: + accessType = PARObjectWrite + default: + return nil, cerrors.Newf(cerrors.InvalidArgument, "method must be GET or PUT, got %q", req.Method) + } + + expiresIn := req.ExpiresIn + if expiresIn <= 0 { + expiresIn = defaultPresignLifetime + } + + expires := m.opts.Clock.Now().UTC().Add(expiresIn) + + par, err := m.CreatePAR(ctx, req.Bucket, PARSpec{ + Name: "presigned-" + req.Key, + ObjectName: req.Key, + AccessType: accessType, + TimeExpires: expires, + }) + if err != nil { + return nil, err + } + + return &driver.PresignedURL{ + URL: "https://objectstorage." + m.opts.OCIRegion() + ".oraclecloud.com" + par.AccessURI, + Method: req.Method, + ExpiresAt: expires, + }, nil +} diff --git a/providers/oci/objectstorage/race_test.go b/providers/oci/objectstorage/race_test.go new file mode 100644 index 000000000..3e5412993 --- /dev/null +++ b/providers/oci/objectstorage/race_test.go @@ -0,0 +1,99 @@ +package objectstorage_test + +import ( + "context" + "fmt" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/stackshy/cloudemu/v2/config" + "github.com/stackshy/cloudemu/v2/providers/oci/objectstorage" + "github.com/stackshy/cloudemu/v2/services/storage/driver" +) + +// TestConcurrentObjectOperations exercises the paths that read one store while +// writing another, which is what m.mu spans. +func TestConcurrentObjectOperations(t *testing.T) { + m := objectstorage.New(config.NewOptions( + config.WithRegion("us-ashburn-1"), + config.WithCompartmentID(testCompartment), + )) + ctx := context.Background() + + _, err := m.CreateBucketWith(ctx, objectstorage.BucketSpec{ + Name: testBucket, CompartmentID: testCompartment, Versioning: objectstorage.VersioningEnabled, + }) + require.NoError(t, err) + + const workers = 8 + + var wg sync.WaitGroup + + for i := range workers { + wg.Add(1) + + go func(n int) { + defer wg.Done() + + key := fmt.Sprintf("k-%d", n) + _ = m.PutObject(ctx, testBucket, key, []byte("v"), "text/plain", nil) + _, _ = m.GetObject(ctx, testBucket, key) + _, _ = m.HeadObject(ctx, testBucket, key) + _, _ = m.ListObjects(ctx, testBucket, driver.ListOptions{}) + _, _ = m.ListObjectVersions(ctx, testBucket, driver.ListOptions{}) + _, _ = m.BucketDetails(ctx, testBucket) + _, _ = m.ListBucketsIn(ctx, testCompartment) + _, _ = m.GeneratePresignedURL(ctx, driver.PresignedURLRequest{ + Bucket: testBucket, Key: key, Method: "GET", + }) + _, _ = m.ListPARs(ctx, testBucket, "") + _, _, _ = m.DeleteObjectVersion(ctx, testBucket, key, "") + }(i) + } + + wg.Wait() +} + +// TestConcurrentMultipartUploads runs several uploads in parallel over the +// shared multipart store. +func TestConcurrentMultipartUploads(t *testing.T) { + m := objectstorage.New(config.NewOptions( + config.WithRegion("us-ashburn-1"), + config.WithCompartmentID(testCompartment), + )) + ctx := context.Background() + + _, err := m.CreateBucketWith(ctx, objectstorage.BucketSpec{ + Name: testBucket, CompartmentID: testCompartment, + }) + require.NoError(t, err) + + const workers = 8 + + var wg sync.WaitGroup + + for i := range workers { + wg.Add(1) + + go func(n int) { + defer wg.Done() + + key := fmt.Sprintf("big-%d", n) + + up, upErr := m.CreateMultipartUploadWith(ctx, testBucket, objectstorage.MultipartUploadSpec{Object: key}) + if upErr != nil { + return + } + + _, _ = m.UploadPart(ctx, testBucket, key, up.UploadID, 1, []byte("aaa")) + _, _ = m.ListParts(ctx, testBucket, key, up.UploadID) + _, _ = m.ListMultipartUploads(ctx, testBucket) + _ = m.CompleteMultipartUpload(ctx, testBucket, key, up.UploadID, + []driver.UploadPart{{PartNumber: 1}}) + }(i) + } + + wg.Wait() +} diff --git a/providers/oci/objectstorage/retention.go b/providers/oci/objectstorage/retention.go new file mode 100644 index 000000000..530fc80bb --- /dev/null +++ b/providers/oci/objectstorage/retention.go @@ -0,0 +1,322 @@ +package objectstorage + +import ( + "context" + "sort" + "strings" + "time" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/idgen" +) + +const ( + hoursPerDay = 24 + daysPerYear = 365 +) + +// RetentionDuration is how long a rule retains an object after its last +// modification. +type RetentionDuration struct { + TimeAmount int64 + TimeUnit string +} + +// RetentionRule is a bucket retention rule. A rule with no duration is an +// indefinite hold on the whole bucket; a locked rule cannot be shortened or +// deleted, which is what makes OCI retention a compliance control. +type RetentionRule struct { + ID string + DisplayName string + Duration *RetentionDuration + TimeRuleLocked string + TimeCreated string + TimeModified string + ETag string +} + +// RetentionRuleSpec is a rule to create or update. +type RetentionRuleSpec struct { + DisplayName string + Duration *RetentionDuration + TimeRuleLocked *time.Time +} + +type retentionRuleData struct { + ID string + DisplayName string + Duration *RetentionDuration + TimeRuleLocked string + TimeCreated string + TimeModified string + ETag string +} + +func (d RetentionDuration) span() (time.Duration, error) { + switch strings.ToUpper(d.TimeUnit) { + case RetentionDays: + return time.Duration(d.TimeAmount) * hoursPerDay * time.Hour, nil + case RetentionYears: + return time.Duration(d.TimeAmount) * daysPerYear * hoursPerDay * time.Hour, nil + default: + return 0, cerrors.Newf(cerrors.InvalidArgument, "unsupported timeUnit %q, want DAYS or YEARS", d.TimeUnit) + } +} + +// CreateRetentionRule adds a retention rule to a bucket. +func (m *Mock) CreateRetentionRule(_ context.Context, bucket string, spec RetentionRuleSpec) (*RetentionRule, error) { + if err := validateRule(spec); err != nil { + return nil, err + } + + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + now := m.now() + rule := &retentionRuleData{ + ID: idgen.OCID(typeRetentionRule, m.opts.Realm, m.opts.OCIRegion()), + DisplayName: spec.DisplayName, + Duration: spec.Duration, + TimeCreated: now, + TimeModified: now, + ETag: newETag(), + } + + if spec.TimeRuleLocked != nil { + rule.TimeRuleLocked = spec.TimeRuleLocked.UTC().Format(timeFormat) + } + + bkt.retention.Set(rule.ID, rule) + + return projectRule(rule), nil +} + +func validateRule(spec RetentionRuleSpec) error { + if spec.Duration == nil { + return nil + } + + if spec.Duration.TimeAmount <= 0 { + return cerrors.New(cerrors.InvalidArgument, "timeAmount must be positive") + } + + _, err := spec.Duration.span() + + return err +} + +// GetRetentionRule returns one rule. +func (m *Mock) GetRetentionRule(_ context.Context, bucket, ruleID string) (*RetentionRule, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + rule, ok := bkt.retention.Get(ruleID) + if !ok { + return nil, cerrors.Newf(cerrors.NotFound, "retention rule %q not found in bucket %q", ruleID, bucket) + } + + return projectRule(rule), nil +} + +// ListRetentionRules returns a bucket's rules, ordered by id. +func (m *Mock) ListRetentionRules(_ context.Context, bucket string) ([]RetentionRule, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + ids := bkt.retention.Keys() + sort.Strings(ids) + + out := make([]RetentionRule, 0, len(ids)) + + for _, id := range ids { + if rule, ok := bkt.retention.Get(id); ok { + out = append(out, *projectRule(rule)) + } + } + + return out, nil +} + +// UpdateRetentionRule replaces a rule. A locked rule may only be extended, +// which is the whole point of locking one. +func (m *Mock) UpdateRetentionRule( + _ context.Context, bucket, ruleID string, spec RetentionRuleSpec, +) (*RetentionRule, error) { + if err := validateRule(spec); err != nil { + return nil, err + } + + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + rule, ok := bkt.retention.Get(ruleID) + if !ok { + return nil, cerrors.Newf(cerrors.NotFound, "retention rule %q not found in bucket %q", ruleID, bucket) + } + + if err := checkLockedUpdate(rule, spec, m.opts.Clock.Now()); err != nil { + return nil, err + } + + rule.DisplayName = spec.DisplayName + rule.Duration = spec.Duration + rule.TimeModified = m.now() + rule.ETag = newETag() + + if spec.TimeRuleLocked != nil { + rule.TimeRuleLocked = spec.TimeRuleLocked.UTC().Format(timeFormat) + } + + return projectRule(rule), nil +} + +// checkLockedUpdate refuses a change that would weaken an active locked rule. +func checkLockedUpdate(rule *retentionRuleData, spec RetentionRuleSpec, now time.Time) error { + if !ruleLocked(rule, now) { + return nil + } + + if spec.Duration == nil { + return cerrors.Newf(cerrors.FailedPrecondition, + "retention rule %q is locked; its duration cannot be removed", rule.ID) + } + + current, err := rule.Duration.span() + if err != nil { + return err + } + + next, err := spec.Duration.span() + if err != nil { + return err + } + + if next < current { + return cerrors.Newf(cerrors.FailedPrecondition, + "retention rule %q is locked; its duration can only be extended", rule.ID) + } + + return nil +} + +// DeleteRetentionRule removes a rule. A locked rule cannot be deleted. +func (m *Mock) DeleteRetentionRule(_ context.Context, bucket, ruleID string) error { + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return err + } + + rule, ok := bkt.retention.Get(ruleID) + if !ok { + return cerrors.Newf(cerrors.NotFound, "retention rule %q not found in bucket %q", ruleID, bucket) + } + + if ruleLocked(rule, m.opts.Clock.Now()) { + return cerrors.Newf(cerrors.FailedPrecondition, "retention rule %q is locked and cannot be deleted", rule.ID) + } + + bkt.retention.Delete(ruleID) + + return nil +} + +// ruleLocked reports whether a rule's lock has taken effect. OCI gives the +// caller a grace period between requesting the lock and it engaging. +func ruleLocked(rule *retentionRuleData, now time.Time) bool { + if rule.TimeRuleLocked == "" { + return false + } + + at, err := time.Parse(timeFormat, rule.TimeRuleLocked) + if err != nil { + return false + } + + return !now.Before(at) +} + +// retentionBlocksLocked refuses a delete or overwrite that an active retention +// rule protects. A rule with no duration holds every object indefinitely; a +// rule with one holds an object until its last modification ages out. +// Callers hold mu. +func retentionBlocksLocked(bkt *bucketData, name string, now time.Time) error { + if bkt.retention.Len() == 0 { + return nil + } + + obj, ok := bkt.objects.Get(name) + if !ok { + return nil + } + + modified, err := time.Parse(timeFormat, obj.TimeModified) + if err != nil { + return nil //nolint:nilerr // an unparseable timestamp cannot prove the object is retained + } + + for _, id := range bkt.retention.Keys() { + rule, exists := bkt.retention.Get(id) + if !exists { + continue + } + + if rule.Duration == nil { + return cerrors.Newf(cerrors.FailedPrecondition, + "object %q is held indefinitely by retention rule %q", name, rule.ID) + } + + span, spanErr := rule.Duration.span() + if spanErr != nil { + continue + } + + if now.Before(modified.Add(span)) { + return cerrors.Newf(cerrors.FailedPrecondition, + "object %q is retained until %s by retention rule %q", + name, modified.Add(span).UTC().Format(timeFormat), rule.ID) + } + } + + return nil +} + +func projectRule(rule *retentionRuleData) *RetentionRule { + out := &RetentionRule{ + ID: rule.ID, + DisplayName: rule.DisplayName, + TimeRuleLocked: rule.TimeRuleLocked, + TimeCreated: rule.TimeCreated, + TimeModified: rule.TimeModified, + ETag: rule.ETag, + } + + if rule.Duration != nil { + d := *rule.Duration + out.Duration = &d + } + + return out +} diff --git a/providers/oci/objectstorage/review_test.go b/providers/oci/objectstorage/review_test.go new file mode 100644 index 000000000..1617d165f --- /dev/null +++ b/providers/oci/objectstorage/review_test.go @@ -0,0 +1,272 @@ +package objectstorage_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/providers/oci/objectstorage" + "github.com/stackshy/cloudemu/v2/services/storage/driver" +) + +func codeOf(t *testing.T, err error) string { + t.Helper() + + var se *objectstorage.ServiceError + + require.True(t, errors.As(err, &se), "want a ServiceError, got %v", err) + assert.NotEmpty(t, se.Error()) + + return se.Code +} + +// A rename moves the bucket, its objects, versions and PARs, and with an engine +// wired the bytes follow to the new name's references. +func TestRenameBucket(t *testing.T) { + ctx := context.Background() + + for _, wired := range []bool{false, true} { + eng := newFakeStorageEngine() + + m := newMock(t) + if wired { + m = newEngineMock(t, eng) + } + + newBucket(t, m, "old") + newBucket(t, m, "taken") + require.NoError(t, m.SetVersioningStatus(ctx, "old", objectstorage.VersioningEnabled)) + require.NoError(t, m.PutObject(ctx, "old", "k", []byte("v1"), "text/plain", nil)) + require.NoError(t, m.PutObject(ctx, "old", "k", []byte("v2"), "text/plain", nil)) + + par, err := m.CreatePAR(ctx, "old", objectstorage.PARSpec{ + Name: "p", ObjectName: "k", AccessType: objectstorage.PARObjectRead, TimeExpires: inAnHour(), + }) + require.NoError(t, err) + + taken := "taken" + _, err = m.UpdateBucket(ctx, "old", objectstorage.BucketUpdate{Name: &taken}) + require.Error(t, err) + assert.Equal(t, objectstorage.CodeBucketAlreadyExists, codeOf(t, err)) + + bad := "bad name!" + _, err = m.UpdateBucket(ctx, "old", objectstorage.BucketUpdate{Name: &bad}) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err)) + + next := "new" + got, err := m.UpdateBucket(ctx, "old", objectstorage.BucketUpdate{Name: &next}) + require.NoError(t, err) + assert.Equal(t, "new", got.Name) + + _, err = m.BucketDetails(ctx, "old") + require.Error(t, err) + + obj, err := m.GetObject(ctx, "new", "k") + require.NoError(t, err) + assert.Equal(t, []byte("v2"), obj.Data, "engine=%v", wired) + + list, err := m.ListObjectVersions(ctx, "new", driver.ListOptions{}) + require.NoError(t, err) + require.Len(t, list.Versions, 2) + + old, err := m.GetObjectVersion(ctx, "new", "k", list.Versions[1].VersionID) + require.NoError(t, err) + assert.Equal(t, []byte("v1"), old.Data, "engine=%v", wired) + + resolved, err := m.ResolvePAR(ctx, tokenFrom(t, par.AccessURI)) + require.NoError(t, err) + assert.Equal(t, "new", resolved.Bucket, "PARs follow the bucket") + + if wired { + assert.False(t, eng.has("old", "k", list.Versions[0].VersionID), "no bytes left under the old name") + } + } +} + +func TestBucketAndObjectPreconditions(t *testing.T) { + ctx := context.Background() + m := newMock(t) + b := newBucket(t, m, testBucket) + + _, err := m.UpdateBucket(ctx, testBucket, objectstorage.BucketUpdate{IfMatch: "stale"}) + require.Error(t, err) + assert.Equal(t, objectstorage.CodeIfMatchFailed, codeOf(t, err)) + + _, err = m.UpdateBucket(ctx, testBucket, objectstorage.BucketUpdate{IfMatch: b.ETag}) + require.NoError(t, err) + + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v"), "text/plain", nil)) + + err = m.DeleteBucket(ctx, testBucket) + require.Error(t, err) + assert.Equal(t, objectstorage.CodeBucketNotEmpty, codeOf(t, err)) + assert.Equal(t, cerrors.FailedPrecondition, cerrors.GetCode(err), "the portable code still applies") + + _, err = m.PutObjectWith(ctx, testBucket, "k", []byte("x"), objectstorage.PutOptions{IfNoneMatch: "some-etag"}) + require.Error(t, err) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err), "a write's if-none-match supports only *") + + _, err = m.PutObjectWith(ctx, testBucket, "k", []byte("x"), objectstorage.PutOptions{IfMatch: "*"}) + require.NoError(t, err, "if-match * matches any existing object") + + head, err := m.HeadObject(ctx, testBucket, "k") + require.NoError(t, err) + + _, _, err = m.DeleteObjectIf(ctx, testBucket, "k", "", "stale") + require.Error(t, err) + assert.Equal(t, objectstorage.CodeIfMatchFailed, codeOf(t, err)) + + _, _, err = m.DeleteObjectIf(ctx, testBucket, "k", "", head.ETag) + require.NoError(t, err) + + err = m.DeleteBucketIf(ctx, testBucket, "stale") + require.Error(t, err) + assert.Equal(t, objectstorage.CodeIfMatchFailed, codeOf(t, err)) +} + +func TestDeleteVersionIfMatch(t *testing.T) { + ctx := context.Background() + m := newMock(t) + newBucket(t, m, testBucket) + require.NoError(t, m.SetVersioningStatus(ctx, testBucket, objectstorage.VersioningEnabled)) + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v1"), "text/plain", nil)) + + first, err := m.HeadObject(ctx, testBucket, "k") + require.NoError(t, err) + require.NoError(t, m.PutObject(ctx, testBucket, "k", []byte("v2"), "text/plain", nil)) + + _, _, err = m.DeleteObjectIf(ctx, testBucket, "k", first.VersionID, "stale") + require.Error(t, err, "if-match is checked against the version being deleted") + + _, _, err = m.DeleteObjectIf(ctx, testBucket, "k", "nope", "anything") + require.Error(t, err) + + _, _, err = m.DeleteObjectIf(ctx, testBucket, "k", first.VersionID, first.ETag) + require.NoError(t, err) +} + +func TestCopyObjectWithProvider(t *testing.T) { + ctx := context.Background() + m := newMock(t) + newBucket(t, m, "src") + newBucket(t, m, "dst") + require.NoError(t, m.SetVersioningStatus(ctx, "src", objectstorage.VersioningEnabled)) + require.NoError(t, m.PutObject(ctx, "src", "a", []byte("v1"), "text/plain", map[string]string{"o": "1"})) + + first, err := m.HeadObject(ctx, "src", "a") + require.NoError(t, err) + require.NoError(t, m.PutObject(ctx, "src", "a", []byte("v2"), "text/plain", nil)) + + base := objectstorage.CopySpec{SourceBucket: "src", SourceObject: "a", DestinationBucket: "dst", DestinationObject: "b"} + + for name, mutate := range map[string]func(*objectstorage.CopySpec){ + "bad tier": func(s *objectstorage.CopySpec) { s.StorageTier = "Glacier" }, + "missing src bucket": func(s *objectstorage.CopySpec) { s.SourceBucket = "nope" }, + "missing src object": func(s *objectstorage.CopySpec) { s.SourceObject = "nope" }, + "missing src version": func(s *objectstorage.CopySpec) { s.SourceVersionID = "nope" }, + "stale src if-match": func(s *objectstorage.CopySpec) { s.SourceIfMatch = first.ETag }, + "missing dst bucket": func(s *objectstorage.CopySpec) { s.DestinationBucket = "nope" }, + "dst if-match absent": func(s *objectstorage.CopySpec) { s.DestinationIfMatch = "x" }, + } { + spec := base + mutate(&spec) + require.Error(t, m.CopyObjectWith(ctx, spec), name) + } + + spec := base + spec.SourceVersionID = first.VersionID + require.NoError(t, m.CopyObjectWith(ctx, spec)) + + got, err := m.GetObject(ctx, "dst", "b") + require.NoError(t, err) + assert.Equal(t, []byte("v1"), got.Data) + assert.Equal(t, map[string]string{"o": "1"}, got.Info.Metadata, "nil Metadata keeps the source's") + assert.NotEqual(t, first.ETag, got.Info.ETag, "a copy mints its own ETag") + + spec.DestinationIfNoneMatch = "*" + err = m.CopyObjectWith(ctx, spec) + require.Error(t, err) + assert.Equal(t, objectstorage.CodeIfNoneMatchFailed, codeOf(t, err)) +} + +func TestLifecycleRuleValidation(t *testing.T) { + ctx := context.Background() + m := newMock(t) + newBucket(t, m, testBucket) + + ok := objectstorage.LifecycleRule{ + Name: "r", Action: objectstorage.LifecycleDelete, TimeAmount: 1, TimeUnit: objectstorage.UnitDays, + } + + for name, mutate := range map[string]func(*objectstorage.LifecycleRule){ + "no name": func(r *objectstorage.LifecycleRule) { r.Name = "" }, + "zero amount": func(r *objectstorage.LifecycleRule) { r.TimeAmount = 0 }, + "bad unit": func(r *objectstorage.LifecycleRule) { r.TimeUnit = "days" }, + "bad target": func(r *objectstorage.LifecycleRule) { r.Target = "buckets" }, + "bad action": func(r *objectstorage.LifecycleRule) { r.Action = "MOVE" }, + "abort on objects": func(r *objectstorage.LifecycleRule) { r.Action = objectstorage.LifecycleAbort }, + "archive on uploads": func(r *objectstorage.LifecycleRule) { + r.Action, r.Target = objectstorage.LifecycleArchive, objectstorage.TargetMultipartUploads + }, + "unterminated class": func(r *objectstorage.LifecycleRule) { r.ExclusionPatterns = []string{"[a"} }, + } { + rule := ok + mutate(&rule) + + _, err := m.PutLifecyclePolicy(ctx, testBucket, []objectstorage.LifecycleRule{rule}) + require.Error(t, err, name) + assert.Equal(t, cerrors.InvalidArgument, cerrors.GetCode(err), name) + } + + // Character classes and ? are part of the pattern language. + rule := ok + rule.InclusionPatterns = []string{"log-[0-9]?.txt"} + _, err := m.PutLifecyclePolicy(ctx, testBucket, []objectstorage.LifecycleRule{rule}) + require.NoError(t, err) + + _, err = m.PutLifecyclePolicy(ctx, "missing", []objectstorage.LifecycleRule{ok}) + require.Error(t, err) + + _, err = m.GetLifecyclePolicy(ctx, "missing") + require.Error(t, err) + + // The portable path round-trips through the OCI rules it expands to. + require.NoError(t, m.PutLifecycleConfig(ctx, testBucket, driver.LifecycleConfig{Rules: []driver.LifecycleRule{ + {ID: "all", Prefix: "p/", Enabled: true, ExpirationDays: 30, TransitionDays: 10, + TransitionStorageClass: objectstorage.TierInfrequentAccess, AbortMultipartDays: 7}, + }})) + + policy, err := m.GetLifecyclePolicy(ctx, testBucket) + require.NoError(t, err) + require.Len(t, policy.Rules, 3) + assert.Equal(t, objectstorage.LifecycleInfrequent, policy.Rules[1].Action) + assert.Equal(t, objectstorage.TargetMultipartUploads, policy.Rules[2].Target) + + cfg, err := m.GetLifecycleConfig(ctx, testBucket) + require.NoError(t, err) + require.Len(t, cfg.Rules, 3) + assert.Equal(t, 30, cfg.Rules[0].ExpirationDays) + assert.Equal(t, 10, cfg.Rules[1].TransitionDays) + assert.Equal(t, 7, cfg.Rules[2].AbortMultipartDays) + + // A previous-versions rule has no portable form. + rule = ok + rule.Target = objectstorage.TargetPreviousVersions + _, err = m.PutLifecyclePolicy(ctx, testBucket, []objectstorage.LifecycleRule{rule}) + require.NoError(t, err) + + _, err = m.GetLifecycleConfig(ctx, testBucket) + assert.Equal(t, cerrors.Unimplemented, cerrors.GetCode(err)) +} + +func TestRegion(t *testing.T) { + assert.Equal(t, "us-ashburn-1", newMock(t).Region()) +} + +func inAnHour() time.Time { return time.Now().Add(time.Hour) } diff --git a/providers/oci/objectstorage/snapshot.go b/providers/oci/objectstorage/snapshot.go new file mode 100644 index 000000000..0437d2df4 --- /dev/null +++ b/providers/oci/objectstorage/snapshot.go @@ -0,0 +1,329 @@ +package objectstorage + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/stackshy/cloudemu/v2/internal/memstore" + "github.com/stackshy/cloudemu/v2/internal/snapshot" +) + +var _ snapshot.Snapshottable = (*Mock)(nil) + +// storageSnapshot is the full serialized state of the Object Storage mock: +// every bucket keyed by name. The namespace is derived from the tenancy rather +// than captured, so a restore into a differently configured emulator keeps that +// emulator's namespace instead of resurrecting a stale one. +type storageSnapshot struct { + Buckets map[string]*bucketSnapshot `json:"buckets,omitempty"` +} + +// bucketSnapshot captures a bucket's OCI settings, its current objects, its +// full version history, its pre-authenticated requests, its retention rules and +// its lifecycle policy. Object names, version ids, PAR OCIDs and redemption +// tokens are preserved, so an access URI issued before the snapshot still +// redeems after the restore. In-progress multipart uploads are transient and +// intentionally not captured, matching the other storage providers. +type bucketSnapshot struct { + ID string `json:"id"` + Name string `json:"name"` + CompartmentID string `json:"compartmentId,omitempty"` + CreatedBy string `json:"createdBy,omitempty"` + TimeCreated string `json:"timeCreated,omitempty"` + ETag string `json:"etag,omitempty"` + PublicAccessType string `json:"publicAccessType,omitempty"` + StorageTier string `json:"storageTier,omitempty"` + Versioning string `json:"versioning,omitempty"` + KMSKeyID string `json:"kmsKeyId,omitempty"` + AutoTiering string `json:"autoTiering,omitempty"` + ObjectEventsEnabled bool `json:"objectEventsEnabled,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + FreeformTags map[string]string `json:"freeformTags,omitempty"` + DefinedTags map[string]map[string]string `json:"definedTags,omitempty"` + Lifecycle *LifecyclePolicy `json:"lifecycle,omitempty"` + Objects map[string]*objectSnapshot `json:"objects,omitempty"` + Versions map[string][]*versionSnapshot `json:"versions,omitempty"` + PARs []*parSnapshot `json:"pars,omitempty"` + Retention []*retentionSnapshot `json:"retention,omitempty"` +} + +// objectSnapshot is a current object. Data is omitted in a metadata-only +// (includeAssets=false) snapshot; Size is kept independently so Head and List +// stay correct without it. +type objectSnapshot struct { + Name string `json:"name"` + Data []byte `json:"data,omitempty"` + Size int64 `json:"size"` + ContentType string `json:"contentType,omitempty"` + ContentMD5 string `json:"contentMd5,omitempty"` + ETag string `json:"etag,omitempty"` + TimeCreated string `json:"timeCreated,omitempty"` + TimeModified string `json:"timeModified,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + StorageTier string `json:"storageTier,omitempty"` + VersionID string `json:"versionId,omitempty"` +} + +// versionSnapshot is one entry in a name's version chain, oldest first. Its +// fields mirror the unexported objectVersion, which json.Marshal cannot see. +type versionSnapshot struct { + VersionID string `json:"versionId"` + Data []byte `json:"data,omitempty"` + Size int64 `json:"size"` + ContentType string `json:"contentType,omitempty"` + ContentMD5 string `json:"contentMd5,omitempty"` + ETag string `json:"etag,omitempty"` + TimeModified string `json:"timeModified,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + StorageTier string `json:"storageTier,omitempty"` + DeleteMarker bool `json:"deleteMarker,omitempty"` +} + +// parSnapshot carries the redemption token as well as the OCID, so an access +// URI handed out before the snapshot still resolves afterwards. +type parSnapshot struct { + ID string `json:"id"` + Name string `json:"name,omitempty"` + Bucket string `json:"bucket,omitempty"` + ObjectName string `json:"objectName,omitempty"` + AccessType string `json:"accessType,omitempty"` + BucketListingAction string `json:"bucketListingAction,omitempty"` + TimeCreated string `json:"timeCreated,omitempty"` + TimeExpires string `json:"timeExpires,omitempty"` + Token string `json:"token,omitempty"` +} + +type retentionSnapshot struct { + ID string `json:"id"` + DisplayName string `json:"displayName,omitempty"` + Duration *RetentionDuration `json:"duration,omitempty"` + TimeRuleLocked string `json:"timeRuleLocked,omitempty"` + TimeCreated string `json:"timeCreated,omitempty"` + TimeModified string `json:"timeModified,omitempty"` + ETag string `json:"etag,omitempty"` +} + +// Snapshot captures every bucket's state as JSON. When includeAssets is false +// the object and version bytes are omitted, mirroring the persist default that +// keeps snapshot files small. +func (m *Mock) Snapshot(_ context.Context, includeAssets bool) (json.RawMessage, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + snap := storageSnapshot{Buckets: make(map[string]*bucketSnapshot, m.buckets.Len())} + + for name, bkt := range m.buckets.All() { + snap.Buckets[name] = snapshotBucket(bkt, includeAssets) + } + + return json.Marshal(snap) +} + +func snapshotBucket(bkt *bucketData, includeAssets bool) *bucketSnapshot { + bs := &bucketSnapshot{ + ID: bkt.ID, Name: bkt.Name, CompartmentID: bkt.CompartmentID, CreatedBy: bkt.CreatedBy, + TimeCreated: bkt.TimeCreated, ETag: bkt.ETag, PublicAccessType: bkt.PublicAccessType, + StorageTier: bkt.StorageTier, Versioning: bkt.Versioning, KMSKeyID: bkt.KMSKeyID, + AutoTiering: bkt.AutoTiering, ObjectEventsEnabled: bkt.ObjectEventsEnabled, + Metadata: bkt.Metadata, FreeformTags: bkt.FreeformTags, DefinedTags: bkt.DefinedTags, + Lifecycle: bkt.lifecycle, + Objects: make(map[string]*objectSnapshot, bkt.objects.Len()), + } + + for name, obj := range bkt.objects.All() { + bs.Objects[name] = &objectSnapshot{ + Name: obj.Name, Data: assetBytes(obj.Data, includeAssets), Size: obj.Size, + ContentType: obj.ContentType, ContentMD5: obj.ContentMD5, ETag: obj.ETag, + TimeCreated: obj.TimeCreated, TimeModified: obj.TimeModified, + Metadata: obj.Metadata, StorageTier: obj.StorageTier, VersionID: obj.VersionID, + } + } + + snapshotVersions(bkt, bs, includeAssets) + snapshotPARs(bkt, bs) + snapshotRetention(bkt, bs) + + return bs +} + +func snapshotVersions(bkt *bucketData, bs *bucketSnapshot, includeAssets bool) { + if len(bkt.versions) == 0 { + return + } + + bs.Versions = make(map[string][]*versionSnapshot, len(bkt.versions)) + + for name, chain := range bkt.versions { + out := make([]*versionSnapshot, 0, len(chain)) + + for _, v := range chain { + out = append(out, &versionSnapshot{ + VersionID: v.versionID, Data: assetBytes(v.data, includeAssets), Size: v.size, + ContentType: v.contentType, ContentMD5: v.contentMD5, ETag: v.etag, + TimeModified: v.timeModified, Metadata: v.metadata, + StorageTier: v.storageTier, DeleteMarker: v.deleteMarker, + }) + } + + bs.Versions[name] = out + } +} + +func snapshotPARs(bkt *bucketData, bs *bucketSnapshot) { + if bkt.pars.Len() == 0 { + return + } + + bs.PARs = make([]*parSnapshot, 0, bkt.pars.Len()) + + for _, id := range bkt.pars.Keys() { + par, ok := bkt.pars.Get(id) + if !ok { + continue + } + + ps := &parSnapshot{ + ID: par.ID, Name: par.Name, Bucket: par.Bucket, ObjectName: par.ObjectName, + AccessType: par.AccessType, BucketListingAction: par.BucketListingAction, + TimeCreated: par.TimeCreated, Token: par.token, + } + + if !par.TimeExpires.IsZero() { + ps.TimeExpires = par.TimeExpires.UTC().Format(timeFormat) + } + + bs.PARs = append(bs.PARs, ps) + } +} + +func snapshotRetention(bkt *bucketData, bs *bucketSnapshot) { + if bkt.retention.Len() == 0 { + return + } + + bs.Retention = make([]*retentionSnapshot, 0, bkt.retention.Len()) + + for _, id := range bkt.retention.Keys() { + rule, ok := bkt.retention.Get(id) + if !ok { + continue + } + + bs.Retention = append(bs.Retention, &retentionSnapshot{ + ID: rule.ID, DisplayName: rule.DisplayName, Duration: rule.Duration, + TimeRuleLocked: rule.TimeRuleLocked, TimeCreated: rule.TimeCreated, + TimeModified: rule.TimeModified, ETag: rule.ETag, + }) + } +} + +// assetBytes returns data only when assets are included, so a metadata-only +// snapshot omits object and version bodies. +func assetBytes(data []byte, includeAssets bool) []byte { + if !includeAssets { + return nil + } + + return data +} + +// Restore rebuilds every bucket under its original name, with its objects, +// version history, PARs, retention rules and lifecycle policy intact. +func (m *Mock) Restore(_ context.Context, data json.RawMessage) error { + var snap storageSnapshot + if err := json.Unmarshal(data, &snap); err != nil { + return fmt.Errorf("objectstorage: parse snapshot: %w", err) + } + + m.mu.Lock() + defer m.mu.Unlock() + + for name, bs := range snap.Buckets { + m.buckets.Set(name, restoreBucket(bs, m.namespace)) + } + + return nil +} + +func restoreBucket(bs *bucketSnapshot, namespace string) *bucketData { + bkt := &bucketData{ + ID: bs.ID, Name: bs.Name, Namespace: namespace, CompartmentID: bs.CompartmentID, + CreatedBy: bs.CreatedBy, TimeCreated: bs.TimeCreated, ETag: bs.ETag, + PublicAccessType: bs.PublicAccessType, StorageTier: bs.StorageTier, + Versioning: bs.Versioning, KMSKeyID: bs.KMSKeyID, AutoTiering: bs.AutoTiering, + ObjectEventsEnabled: bs.ObjectEventsEnabled, Metadata: bs.Metadata, + FreeformTags: bs.FreeformTags, DefinedTags: bs.DefinedTags, + objects: memstore.New[*objectData](), + multiparts: memstore.New[*multipartUpload](), + pars: memstore.New[*parData](), + retention: memstore.New[*retentionRuleData](), + lifecycle: bs.Lifecycle, + } + + for name, os := range bs.Objects { + bkt.objects.Set(name, &objectData{ + Name: os.Name, Data: os.Data, Size: os.Size, ContentType: os.ContentType, + ContentMD5: os.ContentMD5, ETag: os.ETag, TimeCreated: os.TimeCreated, + TimeModified: os.TimeModified, Metadata: os.Metadata, + StorageTier: os.StorageTier, VersionID: os.VersionID, + }) + } + + restoreVersions(bkt, bs) + restorePARs(bkt, bs) + restoreRetention(bkt, bs) + + return bkt +} + +func restoreVersions(bkt *bucketData, bs *bucketSnapshot) { + if len(bs.Versions) == 0 { + return + } + + bkt.versions = make(map[string][]*objectVersion, len(bs.Versions)) + + for name, chain := range bs.Versions { + out := make([]*objectVersion, 0, len(chain)) + + for _, v := range chain { + out = append(out, &objectVersion{ + versionID: v.VersionID, data: v.Data, size: v.Size, contentType: v.ContentType, + contentMD5: v.ContentMD5, etag: v.ETag, timeModified: v.TimeModified, + metadata: v.Metadata, storageTier: v.StorageTier, deleteMarker: v.DeleteMarker, + }) + } + + bkt.versions[name] = out + } +} + +func restorePARs(bkt *bucketData, bs *bucketSnapshot) { + for _, ps := range bs.PARs { + par := &parData{ + ID: ps.ID, Name: ps.Name, Bucket: ps.Bucket, ObjectName: ps.ObjectName, + AccessType: ps.AccessType, BucketListingAction: ps.BucketListingAction, + TimeCreated: ps.TimeCreated, token: ps.Token, + } + + if ps.TimeExpires != "" { + if t, err := time.Parse(timeFormat, ps.TimeExpires); err == nil { + par.TimeExpires = t + } + } + + bkt.pars.Set(par.ID, par) + } +} + +func restoreRetention(bkt *bucketData, bs *bucketSnapshot) { + for _, rs := range bs.Retention { + bkt.retention.Set(rs.ID, &retentionRuleData{ + ID: rs.ID, DisplayName: rs.DisplayName, Duration: rs.Duration, + TimeRuleLocked: rs.TimeRuleLocked, TimeCreated: rs.TimeCreated, + TimeModified: rs.TimeModified, ETag: rs.ETag, + }) + } +} diff --git a/providers/oci/objectstorage/snapshot_test.go b/providers/oci/objectstorage/snapshot_test.go new file mode 100644 index 000000000..8f79311ad --- /dev/null +++ b/providers/oci/objectstorage/snapshot_test.go @@ -0,0 +1,193 @@ +package objectstorage_test + +import ( + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stackshy/cloudemu/v2/internal/idgen" + "github.com/stackshy/cloudemu/v2/providers/oci/objectstorage" + "github.com/stackshy/cloudemu/v2/services/storage/driver" +) + +// TestSnapshotRestoreRoundTrip seeds a bucket with objects, a version chain, a +// PAR, a retention rule and a lifecycle policy, snapshots, restores into a +// fresh mock and asserts everything comes back under its original identity — +// bucket OCID, object version ids and the PAR's redemption token included. +func TestSnapshotRestoreRoundTrip(t *testing.T) { + ctx := t.Context() + src := newMock(t) + + bucket := newBucket(t, src, testBucket) + + require.NoError(t, src.SetVersioningStatus(ctx, testBucket, objectstorage.VersioningEnabled)) + require.NoError(t, src.PutObject(ctx, testBucket, "logs/a.txt", []byte("v1"), "text/plain", nil)) + + first, err := src.HeadObject(ctx, testBucket, "logs/a.txt") + require.NoError(t, err) + + _, err = src.PutObjectWith(ctx, testBucket, "logs/a.txt", []byte("v2"), objectstorage.PutOptions{ + ContentType: "text/plain", + StorageTier: objectstorage.TierInfrequentAccess, + Metadata: map[string]string{"owner": "ada"}, + }) + require.NoError(t, err) + + par, err := src.CreatePAR(ctx, testBucket, objectstorage.PARSpec{ + Name: "read", ObjectName: "logs/a.txt", AccessType: objectstorage.PARObjectRead, + TimeExpires: time.Now().Add(time.Hour).UTC(), + }) + require.NoError(t, err) + + rule, err := src.CreateRetentionRule(ctx, testBucket, objectstorage.RetentionRuleSpec{ + DisplayName: "hold", + Duration: &objectstorage.RetentionDuration{TimeAmount: 10, TimeUnit: objectstorage.RetentionDays}, + }) + require.NoError(t, err) + + require.NoError(t, src.PutLifecycleConfig(ctx, testBucket, driver.LifecycleConfig{ + Rules: []driver.LifecycleRule{{ID: "expire", Prefix: "logs/", ExpirationDays: 30, Enabled: true}}, + })) + + require.NoError(t, src.PutBucketTagging(ctx, testBucket, map[string]string{"env": "dev"})) + + data, err := src.Snapshot(ctx, true) + require.NoError(t, err) + + dst := newMock(t) + require.NoError(t, dst.Restore(ctx, data)) + + restored, err := dst.BucketDetails(ctx, testBucket) + require.NoError(t, err) + assert.Equal(t, bucket.ID, restored.ID, "the bucket keeps its OCID") + assert.Equal(t, objectstorage.VersioningEnabled, restored.Versioning) + assert.Equal(t, map[string]string{"env": "dev"}, restored.FreeformTags) + + current, err := dst.GetObject(ctx, testBucket, "logs/a.txt") + require.NoError(t, err) + assert.Equal(t, []byte("v2"), current.Data) + assert.Equal(t, map[string]string{"owner": "ada"}, current.Info.Metadata) + + details, err := dst.ObjectDetailsOf(ctx, testBucket, "logs/a.txt") + require.NoError(t, err) + assert.Equal(t, objectstorage.TierInfrequentAccess, details.StorageTier) + assert.Equal(t, int64(2), details.Size) + + old, err := dst.GetObjectVersion(ctx, testBucket, "logs/a.txt", first.VersionID) + require.NoError(t, err) + assert.Equal(t, []byte("v1"), old.Data, "the version chain keeps its ids") + + // The PAR still resolves by its original redemption token. + resolved, err := dst.ResolvePAR(ctx, tokenFrom(t, par.AccessURI)) + require.NoError(t, err) + assert.Equal(t, par.ID, resolved.ID) + assert.Equal(t, objectstorage.PARObjectRead, resolved.AccessType) + + gotRule, err := dst.GetRetentionRule(ctx, testBucket, rule.ID) + require.NoError(t, err) + assert.Equal(t, int64(10), gotRule.Duration.TimeAmount) + + cfg, err := dst.GetLifecycleConfig(ctx, testBucket) + require.NoError(t, err) + require.Len(t, cfg.Rules, 1) + assert.Equal(t, 30, cfg.Rules[0].ExpirationDays) + + // The restored bucket is still writable, and its retention rule still bites. + require.NoError(t, dst.PutObject(ctx, testBucket, "fresh", []byte("x"), "text/plain", nil)) + + err = dst.DeleteObject(ctx, testBucket, "logs/a.txt") + require.Error(t, err, "the restored retention rule still holds the object") +} + +// A metadata-only snapshot keeps every identity and size but drops the bodies, +// which is the persist default. +func TestSnapshotWithoutAssets(t *testing.T) { + ctx := t.Context() + src := newMock(t) + newBucket(t, src, testBucket) + + require.NoError(t, src.PutObject(ctx, testBucket, "k", []byte("hello"), "text/plain", nil)) + + data, err := src.Snapshot(ctx, false) + require.NoError(t, err) + assert.NotContains(t, string(data), "aGVsbG8=", "the body is not captured") + + dst := newMock(t) + require.NoError(t, dst.Restore(ctx, data)) + + head, err := dst.HeadObject(ctx, testBucket, "k") + require.NoError(t, err) + assert.Equal(t, int64(5), head.Size, "the size survives without the bytes") + + list, err := dst.ListObjects(ctx, testBucket, driver.ListOptions{}) + require.NoError(t, err) + require.Len(t, list.Objects, 1) + assert.Equal(t, int64(5), list.Objects[0].Size) +} + +func TestRestoreRejectsMalformedSnapshot(t *testing.T) { + m := newMock(t) + + err := m.Restore(t.Context(), []byte("{")) + require.Error(t, err) + assert.Contains(t, err.Error(), "parse snapshot") +} + +// A persist restart resets the process id counter while restored PARs keep +// their tokens and OCIDs. A PAR minted after the restore must not reuse either, +// or the restored PAR's access URI would resolve to the new one. +func TestPARsDoNotCollideAfterRestore(t *testing.T) { + ctx := t.Context() + + // Both "processes" run the same sequence from a zeroed counter, so the + // counter alone would hand the second PAR the first one's token and OCID. + idgen.Reset() + + src := newMock(t) + newBucket(t, src, "secret") + require.NoError(t, src.PutObject(ctx, "secret", "payroll.csv", []byte("salaries"), "text/csv", nil)) + + old, err := src.CreatePAR(ctx, "secret", objectstorage.PARSpec{ + Name: "payroll", ObjectName: "payroll.csv", AccessType: objectstorage.PARObjectRead, + TimeExpires: time.Now().Add(time.Hour), + }) + require.NoError(t, err) + + data, err := src.Snapshot(ctx, true) + require.NoError(t, err) + + // Restart: the counter starts over, exactly as a fresh process would. + idgen.Reset() + + dst := newMock(t) + require.NoError(t, dst.Restore(ctx, data)) + newBucket(t, dst, "public") + require.NoError(t, dst.PutObject(ctx, "public", "readme.txt", []byte("hello"), "text/plain", nil)) + + fresh, err := dst.CreatePAR(ctx, "public", objectstorage.PARSpec{ + Name: "readme", ObjectName: "readme.txt", AccessType: objectstorage.PARObjectRead, + TimeExpires: time.Now().Add(time.Hour), + }) + require.NoError(t, err) + + assert.NotEqual(t, old.ID, fresh.ID) + assert.NotEqual(t, tokenFrom(t, old.AccessURI), tokenFrom(t, fresh.AccessURI)) + + for _, tc := range []struct { + par *objectstorage.PreauthenticatedRequest + bucket string + object string + }{ + {old, "secret", "payroll.csv"}, + {fresh, "public", "readme.txt"}, + } { + resolved, err := dst.ResolvePAR(ctx, tokenFrom(t, tc.par.AccessURI)) + require.NoError(t, err) + assert.Equal(t, tc.par.ID, resolved.ID) + assert.Equal(t, tc.bucket, resolved.Bucket) + assert.True(t, objectstorage.PARAllows(resolved, http.MethodGet, tc.object)) + } +} diff --git a/providers/oci/objectstorage/unsupported.go b/providers/oci/objectstorage/unsupported.go new file mode 100644 index 000000000..4a34e7152 --- /dev/null +++ b/providers/oci/objectstorage/unsupported.go @@ -0,0 +1,65 @@ +package objectstorage + +import ( + "context" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/services/storage/driver" +) + +// What OCI does instead of the operation being asked for. +const ( + viaIAMPolicy = "OCI grants bucket access through an Identity policy, not a policy document on the bucket" + viaNoCORS = "OCI Object Storage has no per-bucket CORS configuration" + viaObjectMeta = "an OCI object carries opc-meta- user metadata, not tags; use UpdateObjectMetadata" +) + +// unsupported reports an operation with no OCI equivalent. +func unsupported(operation, instead string) error { + return cerrors.Newf(cerrors.Unimplemented, "%s is not an OCI operation: %s", operation, instead) +} + +// PutBucketPolicy is not an OCI operation. +func (*Mock) PutBucketPolicy(_ context.Context, _ string, _ driver.BucketPolicy) error { + return unsupported("PutBucketPolicy", viaIAMPolicy) +} + +// GetBucketPolicy is not an OCI operation. +func (*Mock) GetBucketPolicy(_ context.Context, _ string) (*driver.BucketPolicy, error) { + return nil, unsupported("GetBucketPolicy", viaIAMPolicy) +} + +// DeleteBucketPolicy is not an OCI operation. +func (*Mock) DeleteBucketPolicy(_ context.Context, _ string) error { + return unsupported("DeleteBucketPolicy", viaIAMPolicy) +} + +// PutCORSConfig is not an OCI operation. +func (*Mock) PutCORSConfig(_ context.Context, _ string, _ driver.CORSConfig) error { + return unsupported("PutCORSConfig", viaNoCORS) +} + +// GetCORSConfig is not an OCI operation. +func (*Mock) GetCORSConfig(_ context.Context, _ string) (*driver.CORSConfig, error) { + return nil, unsupported("GetCORSConfig", viaNoCORS) +} + +// DeleteCORSConfig is not an OCI operation. +func (*Mock) DeleteCORSConfig(_ context.Context, _ string) error { + return unsupported("DeleteCORSConfig", viaNoCORS) +} + +// PutObjectTagging is not an OCI operation. +func (*Mock) PutObjectTagging(_ context.Context, _, _ string, _ map[string]string) error { + return unsupported("PutObjectTagging", viaObjectMeta) +} + +// GetObjectTagging is not an OCI operation. +func (*Mock) GetObjectTagging(_ context.Context, _, _ string) (map[string]string, error) { + return nil, unsupported("GetObjectTagging", viaObjectMeta) +} + +// DeleteObjectTagging is not an OCI operation. +func (*Mock) DeleteObjectTagging(_ context.Context, _, _ string) error { + return unsupported("DeleteObjectTagging", viaObjectMeta) +} diff --git a/providers/oci/objectstorage/versioning.go b/providers/oci/objectstorage/versioning.go new file mode 100644 index 000000000..17fec3d1b --- /dev/null +++ b/providers/oci/objectstorage/versioning.go @@ -0,0 +1,522 @@ +package objectstorage + +import ( + "context" + "sort" + "strings" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/idgen" + "github.com/stackshy/cloudemu/v2/services/storage/driver" + "github.com/stackshy/cloudemu/v2/services/storage/storageengine" +) + +// newVersionID mints an object version id. OCI version ids are opaque. +func newVersionID() string { return idgen.GenerateID("") } + +// setVersioningLocked applies a versioning state to a bucket. Enabling it +// seeds every current object that has no history yet into its chain, so the +// first overwrite after enabling keeps the original as a prior version. +// Callers hold mu for writing. +func (m *Mock) setVersioningLocked(ctx context.Context, bucket string, bkt *bucketData, status string) error { + bkt.Versioning = status + + if status != VersioningEnabled { + return nil + } + + if bkt.versions == nil { + bkt.versions = make(map[string][]*objectVersion) + } + + for _, name := range bkt.objects.Keys() { + obj, ok := bkt.objects.Get(name) + if !ok || len(bkt.versions[name]) > 0 { + continue + } + + if err := m.assignVersionLocked(ctx, bucket, obj); err != nil { + return err + } + + appendVersion(bkt, name, versionOf(obj)) + } + + return nil +} + +// assignVersionLocked gives a pre-versioning object a version id, moving its +// engine bytes to the versioned reference. Callers hold mu for writing. +func (m *Mock) assignVersionLocked(ctx context.Context, bucket string, obj *objectData) error { + prev := obj.VersionID + next := newVersionID() + + if m.engineWired() { + if err := storageengine.Copy(ctx, m.opts.StorageEngine, + engineRef(bucket, obj.Name, next), engineRef(bucket, obj.Name, prev)); err != nil { + return err + } + + _ = storageengine.Delete(ctx, m.opts.StorageEngine, engineRef(bucket, obj.Name, prev)) + } + + obj.VersionID = next + + return nil +} + +// storeObjectLocked writes an object as the bucket's current version and, on a +// versioned bucket, records it in history. Enabled appends a fresh version; +// Suspended overwrites the reusable "null" version; a bucket that never had +// versioning keeps no history. Callers hold mu. +func storeObjectLocked(bkt *bucketData, obj *objectData) { + switch bkt.Versioning { + case VersioningEnabled: + obj.VersionID = newVersionID() + appendVersion(bkt, obj.Name, versionOf(obj)) + case VersioningSuspended: + obj.VersionID = nullVersionID + replaceNullVersion(bkt, obj.Name, versionOf(obj)) + } + + bkt.objects.Set(obj.Name, obj) +} + +// deleteCurrentLocked applies a delete with no version id. Enabled appends a +// delete marker, Suspended replaces the null version with one, and an +// unversioned bucket removes the object outright. Callers hold mu. +func (m *Mock) deleteCurrentLocked(bkt *bucketData, name string) (versionID string, deleteMarker, existed bool) { + now := m.now() + + switch bkt.Versioning { + case VersioningEnabled: + vid := newVersionID() + appendVersion(bkt, name, &objectVersion{versionID: vid, deleteMarker: true, timeModified: now}) + bkt.objects.Delete(name) + + return vid, true, true + case VersioningSuspended: + replaceNullVersion(bkt, name, &objectVersion{versionID: nullVersionID, deleteMarker: true, timeModified: now}) + bkt.objects.Delete(name) + + return nullVersionID, true, true + default: + if !bkt.objects.Has(name) { + return "", false, false + } + + bkt.objects.Delete(name) + + return "", false, true + } +} + +func appendVersion(bkt *bucketData, name string, v *objectVersion) { + if bkt.versions == nil { + bkt.versions = make(map[string][]*objectVersion) + } + + bkt.versions[name] = append(bkt.versions[name], v) +} + +func replaceNullVersion(bkt *bucketData, name string, v *objectVersion) { + if bkt.versions == nil { + bkt.versions = make(map[string][]*objectVersion) + } + + kept := make([]*objectVersion, 0, len(bkt.versions[name])+1) + + for _, ex := range bkt.versions[name] { + if ex.versionID != nullVersionID { + kept = append(kept, ex) + } + } + + bkt.versions[name] = append(kept, v) +} + +func versionOf(obj *objectData) *objectVersion { + return &objectVersion{ + versionID: obj.VersionID, + data: obj.Data, + size: obj.Size, + contentType: obj.ContentType, + contentMD5: obj.ContentMD5, + etag: obj.ETag, + timeModified: obj.TimeModified, + metadata: obj.Metadata, + storageTier: obj.StorageTier, + } +} + +func objectOfVersion(name string, v *objectVersion) *objectData { + return &objectData{ + Name: name, + Data: v.data, + Size: v.size, + ContentType: v.contentType, + ContentMD5: v.contentMD5, + ETag: v.etag, + TimeCreated: v.timeModified, + TimeModified: v.timeModified, + Metadata: cloneMeta(v.metadata), + StorageTier: v.storageTier, + VersionID: v.versionID, + } +} + +func infoOfVersion(name string, v *objectVersion) driver.ObjectInfo { + return driver.ObjectInfo{ + Key: name, + Size: v.size, + ContentType: v.contentType, + ETag: v.etag, + LastModified: v.timeModified, + Metadata: cloneMeta(v.metadata), + VersionID: v.versionID, + DeleteMarker: v.deleteMarker, + } +} + +// SetBucketVersioning enables versioning, or suspends it when disabling. OCI +// never returns a bucket to Disabled once it has been enabled; use +// SetVersioningStatus for the full tri-state. +func (m *Mock) SetBucketVersioning(ctx context.Context, bucket string, enabled bool) error { + status := VersioningSuspended + if enabled { + status = VersioningEnabled + } + + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return err + } + + return m.setVersioningLocked(ctx, bucket, bkt, status) +} + +func (m *Mock) GetBucketVersioning(_ context.Context, bucket string) (bool, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return false, err + } + + return bkt.Versioning == VersioningEnabled, nil +} + +// SetVersioningStatus sets the bucket's versioning state. OCI's Disabled is +// accepted only while the bucket has never been versioned. +func (m *Mock) SetVersioningStatus(ctx context.Context, bucket, status string) error { + if !validVersioning(status) { + return cerrors.Newf(cerrors.InvalidArgument, "invalid versioning status %q", status) + } + + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return err + } + + if status == VersioningDisabled && bkt.Versioning != VersioningDisabled { + return cerrors.New(cerrors.InvalidArgument, + "versioning cannot be set back to Disabled once enabled; use Suspended") + } + + return m.setVersioningLocked(ctx, bucket, bkt, status) +} + +// VersioningStatus returns "Disabled", "Enabled" or "Suspended". +func (m *Mock) VersioningStatus(_ context.Context, bucket string) (string, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return "", err + } + + return bkt.Versioning, nil +} + +// GetObjectVersion returns a specific version, or the current object when +// versionID is empty. A delete marker reports NotFound. +func (m *Mock) GetObjectVersion(ctx context.Context, bucket, key, versionID string) (*driver.Object, error) { + if versionID == "" { + return m.GetObject(ctx, bucket, key) + } + + m.mu.RLock() + defer m.mu.RUnlock() + + v, err := m.findVersionLocked(bucket, key, versionID) + if err != nil { + return nil, err + } + + data, err := m.engineLoad(ctx, engineRef(bucket, key, versionID), v.data) + if err != nil { + return nil, err + } + + return &driver.Object{Info: infoOfVersion(key, v), Data: cloneBytes(data)}, nil +} + +// HeadObjectVersion returns metadata for a specific version. +func (m *Mock) HeadObjectVersion(ctx context.Context, bucket, key, versionID string) (*driver.ObjectInfo, error) { + if versionID == "" { + return m.HeadObject(ctx, bucket, key) + } + + m.mu.RLock() + defer m.mu.RUnlock() + + v, err := m.findVersionLocked(bucket, key, versionID) + if err != nil { + return nil, err + } + + info := infoOfVersion(key, v) + + return &info, nil +} + +// findVersionLocked resolves a stored (non-delete-marker) version. Callers +// hold mu. +func (m *Mock) findVersionLocked(bucket, key, versionID string) (*objectVersion, error) { + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + for _, v := range bkt.versions[key] { + if v.versionID != versionID { + continue + } + + if v.deleteMarker { + return nil, cerrors.Newf(cerrors.NotFound, "version %q of %q is a delete marker", versionID, key) + } + + return v, nil + } + + return nil, cerrors.Newf(cerrors.NotFound, "version %q of %q not found", versionID, key) +} + +// DeleteObjectVersion removes one version, or performs a top-level delete when +// versionID is empty. A top-level delete that finds nothing is NotFound, as +// DeleteObject is; a versioned bucket always records a delete marker. +func (m *Mock) DeleteObjectVersion( + ctx context.Context, bucket, key, versionID string, +) (deletedVersionID string, deleteMarker bool, err error) { + return m.DeleteObjectIf(ctx, bucket, key, versionID, "") +} + +// DeleteObjectIf is DeleteObjectVersion guarded by an if-match ETag, checked +// against the version being deleted: the current object when versionID is +// empty. +func (m *Mock) DeleteObjectIf( + ctx context.Context, bucket, key, versionID, ifMatch string, +) (deletedVersionID string, deleteMarker bool, err error) { + m.mu.Lock() + defer m.mu.Unlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return "", false, err + } + + if ifMatch != "" { + if matchErr := checkETag(targetETagLocked(bkt, key, versionID), ifMatch, ""); matchErr != nil { + return "", false, matchErr + } + } + + if holdErr := retentionBlocksLocked(bkt, key, m.opts.Clock.Now()); holdErr != nil { + return "", false, holdErr + } + + if versionID == "" { + vid, marker, existed := m.deleteCurrentLocked(bkt, key) + if !existed { + return "", false, cerrors.Newf(cerrors.NotFound, "object %q not found in bucket %q", key, bucket) + } + + m.purgeLocked(ctx, bucket, key, vid, marker) + + return vid, marker, nil + } + + removed, err := removeVersionLocked(bkt, key, versionID) + if err != nil { + return "", false, err + } + + m.purgeLocked(ctx, bucket, key, versionID, removed.deleteMarker) + + return versionID, removed.deleteMarker, nil +} + +// removeVersionLocked drops one version from a name's chain and recomputes the +// current object. Callers hold mu. +func removeVersionLocked(bkt *bucketData, key, versionID string) (*objectVersion, error) { + chain := bkt.versions[key] + + for i, v := range chain { + if v.versionID != versionID { + continue + } + + bkt.versions[key] = append(chain[:i], chain[i+1:]...) + if len(bkt.versions[key]) == 0 { + delete(bkt.versions, key) + } + + recomputeCurrentLocked(bkt, key) + + return v, nil + } + + return nil, cerrors.Newf(cerrors.NotFound, "version %q of %q not found", versionID, key) +} + +// targetETagLocked is the ETag of the version a delete addresses, or empty when +// it does not exist. Callers hold mu. +func targetETagLocked(bkt *bucketData, key, versionID string) string { + if versionID == "" { + if obj, ok := bkt.objects.Get(key); ok { + return obj.ETag + } + + return "" + } + + for _, v := range bkt.versions[key] { + if v.versionID == versionID && !v.deleteMarker { + return v.etag + } + } + + return "" +} + +// recomputeCurrentLocked resets a name's current object to its newest stored +// version, removing it when the newest is a delete marker or none remain. +// Callers hold mu. +func recomputeCurrentLocked(bkt *bucketData, name string) { + chain := bkt.versions[name] + if len(chain) == 0 { + bkt.objects.Delete(name) + return + } + + latest := chain[len(chain)-1] + if latest.deleteMarker { + bkt.objects.Delete(name) + return + } + + bkt.objects.Set(name, objectOfVersion(name, latest)) +} + +// ListObjectVersions returns every version and delete marker matching opts, +// newest first within each name. +func (m *Mock) ListObjectVersions( + _ context.Context, bucket string, opts driver.ListOptions, +) (*driver.VersionListResult, error) { + m.mu.RLock() + defer m.mu.RUnlock() + + bkt, err := m.bucketLocked(bucket) + if err != nil { + return nil, err + } + + result := &driver.VersionListResult{} + prefixSet := make(map[string]struct{}) + + for _, name := range versionedNamesLocked(bkt) { + if opts.Prefix != "" && !strings.HasPrefix(name, opts.Prefix) { + continue + } + + if opts.Delimiter != "" { + rest := name[len(opts.Prefix):] + if idx := strings.Index(rest, opts.Delimiter); idx >= 0 { + prefixSet[opts.Prefix+rest[:idx+len(opts.Delimiter)]] = struct{}{} + continue + } + } + + result.Versions = append(result.Versions, versionsOfLocked(bkt, name)...) + } + + for p := range prefixSet { + result.CommonPrefixes = append(result.CommonPrefixes, p) + } + + sort.Strings(result.CommonPrefixes) + + return result, nil +} + +// versionedNamesLocked is the union of names with history and names present +// only as a current object. Callers hold mu. +func versionedNamesLocked(bkt *bucketData) []string { + set := make(map[string]struct{}, len(bkt.versions)) + for n := range bkt.versions { + set[n] = struct{}{} + } + + for _, n := range bkt.objects.Keys() { + set[n] = struct{}{} + } + + names := make([]string, 0, len(set)) + for n := range set { + names = append(names, n) + } + + sort.Strings(names) + + return names +} + +// versionsOfLocked projects one name's chain newest-first. A name with no +// history is reported as its single "null" version. Callers hold mu. +func versionsOfLocked(bkt *bucketData, name string) []driver.ObjectVersion { + chain := bkt.versions[name] + if len(chain) == 0 { + obj, ok := bkt.objects.Get(name) + if !ok { + return nil + } + + return []driver.ObjectVersion{{ + Key: name, VersionID: nullVersionID, IsLatest: true, + Size: obj.Size, ETag: obj.ETag, + ContentType: obj.ContentType, LastModified: obj.TimeModified, + }} + } + + out := make([]driver.ObjectVersion, 0, len(chain)) + + for i := len(chain) - 1; i >= 0; i-- { + v := chain[i] + out = append(out, driver.ObjectVersion{ + Key: name, VersionID: v.versionID, IsLatest: i == len(chain)-1, + DeleteMarker: v.deleteMarker, Size: v.size, ETag: v.etag, + ContentType: v.contentType, LastModified: v.timeModified, + }) + } + + return out +} diff --git a/providers/oci/oci.go b/providers/oci/oci.go index 76e721a43..497164dd2 100644 --- a/providers/oci/oci.go +++ b/providers/oci/oci.go @@ -6,6 +6,7 @@ import ( "github.com/stackshy/cloudemu/v2/internal/snapshot" "github.com/stackshy/cloudemu/v2/providers/oci/identity" "github.com/stackshy/cloudemu/v2/providers/oci/monitoring" + "github.com/stackshy/cloudemu/v2/providers/oci/objectstorage" vcnprovider "github.com/stackshy/cloudemu/v2/providers/oci/vcn" cachedriver "github.com/stackshy/cloudemu/v2/services/cache/driver" computedriver "github.com/stackshy/cloudemu/v2/services/compute/driver" @@ -75,6 +76,7 @@ func New(opts ...config.Option) *Provider { Region: o.OCIRegion(), } p.Identity = identity.New(o) + p.ObjectStorage = objectstorage.New(o) p.VCN = vcnprovider.New(o) p.Monitoring = monitoring.New(o) diff --git a/server/oci/objectstorage/bucket.go b/server/oci/objectstorage/bucket.go new file mode 100644 index 000000000..e2e949f8e --- /dev/null +++ b/server/oci/objectstorage/bucket.go @@ -0,0 +1,220 @@ +package objectstorage + +import ( + "net/http" + + osprovider "github.com/stackshy/cloudemu/v2/providers/oci/objectstorage" + "github.com/stackshy/cloudemu/v2/server/wire/ocirest" +) + +// getNamespace serves GET /n. Real OCI returns the namespace as a bare JSON +// string. +func (h *Handler) getNamespace(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + methodNotAllowed(w, r) + return + } + + ocirest.WriteJSON(w, r, http.StatusOK, h.extras.Namespace()) +} + +// namespaceMetadata serves GET /n/{ns}. +func (h *Handler) namespaceMetadata(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + methodNotAllowed(w, r) + return + } + + md := h.extras.Metadata(r.Context()) + ocirest.WriteJSON(w, r, http.StatusOK, namespaceMetadataBody{ + Namespace: md.Namespace, + DefaultS3CompartmentID: md.DefaultS3CompartmentID, + DefaultSwiftCompartmentID: md.DefaultSwiftCompartmentID, + }) +} + +func (h *Handler) createBucket(w http.ResponseWriter, r *http.Request) { + var req createBucketBody + + if !ocirest.DecodeJSON(w, r, &req) { + return + } + + if req.Name == "" { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, "name is required") + return + } + + if req.CompartmentID == "" { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, "compartmentId is required") + return + } + + if !h.requireCompartment(w, r, req.CompartmentID) { + return + } + + bkt, err := h.extras.CreateBucketWith(r.Context(), osprovider.BucketSpec{ + Name: req.Name, + CompartmentID: req.CompartmentID, + PublicAccessType: req.PublicAccessType, + StorageTier: req.StorageTier, + Versioning: req.Versioning, + KMSKeyID: req.KMSKeyID, + AutoTiering: req.AutoTiering, + ObjectEventsEnabled: req.ObjectEventsEnabled, + Metadata: req.Metadata, + FreeformTags: req.FreeformTags, + DefinedTags: req.DefinedTags, + }) + if err != nil { + writeDriverError(w, r, err) + return + } + + w.Header().Set("ETag", bkt.ETag) + ocirest.WriteJSON(w, r, http.StatusOK, toBucketBody(bkt)) +} + +func (h *Handler) listBuckets(w http.ResponseWriter, r *http.Request) { + compartmentID, ok := ocirest.RequireCompartmentID(w, r) + if !ok { + return + } + + buckets, err := h.extras.ListBucketsIn(r.Context(), compartmentID) + if err != nil { + writeDriverError(w, r, err) + return + } + + out := make([]bucketSummaryBody, 0, len(buckets)) + + for i := range buckets { + b := &buckets[i] + out = append(out, bucketSummaryBody{ + Namespace: b.Namespace, + Name: b.Name, + CompartmentID: b.CompartmentID, + CreatedBy: b.CreatedBy, + TimeCreated: b.TimeCreated, + ETag: b.ETag, + FreeformTags: b.FreeformTags, + DefinedTags: b.DefinedTags, + }) + } + + writePage(w, r, out) +} + +// serveBucketItem serves GET/HEAD/POST/DELETE on one bucket. +func (h *Handler) serveBucketItem(w http.ResponseWriter, r *http.Request, bucket string) { + switch r.Method { + case http.MethodGet: + h.getBucket(w, r, bucket) + case http.MethodHead: + h.headBucket(w, r, bucket) + case http.MethodPost: + h.updateBucket(w, r, bucket) + case http.MethodDelete: + h.deleteBucket(w, r, bucket) + default: + methodNotAllowed(w, r) + } +} + +func (h *Handler) getBucket(w http.ResponseWriter, r *http.Request, bucket string) { + bkt, err := h.extras.BucketDetails(r.Context(), bucket) + if err != nil { + writeDriverError(w, r, err) + return + } + + w.Header().Set("ETag", bkt.ETag) + ocirest.WriteJSON(w, r, http.StatusOK, toBucketBody(bkt)) +} + +func (h *Handler) headBucket(w http.ResponseWriter, r *http.Request, bucket string) { + bkt, err := h.extras.BucketDetails(r.Context(), bucket) + if err != nil { + writeDriverError(w, r, err) + return + } + + w.Header().Set("ETag", bkt.ETag) + ocirest.WriteJSON(w, r, http.StatusOK, nil) +} + +func (h *Handler) updateBucket(w http.ResponseWriter, r *http.Request, bucket string) { + var req updateBucketBody + + if !ocirest.DecodeJSON(w, r, &req) { + return + } + + if req.Namespace != "" && req.Namespace != h.extras.Namespace() { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, + "moving a bucket to another namespace is not emulated; namespace must be "+h.extras.Namespace()) + + return + } + + if req.CompartmentID != nil && !h.requireCompartment(w, r, *req.CompartmentID) { + return + } + + bkt, err := h.extras.UpdateBucket(r.Context(), bucket, osprovider.BucketUpdate{ + Name: req.Name, + IfMatch: r.Header.Get("If-Match"), + CompartmentID: req.CompartmentID, + PublicAccessType: req.PublicAccessType, + Versioning: req.Versioning, + KMSKeyID: req.KMSKeyID, + AutoTiering: req.AutoTiering, + ObjectEventsEnabled: req.ObjectEventsEnabled, + Metadata: req.Metadata, + FreeformTags: req.FreeformTags, + DefinedTags: req.DefinedTags, + }) + if err != nil { + writeDriverError(w, r, err) + return + } + + w.Header().Set("ETag", bkt.ETag) + ocirest.WriteJSON(w, r, http.StatusOK, toBucketBody(bkt)) +} + +func (h *Handler) deleteBucket(w http.ResponseWriter, r *http.Request, bucket string) { + if err := h.extras.DeleteBucketIf(r.Context(), bucket, r.Header.Get("If-Match")); err != nil { + writeDriverError(w, r, err) + return + } + + ocirest.WriteJSON(w, r, http.StatusNoContent, nil) +} + +func toBucketBody(b *osprovider.Bucket) bucketBody { + return bucketBody{ + ID: b.ID, + Namespace: b.Namespace, + Name: b.Name, + CompartmentID: b.CompartmentID, + CreatedBy: b.CreatedBy, + TimeCreated: b.TimeCreated, + ETag: b.ETag, + PublicAccessType: b.PublicAccessType, + StorageTier: b.StorageTier, + Versioning: b.Versioning, + KMSKeyID: b.KMSKeyID, + AutoTiering: b.AutoTiering, + ObjectEventsEnabled: b.ObjectEventsEnabled, + ReplicationEnabled: b.ReplicationEnabled, + IsReadOnly: b.IsReadOnly, + Metadata: b.Metadata, + FreeformTags: b.FreeformTags, + DefinedTags: b.DefinedTags, + ApproximateCount: b.ApproximateCount, + ApproximateSize: b.ApproximateSize, + } +} diff --git a/server/oci/objectstorage/conditional_test.go b/server/oci/objectstorage/conditional_test.go new file mode 100644 index 000000000..26107c303 --- /dev/null +++ b/server/oci/objectstorage/conditional_test.go @@ -0,0 +1,383 @@ +package objectstorage_test + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// doWith issues a request carrying extra headers. +func (f fixture) doWith(t *testing.T, method, path string, body []byte, headers map[string]string) *httptest.ResponseRecorder { + t.Helper() + + req := httptest.NewRequest(method, path, bytes.NewReader(body)) + for k, v := range headers { + req.Header.Set(k, v) + } + + rec := httptest.NewRecorder() + f.handler.ServeHTTP(rec, req) + + return rec +} + +func errorCode(t *testing.T, rec *httptest.ResponseRecorder) string { + t.Helper() + + var body struct { + Code string `json:"code"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body), rec.Body.String()) + + return body.Code +} + +func TestObjectConditionalRequests(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + + key := f.bucketPath("photos") + "/o/k" + + // if-none-match: * creates only when absent. + rec := f.doWith(t, http.MethodPut, key, []byte("v1"), map[string]string{"if-none-match": "*"}) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + first := rec.Header().Get("ETag") + + rec = f.doWith(t, http.MethodPut, key, []byte("clobber"), map[string]string{"if-none-match": "*"}) + require.Equal(t, http.StatusPreconditionFailed, rec.Code) + assert.Equal(t, "IfNoneMatchFailed", errorCode(t, rec)) + + rec = f.doWith(t, http.MethodPut, key, []byte("x"), map[string]string{"if-none-match": first}) + assert.Equal(t, http.StatusBadRequest, rec.Code, "a write's if-none-match supports only *") + + // if-match must name the current ETag. + rec = f.doWith(t, http.MethodPut, key, []byte("clobber"), map[string]string{"if-match": "bogus"}) + require.Equal(t, http.StatusPreconditionFailed, rec.Code) + assert.Equal(t, "IfMatchFailed", errorCode(t, rec)) + + rec = f.do(t, http.MethodGet, key, nil) + assert.Equal(t, "v1", rec.Body.String(), "a failed precondition must not write") + + // Identical bytes still mint a new ETag: it is opaque, not a content hash. + rec = f.doWith(t, http.MethodPut, key, []byte("v1"), map[string]string{"if-match": first}) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + second := rec.Header().Get("ETag") + assert.NotEqual(t, first, second) + + rec = f.doWith(t, http.MethodPut, key, []byte("v2"), map[string]string{"if-match": first}) + assert.Equal(t, http.StatusPreconditionFailed, rec.Code, "the old ETag no longer matches") + + // Reads: a stale if-match is 412, a matching if-none-match is 304. + for _, method := range []string{http.MethodGet, http.MethodHead} { + rec = f.doWith(t, method, key, nil, map[string]string{"if-match": first}) + assert.Equal(t, http.StatusPreconditionFailed, rec.Code, method) + + rec = f.doWith(t, method, key, nil, map[string]string{"if-match": second}) + assert.Equal(t, http.StatusOK, rec.Code, method) + + rec = f.doWith(t, method, key, nil, map[string]string{"if-none-match": second}) + assert.Equal(t, http.StatusNotModified, rec.Code, method) + assert.Empty(t, rec.Body.String()) + + rec = f.doWith(t, method, key, nil, map[string]string{"if-none-match": first}) + assert.Equal(t, http.StatusOK, rec.Code, method) + } + + // Delete honors if-match. + rec = f.doWith(t, http.MethodDelete, key, nil, map[string]string{"if-match": first}) + require.Equal(t, http.StatusPreconditionFailed, rec.Code) + assert.Equal(t, "IfMatchFailed", errorCode(t, rec)) + + rec = f.doWith(t, http.MethodDelete, key, nil, map[string]string{"if-match": second}) + assert.Equal(t, http.StatusNoContent, rec.Code) + + rec = f.doWith(t, http.MethodPut, key, []byte("x"), map[string]string{"if-match": second}) + assert.Equal(t, http.StatusPreconditionFailed, rec.Code, "if-match on an absent object fails") +} + +func TestBucketConditionalRequests(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + + rec := f.do(t, http.MethodGet, f.bucketPath("photos"), nil) + etag := rec.Header().Get("ETag") + require.NotEmpty(t, etag) + + rec = f.doWith(t, http.MethodPost, f.bucketPath("photos"), []byte(`{"freeformTags":{"a":"b"}}`), + map[string]string{"if-match": "stale"}) + require.Equal(t, http.StatusPreconditionFailed, rec.Code) + assert.Equal(t, "IfMatchFailed", errorCode(t, rec)) + + rec = f.doWith(t, http.MethodPost, f.bucketPath("photos"), []byte(`{"freeformTags":{"a":"b"}}`), + map[string]string{"if-match": etag}) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + updated := rec.Header().Get("ETag") + + rec = f.doWith(t, http.MethodDelete, f.bucketPath("photos"), nil, map[string]string{"if-match": etag}) + require.Equal(t, http.StatusPreconditionFailed, rec.Code) + + rec = f.doWith(t, http.MethodDelete, f.bucketPath("photos"), nil, map[string]string{"if-match": updated}) + assert.Equal(t, http.StatusNoContent, rec.Code) +} + +func TestGetObjectRange(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + + key := f.bucketPath("photos") + "/o/k" + rec := f.doWith(t, http.MethodPut, key, []byte("0123456789"), map[string]string{"Content-Type": "text/plain"}) + require.Equal(t, http.StatusOK, rec.Code) + + tests := []struct { + spec, body, contentRange string + }{ + {"bytes=0-2", "012", "bytes 0-2/10"}, + {"bytes=7-", "789", "bytes 7-9/10"}, + {"bytes=-4", "6789", "bytes 6-9/10"}, + {"bytes=8-100", "89", "bytes 8-9/10"}, + } + + for _, tc := range tests { + t.Run(tc.spec, func(t *testing.T) { + rec := f.doWith(t, http.MethodGet, key, nil, map[string]string{"Range": tc.spec}) + require.Equal(t, http.StatusPartialContent, rec.Code, rec.Body.String()) + assert.Equal(t, tc.body, rec.Body.String()) + assert.Equal(t, tc.contentRange, rec.Header().Get("Content-Range")) + assert.Equal(t, "text/plain", rec.Header().Get("Content-Type")) + }) + } + + rec = f.doWith(t, http.MethodGet, key, nil, map[string]string{"Range": "bytes=20-30"}) + require.Equal(t, http.StatusRequestedRangeNotSatisfiable, rec.Code) + assert.Equal(t, "bytes */10", rec.Header().Get("Content-Range")) + + rec = f.doWith(t, http.MethodGet, key, nil, map[string]string{"Range": "bytes=0-1,4-5"}) + assert.Equal(t, http.StatusBadRequest, rec.Code, "a multi-range request is refused, not answered in full") + + rec = f.do(t, http.MethodGet, key, nil) + assert.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "bytes", rec.Header().Get("Accept-Ranges")) +} + +func TestBucketRenameAndErrorCodes(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "tfb") + f.createBucket(t, "taken") + require.NoError(t, f.mock.PutObject(t.Context(), "tfb", "k", []byte("v"), "text/plain", nil)) + + rec := f.do(t, http.MethodPost, "/n/"+f.ns+"/b", map[string]any{"name": "tfb", "compartmentId": testCompartment}) + require.Equal(t, http.StatusConflict, rec.Code) + assert.Equal(t, "BucketAlreadyExists", errorCode(t, rec)) + + rec = f.do(t, http.MethodDelete, f.bucketPath("tfb"), nil) + require.Equal(t, http.StatusConflict, rec.Code) + assert.Equal(t, "BucketNotEmpty", errorCode(t, rec)) + + rec = f.do(t, http.MethodPost, f.bucketPath("tfb"), map[string]any{"name": "taken"}) + require.Equal(t, http.StatusConflict, rec.Code) + assert.Equal(t, "BucketAlreadyExists", errorCode(t, rec)) + + rec = f.do(t, http.MethodPost, f.bucketPath("tfb"), map[string]any{"name": "bad name!"}) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPost, f.bucketPath("tfb"), map[string]any{"namespace": "elsewhere"}) + assert.Equal(t, http.StatusBadRequest, rec.Code, "a cross-namespace move is refused, not ignored") + + rec = f.do(t, http.MethodPost, f.bucketPath("tfb"), map[string]any{"name": "tfb2"}) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), `"name":"tfb2"`) + + rec = f.do(t, http.MethodGet, f.bucketPath("tfb"), nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodGet, f.bucketPath("tfb2")+"/o/k", nil) + require.Equal(t, http.StatusOK, rec.Code, "objects follow the bucket") + assert.Equal(t, "v", rec.Body.String()) +} + +func TestBucketNamesAreValidated(t *testing.T) { + f := newFixture(t) + + for _, name := range []string{"bad name!", "slash/name", strings.Repeat("a", 257)} { + rec := f.do(t, http.MethodPost, "/n/"+f.ns+"/b", map[string]any{"name": name, "compartmentId": testCompartment}) + assert.Equal(t, http.StatusBadRequest, rec.Code, name) + } + + for _, name := range []string{"ok-name_1.2", strings.Repeat("a", 256)} { + rec := f.do(t, http.MethodPost, "/n/"+f.ns+"/b", map[string]any{"name": name, "compartmentId": testCompartment}) + assert.Equal(t, http.StatusOK, rec.Code, name) + } +} + +func TestCompartmentCheckerGatesCreateAndMove(t *testing.T) { + f := newFixture(t) + f.handler.SetCompartmentChecker(func(id string) bool { return id == testCompartment }) + + rec := f.do(t, http.MethodPost, "/n/"+f.ns+"/b", map[string]any{ + "name": "nope", "compartmentId": "ocid1.compartment.oc1..doesnotexist", + }) + require.Equal(t, http.StatusNotFound, rec.Code) + assert.Equal(t, "NotAuthorizedOrNotFound", errorCode(t, rec)) + + f.createBucket(t, "photos") + + rec = f.do(t, http.MethodPost, f.bucketPath("photos"), map[string]any{ + "compartmentId": "ocid1.compartment.oc1..doesnotexist", + }) + require.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos"), nil) + assert.Contains(t, rec.Body.String(), testCompartment, "a refused move leaves the bucket where it was") +} + +// Every Object Storage list pages by limit and page and stamps opc-next-page. +func TestListsPaginate(t *testing.T) { + f := newFixture(t) + + for _, b := range []string{"b1", "b2", "b3"} { + f.createBucket(t, b) + } + + ctx := t.Context() + require.NoError(t, f.mock.SetVersioningStatus(ctx, "b1", "Enabled")) + + for range 3 { + require.NoError(t, f.mock.PutObject(ctx, "b1", "k", []byte("v"), "text/plain", nil)) + } + + for i := range 3 { + rec := f.do(t, http.MethodPost, f.bucketPath("b1")+"/u", map[string]any{"object": "big" + string(rune('a'+i))}) + require.Equal(t, http.StatusOK, rec.Code) + + rec = f.do(t, http.MethodPost, f.bucketPath("b1")+"/p", map[string]any{ + "name": "p", "accessType": "AnyObjectRead", "timeExpires": inAnHour(), + }) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + } + + countOf := func(raw []byte) int { + var arr []json.RawMessage + if json.Unmarshal(raw, &arr) == nil { + return len(arr) + } + + var env struct { + Items []json.RawMessage `json:"items"` + } + + require.NoError(t, json.Unmarshal(raw, &env)) + + return len(env.Items) + } + + for _, path := range []string{ + "/n/" + f.ns + "/b?compartmentId=" + testCompartment, + f.bucketPath("b1") + "/objectversions", + f.bucketPath("b1") + "/p", + f.bucketPath("b1") + "/u", + } { + t.Run(path, func(t *testing.T) { + sep := "?" + if strings.Contains(path, "?") { + sep = "&" + } + + seen := 0 + next := "" + + for range 5 { + url := path + sep + "limit=1" + if next != "" { + url += "&page=" + next + } + + rec := f.do(t, http.MethodGet, url, nil) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.Equal(t, 1, countOf(rec.Body.Bytes())) + + seen++ + next = rec.Header().Get("opc-next-page") + + if next == "" { + break + } + } + + assert.Equal(t, 3, seen, "three items one page at a time, the last with no opc-next-page") + }) + } +} + +// copyObject applies or refuses every CopyObjectDetails field rather than +// dropping one. +func TestCopyObjectHonoursEveryField(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "src") + f.createBucket(t, "dst") + + ctx := t.Context() + require.NoError(t, f.mock.SetVersioningStatus(ctx, "src", "Enabled")) + + rec := f.doWith(t, http.MethodPut, f.bucketPath("src")+"/o/a", []byte("v1"), map[string]string{"opc-meta-owner": "ada"}) + require.Equal(t, http.StatusOK, rec.Code) + v1 := rec.Header().Get("version-id") + srcETag := rec.Header().Get("ETag") + + rec = f.do(t, http.MethodPut, f.bucketPath("src")+"/o/a", []byte("v2")) + require.Equal(t, http.StatusOK, rec.Code) + + copyTo := func(extra map[string]any) *httptest.ResponseRecorder { + body := map[string]any{ + "sourceObjectName": "a", "destinationRegion": "us-ashburn-1", "destinationNamespace": f.ns, + "destinationBucket": "dst", "destinationObjectName": "b", + } + for k, v := range extra { + body[k] = v + } + + return f.do(t, http.MethodPost, f.bucketPath("src")+"/actions/copyObject", body) + } + + rec = copyTo(map[string]any{"destinationRegion": "eu-frankfurt-1"}) + require.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "cross-region") + + rec = copyTo(map[string]any{"destinationRegion": ""}) + assert.Equal(t, http.StatusBadRequest, rec.Code, "destinationRegion is required") + + rec = copyTo(map[string]any{"sourceObjectIfMatchETag": srcETag}) + require.Equal(t, http.StatusPreconditionFailed, rec.Code, "the source has moved on from that ETag") + + rec = copyTo(map[string]any{"destinationObjectStorageTier": "Glacier"}) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + // A source version, replacement metadata and a storage tier all apply. + rec = copyTo(map[string]any{ + "sourceVersionId": v1, + "destinationObjectMetadata": map[string]string{"owner": "bo"}, + "destinationObjectStorageTier": "InfrequentAccess", + }) + require.Equal(t, http.StatusAccepted, rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodGet, f.bucketPath("dst")+"/o/b", nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "v1", rec.Body.String()) + assert.Equal(t, "bo", rec.Header().Get("opc-meta-owner")) + assert.Equal(t, "InfrequentAccess", rec.Header().Get("storage-tier")) + + // The destination preconditions apply too. + rec = copyTo(map[string]any{"destinationObjectIfNoneMatchETag": "*"}) + require.Equal(t, http.StatusPreconditionFailed, rec.Code) + assert.Equal(t, "IfNoneMatchFailed", errorCode(t, rec)) + + rec = copyTo(map[string]any{"destinationObjectIfMatchETag": "stale"}) + require.Equal(t, http.StatusPreconditionFailed, rec.Code) + assert.Equal(t, "IfMatchFailed", errorCode(t, rec)) +} diff --git a/server/oci/objectstorage/errors.go b/server/oci/objectstorage/errors.go new file mode 100644 index 000000000..479905a90 --- /dev/null +++ b/server/oci/objectstorage/errors.go @@ -0,0 +1,63 @@ +package objectstorage + +import ( + "errors" + "net/http" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + "github.com/stackshy/cloudemu/v2/internal/pagination" + osprovider "github.com/stackshy/cloudemu/v2/providers/oci/objectstorage" + "github.com/stackshy/cloudemu/v2/server/wire/ocirest" +) + +// serviceStatus is the HTTP status of each Object Storage-specific code. +// +//nolint:gochecknoglobals // a fixed lookup table +var serviceStatus = map[string]int{ + osprovider.CodeBucketAlreadyExists: http.StatusConflict, + osprovider.CodeBucketNotEmpty: http.StatusConflict, + osprovider.CodeIfMatchFailed: http.StatusPreconditionFailed, + osprovider.CodeIfNoneMatchFailed: http.StatusPreconditionFailed, +} + +// writeDriverError reports the Object Storage code a provider error carries, +// falling back to the shared OCI codec for portable errors. +func writeDriverError(w http.ResponseWriter, r *http.Request, err error) { + var se *osprovider.ServiceError + if errors.As(err, &se) { + if status, ok := serviceStatus[se.Code]; ok { + ocirest.WriteError(w, r, status, se.Code, cerrors.Message(err)) + return + } + } + + ocirest.WriteDriverError(w, r, err) +} + +// writePage pages a full listing by OCI's limit and page parameters, stamping +// opc-next-page when more remain. An absent limit takes Object Storage's own +// page size of 1000, as ListObjects does. +func writePage[T any](w http.ResponseWriter, r *http.Request, items []T) { + writePageAs(w, r, items, func(page []T) any { return page }) +} + +// writePageAs is writePage for a listing wrapped in an envelope. +func writePageAs[T any](w http.ResponseWriter, r *http.Request, items []T, wrap func([]T) any) { + limit := listLimit(r) + if limit <= 0 { + limit = ocirest.MaxLimit + } + + page, err := pagination.Paginate(items, ocirest.Page(r), limit) + if err != nil { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, "invalid page token: "+err.Error()) + return + } + + if page.Items == nil { + page.Items = []T{} + } + + ocirest.SetNextPage(w, page.NextPageToken) + ocirest.WriteJSON(w, r, http.StatusOK, wrap(page.Items)) +} diff --git a/server/oci/objectstorage/handler.go b/server/oci/objectstorage/handler.go new file mode 100644 index 000000000..54eccd7c4 --- /dev/null +++ b/server/oci/objectstorage/handler.go @@ -0,0 +1,399 @@ +// Package objectstorage implements OCI's Object Storage REST API against a +// CloudEmu storage driver. Real github.com/oracle/oci-go-sdk objectstorage +// clients hit this handler the same way they hit +// objectstorage..oraclecloud.com. +// +// Object Storage carries no API-version prefix; every path is rooted at the +// tenancy namespace, so Matches claims /n and the pre-authenticated request +// redemption prefix and nothing else: +// +// GET /n — get namespace +// GET /n/{ns} — namespace metadata +// POST/GET /n/{ns}/b — create, list (by compartmentId) +// GET/POST/DELETE/HEAD /n/{ns}/b/{bucket} — get, update, delete, head +// GET /n/{ns}/b/{bucket}/o — list objects +// PUT/GET/HEAD/DELETE /n/{ns}/b/{bucket}/o/{object} — put, get, head, delete +// GET /n/{ns}/b/{bucket}/objectversions — list object versions +// POST /n/{ns}/b/{bucket}/actions/renameObject +// POST /n/{ns}/b/{bucket}/actions/copyObject — async, work request +// POST /n/{ns}/b/{bucket}/actions/updateObjectStorageTier +// POST/GET /n/{ns}/b/{bucket}/u — multipart create, list +// PUT/POST/GET/DELETE /n/{ns}/b/{bucket}/u/{object} — upload, commit, list parts, abort +// POST/GET /n/{ns}/b/{bucket}/p — pre-authenticated requests +// GET/DELETE /n/{ns}/b/{bucket}/p/{parId} +// POST/GET /n/{ns}/b/{bucket}/retentionRules[/{id}] +// PUT/GET/DELETE /n/{ns}/b/{bucket}/l — object lifecycle policy +// GET/PUT /p/{par}/n/{ns}/b/{bucket}/o/{object} — redeem a PAR +// +// Only ListBuckets requires compartmentId: it is the one collection OCI scopes +// by compartment. Every other list here is scoped by the bucket, which already +// belongs to a compartment, so requiring the parameter would reject calls real +// OCI accepts. +// +// Not emulated: /actions/reencrypt and /actions/restoreObjects, which need +// per-object key material and an archive-retrieval lifecycle the storage driver +// has no shape for — the handler claims them so a caller is told why rather +// than left with a bare 404. +package objectstorage + +import ( + "context" + "net/http" + "strings" + + cerrors "github.com/stackshy/cloudemu/v2/errors" + osprovider "github.com/stackshy/cloudemu/v2/providers/oci/objectstorage" + "github.com/stackshy/cloudemu/v2/server/oci/workrequest" + "github.com/stackshy/cloudemu/v2/server/wire/ocirest" + "github.com/stackshy/cloudemu/v2/services/scope" + "github.com/stackshy/cloudemu/v2/services/storage/driver" +) + +// Path segments this handler claims. +const ( + segNamespace = "n" + segPAR = "p" + segBuckets = "b" + + subObjects = "o" + subUploads = "u" + subPARs = "p" + subActions = "actions" + subRetentionRules = "retentionRules" + subObjectVersions = "objectversions" + subLifecycle = "l" +) + +// Actions on a bucket. +const ( + actionRename = "renameObject" + actionCopy = "copyObject" + actionUpdateTier = "updateObjectStorageTier" + actionReencrypt = "reencrypt" + actionRestoreObjects = "restoreObjects" +) + +// Error codes the handler raises itself. +const ( + codeInvalidParameter = "InvalidParameter" + codeMethodNotAllowed = "MethodNotAllowed" + codeNotImplemented = "NotImplemented" + codeNotFound = "NotAuthorizedOrNotFound" + codeNotAuthorized = "NotAuthenticated" + codeInvalidRange = "InvalidRange" +) + +// operationCopy is the work request a copyObject records. +const operationCopy = "COPY_OBJECT" + +// Extras is the OCI-only surface the portable storage driver cannot express: +// the tenancy namespace, compartments, OCI's bucket settings, object rename +// and storage tiers, retention rules and pre-authenticated requests. +// *providers/oci/objectstorage.Mock satisfies it; any driver that does not is +// served 501 for every path this handler claims. +type Extras interface { + Namespace() string + Region() string + Metadata(ctx context.Context) osprovider.NamespaceMetadata + Scope(bucket string) scope.Scope + + CreateBucketWith(ctx context.Context, spec osprovider.BucketSpec) (*osprovider.Bucket, error) + BucketDetails(ctx context.Context, name string) (*osprovider.Bucket, error) + UpdateBucket(ctx context.Context, name string, upd osprovider.BucketUpdate) (*osprovider.Bucket, error) + ListBucketsIn(ctx context.Context, compartmentID string) ([]osprovider.Bucket, error) + + PutObjectWith( + ctx context.Context, bucket, key string, data []byte, opts osprovider.PutOptions, + ) (*osprovider.ObjectDetails, error) + ObjectDetailsOf(ctx context.Context, bucket, key string) (*osprovider.ObjectDetails, error) + ListObjectDetails( + ctx context.Context, bucket string, opts driver.ListOptions, + ) ([]osprovider.ObjectDetails, []string, string, error) + RenameObject(ctx context.Context, bucket, sourceName, newName string) (*osprovider.ObjectDetails, error) + UpdateObjectStorageTier(ctx context.Context, bucket, key, tier string) error + + CreateMultipartUploadWith( + ctx context.Context, bucket string, spec osprovider.MultipartUploadSpec, + ) (*driver.MultipartUpload, error) + + CreateRetentionRule( + ctx context.Context, bucket string, spec osprovider.RetentionRuleSpec, + ) (*osprovider.RetentionRule, error) + GetRetentionRule(ctx context.Context, bucket, ruleID string) (*osprovider.RetentionRule, error) + ListRetentionRules(ctx context.Context, bucket string) ([]osprovider.RetentionRule, error) + UpdateRetentionRule( + ctx context.Context, bucket, ruleID string, spec osprovider.RetentionRuleSpec, + ) (*osprovider.RetentionRule, error) + DeleteRetentionRule(ctx context.Context, bucket, ruleID string) error + + CreatePAR(ctx context.Context, bucket string, spec osprovider.PARSpec) (*osprovider.PreauthenticatedRequest, error) + GetPAR(ctx context.Context, bucket, parID string) (*osprovider.PreauthenticatedRequest, error) + ListPARs(ctx context.Context, bucket, objectNamePrefix string) ([]osprovider.PreauthenticatedRequest, error) + DeletePAR(ctx context.Context, bucket, parID string) error + ResolvePAR(ctx context.Context, token string) (*osprovider.PreauthenticatedRequest, error) + + PutLifecyclePolicy( + ctx context.Context, bucket string, rules []osprovider.LifecycleRule, + ) (*osprovider.LifecyclePolicy, error) + GetLifecyclePolicy(ctx context.Context, bucket string) (*osprovider.LifecyclePolicy, error) + DeleteLifecyclePolicy(ctx context.Context, bucket string) error + + DeleteBucketIf(ctx context.Context, name, ifMatch string) error + CopyObjectWith(ctx context.Context, spec osprovider.CopySpec) error + DeleteObjectIf(ctx context.Context, bucket, key, versionID, ifMatch string) (string, bool, error) +} + +// Handler serves OCI Object Storage against a storage driver. +type Handler struct { + store driver.Bucket + extras Extras + versioned driver.VersionedBucket + work *workrequest.Store + + // compartmentExists reports whether a compartment OCID exists. It is nil + // unless SetCompartmentChecker wires it from Identity; a nil checker skips + // the check so handlers built without identity keep working. + compartmentExists func(id string) bool +} + +// New returns an Object Storage handler. work records the asynchronous copy; +// a nil store leaves that path unserved. +func New(b driver.Bucket, work *workrequest.Store) *Handler { + extras, _ := b.(Extras) + versioned, _ := b.(driver.VersionedBucket) + + return &Handler{store: b, extras: extras, versioned: versioned, work: work} +} + +// SetCompartmentChecker wires a compartment-existence check so a bucket created +// in, or moved to, a compartment that does not exist is rejected with +// 404 NotAuthorizedOrNotFound, as real OCI does. When unset (nil) the check is +// skipped, so handlers constructed without identity keep working. +func (h *Handler) SetCompartmentChecker(fn func(id string) bool) { h.compartmentExists = fn } + +// requireCompartment reports whether compartmentID exists; if not it writes +// the OCI 404 NotAuthorizedOrNotFound and returns false. A nil checker (no +// identity wired) is a no-op that allows the request. +func (h *Handler) requireCompartment(w http.ResponseWriter, r *http.Request, compartmentID string) bool { + if h.compartmentExists == nil || compartmentID == "" || h.compartmentExists(compartmentID) { + return true + } + + ocirest.WriteError(w, r, http.StatusNotFound, codeNotFound, compartmentID+" not found") + + return false +} + +// route is a parsed Object Storage path. +type route struct { + // PARToken is the redemption token of a /p/{token}/… request. + PARToken string + Namespace string + Bucket string + // Sub is the collection under a bucket: o, u, p, actions, retentionRules, + // objectversions or l. + Sub string + // Rest is everything after Sub: an object name (which may contain slashes), + // a PAR OCID, a retention rule OCID or an action name. + Rest string + // HasBucketSeg records whether a /b segment was present. It is what + // separates /n/{ns}/b from /n/{ns}; sniffing the raw path for "/b" instead + // misreads a namespace that itself starts with b. + HasBucketSeg bool +} + +// Matches claims the namespace-rooted Object Storage paths and the PAR +// redemption prefix, and nothing else. +func (*Handler) Matches(r *http.Request) bool { + _, ok := parsePath(r.URL.Path) + + return ok +} + +// parsePath splits an Object Storage path. It accepts the namespace root and +// everything under it, plus a /p/{token}/n/… PAR redemption. +func parsePath(p string) (route, bool) { + var rt route + + rest := p + + if strings.HasPrefix(rest, "/"+segPAR+"/") { + rem := rest[len("/"+segPAR+"/"):] + + idx := strings.Index(rem, "/"+segNamespace+"/") + if idx <= 0 { + return rt, false + } + + rt.PARToken = rem[:idx] + rest = rem[idx:] + } + + if rest == "/"+segNamespace || rest == "/"+segNamespace+"/" { + return rt, true + } + + if !strings.HasPrefix(rest, "/"+segNamespace+"/") { + return rt, false + } + + return parseNamespaced(rt, rest[len("/"+segNamespace+"/"):]) +} + +// parseNamespaced parses everything after /n/: the namespace, then the bucket +// collection under it. +// +//nolint:gocritic // route is built up and returned by value; the caller owns it. +func parseNamespaced(rt route, rem string) (route, bool) { + rt.Namespace, rem, _ = strings.Cut(rem, "/") + if rem == "" { + return rt, true + } + + var seg string + + seg, rem, _ = strings.Cut(rem, "/") + if seg != segBuckets { + return route{}, false + } + + rt.HasBucketSeg = true + + if rem == "" { + return rt, true + } + + rt.Bucket, rem, _ = strings.Cut(rem, "/") + if rem == "" { + return rt, true + } + + rt.Sub, rt.Rest, _ = strings.Cut(rem, "/") + + return rt, true +} + +// ServeHTTP routes on the path shape, then on method. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + rt, ok := parsePath(r.URL.Path) + if !ok { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, "malformed Object Storage path") + return + } + + if h.extras == nil { + ocirest.WriteError(w, r, http.StatusNotImplemented, codeNotImplemented, + "the wired storage driver does not implement OCI namespaces and compartments") + + return + } + + if rt.PARToken != "" { + h.servePAR(w, r, &rt) + return + } + + if rt.Namespace == "" { + h.getNamespace(w, r) + return + } + + if !h.namespaceOK(w, r, rt.Namespace) { + return + } + + if rt.Bucket == "" { + h.serveBucketCollection(w, r, &rt) + return + } + + h.serveBucket(w, r, &rt) +} + +// namespaceOK rejects a namespace that is not this tenancy's. Real OCI reports +// the same 404 it reports for a missing bucket. +func (h *Handler) namespaceOK(w http.ResponseWriter, r *http.Request, namespace string) bool { + if namespace == h.extras.Namespace() { + return true + } + + ocirest.WriteError(w, r, http.StatusNotFound, codeNotFound, "namespace "+namespace+" not found") + + return false +} + +// serveBucketCollection serves /n/{ns} and /n/{ns}/b. +func (h *Handler) serveBucketCollection(w http.ResponseWriter, r *http.Request, rt *route) { + if !rt.HasBucketSeg { + h.namespaceMetadata(w, r) + return + } + + switch r.Method { + case http.MethodPost: + h.createBucket(w, r) + case http.MethodGet: + h.listBuckets(w, r) + default: + methodNotAllowed(w, r) + } +} + +// serveBucket dispatches everything addressed at one bucket. +func (h *Handler) serveBucket(w http.ResponseWriter, r *http.Request, rt *route) { + switch rt.Sub { + case "": + h.serveBucketItem(w, r, rt.Bucket) + case subObjects: + h.serveObjects(w, r, rt) + case subUploads: + h.serveUploads(w, r, rt) + case subPARs: + h.servePARs(w, r, rt) + case subActions: + h.serveAction(w, r, rt) + case subRetentionRules: + h.serveRetentionRules(w, r, rt) + case subObjectVersions: + h.listObjectVersions(w, r, rt.Bucket) + case subLifecycle: + h.serveLifecycle(w, r, rt.Bucket) + default: + ocirest.WriteError(w, r, http.StatusNotFound, codeNotFound, "unknown collection "+rt.Sub) + } +} + +// serveAction dispatches the bucket-level actions. +func (h *Handler) serveAction(w http.ResponseWriter, r *http.Request, rt *route) { + if r.Method != http.MethodPost { + methodNotAllowed(w, r) + return + } + + switch rt.Rest { + case actionRename: + h.renameObject(w, r, rt.Bucket) + case actionCopy: + h.copyObject(w, r, rt.Bucket) + case actionUpdateTier: + h.updateStorageTier(w, r, rt.Bucket) + case actionReencrypt: + ocirest.WriteError(w, r, http.StatusNotImplemented, codeNotImplemented, + "reencrypt is not emulated; CloudEmu holds no per-object key material to re-wrap") + case actionRestoreObjects: + ocirest.WriteError(w, r, http.StatusNotImplemented, codeNotImplemented, + "restoreObjects is not emulated; archived objects are readable directly") + default: + ocirest.WriteError(w, r, http.StatusNotFound, codeNotFound, "unknown action "+rt.Rest) + } +} + +func methodNotAllowed(w http.ResponseWriter, r *http.Request) { + ocirest.WriteError(w, r, http.StatusMethodNotAllowed, codeMethodNotAllowed, + r.Method+" is not allowed on "+r.URL.Path) +} + +// errVersioningUnsupported reports a driver that keeps no version history. +func errVersioningUnsupported() error { + return cerrors.New(cerrors.Unimplemented, "the wired storage driver does not retain object versions") +} diff --git a/server/oci/objectstorage/handler_test.go b/server/oci/objectstorage/handler_test.go new file mode 100644 index 000000000..d1548ea80 --- /dev/null +++ b/server/oci/objectstorage/handler_test.go @@ -0,0 +1,1182 @@ +package objectstorage_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stackshy/cloudemu/v2/config" + osprovider "github.com/stackshy/cloudemu/v2/providers/oci/objectstorage" + ociobjectstorage "github.com/stackshy/cloudemu/v2/server/oci/objectstorage" + "github.com/stackshy/cloudemu/v2/server/oci/workrequest" + "github.com/stackshy/cloudemu/v2/services/storage/driver" +) + +const testCompartment = "ocid1.compartment.oc1..aaaaaaaatest" + +// The mock must satisfy the handler's OCI-only capability interface. +var _ ociobjectstorage.Extras = (*osprovider.Mock)(nil) + +type fixture struct { + handler *ociobjectstorage.Handler + mock *osprovider.Mock + ns string +} + +func newFixture(t *testing.T) fixture { + t.Helper() + + opts := config.NewOptions( + config.WithRegion("us-ashburn-1"), + config.WithCompartmentID(testCompartment), + ) + mock := osprovider.New(opts) + + return fixture{ + handler: ociobjectstorage.New(mock, workrequest.New(opts)), + mock: mock, + ns: mock.Namespace(), + } +} + +func (f fixture) do(t *testing.T, method, path string, body any) *httptest.ResponseRecorder { + t.Helper() + + var reader io.Reader + + switch b := body.(type) { + case nil: + case []byte: + reader = bytes.NewReader(b) + default: + raw, err := json.Marshal(b) + require.NoError(t, err) + reader = bytes.NewReader(raw) + } + + req := httptest.NewRequest(method, path, reader) + rec := httptest.NewRecorder() + f.handler.ServeHTTP(rec, req) + + return rec +} + +// inAnHour is a timeExpires OCI accepts: required, and in the future. +func inAnHour() string { return time.Now().Add(time.Hour).UTC().Format(time.RFC3339) } + +func (f fixture) bucketPath(bucket string) string { + return "/n/" + f.ns + "/b/" + bucket +} + +// createBucket creates a bucket over the wire and fails if the handler refuses. +func (f fixture) createBucket(t *testing.T, name string) { + t.Helper() + + rec := f.do(t, http.MethodPost, "/n/"+f.ns+"/b", map[string]any{ + "name": name, "compartmentId": testCompartment, + }) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) +} + +func TestMatches(t *testing.T) { + f := newFixture(t) + + tests := []struct { + name string + path string + expect bool + }{ + {name: "namespace root", path: "/n", expect: true}, + {name: "namespace", path: "/n/axaxnpcrorw5", expect: true}, + {name: "bucket collection", path: "/n/axaxnpcrorw5/b", expect: true}, + {name: "bucket", path: "/n/axaxnpcrorw5/b/photos", expect: true}, + {name: "object", path: "/n/axaxnpcrorw5/b/photos/o/dir/a.jpg", expect: true}, + {name: "multipart", path: "/n/axaxnpcrorw5/b/photos/u/big", expect: true}, + {name: "par redemption", path: "/p/tok/n/axaxnpcrorw5/b/photos/o/a.jpg", expect: true}, + {name: "retention rules", path: "/n/axaxnpcrorw5/b/photos/retentionRules", expect: true}, + + {name: "vcn collection", path: "/20160918/vcns", expect: false}, + {name: "vcn subnet", path: "/20160918/subnets/ocid1.subnet.oc1.iad.a", expect: false}, + {name: "work requests", path: "/20160918/workRequests", expect: false}, + {name: "identity users", path: "/20160918/users", expect: false}, + {name: "monitoring", path: "/20180401/metrics", expect: false}, + {name: "namespaces is not the namespace root", path: "/namespaces", expect: false}, + {name: "nodes is not the namespace root", path: "/nodes/n", expect: false}, + {name: "root", path: "/", expect: false}, + {name: "par without a namespace", path: "/p/tok/o/a.jpg", expect: false}, + {name: "namespace with a foreign collection", path: "/n/axaxnpcrorw5/vcns", expect: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, tc.path, nil) + assert.Equal(t, tc.expect, f.handler.Matches(req)) + }) + } +} + +func TestExtrasAbsentServes501(t *testing.T) { + h := ociobjectstorage.New(bareBucket{}, nil) + req := httptest.NewRequest(http.MethodGet, "/n", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + assert.Equal(t, http.StatusNotImplemented, rec.Code) + assert.Contains(t, rec.Body.String(), "namespaces") +} + +func TestNamespaceEndpoints(t *testing.T) { + f := newFixture(t) + + rec := f.do(t, http.MethodGet, "/n", nil) + require.Equal(t, http.StatusOK, rec.Code) + + var ns string + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &ns)) + assert.Equal(t, f.ns, ns) + assert.NotEmpty(t, rec.Header().Get("opc-request-id")) + + rec = f.do(t, http.MethodGet, "/n/"+f.ns, nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Contains(t, rec.Body.String(), `"defaultS3CompartmentId"`) + + rec = f.do(t, http.MethodGet, "/n/wrongns/b", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) +} + +// A namespace beginning with b must still route GET /n/{ns} to the metadata +// endpoint: the /b bucket collection is a path segment, not a substring. +func TestNamespaceMetadataWithBPrefixedNamespace(t *testing.T) { + opts := config.NewOptions( + config.WithRegion("us-ashburn-1"), + config.WithCompartmentID(testCompartment), + config.WithTenancyOCID("ocid1.tenancy.oc1..probe0100"), + ) + mock := osprovider.New(opts) + f := fixture{handler: ociobjectstorage.New(mock, workrequest.New(opts)), mock: mock, ns: mock.Namespace()} + + require.True(t, strings.HasPrefix(f.ns, "b"), "fixture namespace must start with b, got %q", f.ns) + + rec := f.do(t, http.MethodGet, "/n/"+f.ns, nil) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + var meta osprovider.NamespaceMetadata + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &meta)) + assert.Equal(t, f.ns, meta.Namespace) + + // The bucket collection under the same namespace still lists. + rec = f.do(t, http.MethodGet, "/n/"+f.ns+"/b?compartmentId="+testCompartment, nil) + assert.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) +} + +func TestBucketWire(t *testing.T) { + f := newFixture(t) + + rec := f.do(t, http.MethodPost, "/n/"+f.ns+"/b", map[string]any{ + "name": "photos", "compartmentId": testCompartment, "publicAccessType": "ObjectRead", + }) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.NotEmpty(t, rec.Header().Get("ETag")) + + var created map[string]any + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &created)) + assert.Equal(t, "ObjectRead", created["publicAccessType"]) + assert.Contains(t, created["id"], "ocid1.bucket.oc1.iad.") + + rec = f.do(t, http.MethodPost, "/n/"+f.ns+"/b", map[string]any{ + "name": "photos", "compartmentId": testCompartment, + }) + assert.Equal(t, http.StatusConflict, rec.Code) + + rec = f.do(t, http.MethodPost, "/n/"+f.ns+"/b", map[string]any{"name": "nocompartment"}) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodGet, "/n/"+f.ns+"/b", nil) + assert.Equal(t, http.StatusBadRequest, rec.Code, "compartmentId is required on ListBuckets") + + rec = f.do(t, http.MethodGet, "/n/"+f.ns+"/b?compartmentId="+testCompartment, nil) + require.Equal(t, http.StatusOK, rec.Code) + + var summaries []map[string]any + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &summaries)) + require.Len(t, summaries, 1) + + rec = f.do(t, http.MethodGet, "/n/"+f.ns+"/b?compartmentId=ocid1.compartment.oc1..other", nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.JSONEq(t, `[]`, rec.Body.String()) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos"), nil) + assert.Equal(t, http.StatusOK, rec.Code) + + rec = f.do(t, http.MethodGet, f.bucketPath("missing"), nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodPost, f.bucketPath("photos"), map[string]any{"versioning": "Enabled"}) + require.Equal(t, http.StatusOK, rec.Code) + assert.Contains(t, rec.Body.String(), `"versioning":"Enabled"`) + + rec = f.do(t, http.MethodDelete, f.bucketPath("photos"), nil) + assert.Equal(t, http.StatusNoContent, rec.Code) + + rec = f.do(t, http.MethodPatch, f.bucketPath("photos"), nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) +} + +func TestObjectWire(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + + req := httptest.NewRequest(http.MethodPut, f.bucketPath("photos")+"/o/dir/a.txt", bytes.NewReader([]byte("hello"))) + req.Header.Set("Content-Type", "text/plain") + req.Header.Set("opc-meta-owner", "ada") + rec := httptest.NewRecorder() + f.handler.ServeHTTP(rec, req) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.NotEmpty(t, rec.Header().Get("ETag")) + assert.NotEmpty(t, rec.Header().Get("opc-content-md5")) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/o/dir/a.txt", nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "hello", rec.Body.String()) + assert.Equal(t, "text/plain", rec.Header().Get("Content-Type")) + assert.Equal(t, "ada", rec.Header().Get("opc-meta-owner")) + + rec = f.do(t, http.MethodHead, f.bucketPath("photos")+"/o/dir/a.txt", nil) + assert.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "5", rec.Header().Get("Content-Length"), "a HEAD carries no body, so it must report the size") + assert.Equal(t, "text/plain", rec.Header().Get("Content-Type")) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/o/nope.txt", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/o?prefix=dir/&delimiter=/", nil) + require.Equal(t, http.StatusOK, rec.Code) + + var list struct { + Objects []struct { + Name string `json:"name"` + Size int64 `json:"size"` + } `json:"objects"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &list)) + require.Len(t, list.Objects, 1) + assert.Equal(t, "dir/a.txt", list.Objects[0].Name) + assert.Equal(t, int64(5), list.Objects[0].Size) + + rec = f.do(t, http.MethodDelete, f.bucketPath("photos")+"/o/dir/a.txt", nil) + assert.Equal(t, http.StatusNoContent, rec.Code) + + rec = f.do(t, http.MethodDelete, f.bucketPath("photos")+"/o/dir/a.txt", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) +} + +func TestObjectActions(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "src") + f.createBucket(t, "dst") + require.NoError(t, f.mock.PutObject(context.Background(), "src", "old", []byte("v"), "text/plain", nil)) + + rec := f.do(t, http.MethodPost, f.bucketPath("src")+"/actions/renameObject", map[string]any{ + "sourceName": "old", "newName": "new", + }) + assert.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodPost, f.bucketPath("src")+"/actions/renameObject", map[string]any{ + "sourceName": "old", "newName": "other", + }) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodPost, f.bucketPath("src")+"/actions/copyObject", map[string]any{ + "sourceObjectName": "new", "destinationRegion": "us-ashburn-1", + "destinationBucket": "dst", "destinationObjectName": "copied", + }) + require.Equal(t, http.StatusAccepted, rec.Code, rec.Body.String()) + assert.NotEmpty(t, rec.Header().Get("opc-work-request-id")) + + obj, err := f.mock.GetObject(context.Background(), "dst", "copied") + require.NoError(t, err) + assert.Equal(t, []byte("v"), obj.Data) + + rec = f.do(t, http.MethodPost, f.bucketPath("src")+"/actions/copyObject", map[string]any{ + "sourceObjectName": "new", "destinationBucket": "dst", + }) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPost, f.bucketPath("src")+"/actions/updateObjectStorageTier", map[string]any{ + "objectName": "new", "storageTier": "Archive", + }) + assert.Equal(t, http.StatusOK, rec.Code) + + rec = f.do(t, http.MethodPost, f.bucketPath("src")+"/actions/reencrypt", nil) + assert.Equal(t, http.StatusNotImplemented, rec.Code) + + rec = f.do(t, http.MethodPost, f.bucketPath("src")+"/actions/restoreObjects", nil) + assert.Equal(t, http.StatusNotImplemented, rec.Code) + + rec = f.do(t, http.MethodPost, f.bucketPath("src")+"/actions/teleport", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) +} + +func TestMultipartWire(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + + rec := f.do(t, http.MethodPost, f.bucketPath("photos")+"/u", map[string]any{"object": "big"}) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + var up struct { + UploadID string `json:"uploadId"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &up)) + require.NotEmpty(t, up.UploadID) + + base := f.bucketPath("photos") + "/u/big?uploadId=" + up.UploadID + + rec = f.do(t, http.MethodPut, base+"&uploadPartNum=1", []byte("aaa")) + require.Equal(t, http.StatusOK, rec.Code) + etag1 := rec.Header().Get("ETag") + + rec = f.do(t, http.MethodPut, base+"&uploadPartNum=2", []byte("bbb")) + require.Equal(t, http.StatusOK, rec.Code) + etag2 := rec.Header().Get("ETag") + + rec = f.do(t, http.MethodPut, base+"&uploadPartNum=notanumber", []byte("x")) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodGet, base, nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Contains(t, rec.Body.String(), `"partNumber":1`) + + rec = f.do(t, http.MethodPost, base, map[string]any{ + "partsToCommit": []map[string]any{{"partNum": 1}, {"partNum": 2}}, + "partsToExclude": []int{3}, + }) + assert.Equal(t, http.StatusNotImplemented, rec.Code, "partsToExclude must be rejected, not dropped") + + rec = f.do(t, http.MethodPost, base, map[string]any{ + "partsToCommit": []map[string]any{{"partNum": 1}, {"partNum": 2}}, + }) + assert.Equal(t, http.StatusBadRequest, rec.Code, "each committed part needs its etag") + + rec = f.do(t, http.MethodPost, base, map[string]any{ + "partsToCommit": []map[string]any{{"partNum": 1, "etag": etag1}, {"partNum": 2, "etag": "wrong-etag"}}, + }) + assert.Equal(t, http.StatusBadRequest, rec.Code, "a mismatched part etag is refused") + + rec = f.do(t, http.MethodPost, base, map[string]any{ + "partsToCommit": []map[string]any{{"partNum": 1, "etag": etag1}, {"partNum": 2, "etag": etag2}}, + }) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + obj, err := f.mock.GetObject(context.Background(), "photos", "big") + require.NoError(t, err) + assert.Equal(t, []byte("aaabbb"), obj.Data) + + rec = f.do(t, http.MethodDelete, f.bucketPath("photos")+"/u/big?uploadId="+up.UploadID, nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodPut, f.bucketPath("photos")+"/u/big", []byte("x")) + assert.Equal(t, http.StatusBadRequest, rec.Code, "uploadId is required") +} + +func TestVersioningWire(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + + rec := f.do(t, http.MethodPost, f.bucketPath("photos"), map[string]any{"versioning": "Enabled"}) + require.Equal(t, http.StatusOK, rec.Code) + + ctx := context.Background() + require.NoError(t, f.mock.PutObject(ctx, "photos", "k", []byte("v1"), "text/plain", nil)) + require.NoError(t, f.mock.PutObject(ctx, "photos", "k", []byte("v2"), "text/plain", nil)) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/objectversions", nil) + require.Equal(t, http.StatusOK, rec.Code) + + var versions struct { + Items []struct { + VersionID string `json:"versionId"` + Size int64 `json:"size"` + } `json:"items"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &versions)) + require.Len(t, versions.Items, 2) + + oldest := versions.Items[1].VersionID + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/o/k?versionId="+oldest, nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "v1", rec.Body.String()) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/o/k?versionId=bogus", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodDelete, f.bucketPath("photos")+"/o/k", nil) + require.Equal(t, http.StatusNoContent, rec.Code) + assert.Equal(t, "true", rec.Header().Get("is-delete-marker")) +} + +func TestPARWire(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + require.NoError(t, f.mock.PutObject(context.Background(), "photos", "a.txt", []byte("hi"), "text/plain", nil)) + + rec := f.do(t, http.MethodPost, f.bucketPath("photos")+"/p", map[string]any{ + "name": "read-a", "objectName": "a.txt", "accessType": "ObjectRead", "timeExpires": inAnHour(), + }) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + var par struct { + ID string `json:"id"` + AccessURI string `json:"accessUri"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &par)) + require.NotEmpty(t, par.AccessURI) + assert.Contains(t, par.ID, "ocid1.preauthenticatedrequest.") + + rec = f.do(t, http.MethodGet, par.AccessURI, nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "hi", rec.Body.String()) + + rec = f.do(t, http.MethodPut, par.AccessURI, []byte("nope")) + assert.Equal(t, http.StatusForbidden, rec.Code, "a read PAR must not authorize a write") + + rec = f.do(t, http.MethodPost, f.bucketPath("photos")+"/p", map[string]any{ + "name": "bad", "accessType": "ObjectRead", + }) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/p", nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Contains(t, rec.Body.String(), par.ID) + + rec = f.do(t, http.MethodDelete, f.bucketPath("photos")+"/p/"+par.ID, nil) + assert.Equal(t, http.StatusNoContent, rec.Code) + + rec = f.do(t, http.MethodGet, par.AccessURI, nil) + assert.Equal(t, http.StatusNotFound, rec.Code, "a revoked PAR stops working") +} + +func TestRetentionWire(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + + rec := f.do(t, http.MethodPost, f.bucketPath("photos")+"/retentionRules", map[string]any{ + "displayName": "thirty", + "duration": map[string]any{"timeAmount": 30, "timeUnit": "DAYS"}, + }) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + var rule struct { + ID string `json:"id"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &rule)) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/retentionRules", nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Contains(t, rec.Body.String(), rule.ID) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/retentionRules/"+rule.ID, nil) + assert.Equal(t, http.StatusOK, rec.Code) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/retentionRules/missing", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodPost, f.bucketPath("photos")+"/retentionRules", map[string]any{ + "duration": map[string]any{"timeAmount": 1, "timeUnit": "FORTNIGHTS"}, + }) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodDelete, f.bucketPath("photos")+"/retentionRules/"+rule.ID, nil) + assert.Equal(t, http.StatusNoContent, rec.Code) +} + +func TestLifecycleWire(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + + rec := f.do(t, http.MethodPut, f.bucketPath("photos")+"/l", map[string]any{ + "items": []map[string]any{{ + "name": "expire-logs", "action": "DELETE", "timeAmount": 30, "timeUnit": "DAYS", + "isEnabled": true, "objectNameFilter": map[string]any{"inclusionPrefixes": []string{"logs/"}}, + }}, + }) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/l", nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Contains(t, rec.Body.String(), `"action":"DELETE"`) + assert.Contains(t, rec.Body.String(), `"logs/"`) + + rec = f.do(t, http.MethodPut, f.bucketPath("photos")+"/l", map[string]any{ + "items": []map[string]any{{"name": "bogus", "action": "TELEPORT", "timeAmount": 1, "timeUnit": "DAYS"}}, + }) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPut, f.bucketPath("photos")+"/l", map[string]any{ + "items": []map[string]any{{ + "name": "multi", "action": "DELETE", "timeAmount": 1, "timeUnit": "DAYS", + "objectNameFilter": map[string]any{"inclusionPrefixes": []string{"a/", "b/"}}, + }}, + }) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), `"inclusionPrefixes":["a/","b/"]`, "every prefix is kept") + + rec = f.do(t, http.MethodDelete, f.bucketPath("photos")+"/l", nil) + assert.Equal(t, http.StatusNoContent, rec.Code) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/l", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) +} + +// bareBucket is a storage driver that implements nothing beyond driver.Bucket, +// standing in for a non-OCI provider wired into the handler. +type bareBucket struct{ driver.Bucket } + +// A driver with no version history is served the same Unimplemented for every +// versioned route rather than a bare 404. +type unversionedStore struct { + *osprovider.Mock +} + +func (unversionedStore) GetObjectVersion(_ context.Context, _, _, _ string) (*driver.Object, error) { + panic("must not be reached: the handler must not discover this capability") +} + +func TestVersioningUnsupportedIsNamed(t *testing.T) { + opts := config.NewOptions(config.WithRegion("us-ashburn-1"), config.WithCompartmentID(testCompartment)) + mock := osprovider.New(opts) + // A store that is a driver.Bucket and the OCI Extras, but not a + // driver.VersionedBucket. + store := struct { + driver.Bucket + ociobjectstorage.Extras + }{Bucket: mock, Extras: mock} + + f := fixture{ + handler: ociobjectstorage.New(store, workrequest.New(opts)), + mock: mock, + ns: mock.Namespace(), + } + f.createBucket(t, "photos") + require.NoError(t, mock.PutObject(t.Context(), "photos", "k", []byte("v"), "text/plain", nil)) + + paths := []struct { + name string + method string + path string + }{ + {"objectversions", http.MethodGet, f.bucketPath("photos") + "/objectversions"}, + {"get by version", http.MethodGet, f.bucketPath("photos") + "/o/k?versionId=abc"}, + {"head by version", http.MethodHead, f.bucketPath("photos") + "/o/k?versionId=abc"}, + {"delete by version", http.MethodDelete, f.bucketPath("photos") + "/o/k?versionId=abc"}, + } + + for _, tc := range paths { + t.Run(tc.name, func(t *testing.T) { + rec := f.do(t, tc.method, tc.path, nil) + assert.Equal(t, http.StatusNotImplemented, rec.Code, rec.Body.String()) + }) + } +} + +func TestBucketItemWire(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + + rec := f.do(t, http.MethodHead, f.bucketPath("photos"), nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.NotEmpty(t, rec.Header().Get("ETag")) + + rec = f.do(t, http.MethodHead, f.bucketPath("missing"), nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodPost, f.bucketPath("photos"), []byte("{")) + assert.Equal(t, http.StatusBadRequest, rec.Code, "a malformed body is refused") + + rec = f.do(t, http.MethodPost, f.bucketPath("photos"), map[string]any{"versioning": "On"}) + assert.Equal(t, http.StatusBadRequest, rec.Code, "an unmodelled versioning value is named") + + rec = f.do(t, http.MethodPost, f.bucketPath("missing"), map[string]any{"publicAccessType": "NoPublicAccess"}) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodPost, f.bucketPath("photos"), map[string]any{ + "compartmentId": "ocid1.compartment.oc1..moved", + "autoTiering": "InfrequentAccess", + "metadata": map[string]string{"team": "infra"}, + "freeformTags": map[string]string{"env": "dev"}, + }) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), `"autoTiering":"InfrequentAccess"`) + + rec = f.do(t, http.MethodDelete, f.bucketPath("missing"), nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodPost, "/n/"+f.ns+"/b", []byte("not json")) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodDelete, "/n/"+f.ns+"/b", nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) + + rec = f.do(t, http.MethodDelete, "/n/"+f.ns, nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code, "the namespace itself takes only GET") + + rec = f.do(t, http.MethodPost, "/n", nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/unknown", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodGet, "/nope", nil) + assert.Equal(t, http.StatusBadRequest, rec.Code) +} + +// An unspecified limit must yield OCI's page size of 1000, not the 100 the +// other OCI services share through ocirest.DefaultLimit. +func TestListObjectsDefaultPageSizeIsOCIs1000(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + + ctx := t.Context() + for i := range 150 { + require.NoError(t, f.mock.PutObject(ctx, "photos", + fmt.Sprintf("k-%03d", i), []byte("v"), "text/plain", nil)) + } + + var list struct { + Objects []map[string]any `json:"objects"` + NextStartWith string `json:"nextStartWith"` + } + + rec := f.do(t, http.MethodGet, f.bucketPath("photos")+"/o", nil) + require.Equal(t, http.StatusOK, rec.Code) + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &list)) + assert.Len(t, list.Objects, 150, "all 150 fit in OCI's default page") + assert.Empty(t, list.NextStartWith) + assert.Empty(t, rec.Header().Get("opc-next-page")) + + // An explicit limit is still honored, and still paginates. + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/o?limit=100", nil) + require.Equal(t, http.StatusOK, rec.Code) + + list.Objects = nil + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &list)) + assert.Len(t, list.Objects, 100) + assert.NotEmpty(t, list.NextStartWith) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/o?start="+list.NextStartWith, nil) + require.Equal(t, http.StatusOK, rec.Code) + + list.Objects = nil + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &list)) + assert.Len(t, list.Objects, 50) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/o?start=%7Bbroken", nil) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodGet, f.bucketPath("missing")+"/o", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodPost, f.bucketPath("photos")+"/o", nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) + + rec = f.do(t, http.MethodPatch, f.bucketPath("photos")+"/o/k-000", nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) +} + +func TestObjectVersionWire(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + + ctx := t.Context() + require.NoError(t, f.mock.SetVersioningStatus(ctx, "photos", osprovider.VersioningEnabled)) + require.NoError(t, f.mock.PutObject(ctx, "photos", "k", []byte("v1"), "text/plain", nil)) + + first, err := f.mock.HeadObject(ctx, "photos", "k") + require.NoError(t, err) + require.NoError(t, f.mock.PutObject(ctx, "photos", "k", []byte("v2"), "text/plain", nil)) + + rec := f.do(t, http.MethodGet, f.bucketPath("photos")+"/o/k?versionId="+first.VersionID, nil) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.Equal(t, "v1", rec.Body.String()) + assert.Equal(t, first.VersionID, rec.Header().Get("version-id")) + + rec = f.do(t, http.MethodHead, f.bucketPath("photos")+"/o/k?versionId="+first.VersionID, nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "2", rec.Header().Get("Content-Length")) + + rec = f.do(t, http.MethodHead, f.bucketPath("photos")+"/o/k?versionId=nope", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/o/k?versionId=nope", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + // A top-level delete on a versioned bucket reports the delete marker. + rec = f.do(t, http.MethodDelete, f.bucketPath("photos")+"/o/k", nil) + require.Equal(t, http.StatusNoContent, rec.Code) + assert.Equal(t, "true", rec.Header().Get("is-delete-marker")) + assert.NotEmpty(t, rec.Header().Get("version-id")) + + rec = f.do(t, http.MethodDelete, f.bucketPath("photos")+"/o/k?versionId="+first.VersionID, nil) + require.Equal(t, http.StatusNoContent, rec.Code) + assert.Equal(t, first.VersionID, rec.Header().Get("version-id")) + + rec = f.do(t, http.MethodDelete, f.bucketPath("photos")+"/o/k?versionId=nope", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodGet, f.bucketPath("photos")+"/objectversions?prefix=k", nil) + assert.Equal(t, http.StatusOK, rec.Code) + + rec = f.do(t, http.MethodGet, f.bucketPath("missing")+"/objectversions", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodPost, f.bucketPath("photos")+"/objectversions", nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) +} + +func TestMultipartWireErrors(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + + base := f.bucketPath("photos") + "/u" + + rec := f.do(t, http.MethodPost, base, []byte("{")) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPost, base, map[string]any{}) + assert.Equal(t, http.StatusBadRequest, rec.Code, "object is required") + + rec = f.do(t, http.MethodPost, base, map[string]any{"object": "big", "storageTier": "Glacier"}) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPost, base, map[string]any{ + "object": "big", "contentType": "text/plain", "metadata": map[string]string{"owner": "ada"}, + }) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + var up struct { + UploadID string `json:"uploadId"` + Namespace string `json:"namespace"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &up)) + assert.Equal(t, f.ns, up.Namespace) + + rec = f.do(t, http.MethodGet, base, nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Contains(t, rec.Body.String(), up.UploadID) + + rec = f.do(t, http.MethodGet, f.bucketPath("missing")+"/u", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodDelete, base, nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) + + rec = f.do(t, http.MethodPut, base+"/big", []byte("aaa")) + assert.Equal(t, http.StatusBadRequest, rec.Code, "uploadId is required") + + item := base + "/big?uploadId=" + up.UploadID + + rec = f.do(t, http.MethodPut, item, []byte("aaa")) + assert.Equal(t, http.StatusBadRequest, rec.Code, "uploadPartNum is required") + + rec = f.do(t, http.MethodPut, item+"&uploadPartNum=abc", []byte("aaa")) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPut, item+"&uploadPartNum=1", []byte("aaa")) + require.Equal(t, http.StatusOK, rec.Code) + partETag := rec.Header().Get("ETag") + assert.NotEmpty(t, partETag) + + rec = f.do(t, http.MethodGet, item, nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Contains(t, rec.Body.String(), `"partNumber":1`) + + rec = f.do(t, http.MethodGet, base+"/big?uploadId=nope", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodPost, item, []byte("{")) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPost, item, map[string]any{ + "partsToCommit": []map[string]any{{"partNum": 1}}, + "partsToExclude": []int{2}, + }) + assert.Equal(t, http.StatusNotImplemented, rec.Code, "partsToExclude is rejected, not dropped") + + rec = f.do(t, http.MethodPost, item, map[string]any{}) + assert.Equal(t, http.StatusBadRequest, rec.Code, "partsToCommit is required") + + rec = f.do(t, http.MethodPost, item, map[string]any{"partsToCommit": []map[string]any{{"partNum": 9, "etag": "x"}}}) + assert.Equal(t, http.StatusBadRequest, rec.Code, "a part that was never uploaded") + + rec = f.do(t, http.MethodPost, item, map[string]any{"partsToCommit": []map[string]any{{"partNum": 1, "etag": partETag}}}) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.NotEmpty(t, rec.Header().Get("ETag")) + + rec = f.do(t, http.MethodDelete, base+"/big?uploadId=nope", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodPatch, item, nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) +} + +func TestPARWireErrors(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + require.NoError(t, f.mock.PutObject(t.Context(), "photos", "a.txt", []byte("v"), "text/plain", nil)) + + base := f.bucketPath("photos") + "/p" + + rec := f.do(t, http.MethodPost, base, []byte("{")) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPost, base, map[string]any{ + "name": "bad-time", "objectName": "a.txt", "accessType": "ObjectRead", "timeExpires": "tomorrow", + }) + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "RFC3339") + + rec = f.do(t, http.MethodPost, base, map[string]any{ + "name": "bad-access", "objectName": "a.txt", "accessType": "ObjectAppend", + }) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPost, base, map[string]any{ + "name": "read", "objectName": "a.txt", "accessType": "ObjectRead", + "timeExpires": time.Now().Add(time.Hour).UTC().Format(time.RFC3339), + }) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + var par struct { + ID string `json:"id"` + AccessURI string `json:"accessUri"` + FullPath string `json:"fullPath"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &par)) + require.NotEmpty(t, par.ID) + assert.Equal(t, par.AccessURI, par.FullPath) + + rec = f.do(t, http.MethodGet, base+"/"+par.ID, nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Contains(t, rec.Body.String(), `"accessType":"ObjectRead"`) + assert.NotContains(t, rec.Body.String(), "accessUri", "a later Get never returns the access URI") + + rec = f.do(t, http.MethodGet, base+"?objectNamePrefix=a", nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Contains(t, rec.Body.String(), par.ID) + + rec = f.do(t, http.MethodGet, f.bucketPath("missing")+"/p", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodGet, base+"/ocid1.preauthenticatedrequest.oc1..missing", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodPatch, base, nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) + + rec = f.do(t, http.MethodPatch, base+"/"+par.ID, nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) + + // Redeem it. + token := par.AccessURI[len("/p/"):] + token = token[:strings.Index(token, "/")] + + rec = f.do(t, http.MethodGet, "/p/"+token+"/n/"+f.ns+"/b/photos/o/a.txt", nil) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.Equal(t, "v", rec.Body.String()) + + rec = f.do(t, http.MethodPut, "/p/"+token+"/n/"+f.ns+"/b/photos/o/a.txt", []byte("nope")) + assert.Equal(t, http.StatusForbidden, rec.Code, "an ObjectRead PAR does not authorize a write") + + rec = f.do(t, http.MethodGet, "/p/"+token+"/n/"+f.ns+"/b/photos/o/other.txt", nil) + assert.Equal(t, http.StatusForbidden, rec.Code, "the PAR is bound to its object") + + rec = f.do(t, http.MethodGet, "/p/"+token+"/n/"+f.ns+"/b/other/o/a.txt", nil) + assert.Equal(t, http.StatusForbidden, rec.Code, "the PAR is bound to its bucket") + + rec = f.do(t, http.MethodGet, "/p/"+token+"/n/"+f.ns+"/b/photos", nil) + assert.Equal(t, http.StatusBadRequest, rec.Code, "a PAR addresses an object under /o/") + + rec = f.do(t, http.MethodGet, "/p/no-such-token/n/"+f.ns+"/b/photos/o/a.txt", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodDelete, base+"/"+par.ID, nil) + require.Equal(t, http.StatusNoContent, rec.Code) + + rec = f.do(t, http.MethodDelete, base+"/"+par.ID, nil) + assert.Equal(t, http.StatusNotFound, rec.Code) +} + +func TestPARWriteRedemption(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + + rec := f.do(t, http.MethodPost, f.bucketPath("photos")+"/p", map[string]any{ + "name": "write", "objectName": "upload.txt", "accessType": "ObjectWrite", "timeExpires": inAnHour(), + }) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + var par struct { + AccessURI string `json:"accessUri"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &par)) + + token := par.AccessURI[len("/p/"):] + token = token[:strings.Index(token, "/")] + + rec = f.do(t, http.MethodPut, "/p/"+token+"/n/"+f.ns+"/b/photos/o/upload.txt", []byte("written")) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + obj, err := f.mock.GetObject(t.Context(), "photos", "upload.txt") + require.NoError(t, err) + assert.Equal(t, []byte("written"), obj.Data) + + rec = f.do(t, http.MethodGet, "/p/"+token+"/n/"+f.ns+"/b/photos/o/upload.txt", nil) + assert.Equal(t, http.StatusForbidden, rec.Code, "an ObjectWrite PAR does not authorize a read") +} + +func TestRetentionWireErrors(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + + base := f.bucketPath("photos") + "/retentionRules" + + rec := f.do(t, http.MethodPost, base, []byte("{")) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPost, base, map[string]any{ + "displayName": "bad-lock", "timeRuleLocked": "next week", + }) + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "RFC3339") + + rec = f.do(t, http.MethodPost, base, map[string]any{ + "displayName": "bad-unit", + "duration": map[string]any{"timeAmount": 1, "timeUnit": "MONTHS"}, + }) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPost, base, map[string]any{ + "displayName": "hold", + "duration": map[string]any{"timeAmount": 10, "timeUnit": "DAYS"}, + "timeRuleLocked": time.Now().Add(48 * time.Hour).UTC().Format(time.RFC3339), + }) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.NotEmpty(t, rec.Header().Get("ETag")) + + var rule struct { + ID string `json:"id"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &rule)) + require.NotEmpty(t, rule.ID) + + rec = f.do(t, http.MethodGet, base, nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Contains(t, rec.Body.String(), rule.ID) + + rec = f.do(t, http.MethodGet, base+"/"+rule.ID, nil) + require.Equal(t, http.StatusOK, rec.Code) + assert.Contains(t, rec.Body.String(), `"timeUnit":"DAYS"`) + + rec = f.do(t, http.MethodPost, base+"/"+rule.ID, map[string]any{ + "displayName": "hold-longer", + "duration": map[string]any{"timeAmount": 20, "timeUnit": "DAYS"}, + }) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + assert.Contains(t, rec.Body.String(), "hold-longer") + + rec = f.do(t, http.MethodPost, base+"/"+rule.ID, []byte("{")) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPost, base+"/ocid1.retentionrule.oc1..missing", map[string]any{"displayName": "x"}) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodGet, base+"/ocid1.retentionrule.oc1..missing", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodGet, f.bucketPath("missing")+"/retentionRules", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodDelete, base+"/ocid1.retentionrule.oc1..missing", nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodDelete, base+"/"+rule.ID, nil) + require.Equal(t, http.StatusNoContent, rec.Code) + + rec = f.do(t, http.MethodPatch, base, nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) + + rec = f.do(t, http.MethodPatch, base+"/"+rule.ID, nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) +} + +func TestLifecycleWireErrors(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "photos") + + path := f.bucketPath("photos") + "/l" + + rec := f.do(t, http.MethodPut, path, []byte("{")) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodGet, path, nil) + assert.Equal(t, http.StatusNotFound, rec.Code, "no policy yet") + + rec = f.do(t, http.MethodDelete, path, nil) + assert.Equal(t, http.StatusNotFound, rec.Code) + + tests := []struct { + name string + item map[string]any + }{ + {"unsupported action", map[string]any{ + "name": "r", "action": "TELEPORT", "timeAmount": 1, "timeUnit": "DAYS", "isEnabled": true, + }}, + {"unsupported time unit", map[string]any{ + "name": "r", "action": "DELETE", "timeAmount": 1, "timeUnit": "MONTHS", "isEnabled": true, + }}, + {"ABORT without the multipart-uploads target", map[string]any{ + "name": "r", "action": "ABORT", "timeAmount": 1, "timeUnit": "DAYS", "isEnabled": true, + }}, + {"DELETE aimed at multipart uploads", map[string]any{ + "name": "r", "action": "DELETE", "timeAmount": 1, "timeUnit": "DAYS", "target": "multipart-uploads", + }}, + {"unsupported target", map[string]any{ + "name": "r", "action": "DELETE", "timeAmount": 1, "timeUnit": "DAYS", "target": "buckets", + }}, + {"unterminated pattern class", map[string]any{ + "name": "r", "action": "DELETE", "timeAmount": 1, "timeUnit": "DAYS", + "objectNameFilter": map[string]any{"inclusionPatterns": []string{"[abc"}}, + }}, + {"missing unit", map[string]any{"name": "r", "action": "DELETE", "timeAmount": 1}}, + {"non-positive amount", map[string]any{"name": "r", "action": "DELETE", "timeAmount": 0, "timeUnit": "DAYS"}}, + {"missing name", map[string]any{"action": "DELETE", "timeAmount": 1, "timeUnit": "DAYS"}}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + rec := f.do(t, http.MethodPut, path, map[string]any{"items": []map[string]any{tc.item}}) + assert.Equal(t, http.StatusBadRequest, rec.Code, rec.Body.String()) + }) + } + + // A policy reads back exactly as written: unit, target and all three + // filter lists, plus the policy's timeCreated. + sent := []map[string]any{ + { + "name": "old-logs", "action": "DELETE", "timeAmount": 1, "timeUnit": "YEARS", "isEnabled": true, + "target": "previous-object-versions", + "objectNameFilter": map[string]any{ + "inclusionPrefixes": []string{"logs/", "audit/"}, + "inclusionPatterns": []string{"*.log"}, + "exclusionPatterns": []string{"*keep*"}, + }, + }, + {"name": "archive", "action": "ARCHIVE", "timeAmount": 30, "timeUnit": "DAYS", "isEnabled": true, "target": "objects"}, + {"name": "infreq", "action": "INFREQUENT_ACCESS", "timeAmount": 10, "timeUnit": "DAYS", "isEnabled": false}, + {"name": "abort", "action": "ABORT", "timeAmount": 7, "timeUnit": "DAYS", "isEnabled": true, "target": "multipart-uploads"}, + } + + rec = f.do(t, http.MethodPut, path, map[string]any{"items": sent}) + require.Equal(t, http.StatusOK, rec.Code, rec.Body.String()) + + rec = f.do(t, http.MethodGet, path, nil) + require.Equal(t, http.StatusOK, rec.Code) + + var body struct { + TimeCreated string `json:"timeCreated"` + Items []map[string]any `json:"items"` + } + + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &body)) + assert.NotEmpty(t, body.TimeCreated) + + want, err := json.Marshal(sent) + require.NoError(t, err) + + got, err := json.Marshal(body.Items) + require.NoError(t, err) + assert.JSONEq(t, string(want), string(got), "the policy must read back exactly as it was written") + + rec = f.do(t, http.MethodDelete, path, nil) + assert.Equal(t, http.StatusNoContent, rec.Code) + + rec = f.do(t, http.MethodPut, f.bucketPath("missing")+"/l", map[string]any{"items": []map[string]any{}}) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodPatch, path, nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) +} + +func TestObjectActionWireErrors(t *testing.T) { + f := newFixture(t) + f.createBucket(t, "src") + + base := f.bucketPath("src") + "/actions" + + rec := f.do(t, http.MethodPost, base+"/renameObject", []byte("{")) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPost, base+"/copyObject", []byte("{")) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPost, base+"/copyObject", map[string]any{ + "sourceObjectName": "a", "destinationRegion": "us-ashburn-1", + "destinationBucket": "dst", "destinationObjectName": "b", "destinationNamespace": "someotherns", + }) + assert.Equal(t, http.StatusBadRequest, rec.Code) + assert.Contains(t, rec.Body.String(), "cross-namespace") + + rec = f.do(t, http.MethodPost, base+"/updateObjectStorageTier", []byte("{")) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPost, base+"/updateObjectStorageTier", map[string]any{"objectName": "a"}) + assert.Equal(t, http.StatusBadRequest, rec.Code) + + rec = f.do(t, http.MethodPost, base+"/updateObjectStorageTier", map[string]any{ + "objectName": "missing", "storageTier": "Archive", + }) + assert.Equal(t, http.StatusNotFound, rec.Code) + + rec = f.do(t, http.MethodGet, base+"/renameObject", nil) + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) +} + +// copyObject needs the shared work-request store; without one it says so +// rather than pretending the copy was accepted. +func TestCopyObjectWithoutWorkRequests(t *testing.T) { + opts := config.NewOptions(config.WithRegion("us-ashburn-1"), config.WithCompartmentID(testCompartment)) + mock := osprovider.New(opts) + f := fixture{handler: ociobjectstorage.New(mock, nil), mock: mock, ns: mock.Namespace()} + f.createBucket(t, "src") + + rec := f.do(t, http.MethodPost, f.bucketPath("src")+"/actions/copyObject", map[string]any{ + "sourceObjectName": "a", "destinationRegion": "us-ashburn-1", + "destinationBucket": "src", "destinationObjectName": "b", + }) + assert.Equal(t, http.StatusNotImplemented, rec.Code) +} diff --git a/server/oci/objectstorage/lifecycle.go b/server/oci/objectstorage/lifecycle.go new file mode 100644 index 000000000..50e82adfe --- /dev/null +++ b/server/oci/objectstorage/lifecycle.go @@ -0,0 +1,109 @@ +package objectstorage + +import ( + "net/http" + + osprovider "github.com/stackshy/cloudemu/v2/providers/oci/objectstorage" + "github.com/stackshy/cloudemu/v2/server/wire/ocirest" +) + +// serveLifecycle routes the object lifecycle policy at /l. +func (h *Handler) serveLifecycle(w http.ResponseWriter, r *http.Request, bucket string) { + switch r.Method { + case http.MethodPut: + h.putLifecycle(w, r, bucket) + case http.MethodGet: + h.getLifecycle(w, r, bucket) + case http.MethodDelete: + h.deleteLifecycle(w, r, bucket) + default: + methodNotAllowed(w, r) + } +} + +func (h *Handler) putLifecycle(w http.ResponseWriter, r *http.Request, bucket string) { + var req lifecycleBody + + if !ocirest.DecodeJSON(w, r, &req) { + return + } + + rules := make([]osprovider.LifecycleRule, 0, len(req.Items)) + for i := range req.Items { + rules = append(rules, toLifecycleRule(&req.Items[i])) + } + + policy, err := h.extras.PutLifecyclePolicy(r.Context(), bucket, rules) + if err != nil { + writeDriverError(w, r, err) + return + } + + ocirest.WriteJSON(w, r, http.StatusOK, toLifecycleBody(policy)) +} + +func (h *Handler) getLifecycle(w http.ResponseWriter, r *http.Request, bucket string) { + policy, err := h.extras.GetLifecyclePolicy(r.Context(), bucket) + if err != nil { + writeDriverError(w, r, err) + return + } + + ocirest.WriteJSON(w, r, http.StatusOK, toLifecycleBody(policy)) +} + +func (h *Handler) deleteLifecycle(w http.ResponseWriter, r *http.Request, bucket string) { + if err := h.extras.DeleteLifecyclePolicy(r.Context(), bucket); err != nil { + writeDriverError(w, r, err) + return + } + + ocirest.WriteJSON(w, r, http.StatusNoContent, nil) +} + +func toLifecycleRule(item *lifecycleRuleBody) osprovider.LifecycleRule { + rule := osprovider.LifecycleRule{ + Name: item.Name, + Action: item.Action, + TimeAmount: item.TimeAmount, + TimeUnit: item.TimeUnit, + Target: item.Target, + IsEnabled: item.IsEnabled, + } + + if f := item.ObjectNameFilter; f != nil { + rule.InclusionPrefixes = f.InclusionPrefixes + rule.InclusionPatterns = f.InclusionPatterns + rule.ExclusionPatterns = f.ExclusionPatterns + } + + return rule +} + +func toLifecycleBody(policy *osprovider.LifecyclePolicy) lifecycleBody { + out := lifecycleBody{TimeCreated: policy.TimeCreated, Items: make([]lifecycleRuleBody, 0, len(policy.Rules))} + + for i := range policy.Rules { + rule := &policy.Rules[i] + item := lifecycleRuleBody{ + Name: rule.Name, + Target: rule.Target, + Action: rule.Action, + TimeAmount: rule.TimeAmount, + TimeUnit: rule.TimeUnit, + IsEnabled: rule.IsEnabled, + } + + if len(rule.InclusionPrefixes)+len(rule.InclusionPatterns)+len(rule.ExclusionPatterns) > 0 { + item.ObjectNameFilter = &lifecycleFilterBody{ + InclusionPrefixes: rule.InclusionPrefixes, + InclusionPatterns: rule.InclusionPatterns, + ExclusionPatterns: rule.ExclusionPatterns, + } + } + + out.Items = append(out.Items, item) + } + + return out +} diff --git a/server/oci/objectstorage/multipart.go b/server/oci/objectstorage/multipart.go new file mode 100644 index 000000000..da34dad05 --- /dev/null +++ b/server/oci/objectstorage/multipart.go @@ -0,0 +1,200 @@ +package objectstorage + +import ( + "net/http" + "strconv" + + osprovider "github.com/stackshy/cloudemu/v2/providers/oci/objectstorage" + "github.com/stackshy/cloudemu/v2/server/wire/ocirest" + "github.com/stackshy/cloudemu/v2/services/storage/driver" +) + +// serveUploads routes /u and /u/{object}. +func (h *Handler) serveUploads(w http.ResponseWriter, r *http.Request, rt *route) { + if rt.Rest == "" { + switch r.Method { + case http.MethodPost: + h.createUpload(w, r, rt.Bucket) + case http.MethodGet: + h.listUploads(w, r, rt.Bucket) + default: + methodNotAllowed(w, r) + } + + return + } + + uploadID := r.URL.Query().Get("uploadId") + if uploadID == "" { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, "uploadId is required") + return + } + + switch r.Method { + case http.MethodPut: + h.uploadPart(w, r, rt.Bucket, rt.Rest, uploadID) + case http.MethodPost: + h.commitUpload(w, r, rt.Bucket, rt.Rest, uploadID) + case http.MethodGet: + h.listParts(w, r, rt.Bucket, rt.Rest, uploadID) + case http.MethodDelete: + h.abortUpload(w, r, rt.Bucket, rt.Rest, uploadID) + default: + methodNotAllowed(w, r) + } +} + +func (h *Handler) createUpload(w http.ResponseWriter, r *http.Request, bucket string) { + var req createUploadBody + + if !ocirest.DecodeJSON(w, r, &req) { + return + } + + if req.Object == "" { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, "object is required") + return + } + + up, err := h.extras.CreateMultipartUploadWith(r.Context(), bucket, osprovider.MultipartUploadSpec{ + Object: req.Object, + ContentType: req.ContentType, + StorageTier: req.StorageTier, + Metadata: req.Metadata, + }) + if err != nil { + writeDriverError(w, r, err) + return + } + + ocirest.WriteJSON(w, r, http.StatusOK, uploadBody{ + Namespace: h.extras.Namespace(), + Bucket: bucket, + Object: up.Key, + UploadID: up.UploadID, + TimeCreated: up.CreatedAt, + }) +} + +func (h *Handler) listUploads(w http.ResponseWriter, r *http.Request, bucket string) { + uploads, err := h.store.ListMultipartUploads(r.Context(), bucket) + if err != nil { + writeDriverError(w, r, err) + return + } + + out := make([]uploadBody, 0, len(uploads)) + namespace := h.extras.Namespace() + + for i := range uploads { + up := &uploads[i] + out = append(out, uploadBody{ + Namespace: namespace, + Bucket: bucket, + Object: up.Key, + UploadID: up.UploadID, + TimeCreated: up.CreatedAt, + }) + } + + writePage(w, r, out) +} + +func (h *Handler) uploadPart(w http.ResponseWriter, r *http.Request, bucket, object, uploadID string) { + raw := r.URL.Query().Get("uploadPartNum") + + partNum, err := strconv.Atoi(raw) + if err != nil { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, + "uploadPartNum must be an integer, got "+strconv.Quote(raw)) + + return + } + + data, ok := readBody(w, r) + if !ok { + return + } + + part, err := h.store.UploadPart(r.Context(), bucket, object, uploadID, partNum, data) + if err != nil { + writeDriverError(w, r, err) + return + } + + w.Header().Set("ETag", part.ETag) + ocirest.WriteJSON(w, r, http.StatusOK, nil) +} + +// commitUpload assembles the named parts. OCI's partsToExclude is rejected +// rather than dropped: excluding a part changes the object that results. +func (h *Handler) commitUpload(w http.ResponseWriter, r *http.Request, bucket, object, uploadID string) { + var req commitUploadBody + + if !ocirest.DecodeJSON(w, r, &req) { + return + } + + if len(req.PartsToExclude) > 0 { + ocirest.WriteError(w, r, http.StatusNotImplemented, codeNotImplemented, + "partsToExclude is not emulated; omit the parts from partsToCommit instead") + + return + } + + if len(req.PartsToCommit) == 0 { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, "partsToCommit is required") + return + } + + parts := make([]driver.UploadPart, 0, len(req.PartsToCommit)) + + for _, p := range req.PartsToCommit { + if p.ETag == "" { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, + "partsToCommit part "+strconv.Itoa(p.PartNum)+" needs the etag UploadPart returned") + + return + } + + parts = append(parts, driver.UploadPart{PartNumber: p.PartNum, ETag: p.ETag}) + } + + if err := h.store.CompleteMultipartUpload(r.Context(), bucket, object, uploadID, parts); err != nil { + writeDriverError(w, r, err) + return + } + + details, err := h.extras.ObjectDetailsOf(r.Context(), bucket, object) + if err != nil { + writeDriverError(w, r, err) + return + } + + stampObjectHeaders(w, details) + ocirest.WriteJSON(w, r, http.StatusOK, nil) +} + +func (h *Handler) listParts(w http.ResponseWriter, r *http.Request, bucket, object, uploadID string) { + parts, err := h.store.ListParts(r.Context(), bucket, object, uploadID) + if err != nil { + writeDriverError(w, r, err) + return + } + + out := make([]partBody, 0, len(parts)) + for _, p := range parts { + out = append(out, partBody{PartNumber: p.PartNumber, ETag: p.ETag, Size: p.Size}) + } + + writePage(w, r, out) +} + +func (h *Handler) abortUpload(w http.ResponseWriter, r *http.Request, bucket, object, uploadID string) { + if err := h.store.AbortMultipartUpload(r.Context(), bucket, object, uploadID); err != nil { + writeDriverError(w, r, err) + return + } + + ocirest.WriteJSON(w, r, http.StatusNoContent, nil) +} diff --git a/server/oci/objectstorage/object.go b/server/oci/objectstorage/object.go new file mode 100644 index 000000000..d45939d8d --- /dev/null +++ b/server/oci/objectstorage/object.go @@ -0,0 +1,607 @@ +package objectstorage + +import ( + "io" + "net/http" + "strconv" + "strings" + + "github.com/stackshy/cloudemu/v2/internal/idgen" + osprovider "github.com/stackshy/cloudemu/v2/providers/oci/objectstorage" + "github.com/stackshy/cloudemu/v2/server/oci/workrequest" + "github.com/stackshy/cloudemu/v2/server/wire/ocirest" + "github.com/stackshy/cloudemu/v2/services/storage/driver" +) + +// metaPrefix is the header prefix carrying an object's user metadata. +const metaPrefix = "opc-meta-" + +// headerStorageTier is the per-object storage tier header. +const headerStorageTier = "Storage-Tier" + +// defaultContentType is what OCI reports for an object stored without one. +const defaultContentType = "application/octet-stream" + +// maxObjectSize bounds a single PutObject body, so a runaway upload cannot +// exhaust the emulator's memory. +const maxObjectSize = 512 << 20 + +// serveObjects routes /o and /o/{object}. +func (h *Handler) serveObjects(w http.ResponseWriter, r *http.Request, rt *route) { + if rt.Rest == "" { + if r.Method != http.MethodGet { + methodNotAllowed(w, r) + return + } + + h.listObjects(w, r, rt.Bucket) + + return + } + + switch r.Method { + case http.MethodPut: + h.putObject(w, r, rt.Bucket, rt.Rest) + case http.MethodGet: + h.getObject(w, r, rt.Bucket, rt.Rest) + case http.MethodHead: + h.headObject(w, r, rt.Bucket, rt.Rest) + case http.MethodDelete: + h.deleteObject(w, r, rt.Bucket, rt.Rest) + default: + methodNotAllowed(w, r) + } +} + +func (h *Handler) putObject(w http.ResponseWriter, r *http.Request, bucket, object string) { + data, ok := readBody(w, r) + if !ok { + return + } + + details, err := h.extras.PutObjectWith(r.Context(), bucket, object, data, osprovider.PutOptions{ + ContentType: r.Header.Get("Content-Type"), + StorageTier: r.Header.Get(headerStorageTier), + Metadata: metadataFrom(r.Header), + IfMatch: r.Header.Get("If-Match"), + IfNoneMatch: r.Header.Get("If-None-Match"), + }) + if err != nil { + writeDriverError(w, r, err) + return + } + + stampObjectHeaders(w, details) + ocirest.WriteJSON(w, r, http.StatusOK, nil) +} + +// readBody reads a request body, refusing one larger than maxObjectSize. +func readBody(w http.ResponseWriter, r *http.Request) ([]byte, bool) { + data, err := io.ReadAll(io.LimitReader(r.Body, maxObjectSize+1)) + if err != nil { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, "cannot read request body: "+err.Error()) + return nil, false + } + + if len(data) > maxObjectSize { + ocirest.WriteError(w, r, http.StatusRequestEntityTooLarge, codeInvalidParameter, + "object exceeds the emulator's "+strconv.Itoa(maxObjectSize)+" byte limit") + + return nil, false + } + + return data, true +} + +func (h *Handler) getObject(w http.ResponseWriter, r *http.Request, bucket, object string) { + versionID := r.URL.Query().Get("versionId") + + obj, err := h.fetchObject(r, bucket, object, versionID) + if err != nil { + writeDriverError(w, r, err) + return + } + + etag := obj.Info.ETag + + details, detailsErr := h.extras.ObjectDetailsOf(r.Context(), bucket, object) + if detailsErr == nil && versionID == "" { + stampObjectHeaders(w, details) + + etag = details.ETag + } else { + stampInfoHeaders(w, &obj.Info) + } + + if !readPreconditionsHold(w, r, etag) { + return + } + + w.Header().Set("Accept-Ranges", "bytes") + + if spec := r.Header.Get("Range"); spec != "" { + writeRange(w, r, spec, obj.Info.ContentType, obj.Data) + return + } + + writeRaw(w, r, obj.Info.ContentType, obj.Data) +} + +// readPreconditionsHold applies if-match and if-none-match to a read. A +// failed if-match is 412 IfMatchFailed; a matching if-none-match is 304, as +// HTTP and OCI's GetObject and HeadObject define it. +func readPreconditionsHold(w http.ResponseWriter, r *http.Request, etag string) bool { + if ifMatch := r.Header.Get("If-Match"); ifMatch != "" && ifMatch != "*" && ifMatch != etag { + ocirest.WriteError(w, r, http.StatusPreconditionFailed, osprovider.CodeIfMatchFailed, + "the if-match ETag "+strconv.Quote(ifMatch)+" does not match the current ETag") + + return false + } + + if ifNoneMatch := r.Header.Get("If-None-Match"); ifNoneMatch != "" && (ifNoneMatch == "*" || ifNoneMatch == etag) { + stampRequestID(w, r) + w.WriteHeader(http.StatusNotModified) + + return false + } + + return true +} + +// writeRange answers a single byte range with 206 and Content-Range. OCI serves +// one range per request, so a multi-range header is refused rather than +// answered with the whole object. +func writeRange(w http.ResponseWriter, r *http.Request, spec, contentType string, data []byte) { + size := int64(len(data)) + + start, end, ok, multi := parseRange(spec, size) + if multi { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, + "only a single byte range is supported, got "+strconv.Quote(spec)) + + return + } + + if !ok { + w.Header().Set("Content-Range", "bytes */"+strconv.FormatInt(size, 10)) + ocirest.WriteError(w, r, http.StatusRequestedRangeNotSatisfiable, codeInvalidRange, + "range "+strconv.Quote(spec)+" is not satisfiable for an object of "+strconv.FormatInt(size, 10)+" bytes") + + return + } + + if contentType == "" { + contentType = defaultContentType + } + + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", strconv.FormatInt(end-start+1, 10)) + w.Header().Set("Content-Range", "bytes "+strconv.FormatInt(start, 10)+"-"+strconv.FormatInt(end, 10)+ + "/"+strconv.FormatInt(size, 10)) + stampRequestID(w, r) + w.WriteHeader(http.StatusPartialContent) + w.Write(data[start : end+1]) //nolint:errcheck // best-effort response +} + +// parseRange reads a bytes=start-end, bytes=start- or bytes=-suffix header +// against an object of size bytes, clamping end to the last byte. +func parseRange(spec string, size int64) (start, end int64, ok, multi bool) { + unit, set, found := strings.Cut(spec, "=") + if !found || strings.TrimSpace(unit) != "bytes" { + return 0, 0, false, false + } + + if strings.Contains(set, ",") { + return 0, 0, false, true + } + + first, last, found := strings.Cut(strings.TrimSpace(set), "-") + if !found || size == 0 { + return 0, 0, false, false + } + + if first == "" { + start, end, ok = suffixRange(last, size) + return start, end, ok, false + } + + start, end, ok = boundedRange(first, last, size) + + return start, end, ok, false +} + +// suffixRange reads bytes=-n: the last n bytes. +func suffixRange(last string, size int64) (start, end int64, ok bool) { + n, err := strconv.ParseInt(last, 10, 64) + if err != nil || n <= 0 { + return 0, 0, false + } + + return max(size-n, 0), size - 1, true +} + +// boundedRange reads bytes=start- and bytes=start-end, clamping end. +func boundedRange(first, last string, size int64) (start, end int64, ok bool) { + start, err := strconv.ParseInt(first, 10, 64) + if err != nil || start < 0 || start >= size { + return 0, 0, false + } + + if last == "" { + return start, size - 1, true + } + + end, err = strconv.ParseInt(last, 10, 64) + if err != nil || end < start { + return 0, 0, false + } + + return start, min(end, size-1), true +} + +// fetchObject reads the current object, or a specific version when the caller +// names one and the driver keeps history. +func (h *Handler) fetchObject(r *http.Request, bucket, object, versionID string) (*driver.Object, error) { + if versionID == "" { + return h.store.GetObject(r.Context(), bucket, object) + } + + if h.versioned == nil { + return nil, errVersioningUnsupported() + } + + return h.versioned.GetObjectVersion(r.Context(), bucket, object, versionID) +} + +func (h *Handler) headObject(w http.ResponseWriter, r *http.Request, bucket, object string) { + versionID := r.URL.Query().Get("versionId") + + if versionID != "" { + if h.versioned == nil { + writeDriverError(w, r, errVersioningUnsupported()) + return + } + + info, err := h.versioned.HeadObjectVersion(r.Context(), bucket, object, versionID) + if err != nil { + writeDriverError(w, r, err) + return + } + + stampInfoHeaders(w, info) + + if readPreconditionsHold(w, r, info.ETag) { + writeHead(w, r, info.Size, info.ContentType) + } + + return + } + + details, err := h.extras.ObjectDetailsOf(r.Context(), bucket, object) + if err != nil { + writeDriverError(w, r, err) + return + } + + stampObjectHeaders(w, details) + + if readPreconditionsHold(w, r, details.ETag) { + writeHead(w, r, details.Size, details.ContentType) + } +} + +// writeHead answers a HeadObject. The response carries no body, so the object's +// own size and type have to be reported in headers rather than inferred — which +// is why this does not go through ocirest.WriteJSON, whose application/json +// would overwrite the object's content type. +func writeHead(w http.ResponseWriter, r *http.Request, size int64, contentType string) { + if contentType == "" { + contentType = defaultContentType + } + + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + w.Header().Set("Accept-Ranges", "bytes") + stampRequestID(w, r) + w.WriteHeader(http.StatusOK) +} + +// deleteObject deletes the current object, or one version, honoring +// if-match and reporting the delete marker OCI stamps when the bucket keeps +// history. +func (h *Handler) deleteObject(w http.ResponseWriter, r *http.Request, bucket, object string) { + versionID := r.URL.Query().Get("versionId") + + if versionID != "" && h.versioned == nil { + writeDriverError(w, r, errVersioningUnsupported()) + return + } + + deleted, marker, err := h.extras.DeleteObjectIf(r.Context(), bucket, object, versionID, r.Header.Get("If-Match")) + if err != nil { + writeDriverError(w, r, err) + return + } + + if deleted != "" { + w.Header().Set("Version-Id", deleted) + } + + if marker { + w.Header().Set("Is-Delete-Marker", "true") + } + + ocirest.WriteJSON(w, r, http.StatusNoContent, nil) +} + +func (h *Handler) listObjects(w http.ResponseWriter, r *http.Request, bucket string) { + opts := listOptions(r) + + objects, prefixes, next, err := h.extras.ListObjectDetails(r.Context(), bucket, opts) + if err != nil { + writeDriverError(w, r, err) + return + } + + out := listObjectsBody{Objects: make([]objectSummaryBody, 0, len(objects)), Prefixes: prefixes} + + for i := range objects { + o := &objects[i] + out.Objects = append(out.Objects, objectSummaryBody{ + Name: o.Name, + Size: o.Size, + MD5: o.MD5, + ETag: o.ETag, + TimeCreated: o.TimeCreated, + TimeModified: o.TimeModified, + StorageTier: o.StorageTier, + }) + } + + out.NextStartWith = next + ocirest.SetNextPage(w, next) + ocirest.WriteJSON(w, r, http.StatusOK, out) +} + +// listOptions reads OCI's list parameters. OCI names the page cursor "start" +// and the page size "limit". +func listOptions(r *http.Request) driver.ListOptions { + q := r.URL.Query() + + return driver.ListOptions{ + Prefix: q.Get("prefix"), + Delimiter: q.Get("delimiter"), + MaxKeys: listLimit(r), + PageToken: q.Get("start"), + } +} + +// listLimit is the requested page size, or zero when the caller named none. +// Object Storage's own default is 1000, not the 100 shared by the other OCI +// services, so an absent limit is left for the provider to fill in. +func listLimit(r *http.Request) int { + if r.URL.Query().Get("limit") == "" { + return 0 + } + + return ocirest.Limit(r) +} + +func (h *Handler) listObjectVersions(w http.ResponseWriter, r *http.Request, bucket string) { + if r.Method != http.MethodGet { + methodNotAllowed(w, r) + return + } + + if h.versioned == nil { + writeDriverError(w, r, errVersioningUnsupported()) + return + } + + result, err := h.versioned.ListObjectVersions(r.Context(), bucket, listOptions(r)) + if err != nil { + writeDriverError(w, r, err) + return + } + + out := listObjectVersionsBody{ + Items: make([]objectVersionBody, 0, len(result.Versions)), + Prefixes: result.CommonPrefixes, + } + + for i := range result.Versions { + v := &result.Versions[i] + out.Items = append(out.Items, objectVersionBody{ + Name: v.Key, + Size: v.Size, + ETag: v.ETag, + TimeModified: v.LastModified, + VersionID: v.VersionID, + IsDeleteMarker: v.DeleteMarker, + }) + } + + writePageAs(w, r, out.Items, func(page []objectVersionBody) any { + return listObjectVersionsBody{Items: page, Prefixes: out.Prefixes} + }) +} + +func (h *Handler) renameObject(w http.ResponseWriter, r *http.Request, bucket string) { + var req renameObjectBody + + if !ocirest.DecodeJSON(w, r, &req) { + return + } + + details, err := h.extras.RenameObject(r.Context(), bucket, req.SourceName, req.NewName) + if err != nil { + writeDriverError(w, r, err) + return + } + + stampObjectHeaders(w, details) + ocirest.WriteJSON(w, r, http.StatusOK, nil) +} + +// copyObject serves the copy action. OCI runs a copy asynchronously, so the +// response is a 202 carrying the work request the caller polls. +func (h *Handler) copyObject(w http.ResponseWriter, r *http.Request, bucket string) { + if h.work == nil { + ocirest.WriteError(w, r, http.StatusNotImplemented, codeNotImplemented, "work requests are not configured") + return + } + + var req copyObjectBody + + if !ocirest.DecodeJSON(w, r, &req) { + return + } + + if msg := h.copyDestinationProblem(&req); msg != "" { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, msg) + return + } + + err := h.extras.CopyObjectWith(r.Context(), osprovider.CopySpec{ + SourceBucket: bucket, + SourceObject: req.SourceObjectName, + SourceVersionID: req.SourceVersionID, + SourceIfMatch: req.SourceObjectIfMatchETag, + DestinationBucket: req.DestinationBucket, + DestinationObject: req.DestinationObjectName, + DestinationIfMatch: req.DestinationObjectIfMatchETag, + DestinationIfNoneMatch: req.DestinationObjectIfNoneMatchETag, + Metadata: req.DestinationObjectMetadata, + StorageTier: req.DestinationObjectStorageTier, + }) + if err != nil { + writeDriverError(w, r, err) + return + } + + id := h.work.Accept(operationCopy, h.extras.Scope(req.DestinationBucket).Compartment, workrequest.Resource{ + EntityType: "object", + ActionType: workrequest.ActionCreated, + Identifier: req.DestinationBucket + "/" + req.DestinationObjectName, + }) + + ocirest.SetWorkRequestID(w, id) + ocirest.WriteJSON(w, r, http.StatusAccepted, nil) +} + +// copyDestinationProblem names what is wrong with a copy's required fields or +// its destination, or returns "" when the copy can proceed. A destination in +// another region or namespace is refused, not copied locally. +func (h *Handler) copyDestinationProblem(req *copyObjectBody) string { + switch { + case req.SourceObjectName == "" || req.DestinationRegion == "" || + req.DestinationBucket == "" || req.DestinationObjectName == "": + return "sourceObjectName, destinationRegion, destinationBucket and destinationObjectName are required" + case req.DestinationRegion != h.extras.Region(): + return "cross-region copy is not emulated; destinationRegion must be " + h.extras.Region() + case req.DestinationNamespace != "" && req.DestinationNamespace != h.extras.Namespace(): + return "cross-namespace copy is not emulated; destinationNamespace must be " + h.extras.Namespace() + default: + return "" + } +} + +func (h *Handler) updateStorageTier(w http.ResponseWriter, r *http.Request, bucket string) { + var req updateTierBody + + if !ocirest.DecodeJSON(w, r, &req) { + return + } + + if req.ObjectName == "" || req.StorageTier == "" { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, + "objectName and storageTier are required") + + return + } + + if err := h.extras.UpdateObjectStorageTier(r.Context(), bucket, req.ObjectName, req.StorageTier); err != nil { + writeDriverError(w, r, err) + return + } + + ocirest.WriteJSON(w, r, http.StatusOK, nil) +} + +// metadataFrom collects the opc-meta- headers into the object's user metadata. +func metadataFrom(header http.Header) map[string]string { + var out map[string]string + + for name, values := range header { + lower := strings.ToLower(name) + if !strings.HasPrefix(lower, metaPrefix) || len(values) == 0 { + continue + } + + if out == nil { + out = make(map[string]string) + } + + out[strings.TrimPrefix(lower, metaPrefix)] = values[0] + } + + return out +} + +func stampObjectHeaders(w http.ResponseWriter, d *osprovider.ObjectDetails) { + w.Header().Set("ETag", d.ETag) + w.Header().Set("Last-Modified", d.TimeModified) + w.Header().Set(headerStorageTier, d.StorageTier) + + if d.MD5 != "" { + w.Header().Set("Opc-Content-Md5", d.MD5) + } + + if d.VersionID != "" { + w.Header().Set("Version-Id", d.VersionID) + } + + for k, v := range d.Metadata { + w.Header().Set(metaPrefix+k, v) + } +} + +func stampInfoHeaders(w http.ResponseWriter, info *driver.ObjectInfo) { + w.Header().Set("ETag", info.ETag) + w.Header().Set("Last-Modified", info.LastModified) + + if info.VersionID != "" { + w.Header().Set("Version-Id", info.VersionID) + } + + for k, v := range info.Metadata { + w.Header().Set(metaPrefix+k, v) + } +} + +// writeRaw writes an object body, echoing the caller's opc-request-id the way +// ocirest's JSON helpers do. +func writeRaw(w http.ResponseWriter, r *http.Request, contentType string, data []byte) { + if contentType == "" { + contentType = defaultContentType + } + + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + stampRequestID(w, r) + w.WriteHeader(http.StatusOK) + w.Write(data) //nolint:errcheck // best-effort response +} + +func stampRequestID(w http.ResponseWriter, r *http.Request) { + if w.Header().Get(ocirest.HeaderRequestID) != "" { + return + } + + if id := r.Header.Get(ocirest.HeaderRequestID); id != "" { + w.Header().Set(ocirest.HeaderRequestID, id) + return + } + + w.Header().Set(ocirest.HeaderRequestID, idgen.GenerateID("cloudemu")) +} diff --git a/server/oci/objectstorage/par.go b/server/oci/objectstorage/par.go new file mode 100644 index 000000000..b004cb90e --- /dev/null +++ b/server/oci/objectstorage/par.go @@ -0,0 +1,158 @@ +package objectstorage + +import ( + "net/http" + "time" + + osprovider "github.com/stackshy/cloudemu/v2/providers/oci/objectstorage" + "github.com/stackshy/cloudemu/v2/server/wire/ocirest" +) + +// servePARs routes /p and /p/{parId} under a bucket. +func (h *Handler) servePARs(w http.ResponseWriter, r *http.Request, rt *route) { + if rt.Rest == "" { + switch r.Method { + case http.MethodPost: + h.createPAR(w, r, rt.Bucket) + case http.MethodGet: + h.listPARs(w, r, rt.Bucket) + default: + methodNotAllowed(w, r) + } + + return + } + + switch r.Method { + case http.MethodGet: + h.getPAR(w, r, rt.Bucket, rt.Rest) + case http.MethodDelete: + h.deletePAR(w, r, rt.Bucket, rt.Rest) + default: + methodNotAllowed(w, r) + } +} + +func (h *Handler) createPAR(w http.ResponseWriter, r *http.Request, bucket string) { + var req createPARBody + + if !ocirest.DecodeJSON(w, r, &req) { + return + } + + spec := osprovider.PARSpec{ + Name: req.Name, + ObjectName: req.ObjectName, + AccessType: req.AccessType, + BucketListingAction: req.BucketListingAction, + } + + if req.TimeExpires != "" { + expires, err := time.Parse(time.RFC3339, req.TimeExpires) + if err != nil { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, + "timeExpires must be an RFC3339 timestamp: "+err.Error()) + + return + } + + spec.TimeExpires = expires + } + + par, err := h.extras.CreatePAR(r.Context(), bucket, spec) + if err != nil { + writeDriverError(w, r, err) + return + } + + body := toPARBody(par) + body.AccessURI = par.AccessURI + body.FullPath = par.AccessURI + + ocirest.WriteJSON(w, r, http.StatusOK, body) +} + +func (h *Handler) listPARs(w http.ResponseWriter, r *http.Request, bucket string) { + pars, err := h.extras.ListPARs(r.Context(), bucket, r.URL.Query().Get("objectNamePrefix")) + if err != nil { + writeDriverError(w, r, err) + return + } + + out := make([]parBody, 0, len(pars)) + for i := range pars { + out = append(out, toPARBody(&pars[i])) + } + + writePage(w, r, out) +} + +func (h *Handler) getPAR(w http.ResponseWriter, r *http.Request, bucket, parID string) { + par, err := h.extras.GetPAR(r.Context(), bucket, parID) + if err != nil { + writeDriverError(w, r, err) + return + } + + ocirest.WriteJSON(w, r, http.StatusOK, toPARBody(par)) +} + +func (h *Handler) deletePAR(w http.ResponseWriter, r *http.Request, bucket, parID string) { + if err := h.extras.DeletePAR(r.Context(), bucket, parID); err != nil { + writeDriverError(w, r, err) + return + } + + ocirest.WriteJSON(w, r, http.StatusNoContent, nil) +} + +// servePAR redeems a pre-authenticated request: /p/{token}/n/{ns}/b/{b}/o/{o}. +// The token stands in for authentication, so only the object read and write +// the request authorizes are served here. +func (h *Handler) servePAR(w http.ResponseWriter, r *http.Request, rt *route) { + par, err := h.extras.ResolvePAR(r.Context(), rt.PARToken) + if err != nil { + writeDriverError(w, r, err) + return + } + + if rt.Sub != subObjects || rt.Rest == "" { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, + "a pre-authenticated request addresses an object under /o/") + + return + } + + if rt.Bucket != par.Bucket { + ocirest.WriteError(w, r, http.StatusForbidden, codeNotAuthorized, + "pre-authenticated request is scoped to bucket "+par.Bucket) + + return + } + + if !osprovider.PARAllows(par, r.Method, rt.Rest) { + ocirest.WriteError(w, r, http.StatusForbidden, codeNotAuthorized, + "pre-authenticated request does not authorize "+r.Method+" on "+rt.Rest) + + return + } + + switch r.Method { + case http.MethodGet, http.MethodHead: + h.getObject(w, r, par.Bucket, rt.Rest) + default: + h.putObject(w, r, par.Bucket, rt.Rest) + } +} + +func toPARBody(par *osprovider.PreauthenticatedRequest) parBody { + return parBody{ + ID: par.ID, + Name: par.Name, + ObjectName: par.ObjectName, + AccessType: par.AccessType, + BucketListingAction: par.BucketListingAction, + TimeCreated: par.TimeCreated, + TimeExpires: par.TimeExpires, + } +} diff --git a/server/oci/objectstorage/retention.go b/server/oci/objectstorage/retention.go new file mode 100644 index 000000000..fcafe3b20 --- /dev/null +++ b/server/oci/objectstorage/retention.go @@ -0,0 +1,153 @@ +package objectstorage + +import ( + "net/http" + "time" + + osprovider "github.com/stackshy/cloudemu/v2/providers/oci/objectstorage" + "github.com/stackshy/cloudemu/v2/server/wire/ocirest" +) + +// serveRetentionRules routes /retentionRules and /retentionRules/{ruleId}. +func (h *Handler) serveRetentionRules(w http.ResponseWriter, r *http.Request, rt *route) { + if rt.Rest == "" { + switch r.Method { + case http.MethodPost: + h.createRetentionRule(w, r, rt.Bucket) + case http.MethodGet: + h.listRetentionRules(w, r, rt.Bucket) + default: + methodNotAllowed(w, r) + } + + return + } + + switch r.Method { + case http.MethodGet: + h.getRetentionRule(w, r, rt.Bucket, rt.Rest) + case http.MethodPost: + h.updateRetentionRule(w, r, rt.Bucket, rt.Rest) + case http.MethodDelete: + h.deleteRetentionRule(w, r, rt.Bucket, rt.Rest) + default: + methodNotAllowed(w, r) + } +} + +func (h *Handler) createRetentionRule(w http.ResponseWriter, r *http.Request, bucket string) { + spec, ok := decodeRule(w, r) + if !ok { + return + } + + rule, err := h.extras.CreateRetentionRule(r.Context(), bucket, spec) + if err != nil { + writeDriverError(w, r, err) + return + } + + w.Header().Set("ETag", rule.ETag) + ocirest.WriteJSON(w, r, http.StatusOK, toRuleBody(rule)) +} + +func (h *Handler) updateRetentionRule(w http.ResponseWriter, r *http.Request, bucket, ruleID string) { + spec, ok := decodeRule(w, r) + if !ok { + return + } + + rule, err := h.extras.UpdateRetentionRule(r.Context(), bucket, ruleID, spec) + if err != nil { + writeDriverError(w, r, err) + return + } + + w.Header().Set("ETag", rule.ETag) + ocirest.WriteJSON(w, r, http.StatusOK, toRuleBody(rule)) +} + +func (h *Handler) getRetentionRule(w http.ResponseWriter, r *http.Request, bucket, ruleID string) { + rule, err := h.extras.GetRetentionRule(r.Context(), bucket, ruleID) + if err != nil { + writeDriverError(w, r, err) + return + } + + ocirest.WriteJSON(w, r, http.StatusOK, toRuleBody(rule)) +} + +func (h *Handler) listRetentionRules(w http.ResponseWriter, r *http.Request, bucket string) { + rules, err := h.extras.ListRetentionRules(r.Context(), bucket) + if err != nil { + writeDriverError(w, r, err) + return + } + + out := retentionRuleListBody{Items: make([]retentionRuleBody, 0, len(rules))} + for i := range rules { + out.Items = append(out.Items, toRuleBody(&rules[i])) + } + + ocirest.WriteJSON(w, r, http.StatusOK, out) +} + +func (h *Handler) deleteRetentionRule(w http.ResponseWriter, r *http.Request, bucket, ruleID string) { + if err := h.extras.DeleteRetentionRule(r.Context(), bucket, ruleID); err != nil { + writeDriverError(w, r, err) + return + } + + ocirest.WriteJSON(w, r, http.StatusNoContent, nil) +} + +func decodeRule(w http.ResponseWriter, r *http.Request) (osprovider.RetentionRuleSpec, bool) { + var req retentionRuleRequestBody + + if !ocirest.DecodeJSON(w, r, &req) { + return osprovider.RetentionRuleSpec{}, false + } + + spec := osprovider.RetentionRuleSpec{DisplayName: req.DisplayName} + + if req.Duration != nil { + spec.Duration = &osprovider.RetentionDuration{ + TimeAmount: req.Duration.TimeAmount, + TimeUnit: req.Duration.TimeUnit, + } + } + + if req.TimeRuleLocked != "" { + locked, err := time.Parse(time.RFC3339, req.TimeRuleLocked) + if err != nil { + ocirest.WriteError(w, r, http.StatusBadRequest, codeInvalidParameter, + "timeRuleLocked must be an RFC3339 timestamp: "+err.Error()) + + return osprovider.RetentionRuleSpec{}, false + } + + spec.TimeRuleLocked = &locked + } + + return spec, true +} + +func toRuleBody(rule *osprovider.RetentionRule) retentionRuleBody { + out := retentionRuleBody{ + ID: rule.ID, + DisplayName: rule.DisplayName, + TimeRuleLocked: rule.TimeRuleLocked, + TimeCreated: rule.TimeCreated, + TimeModified: rule.TimeModified, + ETag: rule.ETag, + } + + if rule.Duration != nil { + out.Duration = &retentionDurationBody{ + TimeAmount: rule.Duration.TimeAmount, + TimeUnit: rule.Duration.TimeUnit, + } + } + + return out +} diff --git a/server/oci/objectstorage/types.go b/server/oci/objectstorage/types.go new file mode 100644 index 000000000..bf026ef10 --- /dev/null +++ b/server/oci/objectstorage/types.go @@ -0,0 +1,250 @@ +package objectstorage + +// namespaceMetadataBody is the response of GetNamespaceMetadata. +type namespaceMetadataBody struct { + Namespace string `json:"namespace"` + DefaultS3CompartmentID string `json:"defaultS3CompartmentId"` + DefaultSwiftCompartmentID string `json:"defaultSwiftCompartmentId"` +} + +// createBucketBody is the CreateBucket request. +type createBucketBody struct { + Name string `json:"name"` + CompartmentID string `json:"compartmentId"` + PublicAccessType string `json:"publicAccessType"` + StorageTier string `json:"storageTier"` + Versioning string `json:"versioning"` + KMSKeyID string `json:"kmsKeyId"` + AutoTiering string `json:"autoTiering"` + ObjectEventsEnabled bool `json:"objectEventsEnabled"` + Metadata map[string]string `json:"metadata"` + FreeformTags map[string]string `json:"freeformTags"` + DefinedTags map[string]map[string]string `json:"definedTags"` +} + +// updateBucketBody is the UpdateBucket request. Pointers distinguish a field +// the caller sent from one it omitted, which is what OCI's partial update +// needs. +type updateBucketBody struct { + Namespace string `json:"namespace"` + Name *string `json:"name"` + CompartmentID *string `json:"compartmentId"` + PublicAccessType *string `json:"publicAccessType"` + Versioning *string `json:"versioning"` + KMSKeyID *string `json:"kmsKeyId"` + AutoTiering *string `json:"autoTiering"` + ObjectEventsEnabled *bool `json:"objectEventsEnabled"` + Metadata map[string]string `json:"metadata"` + FreeformTags map[string]string `json:"freeformTags"` + DefinedTags map[string]map[string]string `json:"definedTags"` +} + +// bucketBody is a bucket as OCI reports it. +type bucketBody struct { + ID string `json:"id"` + Namespace string `json:"namespace"` + Name string `json:"name"` + CompartmentID string `json:"compartmentId"` + CreatedBy string `json:"createdBy"` + TimeCreated string `json:"timeCreated"` + ETag string `json:"etag"` + PublicAccessType string `json:"publicAccessType"` + StorageTier string `json:"storageTier"` + Versioning string `json:"versioning"` + KMSKeyID string `json:"kmsKeyId,omitempty"` + AutoTiering string `json:"autoTiering"` + ObjectEventsEnabled bool `json:"objectEventsEnabled"` + ReplicationEnabled bool `json:"replicationEnabled"` + IsReadOnly bool `json:"isReadOnly"` + Metadata map[string]string `json:"metadata,omitempty"` + FreeformTags map[string]string `json:"freeformTags,omitempty"` + DefinedTags map[string]map[string]string `json:"definedTags,omitempty"` + ApproximateCount int64 `json:"approximateCount"` + ApproximateSize int64 `json:"approximateSize"` +} + +// bucketSummaryBody is one entry of ListBuckets. OCI's summary is deliberately +// thinner than the full bucket. +type bucketSummaryBody struct { + Namespace string `json:"namespace"` + Name string `json:"name"` + CompartmentID string `json:"compartmentId"` + CreatedBy string `json:"createdBy"` + TimeCreated string `json:"timeCreated"` + ETag string `json:"etag"` + FreeformTags map[string]string `json:"freeformTags,omitempty"` + DefinedTags map[string]map[string]string `json:"definedTags,omitempty"` +} + +// objectSummaryBody is one entry of ListObjects. +type objectSummaryBody struct { + Name string `json:"name"` + Size int64 `json:"size"` + MD5 string `json:"md5,omitempty"` + ETag string `json:"etag,omitempty"` + TimeCreated string `json:"timeCreated,omitempty"` + TimeModified string `json:"timeModified,omitempty"` + StorageTier string `json:"storageTier,omitempty"` +} + +// listObjectsBody is the ListObjects response. +type listObjectsBody struct { + Objects []objectSummaryBody `json:"objects"` + Prefixes []string `json:"prefixes,omitempty"` + NextStartWith string `json:"nextStartWith,omitempty"` +} + +// objectVersionBody is one entry of ListObjectVersions. +type objectVersionBody struct { + Name string `json:"name"` + Size int64 `json:"size"` + ETag string `json:"etag,omitempty"` + TimeModified string `json:"timeModified,omitempty"` + VersionID string `json:"versionId"` + IsDeleteMarker bool `json:"isDeleteMarker"` +} + +// listObjectVersionsBody is the ListObjectVersions response. +type listObjectVersionsBody struct { + Items []objectVersionBody `json:"items"` + Prefixes []string `json:"prefixes,omitempty"` +} + +// renameObjectBody is the renameObject action request. +type renameObjectBody struct { + SourceName string `json:"sourceName"` + NewName string `json:"newName"` +} + +// copyObjectBody is the copyObject action request. +type copyObjectBody struct { + SourceObjectName string `json:"sourceObjectName"` + SourceObjectIfMatchETag string `json:"sourceObjectIfMatchETag"` + SourceVersionID string `json:"sourceVersionId"` + DestinationRegion string `json:"destinationRegion"` + DestinationNamespace string `json:"destinationNamespace"` + DestinationBucket string `json:"destinationBucket"` + DestinationObjectName string `json:"destinationObjectName"` + DestinationObjectIfMatchETag string `json:"destinationObjectIfMatchETag"` + DestinationObjectIfNoneMatchETag string `json:"destinationObjectIfNoneMatchETag"` + DestinationObjectMetadata map[string]string `json:"destinationObjectMetadata"` + DestinationObjectStorageTier string `json:"destinationObjectStorageTier"` +} + +// updateTierBody is the updateObjectStorageTier action request. +type updateTierBody struct { + ObjectName string `json:"objectName"` + StorageTier string `json:"storageTier"` +} + +// createUploadBody is the CreateMultipartUpload request. +type createUploadBody struct { + Object string `json:"object"` + ContentType string `json:"contentType"` + StorageTier string `json:"storageTier"` + Metadata map[string]string `json:"metadata"` +} + +// uploadBody is a multipart upload as OCI reports it. +type uploadBody struct { + Namespace string `json:"namespace"` + Bucket string `json:"bucket"` + Object string `json:"object"` + UploadID string `json:"uploadId"` + TimeCreated string `json:"timeCreated"` +} + +// commitPartBody names one part to commit. +type commitPartBody struct { + PartNum int `json:"partNum"` + ETag string `json:"etag"` +} + +// commitUploadBody is the CommitMultipartUpload request. +type commitUploadBody struct { + PartsToCommit []commitPartBody `json:"partsToCommit"` + PartsToExclude []int `json:"partsToExclude"` +} + +// partBody is one entry of ListMultipartUploadParts. +type partBody struct { + PartNumber int `json:"partNumber"` + ETag string `json:"etag"` + Size int64 `json:"size"` +} + +// createPARBody is the CreatePreauthenticatedRequest request. +type createPARBody struct { + Name string `json:"name"` + ObjectName string `json:"objectName"` + AccessType string `json:"accessType"` + BucketListingAction string `json:"bucketListingAction"` + TimeExpires string `json:"timeExpires"` +} + +// parBody is a pre-authenticated request as OCI reports it. AccessURI is +// returned only from the create call. +type parBody struct { + ID string `json:"id"` + Name string `json:"name"` + AccessURI string `json:"accessUri,omitempty"` + ObjectName string `json:"objectName,omitempty"` + AccessType string `json:"accessType"` + BucketListingAction string `json:"bucketListingAction,omitempty"` + TimeCreated string `json:"timeCreated"` + TimeExpires string `json:"timeExpires"` + FullPath string `json:"fullPath,omitempty"` +} + +// retentionDurationBody is a retention rule's duration. +type retentionDurationBody struct { + TimeAmount int64 `json:"timeAmount"` + TimeUnit string `json:"timeUnit"` +} + +// retentionRuleRequestBody is the Create/UpdateRetentionRule request. +type retentionRuleRequestBody struct { + DisplayName string `json:"displayName"` + Duration *retentionDurationBody `json:"duration"` + TimeRuleLocked string `json:"timeRuleLocked"` +} + +// retentionRuleBody is a retention rule as OCI reports it. +type retentionRuleBody struct { + ID string `json:"id"` + DisplayName string `json:"displayName"` + Duration *retentionDurationBody `json:"duration,omitempty"` + TimeRuleLocked string `json:"timeRuleLocked,omitempty"` + TimeCreated string `json:"timeCreated"` + TimeModified string `json:"timeModified"` + ETag string `json:"etag"` +} + +// retentionRuleListBody is the ListRetentionRules response. +type retentionRuleListBody struct { + Items []retentionRuleBody `json:"items"` +} + +// lifecycleFilterBody is a lifecycle rule's object-name filter. +type lifecycleFilterBody struct { + InclusionPrefixes []string `json:"inclusionPrefixes,omitempty"` + InclusionPatterns []string `json:"inclusionPatterns,omitempty"` + ExclusionPatterns []string `json:"exclusionPatterns,omitempty"` +} + +// lifecycleRuleBody is one OCI object lifecycle rule. +type lifecycleRuleBody struct { + Name string `json:"name"` + Target string `json:"target,omitempty"` + Action string `json:"action"` + TimeAmount int64 `json:"timeAmount"` + TimeUnit string `json:"timeUnit"` + IsEnabled bool `json:"isEnabled"` + ObjectNameFilter *lifecycleFilterBody `json:"objectNameFilter,omitempty"` +} + +// lifecycleBody is the object lifecycle policy. +type lifecycleBody struct { + TimeCreated string `json:"timeCreated,omitempty"` + Items []lifecycleRuleBody `json:"items"` +} diff --git a/server/oci/objectstorage_compartment_gate_test.go b/server/oci/objectstorage_compartment_gate_test.go new file mode 100644 index 000000000..ccc792358 --- /dev/null +++ b/server/oci/objectstorage_compartment_gate_test.go @@ -0,0 +1,57 @@ +package oci_test + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stackshy/cloudemu/v2/config" + ociprovider "github.com/stackshy/cloudemu/v2/providers/oci" + ociserver "github.com/stackshy/cloudemu/v2/server/oci" +) + +// The Object Storage handler gets the same identity-backed compartment check +// as VCN: a bucket created in, or moved to, a compartment that does not exist +// is 404 NotAuthorizedOrNotFound. +func TestObjectStorageCompartmentGate(t *testing.T) { + p := ociprovider.New() + ts := httptest.NewServer(ociserver.New(ociserver.DriversFrom(p))) + t.Cleanup(ts.Close) + + resp, err := ts.Client().Get(ts.URL + "/n") + require.NoError(t, err) + + raw, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + + var ns string + + require.NoError(t, json.Unmarshal(raw, &ns)) + + post := func(path string, body map[string]any) int { + b, err := json.Marshal(body) + require.NoError(t, err) + + resp, err := ts.Client().Post(ts.URL+path, "application/json", bytes.NewReader(b)) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + + return resp.StatusCode + } + + root := "/n/" + ns + "/b" + + assert.Equal(t, http.StatusNotFound, + post(root, map[string]any{"name": "ghost", "compartmentId": "ocid1.compartment.oc1..doesnotexist"})) + assert.Equal(t, http.StatusOK, + post(root, map[string]any{"name": "real", "compartmentId": config.DefaultTenancyOCID})) + assert.Equal(t, http.StatusNotFound, + post(root+"/real", map[string]any{"compartmentId": "ocid1.compartment.oc1..doesnotexist"})) +} diff --git a/server/oci/oci.go b/server/oci/oci.go index 20b2d8f31..fc5742f8f 100644 --- a/server/oci/oci.go +++ b/server/oci/oci.go @@ -13,6 +13,7 @@ import ( "github.com/stackshy/cloudemu/v2/server" "github.com/stackshy/cloudemu/v2/server/oci/identity" "github.com/stackshy/cloudemu/v2/server/oci/monitoring" + "github.com/stackshy/cloudemu/v2/server/oci/objectstorage" "github.com/stackshy/cloudemu/v2/server/oci/vcn" "github.com/stackshy/cloudemu/v2/server/oci/workrequest" cachedriver "github.com/stackshy/cloudemu/v2/services/cache/driver" @@ -94,6 +95,20 @@ func New(d Drivers) *server.Server { srv.Register(monitoring.New(d.Monitoring)) } + if d.ObjectStorage != nil { + osHandler := objectstorage.New(d.ObjectStorage, d.WorkRequests) + + if comps, ok := d.Identity.(identity.Compartments); ok { + osHandler.SetCompartmentChecker(func(id string) bool { + _, err := comps.GetCompartment(context.Background(), id) + + return err == nil + }) + } + + srv.Register(osHandler) + } + if d.VCN != nil { vcnHandler := vcn.New(d.VCN, d.WorkRequests) diff --git a/server/oci/workrequest/workrequest.go b/server/oci/workrequest/workrequest.go index 223d1b998..4471c3f07 100644 --- a/server/oci/workrequest/workrequest.go +++ b/server/oci/workrequest/workrequest.go @@ -3,8 +3,10 @@ // Real OCI returns 202 with an opc-work-request-id from most mutating calls, // and SDK waiters poll GET /{version}/workRequests/{id} until the status is // terminal. Each service publishes that endpoint under its own API version -// prefix; CloudEmu collapses every service onto one HTTP server, so this -// handler claims any path ending in workRequests and answers uniformly. +// prefix (Object Storage, which has none, at the root); CloudEmu collapses every +// service onto one HTTP server, so this handler answers them all uniformly. +// workRequests is only recognized at that anchored position: elsewhere in a path +// it is user data, such as an object key or a bucket name. // // Every CloudEmu mutation completes synchronously, so an accepted work request // is already SUCCEEDED. The envelope exists to keep SDK waiters happy and to @@ -143,8 +145,7 @@ type Handler struct{ store *Store } // NewHandler returns the work request handler backed by store. func NewHandler(store *Store) *Handler { return &Handler{store: store} } -// Matches claims GET on any path under a workRequests segment, regardless of -// the service's API version prefix. +// Matches claims GET on /{version}/workRequests[/…] and /workRequests[/…]. func (*Handler) Matches(r *http.Request) bool { if r.Method != http.MethodGet { return false @@ -196,26 +197,45 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } -// parse splits /{version}/workRequests[/{id}[/{sub}]]. +// parse splits /{version}/workRequests[/{id}[/{sub}]], or the unversioned +// /workRequests[/…] Object Storage uses. The segment must sit directly after a +// date-style version, or lead the path, so a workRequests object key or bucket +// under /n/… is never claimed. func parse(urlPath string) (id, sub string, ok bool) { parts := strings.Split(strings.Trim(urlPath, "/"), "/") - for i, p := range parts { - if p != segment { - continue - } + switch { + case parts[0] == segment: + parts = parts[1:] + case len(parts) > 1 && isAPIVersion(parts[0]) && parts[1] == segment: + parts = parts[2:] + default: + return "", "", false + } + + switch len(parts) { + case 0: + return "", "", true + case 1: + return parts[0], "", true + case 2: //nolint:mnd // an id plus one sub-collection segment + return parts[0], parts[1], true + default: + return "", "", false + } +} + +// isAPIVersion reports whether s is an OCI date-style API version, YYYYMMDD. +func isAPIVersion(s string) bool { + if len(s) != len("20160918") { + return false + } - switch rest := parts[i+1:]; { - case len(rest) == 0: - return "", "", true - case len(rest) == 1: - return rest[0], "", true - case len(rest) == 2: //nolint:mnd // an id plus one sub-collection segment - return rest[0], rest[1], true - default: - return "", "", false + for _, c := range s { + if c < '0' || c > '9' { + return false } } - return "", "", false + return true } diff --git a/server/oci/workrequest/workrequest_test.go b/server/oci/workrequest/workrequest_test.go index 5c169fbd1..8cb3c625a 100644 --- a/server/oci/workrequest/workrequest_test.go +++ b/server/oci/workrequest/workrequest_test.go @@ -106,6 +106,12 @@ func TestHandlerMatches(t *testing.T) { {name: "non-GET is not claimed", method: http.MethodPost, path: "/20160918/workRequests", expect: false}, {name: "unrelated path", method: http.MethodGet, path: "/20160918/instances", expect: false}, {name: "too many trailing segments", method: http.MethodGet, path: "/20160918/workRequests/a/b/c", expect: false}, + {name: "unversioned object storage poll", method: http.MethodGet, path: "/workRequests/ocid1.workrequest.oc1.iad.a", expect: true}, + {name: "object key named workRequests", method: http.MethodGet, path: "/n/ns/b/tfb/o/workRequests/x", expect: false}, + {name: "bucket named workRequests", method: http.MethodGet, path: "/n/ns/b/workRequests", expect: false}, + {name: "PAR redemption of a workRequests key", method: http.MethodGet, path: "/p/tok/n/ns/b/tfb/o/workRequests", expect: false}, + {name: "non-version prefix", method: http.MethodGet, path: "/v1/workRequests/abc", expect: false}, + {name: "workRequests after a resource", method: http.MethodGet, path: "/20160918/vcns/workRequests", expect: false}, } for _, tc := range tests { diff --git a/server/oci/workrequest_collision_test.go b/server/oci/workrequest_collision_test.go new file mode 100644 index 000000000..f936ea6fe --- /dev/null +++ b/server/oci/workrequest_collision_test.go @@ -0,0 +1,90 @@ +package oci_test + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + cloudemu "github.com/stackshy/cloudemu/v2" + ociserver "github.com/stackshy/cloudemu/v2/server/oci" +) + +// The work request poller is registered first, so it must never claim an +// Object Storage path: a key or a bucket named workRequests is user data. +func TestWorkRequestsNamedObjectAndBucketStayReachable(t *testing.T) { + cloud := cloudemu.NewOCI() + ts := httptest.NewServer(ociserver.New(ociserver.DriversFrom(cloud))) + t.Cleanup(ts.Close) + + call := func(method, path string, body []byte) (int, http.Header, []byte) { + t.Helper() + + req, err := http.NewRequestWithContext(t.Context(), method, ts.URL+path, bytes.NewReader(body)) + require.NoError(t, err) + + resp, err := ts.Client().Do(req) + require.NoError(t, err) + + defer resp.Body.Close() + + out, err := io.ReadAll(resp.Body) + require.NoError(t, err) + + return resp.StatusCode, resp.Header, out + } + + _, _, raw := call(http.MethodGet, "/n", nil) + + var ns string + + require.NoError(t, json.Unmarshal(raw, &ns)) + + root := "/n/" + ns + "/b" + + for _, bucket := range []string{"tfb", "workRequests"} { + spec, err := json.Marshal(map[string]string{"name": bucket, "compartmentId": cloud.CompartmentID}) + require.NoError(t, err) + + code, _, body := call(http.MethodPost, root, spec) + require.Equal(t, http.StatusOK, code, string(body)) + } + + code, _, body := call(http.MethodGet, root+"/workRequests", nil) + require.Equal(t, http.StatusOK, code, string(body)) + assert.Contains(t, string(body), `"name":"workRequests"`) + + key := root + "/tfb/o/workRequests/x" + + code, _, body = call(http.MethodPut, key, []byte("payload")) + require.Equal(t, http.StatusOK, code, string(body)) + + code, _, body = call(http.MethodGet, key, nil) + require.Equal(t, http.StatusOK, code, string(body)) + assert.Equal(t, "payload", string(body)) + + // The copy's work request still polls, both unversioned (Object Storage's + // own form) and under a version prefix. + copySpec, err := json.Marshal(map[string]string{ + "sourceObjectName": "workRequests/x", "destinationRegion": cloud.Region, + "destinationBucket": "workRequests", "destinationObjectName": "copied", + }) + require.NoError(t, err) + + code, hdr, body := call(http.MethodPost, root+"/tfb/actions/copyObject", copySpec) + require.Equal(t, http.StatusAccepted, code, string(body)) + + id := hdr.Get("opc-work-request-id") + require.NotEmpty(t, id) + + for _, poll := range []string{"/workRequests/" + id, "/20160918/workRequests/" + id} { + code, _, body = call(http.MethodGet, poll, nil) + require.Equal(t, http.StatusOK, code, string(body)) + assert.Contains(t, string(body), `"status":"SUCCEEDED"`) + } +} diff --git a/server/serveflags/serveflags.go b/server/serveflags/serveflags.go index 6bac061f5..fe5386368 100644 --- a/server/serveflags/serveflags.go +++ b/server/serveflags/serveflags.go @@ -79,6 +79,7 @@ type CommonConfig struct { AccountID string AzureSubscription string + OCITenancy string Region string ProjectID string @@ -138,6 +139,9 @@ func RegisterCommon(fs *flag.FlagSet, c *CommonConfig, getenv func(string) strin fs.StringVar(&c.AccountID, "account-id", "000000000000", "AWS account ID (also GCP/OCI) reported by the emulator") fs.StringVar(&c.AzureSubscription, "azure-subscription", "00000000-0000-0000-0000-000000000000", "Azure subscription id reported by the emulator (a GUID; real Azure SDKs/CLIs require one)") + fs.StringVar(&c.OCITenancy, "oci-tenancy", config.DefaultTenancyOCID, + "OCI tenancy OCID reported by the emulator; it is also the root compartment and "+ + "the Object Storage namespace is derived from it") fs.StringVar(&c.Region, "region", "us-east-1", "default region reported by the emulator") fs.StringVar(&c.ProjectID, "project-id", "cloudemu-local", "GCP project ID reported by the emulator") fs.DurationVar(&c.Latency, "latency", 0, "artificial latency added to every emulated call (e.g. 20ms)") @@ -326,6 +330,7 @@ func (c *CommonConfig) ToServerkitConfig(providers []string) serverkit.Config { config.WithAccountID(c.AccountID), config.WithRegion(c.Region), config.WithProjectID(c.ProjectID), + config.WithTenancyOCID(c.OCITenancy), }, Out: os.Stdout, } diff --git a/server/serveflags/serveflags_test.go b/server/serveflags/serveflags_test.go index 4d68d5fd4..ea7d9f984 100644 --- a/server/serveflags/serveflags_test.go +++ b/server/serveflags/serveflags_test.go @@ -26,7 +26,7 @@ var commonFlagNames = []string{ "account-id", "admin", "admin-token", "admin-token-file", "advertise-host", "async-settle", "aws-port", "azure-port", "azure-subscription", "endpoints-file", "enforce-auth", "gcp-grpc-port", "gcp-port", "host", "init-dir", "k8s-nodes", "k8s-port", - "k8s-progression", "k8s-progression-interval", "latency", "log-requests", "oci-port", + "k8s-progression", "k8s-progression-interval", "latency", "log-requests", "oci-port", "oci-tenancy", "persist", "persist-interval", "persist-metadata-only", "persist-strategy", "project-id", "providers", "quiet", "region", "shutdown-timeout", "state-file", "tick-interval", "tls-cert", "tls-host", "tls-key", "vcr", "vcr-cassette", "vcr-strict", diff --git a/services/storage/storageengine/storageengine.go b/services/storage/storageengine/storageengine.go index af6bcfd39..1caf0afd8 100644 --- a/services/storage/storageengine/storageengine.go +++ b/services/storage/storageengine/storageengine.go @@ -1,8 +1,9 @@ // Package storageengine wires an optional real storage engine into an // object-storage provider's data path. It is shared by every storage provider -// (AWS S3, Azure Blob, GCP GCS) so the put/get/delete/copy hook stays identical -// across clouds and cannot drift. When no engine is configured every call is a -// no-op, leaving the provider's in-memory object bytes untouched. +// (AWS S3, Azure Blob, GCP GCS, OCI Object Storage) so the put/get/delete/copy +// hook stays identical across clouds and cannot drift. When no engine is +// configured every call is a no-op, leaving the provider's in-memory object +// bytes untouched. package storageengine import (