diff --git a/README.md b/README.md index 149e59a46..7fd897d21 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,8 @@ Following extension configurations are supported: - `azurite.disableProductStyleUrl` Force parsing storage account name from request URI path, instead of from request URI host. - `azurite.inMemoryPersistence` Disable persisting any data to disk. If the Azurite process is terminated, all data is lost. - `azurite.extentMemoryLimit` When using in-memory persistence, limit the total size of extents (blob and queue content) to a specific number of megabytes. This does not limit blob, queue, or table metadata. Defaults to 50% of total memory. +- `azurite.accountConfigFilePath` Path to a JSON file containing AccountModel configuration for blob versioning settings. See [Use Blob Versioning](#use-blob-versioning) for details. +- `azurite.accountConfigAsJson` Inline JSON string containing AccountModel configuration for blob versioning settings. See [Use Blob Versioning](#use-blob-versioning) for details. - `azurite.disableTelemetry` Disable telemetry data collection of this Azurite execution. By default, Azurite will collect telemetry data to help improve the product. ### [DockerHub](https://hub.docker.com/_/microsoft-azure-storage-azurite) @@ -240,7 +242,7 @@ docker run -p 10000:10000 -p 10001:10001 -v c:/azurite:/data mcr.microsoft.com/a #### Customize all Azurite V3 supported parameters for docker image ```bash -docker run -p 7777:7777 -p 8888:8888 -p 9999:9999 -v c:/azurite:/workspace mcr.microsoft.com/azure-storage/azurite azurite -l /workspace -d /workspace/debug.log --blobPort 7777 --blobHost 0.0.0.0 --blobKeepAliveTimeout 5 --queuePort 8888 --queueHost 0.0.0.0 --queueKeepAliveTimeout 5 --tablePort 9999 --tableHost 0.0.0.0 --tableKeepAliveTimeout 5 --loose --skipApiVersionCheck --disableProductStyleUrl --disableTelemetry +docker run -p 7777:7777 -p 8888:8888 -p 9999:9999 -v c:/azurite:/workspace mcr.microsoft.com/azure-storage/azurite azurite -l /workspace -d /workspace/debug.log --blobPort 7777 --blobHost 0.0.0.0 --blobKeepAliveTimeout 5 --queuePort 8888 --queueHost 0.0.0.0 --queueKeepAliveTimeout 5 --tablePort 9999 --tableHost 0.0.0.0 --tableKeepAliveTimeout 5 --loose --skipApiVersionCheck --disableProductStyleUrl --accountConfigAsJson "{\"isBlobVersioningEnabled\":true}" --disableTelemetry ``` Above command will try to start Azurite image with configurations: @@ -273,6 +275,10 @@ Above command will try to start Azurite image with configurations: `--disableProductStyleUrl` force parsing storage account name from request URI path, instead of from request URI host. +`--accountConfigFilePath /workspace/accountModel.json` configures blob versioning using an AccountModel JSON file mapped to the docker workspace. See [Use Blob Versioning](#use-blob-versioning) for details. + +`--accountConfigAsJson "{\"isBlobVersioningEnabled\":true}"` configures blob versioning using an inline JSON string. See [Use Blob Versioning](#use-blob-versioning) for details. + `--azurite.disableTelemetry` disable telemetry data collection of this Azurite execution. By default, Azurite will collect telemetry data to help improve the product. > If you use customized azurite parameters for docker image, `--blobHost 0.0.0.0`, `--queueHost 0.0.0.0` are required parameters. @@ -504,6 +510,86 @@ server thus resetting the storage completely. Note that if many hundreds of megabytes of content (queue message or blob content) are stored in-memory, it can take noticeably longer than usual for the process to terminate since all the consumed memory needs to be released. +### Use Blob Versioning + +#### How it works + +Blob Versioning was implemented to follow the exact guidelines outlined in the [Azure Blob Storage versioning documentation](https://learn.microsoft.com/en-us/azure/storage/blobs/versioning-overview), excluding interactions with soft delete, blob expiration, and SAS URIs. For detailed implementation information, see the [blob versioning design document](docs/designs/2025-12-blob-versioning.md). + +#### How to use it + +##### Single Account support + +Optional. By default, this is disabled. To enable it, there are two CLI args you can use: accountConfigFilePath, accountConfigAsJson. + +Blob versioning is enabled by leveraging the [AccountModel](src/common/account/AccountModel.ts). The account model is an abstraction to configure the storage account. Currently, it only supports configuring blob versioning. + +accountConfigFilePath lets you pass in the path to a json file modeled after the AccountModel, which is then used to configure Azurite. + +```bash +azurite --accountConfigFilePath "./myAccountModel.json" +``` + +Example contents of `myAccountModel.json`: + +```json +{ + "isBlobVersioningEnabled": true +} +``` + +accountConfigAsJson allows you to pass a json string as a CLI arg to configure the account as well. + +```bash +azurite --accountConfigAsJson "{ \"isBlobVersioningEnabled\": true }" +``` + +By default, Azurite will always use whatever version of the account model already exists in its databases. However, if any of these parameters are passed in and are valid, the existing account model will be overwritten. + +##### Multi-account AccountModel support + +Both `accountConfigFilePath` and `accountConfigAsJson` support configuring multiple accounts with different versioning settings. + +**Using accountConfigFilePath with multiple accounts:** + +```bash +azurite --accountConfigFilePath "account1:/path/to/config1.json,account2:/path/to/config2.json" +``` + +Where `config1.json` might contain: + +```json +{ + "isBlobVersioningEnabled": true +} +``` + +And `config2.json` might contain: + +```json +{ + "isBlobVersioningEnabled": false +} +``` + +**Using accountConfigAsJson with multiple accounts:** + +```bash +azurite --accountConfigAsJson "account1:{\"isBlobVersioningEnabled\":true},account2:{\"isBlobVersioningEnabled\":false}" +``` + +**Backward compatibility:** + +For single-account configuration, you can omit the account name prefix (defaults to `devstoreaccount1`): + +```bash +azurite --accountConfigFilePath "./myAccountModel.json" +# or +azurite --accountConfigAsJson "{\"isBlobVersioningEnabled\":true}" +``` + +> **Important:** Declaring an account in the AccountModel configuration only sets the versioning behavior for that account. You still need to configure authentication for these accounts using the `AZURITE_ACCOUNTS` environment variable (see [Customized Storage Accounts & Keys](#customized-storage-accounts--keys-1)) to actually use them. Without proper authentication setup, requests to these accounts will fail authentication. **Furthermore, if you want to configure all accounts, you must configure each account individually. If you follow the single-account flow or configure only one account, the other accounts will not be configured.** + ### Command Line Options Differences between Azurite V2 Azurite V3 supports SharedKey, Account Shared Access Signature (SAS), Service SAS, OAuth, and Public Container Access authentications, you can use any Azure Storage SDKs or tools like Storage Explorer to connect Azurite V3 with any authentication strategy. @@ -1039,6 +1125,7 @@ Latest release targets **2025-11-05** API version **blob** service. Detailed support matrix: - Supported Vertical Features + - CORS and Preflight - SharedKey Authentication - OAuth authentication @@ -1046,7 +1133,9 @@ Detailed support matrix: - Shared Access Signature Service Level (Not support response header override in service SAS) - Container Public Access - Blob Tags (preview) + - Blob versioning (Only in LokiDb instances of Azurite, which is the default. Does not support SAS URIs) - Supported REST APIs + - List Containers - Set Service Properties - Get Service Properties @@ -1081,7 +1170,6 @@ Detailed support matrix: - Soft delete & Undelete Blob - Incremental Copy Blob - Blob Query - - Blob Versions - Blob Last Access Time - Concurrent Append - Blob Expiry @@ -1092,6 +1180,7 @@ Detailed support matrix: - Encryption Scope - Get Page Ranges Continuation Token - Blob Immutability Policy and Legal Hold + - SAS URIs for Blob Versions Latest version supports for **2025-11-05** API version **queue** service. Detailed support matrix: @@ -1175,4 +1264,4 @@ provided by the bot. You will only need to do this once across all repos using o This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or -contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. +contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. \ No newline at end of file diff --git a/docs/designs/2025-12-blob-versioning.md b/docs/designs/2025-12-blob-versioning.md new file mode 100644 index 000000000..e18db5db3 --- /dev/null +++ b/docs/designs/2025-12-blob-versioning.md @@ -0,0 +1,100 @@ +# Add Blob Versioning Support + +- Author Name: Rodolfo Orozco Vasquez ([@rorozcov](https://github.com/rorozcov)) +- GitHub Issue: [Azure/Azurite#665](https://github.com/Azure/Azurite/issues/665) + +## Summary + +This design adds support for Azure Blob Storage versioning to Azurite, allowing blobs to maintain previous versions automatically when they are modified or deleted. Versioning is implemented following the [Azure Blob Storage versioning guidelines](https://learn.microsoft.com/en-us/azure/storage/blobs/versioning-overview) as closely as possible, with some limitations based on features not yet supported in Azurite. + +## Motivation + +Blob versioning is a critical feature in Azure Blob Storage that automatically maintains previous versions of a blob. This enables users to: + +- Recover from accidental blob modifications or deletions +- Maintain a history of blob changes over time +- Access and restore previous versions of blobs + +Without blob versioning support in Azurite, developers cannot fully test applications that rely on this feature locally, limiting their ability to validate version-aware workflows before deploying to Azure. + +## Explanation + +### Functional explanation + +Blob versioning in Azurite is controlled through the `AccountModel` abstraction, which allows configuration of storage account-level settings. When enabled, blob versioning automatically creates a new version of a blob whenever it is modified or overwritten. + +Two command line options are available to configure blob versioning: + +1. **`--accountConfigFilePath`** - Path to a JSON configuration file +2. **`--accountConfigAsJson`** - Inline JSON string configuration + +We also support multi-account configuration since Azurite supports multiple accounts. + +### Technical explanation + +Blob versioning is implemented using the `AccountModel` type which is stored in the metadata database: + +```typescript +export interface AccountModel { + key: string; + isBlobVersioningEnabled: boolean; +} +``` + +When versioning is enabled for an account: + +- **For block blobs:** All write operations trigger the creation of a new version, except for the Put Block operation +- **For page blobs and append blobs:** Only a subset of write operations triggers version creation: + - Put Blob + - Put Block List + - Set Blob Metadata + - Copy Blob +- **Operations that do NOT trigger version creation:** + - Put Page (page blob) + - Append Block (append blob) +- Each version is assigned a unique version ID in ISO 8601 date-time format + - Azurite emits the same 7 fractional digits used by Azure. The first 3 digits come from JavaScript millisecond precision, and the final 4 digits provide a per-blob sub-millisecond counter so version IDs remain unique. +- Previous versions are immutable and can be accessed using the version ID +- The `List Blobs` operation can include versions when the `includeVersions` parameter is set to true +- Specific versions can be retrieved, downloaded, or deleted using the `versionId` query parameter + +The configuration is parsed through `EnvironmentFunctions.parseAccountModelFlags()` which supports: + +- Single account configuration +- Multi-account configuration with comma-separated entries +- Both file-based and inline JSON configurations +- Proper JSON parsing with support for nested objects and escaped characters + +### Integration with Authentication + +> **Important:** The `AccountModel` configuration only controls the versioning behavior. To actually use the configured accounts, they must also be set up in the `AZURITE_ACCOUNTS` environment variable for authentication. See [Customized Storage Accounts & Keys](https://github.com/Azure/Azurite#customized-storage-accounts--keys-1) for details. + +### Limitations + +The following Azure Blob Storage versioning features are **not** currently supported: + +- Soft delete integration with versioning +- Blob expiration with versioning +- SAS URIs for specific blob versions +- Version-level immutability policies (Version Level WORM) + +### VS Code Extension Support + +Similar configuration options are available in the VS Code extension settings: + +- `azurite.accountConfigFilePath` - Path to account configuration file +- `azurite.accountConfigAsJson` - Inline JSON configuration string + +## Azure Documentation on Blob Versioning + +This implementation follows the Azure Blob Storage versioning specification as documented in the official Microsoft documentation: + +- [Blob versioning overview](https://learn.microsoft.com/en-us/azure/storage/blobs/versioning-overview) +- [Enable and manage blob versioning](https://learn.microsoft.com/en-us/azure/storage/blobs/versioning-enable) + +The design aligns with Azure's behavior where: + +- Versioning is a storage account-level setting +- Version IDs are automatically assigned timestamps +- Previous versions are immutable +- The current version is mutable diff --git a/package.json b/package.json index 2c8d44869..cd7bc5802 100644 --- a/package.json +++ b/package.json @@ -275,6 +275,16 @@ "type": "boolean", "default": false, "description": "Disable telemetry data collection of this Azurite execution. By default, Azurite will collect telemetry data to help improve the product." + }, + "azurite.accountConfigFilePath": { + "type": "string", + "default": null, + "description": "Path to the account configuration file. This file contains account-specific settings. Mutually exclusive with azurite.accountConfigAsJson." + }, + "azurite.accountConfigAsJson": { + "type": "string", + "default": null, + "description": "Account configuration settings in JSON format. Mutually exclusive with azurite.accountConfigFilePath." } } } diff --git a/src/azurite.ts b/src/azurite.ts index 297b86083..240c74fe5 100644 --- a/src/azurite.ts +++ b/src/azurite.ts @@ -4,6 +4,7 @@ import { dirname, join } from "path"; // Load Environment before BlobServerFactory to make sure args works properly import Environment from "./common/Environment"; +import LokiAccountModelStore from "./common/account/LokiAccountModelStore"; // tslint:disable-next-line:ordered-imports import { BlobServerFactory } from "./blob/BlobServerFactory"; @@ -18,6 +19,7 @@ import { } from "./queue/utils/constants"; import SqlBlobServer from "./blob/SqlBlobServer"; import BlobServer from "./blob/BlobServer"; +import { DEFAULT_ACCOUNT_MODEL_LOKI_DB_PATH } from "./blob/utils/constants"; import TableConfiguration from "./table/TableConfiguration"; import TableServer from "./table/TableServer"; @@ -97,8 +99,16 @@ async function main() { await access(dirname(debugFilePath!)); } + // Create account model store + const accountModels = env.getAccountModels(); + const accountModelStore = new LokiAccountModelStore( + join(location, DEFAULT_ACCOUNT_MODEL_LOKI_DB_PATH), + env.inMemoryPersistence(), + accountModels + ); + const blobServerFactory = new BlobServerFactory(); - const blobServer = await blobServerFactory.createServer(env); + const blobServer = await blobServerFactory.createServer(env, accountModelStore); const blobConfig = blobServer.config; // TODO: Align with blob DEFAULT_BLOB_PERSISTENCE_ARRAY diff --git a/src/blob/BlobConfiguration.ts b/src/blob/BlobConfiguration.ts index b77f94a4d..0d387e33e 100644 --- a/src/blob/BlobConfiguration.ts +++ b/src/blob/BlobConfiguration.ts @@ -1,6 +1,7 @@ import ConfigurationBase from "../common/ConfigurationBase"; import { StoreDestinationArray } from "../common/persistence/IExtentStore"; import { MemoryExtentChunkStore } from "../common/persistence/MemoryExtentStore"; +import IAccountModelStore from "../common/account/IAccountModelStore"; import { DEFAULT_BLOB_EXTENT_LOKI_DB_PATH, DEFAULT_BLOB_LISTENING_PORT, @@ -45,6 +46,7 @@ export default class BlobConfiguration extends ConfigurationBase { disableProductStyleUrl: boolean = false, public readonly isMemoryPersistence: boolean = false, public readonly memoryStore?: MemoryExtentChunkStore, + public readonly accountModelStore?: IAccountModelStore, ) { super( host, @@ -60,7 +62,7 @@ export default class BlobConfiguration extends ConfigurationBase { key, pwd, oauth, - disableProductStyleUrl + disableProductStyleUrl, ); } } diff --git a/src/blob/BlobEnvironment.ts b/src/blob/BlobEnvironment.ts index d046a3773..ea1186448 100644 --- a/src/blob/BlobEnvironment.ts +++ b/src/blob/BlobEnvironment.ts @@ -3,6 +3,8 @@ import { access, ensureDir } from "fs-extra"; import { dirname } from "path"; import IBlobEnvironment from "./IBlobEnvironment"; +import { parseAccountModelFlags } from "../common/EnvironmentFunctions"; +import { AccountModel } from "../common/account/AccountModel"; import { DEFAULT_BLOB_LISTENING_PORT, DEFAULT_BLOB_SERVER_HOST_NAME, @@ -70,6 +72,14 @@ if (!(args as any).config.name) { .option( ["", "disableTelemetry"], "Optional. Disable telemetry data collection of this Azurite execution. By default, Azurite will collect telemetry data to help improve the product." + ) + .option( + ["", "accountConfigFilePath"], + "Optional. Path to the account configuration file" + ) + .option( + ["", "accountConfigAsJson"], + "Optional. Account configuration in JSON format" ); (args as any).config.name = "azurite-blob"; @@ -187,4 +197,8 @@ export default class BlobEnvironment implements IBlobEnvironment { // By default disable debug log } -} + + public getAccountModels(): Map | undefined { + return parseAccountModelFlags(this.flags); + } +} \ No newline at end of file diff --git a/src/blob/BlobServer.ts b/src/blob/BlobServer.ts index fcf5952eb..caaa2d8a6 100644 --- a/src/blob/BlobServer.ts +++ b/src/blob/BlobServer.ts @@ -23,6 +23,7 @@ import BlobGCManager from "./gc/BlobGCManager"; import IBlobMetadataStore from "./persistence/IBlobMetadataStore"; import LokiBlobMetadataStore from "./persistence/LokiBlobMetadataStore"; import StorageError from "./errors/StorageError"; +import IAccountModelStore from "../common/account/IAccountModelStore"; const BEFORE_CLOSE_MESSAGE = `Azurite Blob service is closing...`; const BEFORE_CLOSE_MESSAGE_GC_ERROR = `Azurite Blob service is closing... Critical error happens during GC.`; @@ -47,6 +48,7 @@ export default class BlobServer extends ServerBase implements ICleaner { private readonly extentStore: IExtentStore; private readonly accountDataStore: IAccountDataStore; private readonly gcManager: IGCManager; + private readonly accountModelStore: IAccountModelStore; /** * Creates an instance of Server. @@ -74,12 +76,19 @@ export default class BlobServer extends ServerBase implements ICleaner { httpServer = http.createServer(); } + // Get the account model store from configuration (must be provided by factory) + if (!configuration.accountModelStore) { + throw new Error("Account model store must be provided in BlobConfiguration"); + } + const lokiAccountModelStore = configuration.accountModelStore; + // We can change the persistency layer implementation by // creating a new XXXDataStore class implementing IBlobMetadataStore interface // and replace the default LokiBlobMetadataStore const metadataStore: IBlobMetadataStore = new LokiBlobMetadataStore( configuration.metadataDBPath, - configuration.isMemoryPersistence + configuration.isMemoryPersistence, + lokiAccountModelStore ); const extentMetadataStore: IExtentMetadataStore = @@ -102,6 +111,7 @@ export default class BlobServer extends ServerBase implements ICleaner { logger ); + // IAccountDataStore is used by the request handler for account management const accountDataStore: IAccountDataStore = new AccountDataStore(logger); // We can also change the HTTP framework here by @@ -158,6 +168,7 @@ export default class BlobServer extends ServerBase implements ICleaner { this.extentStore = extentStore; this.accountDataStore = accountDataStore; this.gcManager = gcManager; + this.accountModelStore = lokiAccountModelStore; } /** @@ -181,6 +192,8 @@ export default class BlobServer extends ServerBase implements ICleaner { await this.metadataStore.clean(); } + await this.accountModelStore.clean(); + if (this.accountDataStore !== undefined) { await this.accountDataStore.clean(); } @@ -193,6 +206,10 @@ export default class BlobServer extends ServerBase implements ICleaner { const msg = `Azurite Blob service is starting on ${this.host}:${this.port}`; logger.info(msg); + if (this.accountModelStore.isInitialized() === false) { + await this.accountModelStore.init(); + } + if (this.accountDataStore !== undefined) { await this.accountDataStore.init(); } @@ -244,6 +261,10 @@ export default class BlobServer extends ServerBase implements ICleaner { await this.accountDataStore.close(); } + if (this.accountModelStore.isClosed() === false) { + await this.accountModelStore.close(); + } + logger.info(AFTER_CLOSE_MESSAGE); } -} +} \ No newline at end of file diff --git a/src/blob/BlobServerFactory.ts b/src/blob/BlobServerFactory.ts index 158456476..80cbebe80 100644 --- a/src/blob/BlobServerFactory.ts +++ b/src/blob/BlobServerFactory.ts @@ -13,10 +13,12 @@ import { DEFAULT_BLOB_LOKI_DB_PATH, DEFAULT_BLOB_PERSISTENCE_ARRAY } from "./utils/constants"; +import IAccountModelStore from "../common/account/IAccountModelStore"; export class BlobServerFactory { public async createServer( - blobEnvironment?: IBlobEnvironment + blobEnvironment?: IBlobEnvironment, + accountModelStore?: IAccountModelStore ): Promise { // TODO: Check it's in Visual Studio Code environment or not const isVSC = false; @@ -43,10 +45,29 @@ export class BlobServerFactory { if (isSQL) { if (env.inMemoryPersistence()) { - throw new Error(`The --inMemoryPersistence option is not supported when using SQL-based metadata storage.`) + throw new Error( + `The --inMemoryPersistence option is not supported when using SQL-based metadata storage.` + ); } if (env.extentMemoryLimit() !== undefined) { - throw new Error(`The --extentMemoryLimit option is not supported when using SQL-based metadata storage.`) + throw new Error( + `The --extentMemoryLimit option is not supported when using SQL-based metadata storage.` + ); + } + if (accountModelStore !== undefined) { + if (!accountModelStore.isInitialized()) { + await accountModelStore.init(); + } + const versioningEnabled = + accountModelStore.hasBlobVersioningEnabled(); + if (!accountModelStore.isClosed()) { + await accountModelStore.close(); + } + if (versioningEnabled) { + throw new Error( + "Blob versioning is not supported when using SQL-based metadata storage." + ); + } } const config = new SqlBlobConfiguration( @@ -90,6 +111,8 @@ export class BlobServerFactory { env.oauth(), env.disableProductStyleUrl(), env.inMemoryPersistence(), + undefined, + accountModelStore ); return new BlobServer(config); diff --git a/src/blob/IBlobEnvironment.ts b/src/blob/IBlobEnvironment.ts index a57700759..3e36e1e80 100644 --- a/src/blob/IBlobEnvironment.ts +++ b/src/blob/IBlobEnvironment.ts @@ -1,4 +1,6 @@ -export default interface IBlobEnvironment { +import IAccountModelEnvironment from "../common/IAccountModelEnvironment"; + +export default interface IBlobEnvironment extends IAccountModelEnvironment { blobHost(): string | undefined; blobPort(): number | undefined; blobKeepAliveTimeout(): number | undefined; diff --git a/src/blob/errors/StorageErrorFactory.ts b/src/blob/errors/StorageErrorFactory.ts index a0c4e897b..a417de1da 100644 --- a/src/blob/errors/StorageErrorFactory.ts +++ b/src/blob/errors/StorageErrorFactory.ts @@ -9,6 +9,28 @@ const DefaultID: string = "DefaultBlobRequestID"; * @class StorageErrorFactory */ export default class StorageErrorFactory { + public static getMutuallyExclusiveQueryParameters( + contextID: string = DefaultID + ): StorageError { + return new StorageError( + 400, + "MutuallyExclusiveQueryParameters", + "The query parameter is invalid. Two or more mutually exclusive query parameters were specified.", + contextID + ); + } + + public static getOperationNotAllowedOnRootBlob( + contextID: string = DefaultID + ): StorageError { + return new StorageError( + 403, + "OperationNotAllowedOnRootBlob", + "The specified operation is not allowed on root blob.", + contextID + ); + } + public static getContainerNotFound( contextID: string = DefaultID ): StorageError { @@ -888,4 +910,4 @@ export default class StorageErrorFactory { contextID ); } -} +} \ No newline at end of file diff --git a/src/blob/handlers/AppendBlobHandler.ts b/src/blob/handlers/AppendBlobHandler.ts index 08d24f73b..a21fbdbe6 100644 --- a/src/blob/handlers/AppendBlobHandler.ts +++ b/src/blob/handlers/AppendBlobHandler.ts @@ -79,7 +79,7 @@ export default class AppendBlobHandler extends BaseHandler blobTags: options.blobTagsString === undefined ? undefined : getTagsFromString(options.blobTagsString, context.contextId!), }; - await this.metadataStore.createBlob( + const createdBlob = await this.metadataStore.createBlob( context, blob, options.leaseAccessConditions, @@ -95,7 +95,8 @@ export default class AppendBlobHandler extends BaseHandler version: BLOB_API_VERSION, date, isServerEncrypted: true, - clientRequestId: options.requestId + clientRequestId: options.requestId, + versionId: createdBlob.versionId ? createdBlob.versionId : undefined }; return response; @@ -130,6 +131,7 @@ export default class AppendBlobHandler extends BaseHandler accountName, containerName, blobName, + undefined, undefined ); @@ -252,4 +254,4 @@ export default class AppendBlobHandler extends BaseHandler return response; } -} +} \ No newline at end of file diff --git a/src/blob/handlers/BaseHandler.ts b/src/blob/handlers/BaseHandler.ts index 63e6106c6..22e3fd913 100644 --- a/src/blob/handlers/BaseHandler.ts +++ b/src/blob/handlers/BaseHandler.ts @@ -1,6 +1,7 @@ import IExtentStore from "../../common/persistence/IExtentStore"; import ILogger from "../generated/utils/ILogger"; import IBlobMetadataStore from "../persistence/IBlobMetadataStore"; +import { validateSnapshotAndVersionId } from "../utils/utils"; /** * BaseHandler class should maintain a singleton to persistency layer, such as maintain a database connection pool. @@ -17,4 +18,12 @@ export default class BaseHandler { protected readonly logger: ILogger, protected readonly loose: boolean ) {} + + protected validateVersionId( + snapshot: string | undefined, + versionId: string | undefined, + contextId: string + ): void { + validateSnapshotAndVersionId(snapshot, versionId, contextId); + } } diff --git a/src/blob/handlers/BlobHandler.ts b/src/blob/handlers/BlobHandler.ts index 1ee4b9bc6..ac608a776 100644 --- a/src/blob/handlers/BlobHandler.ts +++ b/src/blob/handlers/BlobHandler.ts @@ -29,7 +29,8 @@ import { deserializePageBlobRangeHeader, deserializeRangeHeader, getBlobTagsCount, - validateBlobTag + parseDateFromAssumedString, + validateBlobTag, } from "../utils/utils"; import BaseHandler from "./BaseHandler"; import IPageBlobRangesManager from "./IPageBlobRangesManager"; @@ -65,6 +66,12 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { options: Models.BlobDownloadOptionalParams, context: Context ): Promise { + this.validateVersionId( + options.snapshot, + options.versionId, + context.contextId! + ); + const blobCtx = new BlobStorageContext(context); const accountName = blobCtx.account!; const containerName = blobCtx.container!; @@ -76,6 +83,7 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { containerName, blobName, options.snapshot, + options.versionId, options.leaseAccessConditions, options.modifiedAccessConditions ); @@ -107,6 +115,12 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { options: Models.BlobGetPropertiesOptionalParams, context: Context ): Promise { + this.validateVersionId( + options.snapshot, + options.versionId, + context.contextId! + ); + const blobCtx = new BlobStorageContext(context); const account = blobCtx.account!; const container = blobCtx.container!; @@ -117,6 +131,7 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { container, blob, options.snapshot, + options.versionId, options.leaseAccessConditions, options.modifiedAccessConditions ); @@ -134,7 +149,8 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { date: context.startTime, clientRequestId: options.requestId, contentLength: res.properties.contentLength, - lastModified: res.properties.lastModified + lastModified: res.properties.lastModified, + versionId: res.versionId ? res.versionId : undefined } : { statusCode: 200, @@ -158,6 +174,7 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { contentLanguage: context.request!.getQuery("rscl") ?? res.properties.contentLanguage, contentType: context.request!.getQuery("rsct") ?? res.properties.contentType, tagCount: res.properties.tagCount, + versionId: res.versionId ? res.versionId : undefined }; return response; @@ -175,6 +192,12 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { options: Models.BlobDeleteMethodOptionalParams, context: Context ): Promise { + this.validateVersionId( + options.snapshot, + options.versionId, + context.contextId! + ); + const blobCtx = new BlobStorageContext(context); const account = blobCtx.account!; const container = blobCtx.container!; @@ -353,7 +376,8 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { requestId: context.contextId, date: context.startTime, version: BLOB_API_VERSION, - clientRequestId: options.requestId + clientRequestId: options.requestId, + versionId: res.versionId ? res.versionId : undefined }; return response; @@ -616,7 +640,8 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { date: context.startTime!, version: BLOB_API_VERSION, snapshot: res.snapshot, - clientRequestId: options.requestId + clientRequestId: options.requestId, + versionId: res.versionId ? res.versionId : undefined }; return response; @@ -649,6 +674,16 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { sourceBlob ] = extractStoragePartsFromPath(url.hostname, url.pathname, blobCtx.disableProductStyleUrl); const snapshot = url.searchParams.get("snapshot") || ""; + const versionId = url.searchParams.get("versionid") || ""; + + this.validateVersionId(snapshot, versionId, context.contextId!); + + if (snapshot && !parseDateFromAssumedString(snapshot)) { + throw StorageErrorFactory.getInvalidQueryParameterValue( + context.contextId!, + "snapshot" + ); + } if ( sourceAccount === undefined || @@ -674,7 +709,8 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { account: sourceAccount, container: sourceContainer, blob: sourceBlob, - snapshot + snapshot: snapshot, + versionId: versionId }, { account, container, blob }, copySource, @@ -692,7 +728,8 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { date: context.startTime, copyId: res.copyId, copyStatus: res.copyStatus, - clientRequestId: options.requestId + clientRequestId: options.requestId, + versionId: res.versionId ? res.versionId : undefined }; return response; @@ -799,6 +836,7 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { containerName, blobName, undefined, + undefined, options.leaseAccessConditions ); @@ -848,6 +886,16 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { sourceBlob ] = extractStoragePartsFromPath(url.hostname, url.pathname, blobCtx.disableProductStyleUrl); const snapshot = url.searchParams.get("snapshot") || ""; + const versionId = url.searchParams.get("versionid") || ""; + + this.validateVersionId(snapshot, versionId, context.contextId!); + + if (snapshot && !parseDateFromAssumedString(snapshot)) { + throw StorageErrorFactory.getInvalidQueryParameterValue( + context.contextId!, + "snapshot" + ); + } if ( sourceAccount === undefined || @@ -877,7 +925,8 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { account: sourceAccount, container: sourceContainer, blob: sourceBlob, - snapshot + snapshot: snapshot, + versionId: versionId }, { account, container, blob }, copySource, @@ -907,10 +956,11 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { date: context.startTime, copyId: res.copyId, copyStatus, + clientRequestId: options.requestId, + versionId: res.versionId ? res.versionId : undefined, // Per the Copy Blob From URL REST contract, echo the source's Content-MD5 // back to the client when it was supplied in x-ms-source-content-md5. - contentMD5: options.sourceContentMD5, - clientRequestId: options.requestId + contentMD5: options.sourceContentMD5 }; return response; @@ -930,6 +980,12 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { options: Models.BlobSetTierOptionalParams, context: Context ): Promise { + this.validateVersionId( + options.snapshot, + options.versionId, + context.contextId! + ); + const blobCtx = new BlobStorageContext(context); const account = blobCtx.account!; const container = blobCtx.container!; @@ -939,6 +995,7 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { account, container, blob, + options.versionId, tier, options.leaseAccessConditions ); @@ -1026,14 +1083,14 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { // Start Range is bigger than blob length if (rangeStart > blob.properties.contentLength!) { - throw StorageErrorFactory.getInvalidPageRange2(context.contextId!,`bytes */${blob.properties.contentLength}`); + throw StorageErrorFactory.getInvalidPageRange2(context.contextId!, `bytes */${blob.properties.contentLength}`); } // Will automatically shift request with longer data end than blob size to blob size if (rangeEnd + 1 >= blob.properties.contentLength!) { // report error is blob size is 0, and rangeEnd is specified but not 0 if (blob.properties.contentLength == 0 && rangeEnd !== 0 && rangeEnd !== Infinity) { - throw StorageErrorFactory.getInvalidPageRange2(context.contextId!,`bytes */${blob.properties.contentLength}`); + throw StorageErrorFactory.getInvalidPageRange2(context.contextId!, `bytes */${blob.properties.contentLength}`); } else { rangeEnd = blob.properties.contentLength! - 1; @@ -1114,7 +1171,7 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { acceptRanges: "bytes", contentLength, contentRange, - contentMD5: contentRange ? (context.request!.getHeader("x-ms-range-get-content-md5") ? contentMD5: undefined) : contentMD5, + contentMD5: contentRange ? (context.request!.getHeader("x-ms-range-get-content-md5") ? contentMD5 : undefined) : contentMD5, tagCount: getBlobTagsCount(blob.blobTags), isServerEncrypted: true, clientRequestId: options.requestId, @@ -1123,6 +1180,7 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { blob.properties.blobType === Models.BlobType.AppendBlob ? (blob.committedBlocksInOrder || []).length : undefined, + versionId: blob.versionId ? blob.versionId : undefined }; return response; @@ -1154,14 +1212,14 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { // Start Range is bigger than blob length if (rangeStart > blob.properties.contentLength!) { - throw StorageErrorFactory.getInvalidPageRange2(context.contextId!,`bytes */${blob.properties.contentLength}`); + throw StorageErrorFactory.getInvalidPageRange2(context.contextId!, `bytes */${blob.properties.contentLength}`); } // Will automatically shift request with longer data end than blob size to blob size if (rangeEnd + 1 >= blob.properties.contentLength!) { // report error is blob size is 0, and rangeEnd is specified but not 0 if (blob.properties.contentLength == 0 && rangeEnd !== 0 && rangeEnd !== Infinity) { - throw StorageErrorFactory.getInvalidPageRange2(context.contextId!,`bytes */${blob.properties.contentLength}`); + throw StorageErrorFactory.getInvalidPageRange2(context.contextId!, `bytes */${blob.properties.contentLength}`); } else { rangeEnd = blob.properties.contentLength! - 1; @@ -1250,12 +1308,13 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { contentType: context.request!.getQuery("rsct") ?? blob.properties.contentType, contentLength, contentRange, - contentMD5: contentRange ? (context.request!.getHeader("x-ms-range-get-content-md5") ? contentMD5: undefined) : contentMD5, + contentMD5: contentRange ? (context.request!.getHeader("x-ms-range-get-content-md5") ? contentMD5 : undefined) : contentMD5, blobContentMD5: blob.properties.contentMD5, tagCount: getBlobTagsCount(blob.blobTags), isServerEncrypted: true, creationTime: blob.properties.creationTime, - clientRequestId: options.requestId + clientRequestId: options.requestId, + versionId: blob.versionId ? blob.versionId : undefined, }; return response; @@ -1272,6 +1331,12 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { options: Models.BlobGetTagsOptionalParams, context: Context ): Promise { + this.validateVersionId( + options.snapshot, + options.versionId, + context.contextId! + ); + const blobCtx = new BlobStorageContext(context); const account = blobCtx.account!; const container = blobCtx.container!; @@ -1282,6 +1347,7 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { container, blob, options.snapshot, + options.versionId, options.leaseAccessConditions, options.modifiedAccessConditions ); @@ -1314,12 +1380,15 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { // Get snapshot (swagger not defined snapshot as parameter, but server support set tag on blob snapshot) let snapshot = context.request!.getQuery("snapshot"); + this.validateVersionId(snapshot, options.versionId, context.contextId!); + await this.metadataStore.setBlobTag( context, account, container, blob, snapshot, + options.versionId, options.leaseAccessConditions, tags, options.modifiedAccessConditions @@ -1340,8 +1409,7 @@ export default class BlobHandler extends BaseHandler implements IBlobHandler { try { return new URL(copySource) } - catch - { + catch { throw StorageErrorFactory.getInvalidHeaderValue( context.contextId, { diff --git a/src/blob/handlers/BlockBlobHandler.ts b/src/blob/handlers/BlockBlobHandler.ts index 666a067f0..107152425 100644 --- a/src/blob/handlers/BlockBlobHandler.ts +++ b/src/blob/handlers/BlockBlobHandler.ts @@ -118,7 +118,6 @@ export default class BlockBlobHandler accessTierInferred: true, accessTierChangeTime: date }, - snapshot: "", isCommitted: true, persistency, blobTags: options.blobTagsString === undefined ? undefined : getTagsFromString(options.blobTagsString, context.contextId!), @@ -136,7 +135,7 @@ export default class BlockBlobHandler } // TODO: Need a lock for multi keys including containerName and blobName // TODO: Provide a specified function. - await this.metadataStore.createBlob( + const createdBlob = await this.metadataStore.createBlob( context, blob, options.leaseAccessConditions, @@ -152,7 +151,8 @@ export default class BlockBlobHandler version: BLOB_API_VERSION, date, isServerEncrypted: true, - clientRequestId: options.requestId + clientRequestId: options.requestId, + versionId: createdBlob.versionId ? createdBlob.versionId : undefined }; return response; @@ -323,8 +323,10 @@ export default class BlockBlobHandler accountName, containerName, name: blobName, - snapshot: "", - blobTags: options.blobTagsString === undefined ? undefined : getTagsFromString(options.blobTagsString, context.contextId!), + blobTags: + options.blobTagsString === undefined + ? undefined + : getTagsFromString(options.blobTagsString, context.contextId!), properties: { lastModified: context.startTime!, creationTime: context.startTime!, @@ -362,7 +364,7 @@ export default class BlockBlobHandler blob.properties.accessTierInferred = true; } - await this.metadataStore.commitBlockList( + const storeResponse = await this.metadataStore.commitBlockList( context, blob, commitBlockList, @@ -381,7 +383,8 @@ export default class BlockBlobHandler version: BLOB_API_VERSION, date: blobCtx.startTime, isServerEncrypted: true, - clientRequestId: options.requestId + clientRequestId: options.requestId, + versionId: storeResponse.versionId ? storeResponse.versionId : undefined }; return response; } @@ -396,6 +399,8 @@ export default class BlockBlobHandler const blobName = blobCtx.blob!; const date = blobCtx.startTime!; + // TODO: Updated generated optional params to support versionId. + // https://learn.microsoft.com/en-us/rest/api/storageservices/get-block-list?tabs=microsoft-entra-id const res = await this.metadataStore.getBlockList( context, accountName, @@ -403,6 +408,7 @@ export default class BlockBlobHandler blobName, options.snapshot, undefined, + undefined, options.leaseAccessConditions, options.modifiedAccessConditions ); diff --git a/src/blob/handlers/ContainerHandler.ts b/src/blob/handlers/ContainerHandler.ts index fe5ec6adb..291fc5a17 100644 --- a/src/blob/handlers/ContainerHandler.ts +++ b/src/blob/handlers/ContainerHandler.ts @@ -644,6 +644,8 @@ export default class ContainerHandler extends BaseHandler let includeUncommittedBlobs: boolean = false; let includeTags: boolean = false; let includeMetadata: boolean = false; + let includeVersions: boolean = false; + let includeDeletedWithVersions: boolean = false; if (options.include !== undefined) { options.include.forEach(element => { if (Models.ListBlobsIncludeItem.Snapshots.toLowerCase() === element.toLowerCase()) { @@ -658,7 +660,13 @@ export default class ContainerHandler extends BaseHandler if (Models.ListBlobsIncludeItem.Metadata.toLowerCase() === element.toLowerCase()) { includeMetadata = true; } - }) + if (Models.ListBlobsIncludeItem.Versions.toLowerCase() === element.toLowerCase()) { + includeVersions = true; + } + if (Models.ListBlobsIncludeItem.Deletedwithversions.toLowerCase() === element.toLowerCase()) { + includeDeletedWithVersions = true; + } + }); } if ( options.maxresults === undefined || @@ -677,7 +685,9 @@ export default class ContainerHandler extends BaseHandler options.maxresults, marker, includeSnapshots, - includeUncommittedBlobs + includeUncommittedBlobs, + includeVersions, + includeDeletedWithVersions ); const serviceEndpoint = `${request.getEndpoint()}/${accountName}`; @@ -706,7 +716,9 @@ export default class ContainerHandler extends BaseHandler tagCount: getBlobTagsCount(item.blobTags), accessTierInferred: item.properties.accessTierInferred === true ? true : undefined - } + }, + versionId: item.versionId ? item.versionId : undefined, + isCurrentVersion: item.isCurrentVersion === true ? true : undefined, }; }) }, @@ -749,6 +761,8 @@ export default class ContainerHandler extends BaseHandler let includeUncommittedBlobs: boolean = false; let includeTags: boolean = false; let includeMetadata: boolean = false; + let includeVersions: boolean = false; + let includeDeletedWithVersions: boolean = false; if (options.include !== undefined) { options.include.forEach(element => { if (Models.ListBlobsIncludeItem.Snapshots.toLowerCase() === element.toLowerCase()) { @@ -763,8 +777,13 @@ export default class ContainerHandler extends BaseHandler if (Models.ListBlobsIncludeItem.Metadata.toLowerCase() === element.toLowerCase()) { includeMetadata = true; } - } - ) + if (Models.ListBlobsIncludeItem.Versions.toLowerCase() === element.toLowerCase()) { + includeVersions = true; + } + if (Models.ListBlobsIncludeItem.Deletedwithversions.toLowerCase() === element.toLowerCase()) { + includeDeletedWithVersions = true; + } + }); } if ( options.maxresults === undefined || @@ -783,7 +802,9 @@ export default class ContainerHandler extends BaseHandler options.maxresults, marker, includeSnapshots, - includeUncommittedBlobs + includeUncommittedBlobs, + includeVersions, + includeDeletedWithVersions ); const serviceEndpoint = `${request.getEndpoint()}/${accountName}`; diff --git a/src/blob/handlers/PageBlobHandler.ts b/src/blob/handlers/PageBlobHandler.ts index 28f79d018..160bffd69 100644 --- a/src/blob/handlers/PageBlobHandler.ts +++ b/src/blob/handlers/PageBlobHandler.ts @@ -138,7 +138,6 @@ export default class PageBlobHandler extends BaseHandler // : Models.AccessTier.P4, // TODO: Infer tier from size // accessTierInferred }, - snapshot: "", isCommitted: true, pageRangesInOrder: [], blobTags: options.blobTagsString === undefined ? undefined : getTagsFromString(options.blobTagsString, context.contextId!), @@ -146,7 +145,7 @@ export default class PageBlobHandler extends BaseHandler // TODO: What's happens when create page blob right before commit block list? Or should we lock // Should we check if there is an uncommitted blob? - await this.metadataStore.createBlob( + const createdBlob = await this.metadataStore.createBlob( context, blob, options.leaseAccessConditions, @@ -162,7 +161,8 @@ export default class PageBlobHandler extends BaseHandler version: BLOB_API_VERSION, date, isServerEncrypted: true, - clientRequestId: options.requestId + clientRequestId: options.requestId, + versionId: createdBlob.versionId ? createdBlob.versionId : undefined }; return response; @@ -193,6 +193,7 @@ export default class PageBlobHandler extends BaseHandler containerName, blobName, undefined, + undefined, options.leaseAccessConditions ); @@ -313,6 +314,7 @@ export default class PageBlobHandler extends BaseHandler containerName, blobName, undefined, + undefined, options.leaseAccessConditions ); @@ -518,4 +520,4 @@ export default class PageBlobHandler extends BaseHandler ): Promise { throw new NotImplementedError(context.contextId); } -} +} \ No newline at end of file diff --git a/src/blob/main.ts b/src/blob/main.ts index f374291fd..bbbc2ee4a 100644 --- a/src/blob/main.ts +++ b/src/blob/main.ts @@ -1,4 +1,5 @@ #!/usr/bin/env node +import { join } from "path"; import * as Logger from "../common/Logger"; import { BlobServerFactory } from "./BlobServerFactory"; import SqlBlobServer from "./SqlBlobServer"; @@ -6,6 +7,8 @@ import BlobServer from "./BlobServer"; import { setExtentMemoryLimit } from "../common/ConfigurationBase"; import BlobEnvironment from "./BlobEnvironment"; import { AzuriteTelemetryClient } from "../common/Telemetry"; +import LokiAccountModelStore from "../common/account/LokiAccountModelStore"; +import { DEFAULT_ACCOUNT_MODEL_LOKI_DB_PATH } from "./utils/constants"; // tslint:disable:no-console @@ -25,8 +28,19 @@ function shutdown(server: BlobServer | SqlBlobServer) { * Entry for Azurite blob service. */ async function main() { + const env = new BlobEnvironment(); + const location = await env.location(); + + // Create account model store + const accountModels = env.getAccountModels(); + const accountModelStore = new LokiAccountModelStore( + join(location, DEFAULT_ACCOUNT_MODEL_LOKI_DB_PATH), + env.inMemoryPersistence(), + accountModels + ); + const blobServerFactory = new BlobServerFactory(); - const server = await blobServerFactory.createServer(); + const server = await blobServerFactory.createServer(env, accountModelStore); const config = server.config; // We use logger singleton as global debugger logger to track detailed outputs cross layers @@ -35,7 +49,6 @@ async function main() { // Enable debug log by default before first release for debugging purpose Logger.configLogger(config.enableDebugLog, config.debugLogFilePath); - let env = new BlobEnvironment(); setExtentMemoryLimit(env, true); // Start server @@ -47,7 +60,6 @@ async function main() { `Azurite Blob service successfully listens on ${server.getHttpServerAddress()}` ); - const location = await env.location(); AzuriteTelemetryClient.init(location, !env.disableTelemetry(), env); await AzuriteTelemetryClient.TraceStartEvent("Blob"); diff --git a/src/blob/persistence/IBlobMetadataStore.ts b/src/blob/persistence/IBlobMetadataStore.ts index fb933f8df..242c017a7 100644 --- a/src/blob/persistence/IBlobMetadataStore.ts +++ b/src/blob/persistence/IBlobMetadataStore.ts @@ -4,7 +4,10 @@ import IDataStore from "../../common/IDataStore"; import IGCExtentProvider from "../../common/IGCExtentProvider"; import * as Models from "../generated/artifacts/models"; import Context from "../generated/Context"; -import { FilterBlobItem } from "../generated/artifacts/models"; +import { + BlobPropertiesInternal, + FilterBlobItem +} from "../generated/artifacts/models"; /** * This model describes a chunk inside a persistency extent for a given extent ID. @@ -128,7 +131,7 @@ interface IPageBlobAdditionalProperties { pageRangesInOrder?: PersistencyPageRange[]; } -interface IBlobAdditionalProperties { +export interface IBlobAdditionalProperties { accountName: string; containerName: string; leaseDurationSeconds?: number; @@ -151,6 +154,7 @@ interface IGetBlobPropertiesRes { properties: Models.BlobPropertiesInternal; metadata?: Models.BlobMetadata; blobCommittedBlockCount?: number; // AppendBlobOnly + versionId?: string; } export type GetBlobPropertiesRes = IGetBlobPropertiesRes; @@ -172,15 +176,36 @@ export type ChangeBlobLeaseResponse = IBlobLeaseResponse; interface ICreateSnapshotResponse { properties: Models.BlobPropertiesInternal; snapshot: string; + versionId?: string; } + export type CreateSnapshotResponse = ICreateSnapshotResponse; +export type SetBlobMetadataResponse = { + versionId?: string; +} & BlobPropertiesInternal; + +export type StartCopyFromURLResponse = { + versionId?: string; +} & BlobPropertiesInternal; + +export type CopyFromURLResponse = { + versionId?: string; +} & BlobPropertiesInternal; + +interface ICommitBlockListResponse { + versionId?: string; +} + +export type CommitBlockListResponse = ICommitBlockListResponse; + // The model contain account name, container name, blob name and snapshot for blob. interface IBlobId { account: string; container: string; blob: string; snapshot?: string; + versionId?: string; } export type BlobId = IBlobId; @@ -495,7 +520,9 @@ export interface IBlobMetadataStore maxResults?: number, marker?: string, includeSnapshots?: boolean, - includeUncommittedBlobs?: boolean + includeUncommittedBlobs?: boolean, + includeVersions?: boolean, + includeDeletedWithVersions?: boolean ): Promise<[BlobModel[], BlobPrefixModel[], string | undefined]>; listAllBlobs( @@ -521,7 +548,7 @@ export interface IBlobMetadataStore * @param {BlobModel} blob * @param {Models.LeaseAccessConditions} [leaseAccessConditions] Optional. Will validate lease if provided * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] - * @returns {Promise} + * @returns {Promise} * @memberof IBlobMetadataStore */ createBlob( @@ -529,7 +556,7 @@ export interface IBlobMetadataStore blob: BlobModel, leaseAccessConditions?: Models.LeaseAccessConditions, modifiedAccessConditions?: Models.ModifiedAccessConditions - ): Promise; + ): Promise; /** * Create snapshot. @@ -563,6 +590,7 @@ export interface IBlobMetadataStore * @param {string} container * @param {string} blob * @param {(string | undefined)} snapshot + * @param {(string | undefined)} versionId * @param {Models.LeaseAccessConditions} [leaseAccessConditions] Optional. Will validate lease if provided * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] * @returns {Promise} @@ -574,6 +602,7 @@ export interface IBlobMetadataStore container: string, blob: string, snapshot: string | undefined, + versionId: string | undefined, leaseAccessConditions?: Models.LeaseAccessConditions, modifiedAccessConditions?: Models.ModifiedAccessConditions ): Promise; @@ -586,6 +615,7 @@ export interface IBlobMetadataStore * @param {string} container * @param {string} blob * @param {(string | undefined)} snapshot + * @param {(string | undefined)} versionId * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] * @returns {Promise} @@ -597,6 +627,7 @@ export interface IBlobMetadataStore container: string, blob: string, snapshot: string | undefined, + versionId: string | undefined, leaseAccessConditions: Models.LeaseAccessConditions | undefined, modifiedAccessConditions?: Models.ModifiedAccessConditions ): Promise; @@ -664,7 +695,7 @@ export interface IBlobMetadataStore leaseAccessConditions: Models.LeaseAccessConditions | undefined, metadata: Models.BlobMetadata | undefined, modifiedAccessConditions?: Models.ModifiedAccessConditions - ): Promise; + ): Promise; /** * Acquire blob lease. @@ -783,6 +814,7 @@ export interface IBlobMetadataStore * @param {string} container * @param {string} blob * @param {string} [snapshot] + * @param {string} [versionId] * @returns {Promise} * @memberof IBlobMetadataStore */ @@ -791,7 +823,8 @@ export interface IBlobMetadataStore account: string, container: string, blob: string, - snapshot?: string + snapshot?: string, + versionId?: string ): Promise; /** @@ -825,7 +858,7 @@ export interface IBlobMetadataStore * @param {(Models.BlobMetadata | undefined)} metadata * @param {(Models.AccessTier | undefined)} tier * @param {Models.BlobStartCopyFromURLOptionalParams} [leaseAccessConditions] - * @returns {Promise} + * @returns {Promise} * @memberof IBlobMetadataStore */ startCopyFromURL( @@ -836,7 +869,7 @@ export interface IBlobMetadataStore metadata: Models.BlobMetadata | undefined, tier: Models.AccessTier | undefined, leaseAccessConditions?: Models.BlobStartCopyFromURLOptionalParams - ): Promise; + ): Promise; /** * Sync copy from Url. @@ -848,7 +881,7 @@ export interface IBlobMetadataStore * @param {(Models.BlobMetadata | undefined)} metadata * @param {(Models.AccessTier | undefined)} tier * @param {Models.BlobCopyFromURLOptionalParams} [leaseAccessConditions] - * @returns {Promise} + * @returns {Promise} * @memberof IBlobMetadataStore */ copyFromURL( @@ -859,7 +892,7 @@ export interface IBlobMetadataStore metadata: Models.BlobMetadata | undefined, tier: Models.AccessTier | undefined, leaseAccessConditions?: Models.BlobCopyFromURLOptionalParams - ): Promise; + ): Promise; /** * Update Tier for a blob. @@ -868,6 +901,7 @@ export interface IBlobMetadataStore * @param {string} account * @param {string} container * @param {string} blob + * @param {string} versionId * @param {Models.AccessTier} tier * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions * @returns {(Promise<200 | 202>)} @@ -878,6 +912,7 @@ export interface IBlobMetadataStore account: string, container: string, blob: string, + versionId: string | undefined, tier: Models.AccessTier, leaseAccessConditions: Models.LeaseAccessConditions | undefined ): Promise<200 | 202>; @@ -933,7 +968,7 @@ export interface IBlobMetadataStore blockList: { blockName: string; blockCommitType: string }[], leaseAccessConditions?: Models.LeaseAccessConditions, modifiedAccessConditions?: Models.ModifiedAccessConditions - ): Promise; + ): Promise; /** * Gets blocks list for a blob from persistency layer by account, container and blob names. @@ -942,6 +977,7 @@ export interface IBlobMetadataStore * @param {string} container * @param {string} blob * @param {string} [snapshot] + * @param {string} [versionId] * @param {(boolean | undefined)} isCommitted * @param {Context} context * @returns {Promise<{ @@ -957,6 +993,7 @@ export interface IBlobMetadataStore container: string, blob: string, snapshot: string | undefined, + versionId: string | undefined, isCommitted: boolean | undefined, leaseAccessConditions: Models.LeaseAccessConditions | undefined, modifiedAccessConditions: Models.ModifiedAccessConditions | undefined @@ -1101,6 +1138,7 @@ export interface IBlobMetadataStore * @param {string} container * @param {string} blob * @param {(string | undefined)} snapshot + * @param {(string | undefined)} versionId * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions * @param {(Models.BlobMetadata | undefined)} metadata * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] @@ -1113,6 +1151,7 @@ export interface IBlobMetadataStore container: string, blob: string, snapshot: string | undefined, + versionId: string | undefined, leaseAccessConditions: Models.LeaseAccessConditions | undefined, tags: Models.BlobTags | undefined, modifiedAccessConditions?: Models.ModifiedAccessConditions @@ -1126,6 +1165,7 @@ export interface IBlobMetadataStore * @param {string} container * @param {string} blob * @param {(string | undefined)} snapshot + * @param {(string | undefined)} versionId * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] * @returns {Promise} @@ -1137,8 +1177,9 @@ export interface IBlobMetadataStore container: string, blob: string, snapshot: string | undefined, + versionId: string | undefined, leaseAccessConditions: Models.LeaseAccessConditions | undefined, - modifiedAccessConditions?: Models.ModifiedAccessConditions, + modifiedAccessConditions?: Models.ModifiedAccessConditions ): Promise; /** diff --git a/src/blob/persistence/LokiBlobMetadataStore.ts b/src/blob/persistence/LokiBlobMetadataStore.ts index d0e3f62d6..4fcfe1a90 100644 --- a/src/blob/persistence/LokiBlobMetadataStore.ts +++ b/src/blob/persistence/LokiBlobMetadataStore.ts @@ -45,13 +45,16 @@ import IBlobMetadataStore, { BreakContainerLeaseResponse, ChangeBlobLeaseResponse, ChangeContainerLeaseResponse, + CommitBlockListResponse, ContainerModel, + CopyFromURLResponse, CreateSnapshotResponse, FilterBlobModel, GetBlobPropertiesRes, GetContainerAccessPolicyResponse, GetContainerPropertiesResponse, GetPageRangeResponse, + IBlobAdditionalProperties, IContainerMetadata, IExtentChunk, PersistencyBlockModel, @@ -60,7 +63,9 @@ import IBlobMetadataStore, { RenewBlobLeaseResponse, RenewContainerLeaseResponse, ServicePropertiesModel, - SetContainerAccessPolicyOptions + SetBlobMetadataResponse, + SetContainerAccessPolicyOptions, + StartCopyFromURLResponse } from "./IBlobMetadataStore"; import PageWithDelimiter from "./PageWithDelimiter"; import FilterBlobPage from "./FilterBlobPage"; @@ -68,8 +73,11 @@ import { generateQueryBlobWithTagsWhereFunction } from "./QueryInterpreter/Query import { getBlobTagsCount, getTagsFromString, + isNullOrWhitespace, + parseDateFromAssumedString, toBlobTags } from "../utils/utils"; +import IAccountModelStore from "../../common/account/IAccountModelStore"; /** * This is a metadata source implementation for blob based on loki DB. @@ -99,13 +107,13 @@ import { * @class LokiBlobMetadataStore */ export default class LokiBlobMetadataStore - implements IBlobMetadataStore, IGCExtentProvider -{ + implements IBlobMetadataStore, IGCExtentProvider { private readonly db: Loki; private initialized: boolean = false; private closed: boolean = true; + private readonly accountModelStore: IAccountModelStore; private readonly SERVICES_COLLECTION = "$SERVICES_COLLECTION$"; private readonly CONTAINERS_COLLECTION = "$CONTAINERS_COLLECTION$"; private readonly BLOBS_COLLECTION = "$BLOBS_COLLECTION$"; @@ -115,22 +123,137 @@ export default class LokiBlobMetadataStore public constructor( public readonly lokiDBPath: string, - inMemory: boolean + inMemory: boolean, + accountModelStore: IAccountModelStore ) { + this.accountModelStore = accountModelStore; this.db = new Loki( lokiDBPath, inMemory ? { - persistenceMethod: "memory" - } + persistenceMethod: "memory" + } : { - persistenceMethod: "fs", - autosave: true, - autosaveInterval: 5000 - } + persistenceMethod: "fs", + autosave: true, + autosaveInterval: 5000 + } ); } + public isBlobVersioningEnabled(accountName: string): boolean { + if (!this.accountModelStore.isInitialized()) { + throw new Error("Account model store is not initialized."); + } + + return this.accountModelStore.isBlobVersioningEnabled(accountName); + } + + private formatVersionId(date: Date): string { + return convertDateTimeStringMsTo7Digital(date.toISOString()); + } + + private generateVersionId( + context: Context, + account: string, + container: string, + blob: string + ): string { + const timestamp = this.formatVersionId(context.startTime ?? new Date()); + const timestampParts = /^(.*\.\d{3})(\d{4})Z$/.exec(timestamp); + if (timestampParts === null) { + throw new Error(`Unable to generate blob version ID from ${timestamp}.`); + } + + const [, millisecondTimestamp] = timestampParts; + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + let highestSubMillisecond = -1; + + for (const doc of coll.find({ + name: blob, + accountName: account, + containerName: container + }) as BlobModel[]) { + for (const existingTimestamp of [doc.versionId, doc.snapshot]) { + const existingParts = + typeof existingTimestamp === "string" + ? /^(.*\.\d{3})(\d{4})Z$/.exec(existingTimestamp) + : null; + if ( + existingParts !== null && + existingParts[1] === millisecondTimestamp + ) { + highestSubMillisecond = Math.max( + highestSubMillisecond, + Number(existingParts[2]) + ); + } + } + } + + if (highestSubMillisecond >= 9999) { + throw new Error( + `Unable to generate more than 10000 blob versions in one millisecond for ${blob}.` + ); + } + + return `${millisecondTimestamp}${String( + highestSubMillisecond + 1 + ).padStart(4, "0")}Z`; + } + + private cloneBlobModel(blob: BlobModel): BlobModel { + const clone: BlobModel = { + ...blob, + properties: { ...blob.properties }, + metadata: + blob.metadata === undefined ? undefined : { ...blob.metadata }, + blobTags: + blob.blobTags === undefined + ? undefined + : { + blobTagSet: blob.blobTags.blobTagSet.map((tag) => ({ ...tag })) + }, + committedBlocksInOrder: + blob.committedBlocksInOrder === undefined + ? undefined + : blob.committedBlocksInOrder.map((block) => ({ ...block })), + pageRangesInOrder: + blob.pageRangesInOrder === undefined + ? undefined + : blob.pageRangesInOrder.map((range) => ({ + ...range, + persistency: { ...range.persistency } + })), + persistency: + blob.persistency === undefined ? undefined : { ...blob.persistency } + }; + delete (clone as any).$loki; + delete (clone as any).meta; + return clone; + } + + private findCurrentBlob( + account: string, + container: string, + blob: string + ): BlobModel | undefined { + return this.db + .getCollection(this.BLOBS_COLLECTION) + .chain() + .find({ + name: blob, + accountName: account, + containerName: container + }) + .where( + (candidate) => + (candidate.snapshot === undefined || candidate.snapshot === "") && + candidate.isCurrentVersion !== false + ) + .data()[0]; + } + public isInitialized(): boolean { return this.initialized; } @@ -177,11 +300,19 @@ export default class LokiBlobMetadataStore } // Create containers collection if not exists - if (this.db.getCollection(this.BLOBS_COLLECTION) === null) { - this.db.addCollection(this.BLOBS_COLLECTION, { - indices: ["accountName", "containerName", "name", "snapshot"] // Optimize for find operation + let blobsCollection = this.db.getCollection(this.BLOBS_COLLECTION); + if (blobsCollection === null) { + blobsCollection = this.db.addCollection(this.BLOBS_COLLECTION, { + indices: [ + "accountName", + "containerName", + "name", + "snapshot", + "versionId" + ] // Optimize for find operation }); } + blobsCollection.ensureIndex("versionId"); // Create blocks collection if not exists if (this.db.getCollection(this.BLOCKS_COLLECTION) === null) { @@ -342,9 +473,9 @@ export default class LokiBlobMetadataStore prefix === "" ? { name: { $gt: marker }, accountName: account } : { - name: { $regex: `^${this.escapeRegex(prefix)}`, $gt: marker }, - accountName: account - }; + name: { $regex: `^${this.escapeRegex(prefix)}`, $gt: marker }, + accountName: account + }; // Workaround for loki which will ignore $gt when providing $regex const query2 = { name: { $gt: marker } }; @@ -766,10 +897,10 @@ export default class LokiBlobMetadataStore const leaseTimeSeconds: number = doc.properties.leaseState === Models.LeaseStateType.Breaking && - doc.leaseBreakTime + doc.leaseBreakTime ? Math.round( - (doc.leaseBreakTime.getTime() - context.startTime!.getTime()) / 1000 - ) + (doc.leaseBreakTime.getTime() - context.startTime!.getTime()) / 1000 + ) : 0; coll.update(doc); @@ -871,6 +1002,9 @@ export default class LokiBlobMetadataStore .where((obj) => { return obj.snapshot === undefined || obj.snapshot === ""; }) + .where((obj) => { + return obj.isCurrentVersion !== false; + }) .sort((obj1, obj2) => { if (obj1.name === obj2.name) return 0; if (obj1.name > obj2.name) return 1; @@ -880,24 +1014,24 @@ export default class LokiBlobMetadataStore .limit(maxResults) .data(); - return doc - .map((item) => { - let blobItem: FilterBlobModel; - blobItem = { - name: item.name, - containerName: item.containerName, - tags: item.blobTags - }; - return blobItem; - }) - .filter((blobItem) => { - const tagsMeetConditions = filterFunction(blobItem); - if (tagsMeetConditions.length !== 0) { - blobItem.tags = { blobTagSet: toBlobTags(tagsMeetConditions) }; - return true; - } - return false; - }); + return doc.map((item) => { + let blobItem: FilterBlobModel; + blobItem = { + name: item.name, + containerName: item.containerName, + tags: item.blobTags, + versionId: item.versionId, + isCurrentVersion: item.isCurrentVersion, + }; + return blobItem; + }).filter((blobItem) => { + const tagsMeetConditions = filterFunction(blobItem); + if (tagsMeetConditions.length !== 0) { + blobItem.tags = { blobTagSet: toBlobTags(tagsMeetConditions) }; + return true; + } + return false; + }); }; const nameItem = (item: FilterBlobModel) => { @@ -919,9 +1053,24 @@ export default class LokiBlobMetadataStore maxResults: number = DEFAULT_LIST_BLOBS_MAX_RESULTS, marker: string = "", includeSnapshots?: boolean, - includeUncommittedBlobs?: boolean + includeUncommittedBlobs?: boolean, + includeVersions?: boolean, + includeDeletedWithVersions?: boolean ): Promise<[BlobModel[], BlobPrefixModel[], string | undefined]> { const query: any = {}; + let markerAsTuple: [string, string]; + + if (!marker) { + markerAsTuple = ["", ""]; + } + else { + markerAsTuple = (marker ? marker.split(PageWithDelimiter.VERSIONING_MARKER) : ["", ""]) as [string, string]; + + if (markerAsTuple.length !== 2 || parseDateFromAssumedString(markerAsTuple[1]) === undefined) { + throw StorageErrorFactory.getInvalidQueryParameterValue(context.contextId); + } + } + if (prefix !== "") { query.name = { $regex: `^${this.escapeRegex(prefix)}` }; } @@ -935,6 +1084,31 @@ export default class LokiBlobMetadataStore query.containerName = container; } + const getTimestampFromBlobModel = (item: BlobModel): string => { + if (item.versionId) { + return item.versionId; + } + + if (item.snapshot && item.snapshot.length !== 0) { + return item.snapshot; + } + + const lastModified = parseDateFromAssumedString( + item.properties.lastModified + ); + if (lastModified === undefined) { + throw new Error( + `Failed to parse lastModified on persisted blob ${item.name}` + ); + } + + return lastModified.toISOString(); + }; + + const getMarkerFromBlobModel = (item: BlobModel): [string, string] => { + return [item.name, getTimestampFromBlobModel(item)]; + }; + const coll = this.db.getCollection(this.BLOBS_COLLECTION); const page = new PageWithDelimiter( maxResults, @@ -942,11 +1116,13 @@ export default class LokiBlobMetadataStore prefix ); const readPage = async (offset: number): Promise => { - return await coll + const queryResult = await coll .chain() .find(query) .where((obj) => { - return obj.name > marker!; + const markerTuple = getMarkerFromBlobModel(obj); + + return PageWithDelimiter.isMarkerLater(markerTuple, markerAsTuple); }) .where((obj) => { return includeSnapshots ? true : obj.snapshot.length === 0; @@ -954,23 +1130,44 @@ export default class LokiBlobMetadataStore .where((obj) => { return includeUncommittedBlobs ? true : obj.isCommitted; }) - .sort((obj1, obj2) => { - if (obj1.name === obj2.name) return 0; - if (obj1.name > obj2.name) return 1; - return -1; + .where((obj) => { + if (obj.snapshot.length !== 0) { + return true; + } + + if (includeDeletedWithVersions) { + return true; + } + + if (includeVersions) { + return true; + } + + return obj.isCurrentVersion !== false; + }) + .sort((doc1, doc2) => { + // Primary sort: by blob name (required for PageWithDelimiter) + if (doc1.name !== doc2.name) { + if (doc1.name > doc2.name) return 1; + return -1; + } + + const doc1Timestamp = getTimestampFromBlobModel(doc1); + const doc2Timestamp = getTimestampFromBlobModel(doc2); + + // Compare timestamps - earliest first (latest goes last) + return doc1Timestamp.localeCompare(doc2Timestamp); }) .offset(offset) .limit(maxResults) .data(); - }; - const nameItem = (item: BlobModel) => { - return item.name; + return queryResult; }; const [blobItems, blobPrefixes, nextMarker] = await page.fill( readPage, - nameItem + getMarkerFromBlobModel ); return [ @@ -996,6 +1193,8 @@ export default class LokiBlobMetadataStore ): Promise<[BlobModel[], string | undefined]> { const coll = this.db.getCollection(this.BLOBS_COLLECTION); + // By default, we include all versions. This method is mostly for + // the GC, so there is no point in adding blob versioning support. const docs = await coll .chain() .where((obj) => { @@ -1034,7 +1233,7 @@ export default class LokiBlobMetadataStore * @param {BlobModel} blob * @param {Models.LeaseAccessConditions} [leaseAccessConditions] * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] - * @returns {Promise} + * @returns {Promise} * @memberof LokiBlobMetadataStore */ public async createBlob( @@ -1042,23 +1241,27 @@ export default class LokiBlobMetadataStore blob: BlobModel, leaseAccessConditions?: Models.LeaseAccessConditions, modifiedAccessConditions?: Models.ModifiedAccessConditions - ): Promise { + ): Promise { await this.checkContainerExist( context, blob.accountName, blob.containerName ); + const coll = this.db.getCollection(this.BLOBS_COLLECTION); - const blobDoc = coll.findOne({ - name: blob.name, - accountName: blob.accountName, - containerName: blob.containerName, - snapshot: blob.snapshot - }); + const blobDoc = this.findBlob( + context, + blob.accountName, + blob.containerName, + blob.name + ); validateWriteConditions(context, modifiedAccessConditions, blobDoc); - // Create if not exists + // If-None-Match: "*" (create only if absent). When blob versioning is enabled we allow + // multiple historical versions, so if a current blob exists we still honor ifNoneMatch="*" + // (service would fail create-on-existing). When versioning is disabled we keep only a single + // base blob (versionId ""). if ( modifiedAccessConditions && modifiedAccessConditions.ifNoneMatch === "*" && @@ -1078,8 +1281,39 @@ export default class LokiBlobMetadataStore ) { throw StorageErrorFactory.getBlobArchived(context.contextId); } - coll.remove(blobDoc); + + if (this.isBlobVersioningEnabled(blob.accountName) || blobDoc.isCurrentVersion) { + if (this.isBlobVersioningEnabled(blob.accountName)) { + blobDoc.versionId = isNullOrWhitespace(blobDoc.versionId) + ? this.formatVersionId(blobDoc.properties.lastModified) + : blobDoc.versionId; + } + + blobDoc.isCurrentVersion = false; + coll.update(blobDoc); + } else { + coll.remove(blobDoc); + } + } + + if (!this.isBlobVersioningEnabled(blob.accountName)) { + blob.versionId = ""; + blob.isCurrentVersion = undefined; + } else { + blob.versionId = this.generateVersionId( + context, + blob.accountName, + blob.containerName, + blob.name + ); + blob.isCurrentVersion = true; } + + // Creating a blob (Put Blob / Commit Block List / Append) never creates a snapshot implicitly. + // Service semantics: absence of snapshot param is represented by empty string internally. + // (A snapshot is produced only via the Create Snapshot API.) + blob.snapshot = ""; + delete (blob as any).$loki; return coll.insert(blob); } @@ -1110,6 +1344,7 @@ export default class LokiBlobMetadataStore container, blob, undefined, + undefined, context, false, true @@ -1164,9 +1399,29 @@ export default class LokiBlobMetadataStore coll.insert(snapshotBlob); + let versionIdHeader: string = ""; + if (this.isBlobVersioningEnabled(snapshotBlob.accountName)) { + // If versioning is enabled, a new version will always be created alongside the snapshot + // and contain the same contents as the snapshot. + const copiedSnapshot = this.cloneBlobModel(snapshotBlob); + copiedSnapshot.snapshot = ""; + const newVersion = await this.createBlob( + context, + copiedSnapshot, + leaseAccessConditions, + modifiedAccessConditions + ); + + versionIdHeader = newVersion.versionId!; + } else if (doc.isCurrentVersion) { + doc.isCurrentVersion = false; + coll.update(doc); + } + return { properties: snapshotBlob.properties, - snapshot: snapshotTime + snapshot: snapshotTime, + versionId: versionIdHeader }; } @@ -1181,6 +1436,7 @@ export default class LokiBlobMetadataStore * @param {string} [snapshot=""] * @param {Models.LeaseAccessConditions} [leaseAccessConditions] * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @param {string} [versionId] * @returns {Promise} * @memberof LokiBlobMetadataStore */ @@ -1190,6 +1446,7 @@ export default class LokiBlobMetadataStore container: string, blob: string, snapshot: string = "", + versionId: string = "", leaseAccessConditions?: Models.LeaseAccessConditions, modifiedAccessConditions?: Models.ModifiedAccessConditions ): Promise { @@ -1198,6 +1455,7 @@ export default class LokiBlobMetadataStore container, blob, snapshot, + versionId, context, false, true @@ -1225,6 +1483,7 @@ export default class LokiBlobMetadataStore * @param {string} container * @param {string} blob * @param {string} [snapshot] + * @param {string} [versionId] * @returns {(Promise)} * @memberof LokiBlobMetadataStore */ @@ -1233,15 +1492,17 @@ export default class LokiBlobMetadataStore account: string, container: string, blob: string, - snapshot: string = "" + snapshot: string = "", + versionId: string = "" ): Promise { - const coll = this.db.getCollection(this.BLOBS_COLLECTION); - const blobDoc = coll.findOne({ - name: blob, - accountName: account, - containerName: container, - snapshot - }); + const blobDoc = this.findBlob( + context, + account, + container, + blob, + snapshot, + versionId + ); if (blobDoc) { const blobModel = blobDoc as BlobModel; @@ -1264,6 +1525,7 @@ export default class LokiBlobMetadataStore * @param {string} [snapshot=""] * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] + * @param {string} [versionId] * @returns {Promise} * @memberof LokiBlobMetadataStore */ @@ -1273,6 +1535,7 @@ export default class LokiBlobMetadataStore container: string, blob: string, snapshot: string = "", + versionId: string = "", leaseAccessConditions: Models.LeaseAccessConditions | undefined, modifiedAccessConditions?: Models.ModifiedAccessConditions ): Promise { @@ -1281,6 +1544,7 @@ export default class LokiBlobMetadataStore container, blob, snapshot, + versionId, context, false, true @@ -1306,7 +1570,8 @@ export default class LokiBlobMetadataStore blobCommittedBlockCount: doc.properties.blobType === Models.BlobType.AppendBlob ? (doc.committedBlocksInOrder || []).length - : undefined + : undefined, + versionId: doc.versionId }; } @@ -1318,6 +1583,7 @@ export default class LokiBlobMetadataStore * @param {string} container * @param {string} blob * @param {Models.BlobDeleteMethodOptionalParams} options + * @param {string} [versionId] * @returns {Promise} * @memberof LokiBlobMetadataStore */ @@ -1331,13 +1597,29 @@ export default class LokiBlobMetadataStore const coll = this.db.getCollection(this.BLOBS_COLLECTION); await this.checkContainerExist(context, account, container); + const versionId = options.versionId ?? ""; + const isVersionProvided = !isNullOrWhitespace(versionId); + + if ( + isVersionProvided && + (!isNullOrWhitespace(options.snapshot) || + options.deleteSnapshots !== undefined) + ) { + throw StorageErrorFactory.getInvalidOperation( + context.contextId!, + "When deleting a blob version, you cannot specify a snapshot or deleteSnapshots option." + ); + } + const doc = await this.getBlobWithLeaseUpdated( account, container, blob, options.snapshot, + versionId, context, - false + false, + undefined ); validateWriteConditions(context, options.modifiedAccessConditions, doc); @@ -1361,22 +1643,46 @@ export default class LokiBlobMetadataStore context ); + if (isVersionProvided) { + if (doc.isCurrentVersion === true) { + throw StorageErrorFactory.getOperationNotAllowedOnRootBlob( + context.contextId! + ); + } + + coll.findAndRemove({ + accountName: account, + containerName: container, + name: blob, + snapshot: "", + versionId: versionId + }); + return; + } + // Scenario: Delete base blob only if (againstBaseBlob && options.deleteSnapshots === undefined) { const count = coll.count({ accountName: account, containerName: container, - name: blob + name: blob, + snapshot: { $gt: "" } // Only count actual snapshots, not empty snapshot (base blob) }); - if (count > 1) { + if (count > 0) { throw StorageErrorFactory.getSnapshotsPresent(context.contextId!); } else { - coll.findAndRemove({ - accountName: account, - containerName: container, - name: blob - }); + // Base blob is always set to previous if it is a versioned blob + // We only check isCurrentVersion because if it was a non-current version, + // it would have been deleted already since you need to explicitly specify versionId to delete it. + if (doc.isCurrentVersion === true) { + doc.isCurrentVersion = false; + coll.update(doc); + } else { + // Blob is not versioned, we can delete it directly. + coll.remove(doc); + } } + return; } // Scenario: Delete one snapshot only @@ -1387,6 +1693,7 @@ export default class LokiBlobMetadataStore name: blob, snapshot: doc.snapshot }); + return; } // Scenario: Delete base blob and snapshots @@ -1394,11 +1701,25 @@ export default class LokiBlobMetadataStore againstBaseBlob && options.deleteSnapshots === Models.DeleteSnapshotsOptionType.Include ) { - coll.findAndRemove({ - accountName: account, - containerName: container, - name: blob - }); + if (!this.isBlobVersioningEnabled(account)) { + // If versioning is not enabled, we can delete the base blob directly + // and all its snapshots. + coll.findAndRemove({ + accountName: account, + containerName: container, + name: blob + }); + } else { + // Remove all snapshots first, then mark base blob as non-current + coll.findAndRemove({ + accountName: account, + containerName: container, + name: blob, + snapshot: { $gt: "" } + }); + doc.isCurrentVersion = false; + coll.update(doc); + } } // Scenario: Delete all snapshots only @@ -1444,6 +1765,7 @@ export default class LokiBlobMetadataStore container, blob, undefined, + undefined, context, false, true @@ -1505,13 +1827,14 @@ export default class LokiBlobMetadataStore leaseAccessConditions: Models.LeaseAccessConditions | undefined, metadata: Models.BlobMetadata | undefined, modifiedAccessConditions?: Models.ModifiedAccessConditions - ): Promise { + ): Promise { const coll = this.db.getCollection(this.BLOBS_COLLECTION); - const doc = await this.getBlobWithLeaseUpdated( + let doc = await this.getBlobWithLeaseUpdated( account, container, blob, undefined, + undefined, context, false, true @@ -1526,11 +1849,67 @@ export default class LokiBlobMetadataStore const lease = new BlobLeaseAdapter(doc); new BlobWriteLeaseValidator(leaseAccessConditions).validate(lease, context); new BlobWriteLeaseSyncer(doc).sync(lease); - doc.metadata = metadata; - doc.properties.etag = newEtag(); - doc.properties.lastModified = context.startTime || new Date(); - coll.update(doc); - return doc.properties; + + if (this.isBlobVersioningEnabled(account)) { + // For versioning: mark old version as not current, create new version + doc.isCurrentVersion = false; + doc.versionId = doc.versionId + ? doc.versionId + : this.formatVersionId(doc.properties.lastModified); + coll.update(doc); + + // Create a deep clone by serializing and deserializing + // This is to prevent modifying the doc that was previously updated after calling .update + const clonedDoc = this.cloneBlobModel(doc); + // Prepare new version + clonedDoc.versionId = this.generateVersionId( + context, + account, + container, + blob + ); + clonedDoc.isCurrentVersion = true; + clonedDoc.metadata = metadata; + clonedDoc.properties.etag = newEtag(); + clonedDoc.properties.lastModified = context.startTime || new Date(); + delete (clonedDoc as any).$loki; + coll.insert(clonedDoc); + doc = clonedDoc; + } else { + // For non-versioning: update existing document in place + if (doc.versionId) { + doc.isCurrentVersion = false; + coll.update(doc); + const clonedDoc = this.cloneBlobModel(doc); + + clonedDoc.versionId = ""; + clonedDoc.isCurrentVersion = undefined; + clonedDoc.metadata = metadata; + clonedDoc.properties.etag = newEtag(); + clonedDoc.properties.lastModified = context.startTime || new Date(); + // We insert here instead of an update, because if the previous version had a versionId, + // and now blob versioning is disabled, we must create a new, non-versioned "version" of the document. + // This is what the real service does. + delete (clonedDoc as any).$loki; + coll.insert(clonedDoc); + doc = clonedDoc; + } else { + doc.metadata = metadata; + doc.properties.etag = newEtag(); + doc.properties.lastModified = context.startTime || new Date(); + + coll.update(doc); + } + } + + if (!doc) { + throw StorageErrorFactory.getInvalidOperation( + context.contextId, + "doc should exist here. must be a bug." + ); + } + + return { versionId: doc.versionId ?? "", ...doc.properties }; } /** @@ -1561,6 +1940,7 @@ export default class LokiBlobMetadataStore container, blob, undefined, + undefined, context, false ); // This may return an uncommitted blob, or undefined for an nonexistent blob @@ -1611,6 +1991,7 @@ export default class LokiBlobMetadataStore container, blob, undefined, + undefined, context, false ); // This may return an uncommitted blob, or undefined for an nonexistent blob @@ -1661,6 +2042,7 @@ export default class LokiBlobMetadataStore container, blob, undefined, + undefined, context, false ); // This may return an uncommitted blob, or undefined for an nonexistent blob @@ -1713,6 +2095,7 @@ export default class LokiBlobMetadataStore container, blob, undefined, + undefined, context, false ); // This may return an uncommitted blob, or undefined for an nonexistent blob @@ -1763,6 +2146,7 @@ export default class LokiBlobMetadataStore container, blob, undefined, + undefined, context, false ); // This may return an uncommitted blob, or undefined for an nonexistent blob @@ -1784,10 +2168,10 @@ export default class LokiBlobMetadataStore const leaseTimeSeconds: number = doc.properties.leaseState === Models.LeaseStateType.Breaking && - doc.leaseBreakTime + doc.leaseBreakTime ? Math.round( - (doc.leaseBreakTime.getTime() - context.startTime!.getTime()) / 1000 - ) + (doc.leaseBreakTime.getTime() - context.startTime!.getTime()) / 1000 + ) : 0; coll.update(doc); @@ -1811,17 +2195,19 @@ export default class LokiBlobMetadataStore account: string, container: string, blob: string, - snapshot: string = "" + snapshot: string = "", + versionId: string = "" ): Promise { await this.checkContainerExist(context, account, container); - const coll = this.db.getCollection(this.BLOBS_COLLECTION); - const doc = coll.findOne({ - name: blob, - accountName: account, - containerName: container, - snapshot - }); + const doc = this.findBlob( + context, + account, + container, + blob, + snapshot, + versionId + ); if (!doc) { const requestId = context ? context.contextId : undefined; @@ -1849,13 +2235,15 @@ export default class LokiBlobMetadataStore ): Promise< { blobType: Models.BlobType | undefined; isCommitted: boolean } | undefined > { - const coll = this.db.getCollection(this.BLOBS_COLLECTION); - const doc = coll.findOne({ - name: blob, - accountName: account, - containerName: container, - snapshot - }); + const doc = + snapshot === "" + ? this.findCurrentBlob(account, container, blob) + : this.db.getCollection(this.BLOBS_COLLECTION).findOne({ + name: blob, + accountName: account, + containerName: container, + snapshot + }); if (!doc) { return undefined; } @@ -1871,8 +2259,8 @@ export default class LokiBlobMetadataStore * @param {string} copySource * @param {(Models.BlobMetadata | undefined)} metadata * @param {(Models.AccessTier | undefined)} tier - * @param {Models.BlobStartCopyFromURLOptionalParams} [leaseAccessConditions] - * @returns {Promise} + * @param {Models.BlobStartCopyFromURLOptionalParams} [options] + * @returns {Promise} * @memberof LokiBlobMetadataStore */ public async startCopyFromURL( @@ -1883,13 +2271,14 @@ export default class LokiBlobMetadataStore metadata: Models.BlobMetadata | undefined, tier: Models.AccessTier | undefined, options: Models.BlobStartCopyFromURLOptionalParams = {} - ): Promise { + ): Promise { const coll = this.db.getCollection(this.BLOBS_COLLECTION); const sourceBlob = await this.getBlobWithLeaseUpdated( source.account, source.container, source.blob, source.snapshot, + source.versionId, context, true, true @@ -1917,6 +2306,7 @@ export default class LokiBlobMetadataStore destination.container, destination.blob, undefined, + undefined, context, false ); @@ -2025,7 +2415,8 @@ export default class LokiBlobMetadataStore blobTags: options.blobTagsString === undefined ? undefined - : getTagsFromString(options.blobTagsString, context.contextId!) + : getTagsFromString(options.blobTagsString, context.contextId!), + versionId: "" }; if ( @@ -2052,10 +2443,29 @@ export default class LokiBlobMetadataStore } if (destBlob) { - coll.remove(destBlob); + if (this.isBlobVersioningEnabled(destination.account)) { + destBlob.isCurrentVersion = false; + destBlob.versionId = + destBlob.versionId ?? + this.formatVersionId(destBlob.properties.lastModified); + coll.update(destBlob); + } else { + coll.remove(destBlob); + } } + + if (this.isBlobVersioningEnabled(destination.account)) { + copiedBlob.isCurrentVersion = true; + copiedBlob.versionId = this.generateVersionId( + context, + destination.account, + destination.container, + destination.blob + ); + } + coll.insert(copiedBlob); - return copiedBlob.properties; + return { ...copiedBlob.properties, versionId: copiedBlob.versionId }; } /** @@ -2067,8 +2477,8 @@ export default class LokiBlobMetadataStore * @param {string} copySource * @param {(Models.BlobMetadata | undefined)} metadata * @param {(Models.AccessTier | undefined)} tier - * @param {Models.BlobCopyFromURLOptionalParams} [leaseAccessConditions] - * @returns {Promise} + * @param {Models.BlobCopyFromURLOptionalParams} [options] + * @returns {Promise} * @memberof LokiBlobMetadataStore */ public async copyFromURL( @@ -2079,13 +2489,14 @@ export default class LokiBlobMetadataStore metadata: Models.BlobMetadata | undefined, tier: Models.AccessTier | undefined, options: Models.BlobCopyFromURLOptionalParams = {} - ): Promise { + ): Promise { const coll = this.db.getCollection(this.BLOBS_COLLECTION); const sourceBlob = await this.getBlobWithLeaseUpdated( source.account, source.container, source.blob, source.snapshot, + source.versionId, context, true, true @@ -2112,6 +2523,7 @@ export default class LokiBlobMetadataStore destination.container, destination.blob, undefined, + undefined, context, false ); @@ -2218,7 +2630,8 @@ export default class LokiBlobMetadataStore ? sourceBlob.blobTags : options.blobTagsString === undefined ? undefined - : getTagsFromString(options.blobTagsString, context.contextId!) + : getTagsFromString(options.blobTagsString, context.contextId!), + versionId: "" }; if ( @@ -2245,10 +2658,29 @@ export default class LokiBlobMetadataStore } if (destBlob) { - coll.remove(destBlob); + if (this.isBlobVersioningEnabled(destination.account)) { + destBlob.isCurrentVersion = false; + destBlob.versionId = + destBlob.versionId ?? + this.formatVersionId(destBlob.properties.lastModified); + coll.update(destBlob); + } else { + coll.remove(destBlob); + } + } + + if (this.isBlobVersioningEnabled(destination.account)) { + copiedBlob.isCurrentVersion = true; + copiedBlob.versionId = this.generateVersionId( + context, + destination.account, + destination.container, + destination.blob + ); } + coll.insert(copiedBlob); - return copiedBlob.properties; + return { versionId: copiedBlob.versionId, ...copiedBlob.properties }; } /** @@ -2258,6 +2690,7 @@ export default class LokiBlobMetadataStore * @param {string} account * @param {string} container * @param {string} blob + * @param {string} versionId * @param {Models.AccessTier} tier * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions * @returns {(Promise<200 | 202>)} @@ -2268,6 +2701,7 @@ export default class LokiBlobMetadataStore account: string, container: string, blob: string, + versionId: string = "", tier: Models.AccessTier, leaseAccessConditions: Models.LeaseAccessConditions | undefined ): Promise<200 | 202> { @@ -2277,6 +2711,7 @@ export default class LokiBlobMetadataStore container, blob, undefined, + versionId, context, true, true @@ -2352,11 +2787,11 @@ export default class LokiBlobMetadataStore ); const blobColl = this.db.getCollection(this.BLOBS_COLLECTION); - const blobDoc = blobColl.findOne({ - name: block.blobName, - accountName: block.accountName, - containerName: block.containerName - }); + const blobDoc = this.findCurrentBlob( + block.accountName, + block.containerName, + block.blobName + ); let blobExist = false; @@ -2375,7 +2810,8 @@ export default class LokiBlobMetadataStore blobType: Models.BlobType.BlockBlob }, snapshot: "", - isCommitted: false + isCommitted: false, + versionId: "" }; blobColl.insert(newBlob); } else { @@ -2436,6 +2872,7 @@ export default class LokiBlobMetadataStore block.containerName, block.blobName, undefined, + undefined, context, false, true @@ -2507,7 +2944,7 @@ export default class LokiBlobMetadataStore * @param {{ blockName: string; blockCommitType: string }[]} blockList * @param {Models.LeaseAccessConditions} [leaseAccessConditions] * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] - * @returns {Promise} + * @returns {Promise} * @memberof LokiBlobMetadataStore */ public async commitBlockList( @@ -2516,13 +2953,14 @@ export default class LokiBlobMetadataStore blockList: { blockName: string; blockCommitType: string }[], leaseAccessConditions?: Models.LeaseAccessConditions, modifiedAccessConditions?: Models.ModifiedAccessConditions - ): Promise { + ): Promise { const coll = this.db.getCollection(this.BLOBS_COLLECTION); const doc = await this.getBlobWithLeaseUpdated( blob.accountName, blob.containerName, blob.name, - blob.snapshot, + undefined, + undefined, context, // XStore allows commit block list with empty block list to create a block blob without stage block call // In this case, there will no existing blob doc exists @@ -2615,35 +3053,73 @@ export default class LokiBlobMetadataStore } } + // We always write to the normal blob, not the snapshots. + blob.snapshot = ""; + if (doc) { - // Commit block list - doc.properties.blobType = blob.properties.blobType; - doc.properties.lastModified = blob.properties.lastModified; - doc.committedBlocksInOrder = selectedBlockList; - doc.isCommitted = true; - doc.metadata = blob.metadata; - doc.properties.accessTier = blob.properties.accessTier; - doc.properties.accessTierInferred = blob.properties.accessTierInferred; - doc.properties.etag = blob.properties.etag; - doc.properties.cacheControl = blob.properties.cacheControl; - doc.properties.contentType = blob.properties.contentType; - doc.properties.contentMD5 = blob.properties.contentMD5; - doc.properties.contentEncoding = blob.properties.contentEncoding; - doc.properties.contentLanguage = blob.properties.contentLanguage; - doc.properties.contentDisposition = blob.properties.contentDisposition; - doc.blobTags = blob.blobTags; - doc.properties.contentLength = selectedBlockList - .map((block) => block.size) - .reduce((total, val) => { - return total + val; - }, 0); + if (this.isBlobVersioningEnabled(blob.accountName) && doc.isCommitted) { + doc.isCurrentVersion = false; + doc.versionId = doc.versionId + ? doc.versionId + : this.formatVersionId(doc.properties.lastModified); + coll.update(doc); + + blob.versionId = this.generateVersionId( + context, + blob.accountName, + blob.containerName, + blob.name + ); + blob.isCurrentVersion = true; + blob.committedBlocksInOrder = selectedBlockList; + blob.properties.contentLength = selectedBlockList + .map((block) => block.size) + .reduce((total, val) => { + return total + val; + }, 0); + coll.insert(blob); + } else { + // Commit block list + doc.properties.blobType = blob.properties.blobType; + doc.properties.lastModified = blob.properties.lastModified; + doc.committedBlocksInOrder = selectedBlockList; + doc.isCommitted = true; + doc.metadata = blob.metadata; + doc.properties.accessTier = blob.properties.accessTier; + doc.properties.accessTierInferred = blob.properties.accessTierInferred; + doc.properties.etag = blob.properties.etag; + doc.properties.cacheControl = blob.properties.cacheControl; + doc.properties.contentType = blob.properties.contentType; + doc.properties.contentMD5 = blob.properties.contentMD5; + doc.properties.contentEncoding = blob.properties.contentEncoding; + doc.properties.contentLanguage = blob.properties.contentLanguage; + doc.properties.contentDisposition = blob.properties.contentDisposition; + doc.blobTags = blob.blobTags; + doc.properties.contentLength = selectedBlockList + .map((block) => block.size) + .reduce((total, val) => { + return total + val; + }, 0); + + // set lease state to available if it's expired + if (lease) { + new BlobWriteLeaseSyncer(doc).sync(lease); + } - // set lease state to available if it's expired - if (lease) { - new BlobWriteLeaseSyncer(doc).sync(lease); - } + // This is for a doc that is not yet committed + if (this.isBlobVersioningEnabled(blob.accountName)) { + doc.isCurrentVersion = true; + doc.versionId = this.generateVersionId( + context, + blob.accountName, + blob.containerName, + blob.name + ); + } - coll.update(doc); + coll.update(doc); + blob = doc; + } } else { blob.committedBlocksInOrder = selectedBlockList; blob.properties.contentLength = selectedBlockList @@ -2651,6 +3127,19 @@ export default class LokiBlobMetadataStore .reduce((total, val) => { return total + val; }, 0); + + if (this.isBlobVersioningEnabled(blob.accountName)) { + blob.isCurrentVersion = true; + blob.versionId = this.generateVersionId( + context, + blob.accountName, + blob.containerName, + blob.name + ); + } else { + blob.versionId = blob.versionId ?? ""; + } + coll.insert(blob); } @@ -2659,6 +3148,8 @@ export default class LokiBlobMetadataStore containerName: blob.containerName, blobName: blob.name }); + + return { versionId: blob.versionId }; } /** @@ -2682,7 +3173,8 @@ export default class LokiBlobMetadataStore account: string, container: string, blob: string, - snapshot: string | undefined, + snapshot: string = "", + versionId: string = "", isCommitted: boolean | undefined, leaseAccessConditions: Models.LeaseAccessConditions | undefined, modifiedAccessConditions: Models.ModifiedAccessConditions | undefined @@ -2696,6 +3188,7 @@ export default class LokiBlobMetadataStore container, blob, snapshot, + versionId, context ); @@ -2774,6 +3267,7 @@ export default class LokiBlobMetadataStore blob.containerName, blob.name, blob.snapshot, + blob.versionId, context!, false, true @@ -2843,6 +3337,7 @@ export default class LokiBlobMetadataStore blob.containerName, blob.name, blob.snapshot, + blob.versionId, context!, false, true @@ -2907,6 +3402,9 @@ export default class LokiBlobMetadataStore container, blob, snapshot, + // The REST API does not allow users to specify the version + // so we default to the "current" version. + undefined, context, false, true @@ -2961,6 +3459,7 @@ export default class LokiBlobMetadataStore container, blob, undefined, + undefined, context, false, true @@ -3032,6 +3531,7 @@ export default class LokiBlobMetadataStore container, blob, undefined, + undefined, context, false, true @@ -3167,7 +3667,6 @@ export default class LokiBlobMetadataStore arr[i] = obj[i]; } - return new Uint8Array(arr); } @@ -3330,6 +3829,7 @@ export default class LokiBlobMetadataStore * @param {Context} context * @param {undefined} [forceExist] * @param {boolean} [forceCommitted] If true, will take uncommitted blob as a non-exist blob and throw exception. + * @param {string} [versionId] Version ID of the blob, used for versioned blob. * @returns {Promise} * @memberof LokiBlobMetadataStore */ @@ -3338,6 +3838,7 @@ export default class LokiBlobMetadataStore container: string, blob: string, snapshot: string | undefined, + versionId: string | undefined, context: Context, forceExist?: true, forceCommitted?: boolean @@ -3354,6 +3855,7 @@ export default class LokiBlobMetadataStore * @param {(string | undefined)} snapshot * @param {Context} context * @param {false} forceExist + * @param {string} [versionId] Version ID of the blob, used for versioned blob. * @param {boolean} [forceCommitted] If true, will take uncommitted blob as a non-exist blob and return undefined. * @returns {(Promise)} * @memberof LokiBlobMetadataStore @@ -3363,6 +3865,7 @@ export default class LokiBlobMetadataStore container: string, blob: string, snapshot: string | undefined, + versionId: string | undefined, context: Context, forceExist: false, forceCommitted?: boolean @@ -3373,22 +3876,23 @@ export default class LokiBlobMetadataStore container: string, blob: string, snapshot: string = "", + versionId: string = "", context: Context, forceExist?: boolean, forceCommitted?: boolean ): Promise { await this.checkContainerExist(context, account, container); + const doc = this.findBlob( + context, + account, + container, + blob, + snapshot, + versionId + ); - const coll = this.db.getCollection(this.BLOBS_COLLECTION); - const doc = coll.findOne({ - name: blob, - accountName: account, - containerName: container, - snapshot - }); - - // Force exist if parameter forceExist is undefined or true if (forceExist === undefined || forceExist === true) { + // Force exist if parameter forceExist is undefined or true if (forceCommitted) { if (!doc || !(doc as BlobModel).isCommitted) { throw StorageErrorFactory.getBlobNotFound(context.contextId); @@ -3446,7 +3950,6 @@ export default class LokiBlobMetadataStore * @param {(string | undefined)} snapshot * @param {(Models.LeaseAccessConditions | undefined)} leaseAccessConditions * @param {(Models.BlobTags | undefined)} tags - * @param {Models.ModifiedAccessConditions} [modifiedAccessConditions] * @returns {Promise} * @memberof LokiBlobMetadataStore */ @@ -3455,10 +3958,10 @@ export default class LokiBlobMetadataStore account: string, container: string, blob: string, - snapshot: string | undefined, + snapshot: string = "", + versionId: string = "", leaseAccessConditions: Models.LeaseAccessConditions | undefined, - tags: Models.BlobTags | undefined, - modifiedAccessConditions?: Models.ModifiedAccessConditions + tags: Models.BlobTags | undefined ): Promise { const coll = this.db.getCollection(this.BLOBS_COLLECTION); const doc = await this.getBlobWithLeaseUpdated( @@ -3466,6 +3969,7 @@ export default class LokiBlobMetadataStore container, blob, snapshot, + versionId, context, false, true @@ -3501,6 +4005,7 @@ export default class LokiBlobMetadataStore container: string, blob: string, snapshot: string = "", + versionId: string = "", leaseAccessConditions: Models.LeaseAccessConditions | undefined, modifiedAccessConditions?: Models.ModifiedAccessConditions ): Promise { @@ -3509,6 +4014,7 @@ export default class LokiBlobMetadataStore container, blob, snapshot, + versionId, context, false, true @@ -3599,4 +4105,127 @@ export default class LokiBlobMetadataStore return doc.properties; } + + private findBlob( + context: Context, + account: string, + container: string, + blob: string, + snapshot: string = "", + versionId: string = "" + ): BlobModel | undefined { + const blobFound = this.findBlobCore( + context, + account, + container, + blob, + snapshot, + versionId + ); + + if (blobFound) { + return this.reviveKnownDateFields(context, blobFound); + } + + return blobFound; + } + + private findBlobCore( + context: Context, + account: string, + container: string, + blob: string, + snapshot: string = "", + versionId: string = "" + ): BlobModel | undefined { + const versionIdProvided = !isNullOrWhitespace(versionId); + const snapshotProvided = !isNullOrWhitespace(snapshot); + + // Cannot specify both versionId and snapshot + if (versionIdProvided && snapshotProvided) { + throw StorageErrorFactory.getMutuallyExclusiveQueryParameters( + context.contextId + ); + } + + const coll = this.db.getCollection(this.BLOBS_COLLECTION); + + const initQuery = { + name: blob, + accountName: account, + containerName: container + }; + let blobDocFindChain = coll.chain().find(initQuery); + + if (versionIdProvided) { + // If versionId is provided, simply find and return that specific version + blobDocFindChain = blobDocFindChain.find({ versionId: versionId }); + return blobDocFindChain.data()[0]; + } else if (snapshotProvided) { + // If snapshot is provided, find that specific snapshot + blobDocFindChain = blobDocFindChain.find({ snapshot: snapshot }); + return blobDocFindChain.data()[0]; + } else { + return this.findCurrentBlob(account, container, blob); + } + } + + /** + * Revive well-known date fields on a single blob document. + * For each targeted field: + * - If it's already a Date, leave it. + * - If it's a non-empty string, attempt to parse as Date. If valid, replace with Date. + * - Ignore null/undefined or whitespace-only strings. + */ + private reviveKnownDateFields(context: Context, blob: BlobModel): BlobModel { + if (!blob || typeof blob !== "object") { + return blob; + } + + const propDateCannotBeUndefined: Array< + keyof Models.BlobPropertiesInternal + > = ["lastModified"]; + + const propDateKeys: Array = [ + "creationTime", + "copyCompletionTime", + "deletedTime", + "accessTierChangeTime", + "lastAccessedOn", + "immutabilityPolicyExpiresOn", + "expiresOn" + ]; + + const topLevelDateKeys: Array = [ + "leaseExpireTime", + "leaseBreakTime" + ]; + + // Helper + + if (blob.properties) { + for (const k of propDateCannotBeUndefined) { + const parsedValue = parseDateFromAssumedString(blob.properties[k]); + if (parsedValue) { + (blob.properties[k] as Date) = parsedValue; + } else { + // This should not happen but we ar not throwing for back compat reasons. + console.log("[Warning][" + context.contextId + `] Failed to parse date field ${k} on blob ${blob.name}`); + } + } + + // For properties that can be undefined, we parse and set them if valid + for (const k of propDateKeys) { + (blob.properties[k] as Date | undefined) = parseDateFromAssumedString( + blob.properties[k] + ); + } + } + + for (const k of topLevelDateKeys) { + (blob[k] as Date | undefined) = parseDateFromAssumedString(blob[k]); + } + + return blob; + } } diff --git a/src/blob/persistence/PageWithDelimiter.ts b/src/blob/persistence/PageWithDelimiter.ts index 05e3bf280..006d6d466 100644 --- a/src/blob/persistence/PageWithDelimiter.ts +++ b/src/blob/persistence/PageWithDelimiter.ts @@ -1,5 +1,7 @@ import { BlobPrefixModel } from "./IBlobMetadataStore"; +export type PageMarkerMode = "name" | "nameAndTimestamp"; + /** * This implements a page of blob results taking delimiters into account. * @@ -11,6 +13,24 @@ import { BlobPrefixModel } from "./IBlobMetadataStore"; * @class PageWithDelimiter */ export default class PageWithDelimiter { + public static readonly VERSIONING_MARKER = "__version_marker__"; + + /** + * Compare two markers and return true if the first marker is later (greater) than the second + * @param marker1 First marker [name, timestamp] + * @param marker2 Second marker [name, timestamp] + * @returns true if marker1 is later than marker2, false otherwise + */ + public static isMarkerLater(marker1: [string, string], marker2: [string, string]): boolean { + if (marker1[0] > marker2[0]) { + return true; // First marker has greater name + } else if (marker1[0] === marker2[0]) { + return marker1[1] > marker2[1]; // Same name, compare timestamps + } else { + return false; // First marker has lesser name + } + } + readonly delimiter: string | undefined; readonly maxResults: number; readonly prefix: string | undefined; @@ -18,7 +38,7 @@ export default class PageWithDelimiter { blobItems: BlobType[] = []; blobPrefixes: Set = new Set(); - latestMarker: string = ""; + latestMarker: [string, string] = ["", ""]; // [name, timestamp] // isFull indicates we could only (maybe) add a prefix private isFull: boolean = false; @@ -26,7 +46,12 @@ export default class PageWithDelimiter { // isExhausted indicates nothing more should be added private isExhausted: boolean = false; - constructor(maxResults: number, delimiter?: string, prefix?: string) { + constructor( + maxResults: number, + delimiter?: string, + prefix?: string, + private readonly markerMode: PageMarkerMode = "nameAndTimestamp" + ) { this.maxResults = maxResults; if (delimiter !== undefined) { this.delimiter = delimiter; @@ -46,7 +71,7 @@ export default class PageWithDelimiter { this.blobPrefixes.clear(); this.isFull = false; this.isExhausted = false; - this.latestMarker = ""; + this.latestMarker = ["", ""]; } private updateFull() { @@ -104,14 +129,31 @@ export default class PageWithDelimiter { * * Return the number of items added */ - private add(name: string, item: BlobType): boolean { + private add([name, timestamp]: [string, string], item: BlobType): boolean { if (this.isExhausted) { return false; } - if (name < this.latestMarker) { + + if (name < this.latestMarker[0]) { throw new Error("add received unsorted item. add must be called on sorted data"); } - const marker = (name > this.latestMarker) ? name : this.latestMarker; + + if ( + this.markerMode === "nameAndTimestamp" && + name === this.latestMarker[0] && + timestamp <= this.latestMarker[1] + ) { + throw new Error("add received unsorted item. Blobs with same name must be added in timestamp order"); + } + + const currentMarker: [string, string] = [ + name, + this.markerMode === "name" ? "" : timestamp + ]; + const marker = PageWithDelimiter.isMarkerLater(currentMarker, this.latestMarker) + ? currentMarker + : this.latestMarker; + let added: boolean = false; if (this.delimiter !== undefined) { const delimiterPosAfterPrefix = name.indexOf( @@ -137,10 +179,10 @@ export default class PageWithDelimiter { /** * Iterate over an array blobs read from a source and add them until the page cannot accept new items */ - private processList(docs: BlobType[], nameFn: (item: BlobType) => string): number { + private processList(docs: BlobType[], markerFunc: (item: BlobType) => [string, string]): number { let added: number = 0; for (const item of docs) { - if (this.add(nameFn(item), item)) { + if (this.add(markerFunc(item), item)) { added++; } if (this.isExhausted) break; @@ -161,13 +203,13 @@ export default class PageWithDelimiter { */ public async fill( reader: (offset: number) => Promise, - namer: (item: BlobType) => string, + markerFunc: (item: BlobType) => [string, string], ): Promise<[BlobType[], BlobPrefixModel[], string]> { let offset: number = 0; let docs = await reader(offset); let added: number = 0; while (docs.length) { - added = this.processList(docs, namer); + added = this.processList(docs, markerFunc); offset += added; if (added < this.maxResults) { break; @@ -177,7 +219,11 @@ export default class PageWithDelimiter { return [ this.blobItems, this.prefixes(), - added < docs.length ? this.latestMarker : "" + added < docs.length + ? this.markerMode === "name" + ? this.latestMarker[0] + : this.latestMarker.join(PageWithDelimiter.VERSIONING_MARKER) + : "" ]; } diff --git a/src/blob/persistence/SqlBlobMetadataStore.ts b/src/blob/persistence/SqlBlobMetadataStore.ts index 8443360b7..504572d9e 100644 --- a/src/blob/persistence/SqlBlobMetadataStore.ts +++ b/src/blob/persistence/SqlBlobMetadataStore.ts @@ -52,6 +52,7 @@ import IBlobMetadataStore, { BreakContainerLeaseResponse, ChangeBlobLeaseResponse, ChangeContainerLeaseResponse, + CommitBlockListResponse, ContainerModel, CreateSnapshotResponse, FilterBlobModel, @@ -125,6 +126,11 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { this.sequelize = new Sequelize(connectionURI, sequelizeOptions); } + // Blob versioning is not supported in SQL. + public isBlobVersioningEnabled(): boolean { + return false; + } + public async init(): Promise { await this.sequelize.authenticate(); @@ -1107,7 +1113,15 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { blob: BlobModel, leaseAccessConditions?: Models.LeaseAccessConditions, modifiedAccessConditions?: Models.ModifiedAccessConditions - ): Promise { + ): Promise { + if (blob.versionId && blob.versionId !== "") { + // SQL metadata store doesn't support versioning + throw StorageErrorFactory.getInvalidOperation( + context.contextId, + "Blob versioning is not supported in SQL metadata store." + ); + } + return this.sequelize.transaction(async (t) => { await this.assertContainerExists( context, @@ -1146,9 +1160,8 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { } if (blobFindResult) { - const blobModel: BlobModel = this.convertDbModelToBlobModel( - blobFindResult - ); + const blobModel: BlobModel = + this.convertDbModelToBlobModel(blobFindResult); LeaseFactory.createLeaseState(new BlobLeaseAdapter(blobModel), context) .validate(new BlobWriteLeaseValidator(leaseAccessConditions)) @@ -1165,6 +1178,8 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { await BlobsModel.upsert(this.convertBlobModelToDbModel(blob), { transaction: t }); + + return blob; // Return the input blob model (now persisted) }); } @@ -1174,9 +1189,16 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { container: string, blob: string, snapshot: string = "", + versionId: string = "", leaseAccessConditions?: Models.LeaseAccessConditions, modifiedAccessConditions?: Models.ModifiedAccessConditions ): Promise { + if (versionId && versionId !== "") { + throw StorageErrorFactory.getInvalidOperation( + context.contextId, + "Blob versioning is not supported in SQL metadata store." + ); + } return this.sequelize.transaction(async (t) => { await this.assertContainerExists(context, account, container, t); @@ -1300,7 +1322,9 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { maxResults: number = DEFAULT_LIST_BLOBS_MAX_RESULTS, marker?: string, includeSnapshots?: boolean, - includeUncommittedBlobs?: boolean + includeUncommittedBlobs?: boolean, + includeVersions?: boolean, + includeDeletedWithVersions?: boolean ): Promise<[BlobModel[], BlobPrefixModel[], any | undefined]> { return this.sequelize.transaction(async (t) => { await this.assertContainerExists(context, account, container, t); @@ -1346,10 +1370,15 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { }; // fill the page by possibly querying multiple times - const page = new PageWithDelimiter(maxResults, delimiter, prefix); + const page = new PageWithDelimiter( + maxResults, + delimiter, + prefix, + "name" + ); - const nameItem = (item: BlobsModel): string => { - return this.getModelValue(item, "blobName", true); + const nameItem = (item: BlobsModel): [string, string] => { + return [this.getModelValue(item, "blobName", true), ""]; }; const readPage = async (off: number): Promise => { @@ -1509,10 +1538,17 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { container: string, blob: string, snapshot: string = "", - isCommitted?: boolean, - leaseAccessConditions?: Models.LeaseAccessConditions, - modifiedAccessConditions?: Models.ModifiedAccessConditions + versionId: string = "", + isCommitted: boolean | undefined, + leaseAccessConditions: Models.LeaseAccessConditions | undefined, + modifiedAccessConditions: Models.ModifiedAccessConditions | undefined ): Promise { + if (versionId && versionId !== "") { + throw StorageErrorFactory.getInvalidOperation( + context.contextId, + "Blob versioning is not supported in SQL metadata store." + ); + } return this.sequelize.transaction(async (t) => { await this.assertContainerExists(context, account, container, t); @@ -1587,7 +1623,7 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { blockList: { blockName: string; blockCommitType: string }[], leaseAccessConditions?: Models.LeaseAccessConditions, modifiedAccessConditions?: Models.ModifiedAccessConditions - ): Promise { + ): Promise { await this.sequelize.transaction(async (t) => { await this.assertContainerExists( context, @@ -1754,6 +1790,9 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { } ); }); + + // SQL does not support versioning + return { versionId: undefined }; } public async getBlobProperties( @@ -1762,9 +1801,16 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { container: string, blob: string, snapshot: string = "", + versionId: string = "", leaseAccessConditions?: Models.LeaseAccessConditions, modifiedAccessConditions?: Models.ModifiedAccessConditions ): Promise { + if (versionId && versionId !== "") { + throw StorageErrorFactory.getInvalidOperation( + context.contextId, + "Blob versioning is not supported in SQL metadata store." + ); + } return this.sequelize.transaction(async (t) => { await this.assertContainerExists(context, account, container, t); @@ -1903,6 +1949,12 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { blob: string, options: Models.BlobDeleteMethodOptionalParams ): Promise { + if (options.versionId !== undefined && options.versionId !== "") { + throw StorageErrorFactory.getInvalidOperation( + context.contextId, + "Blob versioning is not supported in SQL metadata store." + ); + } await this.sequelize.transaction(async (t) => { await this.assertContainerExists(context, account, container, t); @@ -2480,8 +2532,15 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { account: string, container: string, blob: string, - snapshot?: string | undefined + snapshot?: string | undefined, + versionId: string = "" ): Promise { + if (versionId && versionId !== "") { + throw StorageErrorFactory.getInvalidOperation( + context.contextId, + "Blob versioning is not supported in SQL metadata store." + ); + } await this.sequelize.transaction(async (t) => { await this.assertContainerExists(context, account, container, t); @@ -2506,10 +2565,18 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { account: string, container: string, blob: string, - snapshot?: string | undefined + snapshot?: string | undefined, + versionId: string = "" ): Promise< { blobType: Models.BlobType | undefined; isCommitted: boolean } | undefined > { + if (versionId && versionId !== "") { + // SQL path has no context; return undefined to mimic not found for version requests or could throw. + throw StorageErrorFactory.getInvalidOperation( + undefined, + "Blob versioning is not supported in SQL metadata store." + ); + } const res = await BlobsModel.findOne({ where: { accountName: account, @@ -2715,9 +2782,14 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { account: string, container: string, blob: string, + versionId: string | undefined, tier: Models.AccessTier, leaseAccessConditions?: Models.LeaseAccessConditions ): Promise<200 | 202> { + if (versionId !== undefined && versionId !== "") { + throw new NotImplementedinSQLError(context.contextId); + } + return this.sequelize.transaction(async (t) => { await this.assertContainerExists(context, account, container, t); @@ -3370,11 +3442,18 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { account: string, container: string, blob: string, - snapshot: string | undefined, + snapshot: string = "", + versionId: string = "", leaseAccessConditions: Models.LeaseAccessConditions | undefined, tags: Models.BlobTags | undefined, modifiedAccessConditions?: Models.ModifiedAccessConditions ): Promise { + if (versionId && versionId !== "") { + throw StorageErrorFactory.getInvalidOperation( + context.contextId, + "Blob versioning is not supported in SQL metadata store." + ); + } return this.sequelize.transaction(async (t) => { await this.assertContainerExists(context, account, container, t); @@ -3425,9 +3504,16 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { container: string, blob: string, snapshot: string = "", - leaseAccessConditions?: Models.LeaseAccessConditions, + versionId: string = "", + leaseAccessConditions: Models.LeaseAccessConditions | undefined, modifiedAccessConditions?: Models.ModifiedAccessConditions ): Promise { + if (versionId && versionId !== "") { + throw StorageErrorFactory.getInvalidOperation( + context.contextId, + "Blob versioning is not supported in SQL metadata store." + ); + } return this.sequelize.transaction(async (t) => { await this.assertContainerExists(context, account, container, t); @@ -3576,4 +3662,4 @@ export default class SqlBlobMetadataStore implements IBlobMetadataStore { ): Promise { throw new NotImplementedinSQLError(context.contextId); } -} +} \ No newline at end of file diff --git a/src/blob/utils/constants.ts b/src/blob/utils/constants.ts index 5a93a7dcd..335445b44 100644 --- a/src/blob/utils/constants.ts +++ b/src/blob/utils/constants.ts @@ -11,6 +11,7 @@ export const IS_PRODUCTION = process.env.NODE_ENV === "production"; export const DEFAULT_BLOB_LOKI_DB_PATH = "__azurite_db_blob__.json"; export const DEFAULT_BLOB_EXTENT_LOKI_DB_PATH = "__azurite_db_blob_extent__.json"; +export const DEFAULT_ACCOUNT_MODEL_LOKI_DB_PATH = "__azurite_db_account_models__.json"; export const DEFAULT_BLOB_PERSISTENCE_PATH = "__blobstorage__"; export const DEFAULT_DEBUG_LOG_PATH = "./debug.log"; export const DEFAULT_ENABLE_DEBUG_LOG = true; diff --git a/src/blob/utils/utils.ts b/src/blob/utils/utils.ts index 53e00e1a4..1d12dd9e8 100644 --- a/src/blob/utils/utils.ts +++ b/src/blob/utils/utils.ts @@ -142,6 +142,73 @@ export async function computeAndValidateTransactionalChecksums( return calculated; } +/** + * Parses the incoming value into a Date. + * Values unable to be parsed will result in undefined. + * Accepts ISO 8601 timestamps with 3 to 7 fractional-second digits. + * + * @export + * @param {any} [value] + * @returns {Date | undefined} + */ +export function parseDateFromAssumedString(value: any): Date | undefined { + if (value === undefined) { + return undefined; + } + + if (value instanceof Date) { + return value; + } + + if (typeof value === "string" && !isNullOrWhitespace(value)) { + // Validate ISO 8601 format: YYYY-MM-DDTHH:mm:ss.fffffffZ (3-7 decimal places) + const iso8601Regex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3,7}Z$/; + + if (!iso8601Regex.test(value)) { + return undefined; + } + + const d = new Date(value); + if (!isNaN(d.getTime())) { + return d; + } + } + + return undefined; +} + +export function validateSnapshotAndVersionId( + snapshot?: string, + versionId?: string, + contextId?: string +): void { + if ( + versionId !== undefined && + versionId !== "" && + !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{7}Z$/.test(versionId) + ) { + throw StorageErrorFactory.getInvalidQueryParameterValue( + contextId, + "versionid", + versionId, + "The version ID is not a valid RFC 3339 timestamp with 7 digit fractional seconds." + ); + } + + if ( + snapshot !== undefined && + snapshot !== "" && + versionId !== undefined && + versionId !== "" + ) { + throw StorageErrorFactory.getMutuallyExclusiveQueryParameters(contextId); + } +} + +export function isNullOrWhitespace(str: string | null | undefined): boolean { + return !str?.trim(); +} + export function checkApiVersion( inputApiVersion: string, validApiVersions: Array, @@ -393,4 +460,4 @@ export function toBlobTags(input: TagContent[]): BlobTag[] { value: value } }); -} +} \ No newline at end of file diff --git a/src/common/ConfigurationBase.ts b/src/common/ConfigurationBase.ts index be3f0928b..5547d6e3b 100644 --- a/src/common/ConfigurationBase.ts +++ b/src/common/ConfigurationBase.ts @@ -104,4 +104,4 @@ export default abstract class ConfigurationBase { return `http${this.hasCert() === CertOptions.Default ? "" : "s"}://${this.host }:${this.port}`; } -} +} \ No newline at end of file diff --git a/src/common/Environment.ts b/src/common/Environment.ts index 44c25c237..7cae1a5b3 100644 --- a/src/common/Environment.ts +++ b/src/common/Environment.ts @@ -19,6 +19,8 @@ import { } from "../table/utils/constants"; import IEnvironment from "./IEnvironment"; +import { AccountModel } from "./account/AccountModel"; +import { parseAccountModelFlags } from "./EnvironmentFunctions"; import { shouldSkipApiVersionCheck } from "./utils/environment"; args @@ -111,6 +113,14 @@ args .option( ["", "disableTelemetry"], "Optional. Disable telemtry collection of Azurite. If not specify this parameter Azurite will collect telemetry data by default." + ) + .option( + ["", "accountConfigFilePath"], + "Optional. Path to the account configuration file" + ) + .option( + ["", "accountConfigAsJson"], + "Optional. Account configuration in JSON format" ); (args as any).config.name = "azurite"; @@ -245,4 +255,8 @@ export default class Environment implements IEnvironment { // By default disable debug log } -} + + public getAccountModels(): Map | undefined { + return parseAccountModelFlags(this.flags); + } +} \ No newline at end of file diff --git a/src/common/EnvironmentFunctions.ts b/src/common/EnvironmentFunctions.ts new file mode 100644 index 000000000..67dd9a88c --- /dev/null +++ b/src/common/EnvironmentFunctions.ts @@ -0,0 +1,294 @@ +import { readFileSync, existsSync } from "fs"; +import { + AccountModel, + normalizeAccountName +} from "./account/AccountModel"; +import { EMULATOR_ACCOUNT_NAME } from "./utils/constants"; + +/** + * Parses account model flags and returns a map of account models. + * + * Supports multiple formats: + * 1. Multi-account with paths: "accountName1:/path/to/config1.json,accountName2:/path/to/config2.json" + * 2. Multi-account with JSON: "accountName1:{\"isBlobVersioningEnabled\":true},accountName2:{\"isBlobVersioningEnabled\":false}" + * 3. Single account path (backward compatible): "/path/to/config.json" - uses default emulator account name + * 4. Single account JSON (backward compatible): "{\"isBlobVersioningEnabled\":true}" - uses default emulator account name + * + * @param flags - Configuration flags object + * @returns Map of account name to AccountModel, or undefined if no configuration is provided + */ +export function parseAccountModelFlags(flags: { + [key: string]: any; +}): Map | undefined { + const configFilePath = flags?.accountConfigFilePath; + const configAsJson = flags?.accountConfigAsJson; + + if (!configFilePath && !configAsJson) { + // If neither is specified, return undefined + return undefined; + } + + if (configFilePath && configAsJson) { + // If both are specified, throw an error + throw new Error("Specify either accountConfigFilePath or accountConfigAsJson, not both."); + } + + // First, split entries to count them + const configString = configFilePath || configAsJson!; + const entries = splitAccountEntries(configString); + + if (entries.length === 0) { + throw new Error("Account configuration was specified but no valid accounts were found"); + } + + const accountModels = new Map(); + + if (configFilePath) { + // Check if this is single-account mode (no colon prefix) or multi-account mode + if ( + entries.length === 1 && + (!entries[0].includes(":") || /^[a-zA-Z]:[\\/]/.test(entries[0])) + ) { + // Single account mode: just a path without account name prefix + parseSingleAccountConfigFromPath(entries[0], accountModels); + } else { + // Multi-account mode: "accountName1:/path/to/config1.json,accountName2:/path/to/config2.json" + parseAccountConfigFromPaths(entries, accountModels); + } + } else if (configAsJson) { + // Check if this is single-account mode (starts with '{') or multi-account mode + if (entries.length === 1 && entries[0].trim().startsWith('{')) { + // Single account mode: just JSON without account name prefix + parseSingleAccountConfigFromJson(entries[0], accountModels); + } else { + // Multi-account mode: "accountName1:{...},accountName2:{...}" + parseAccountConfigFromJson(entries, accountModels); + } + } + + return accountModels; +} + +/** + * Parses a single account configuration from a file path (backward compatible mode). + * Format: "/path/to/config.json" - uses default emulator account name + */ +function parseSingleAccountConfigFromPath( + filePath: string, + accountModels: Map +): void { + const trimmedPath = filePath.trim(); + + if (!existsSync(trimmedPath)) { + throw new Error(`Account configuration file not found: ${trimmedPath}`); + } + + let json: string; + try { + json = readFileSync(trimmedPath, "utf-8"); + } catch (error) { + throw new Error(`Failed to read account configuration file: ${error}`, { + cause: error + }); + } + + if (!json || json.trim() === "") { + throw new Error(`Account configuration file is empty: ${trimmedPath}`); + } + + const accountModel = parseAccountModelJson(EMULATOR_ACCOUNT_NAME, json); + addAccountModel(accountModels, accountModel); +} + +/** + * Parses a single account configuration from a JSON string (backward compatible mode). + * Format: "{\"isBlobVersioningEnabled\":true}" - uses default emulator account name + */ +function parseSingleAccountConfigFromJson( + jsonString: string, + accountModels: Map +): void { + const trimmedJson = jsonString.trim(); + + if (!trimmedJson) { + throw new Error("Account configuration JSON is empty"); + } + + const accountModel = parseAccountModelJson(EMULATOR_ACCOUNT_NAME, trimmedJson); + addAccountModel(accountModels, accountModel); +} + +/** + * Parses account configuration from file paths. + * Format: "accountName1:/path/to/config1.json,accountName2:/path/to/config2.json" + */ +function parseAccountConfigFromPaths( + entries: string[], + accountModels: Map +): void { + for (const entry of entries) { + const { accountName, value } = parseAccountEntry(entry); + + if (!existsSync(value)) { + throw new Error(`Account configuration file not found for account '${accountName}': ${value}`); + } + + let json: string; + try { + json = readFileSync(value, "utf-8"); + } catch (error) { + throw new Error( + `Failed to read account configuration file for account '${accountName}': ${error}`, + { cause: error } + ); + } + + if (!json || json.trim() === "") { + throw new Error(`Account configuration file is empty for account '${accountName}': ${value}`); + } + + const accountModel = parseAccountModelJson(accountName, json); + addAccountModel(accountModels, accountModel); + } +} + +/** + * Parses account configuration from JSON strings. + * Format: "accountName1:{\"isBlobVersioningEnabled\":true},accountName2:{\"isBlobVersioningEnabled\":false}" + */ +function parseAccountConfigFromJson( + entries: string[], + accountModels: Map +): void { + for (const entry of entries) { + const { accountName, value } = parseAccountEntry(entry); + + if (!value || value.trim() === "") { + throw new Error(`Account configuration is empty for account '${accountName}'`); + } + + const accountModel = parseAccountModelJson(accountName, value); + addAccountModel(accountModels, accountModel); + } +} + +/** + * Splits the configuration string into individual account entries. + * Handles commas that might be inside JSON objects. + */ +function splitAccountEntries(config: string): string[] { + const entries: string[] = []; + let currentEntry = ""; + let braceDepth = 0; + let inQuotes = false; + + for (let i = 0; i < config.length; i++) { + const char = config[i]; + const prevChar = i > 0 ? config[i - 1] : ""; + + if (char === '"' && prevChar !== '\\') { + inQuotes = !inQuotes; + } else if (!inQuotes) { + if (char === '{') { + braceDepth++; + } else if (char === '}') { + braceDepth--; + } else if (char === ',' && braceDepth === 0) { + // This comma is a separator between accounts + if (currentEntry.trim()) { + entries.push(currentEntry.trim()); + } + currentEntry = ""; + continue; + } + } + + currentEntry += char; + } + + // Add the last entry + if (currentEntry.trim()) { + entries.push(currentEntry.trim()); + } + + return entries; +} + +/** + * Parses a single account entry in the format "accountName:value" + */ +function parseAccountEntry(entry: string): { accountName: string; value: string } { + const colonIndex = entry.indexOf(':'); + + if (colonIndex === -1) { + throw new Error(`Invalid account configuration format. Expected 'accountName:value', got: ${entry}`); + } + + const accountName = normalizeAccountName( + entry.substring(0, colonIndex) + ); + const value = entry.substring(colonIndex + 1).trim(); + + if (!accountName) { + throw new Error(`Account name is missing in configuration entry: ${entry}`); + } + + if (!value) { + throw new Error(`Configuration value is missing for account '${accountName}'`); + } + + return { accountName, value }; +} + +/** + * Parses JSON string into an AccountModel. + */ +function parseAccountModelJson(accountName: string, json: string): AccountModel { + let parsed: any; + + try { + parsed = JSON.parse(json); + } catch (error) { + throw new Error( + `Invalid JSON in account configuration for account '${accountName}': ${error}`, + { cause: error } + ); + } + + if (!parsed || typeof parsed !== "object") { + throw new Error(`Account configuration must be a JSON object for account '${accountName}'`); + } + + const isBlobVersioningEnabled = + parsed.isBlobVersioningEnabled === undefined + ? false + : parsed.isBlobVersioningEnabled; + + if (typeof isBlobVersioningEnabled !== "boolean") { + throw new Error(`Account configuration value 'isBlobVersioningEnabled' must be a boolean for account '${accountName}'`); + } + + const accountModel: AccountModel = { + key: normalizeAccountName(accountName), + isBlobVersioningEnabled + }; + + return accountModel; +} + +function addAccountModel( + accountModels: Map, + accountModel: AccountModel +): void { + const accountName = normalizeAccountName(accountModel.key); + if (accountModels.has(accountName)) { + throw new Error( + `Account configuration contains duplicate account '${accountName}'` + ); + } + + accountModels.set(accountName, { + ...accountModel, + key: accountName + }); +} diff --git a/src/common/IAccountModelEnvironment.ts b/src/common/IAccountModelEnvironment.ts new file mode 100644 index 000000000..57ad49b7a --- /dev/null +++ b/src/common/IAccountModelEnvironment.ts @@ -0,0 +1,19 @@ +import { AccountModel } from "./account/AccountModel"; + +/** + * Interface for environments that provide account-level configuration. + * This allows different accounts to have different settings. + * + * @export + * @interface IAccountModelEnvironment + */ +export default interface IAccountModelEnvironment { + /** + * Gets the account models configuration from environment flags. + * Returns a map of account name to AccountModel, or undefined if no account configuration is provided. + * + * @returns {(Map | undefined)} + * @memberof IAccountDataEnvironment + */ + getAccountModels(): Map | undefined; +} diff --git a/src/common/IEnvironment.ts b/src/common/IEnvironment.ts index e307b0a32..d16ecceda 100644 --- a/src/common/IEnvironment.ts +++ b/src/common/IEnvironment.ts @@ -1,8 +1,10 @@ import IBlobEnvironment from "../blob/IBlobEnvironment"; import IQueueEnvironment from "../queue/IQueueEnvironment"; import ITableEnvironment from "../table/ITableEnvironment"; +import IAccountModelEnvironment from "./IAccountModelEnvironment"; export default interface IEnvironment extends IBlobEnvironment, IQueueEnvironment, - ITableEnvironment { } + ITableEnvironment, + IAccountModelEnvironment { } diff --git a/src/common/VSCEnvironment.ts b/src/common/VSCEnvironment.ts index 0bcff08f5..d3036f4b0 100644 --- a/src/common/VSCEnvironment.ts +++ b/src/common/VSCEnvironment.ts @@ -3,6 +3,8 @@ import { isAbsolute, resolve } from "path"; import { window, workspace, WorkspaceFolder } from "vscode"; import IEnvironment from "./IEnvironment"; +import { parseAccountModelFlags } from "./EnvironmentFunctions"; +import { AccountModel } from "./account/AccountModel"; export default class VSCEnvironment implements IEnvironment { public workspaceConfiguration = workspace.getConfiguration("azurite"); @@ -135,4 +137,13 @@ export default class VSCEnvironment implements IEnvironment { this.workspaceConfiguration.get("disableTelemetry") || false ); } -} + + public getAccountModels(): Map | undefined { + const accountConfigFilePath = this.workspaceConfiguration.get("accountConfigFilePath"); + const accountConfigAsJson = this.workspaceConfiguration.get("accountConfigAsJson"); + return parseAccountModelFlags({ + accountConfigFilePath: accountConfigFilePath, + accountConfigAsJson: accountConfigAsJson + }); + } +} \ No newline at end of file diff --git a/src/common/VSCServerManagerBlob.ts b/src/common/VSCServerManagerBlob.ts index 1a3555ca6..51b68df3a 100644 --- a/src/common/VSCServerManagerBlob.ts +++ b/src/common/VSCServerManagerBlob.ts @@ -1,13 +1,8 @@ import { join } from "path"; -import BlobConfiguration from "../blob/BlobConfiguration"; -import BlobServer from "../blob/BlobServer"; -import { - DEFAULT_BLOB_EXTENT_LOKI_DB_PATH, - DEFAULT_BLOB_LOKI_DB_PATH, - DEFAULT_BLOB_PERSISTENCE_ARRAY, - DEFAULT_BLOB_PERSISTENCE_PATH -} from "../blob/utils/constants"; +import { BlobServerFactory } from "../blob/BlobServerFactory"; +import LokiAccountModelStore from "./account/LokiAccountModelStore"; +import { DEFAULT_ACCOUNT_MODEL_LOKI_DB_PATH } from "../blob/utils/constants"; import * as Logger from "./Logger"; import NoLoggerStrategy from "./NoLoggerStrategy"; import VSCChannelLoggerStrategy from "./VSCChannelLoggerStrategy"; @@ -42,11 +37,25 @@ export default class VSCServerManagerBlob extends VSCServerManagerBase { } public async createImpl(): Promise { - const config = await this.getConfiguration(); + const env = new VSCEnvironment(); + const location = await env.location(); + + // Create account model store + const accountModels = env.getAccountModels(); + const accountModelStore = new LokiAccountModelStore( + join(location, DEFAULT_ACCOUNT_MODEL_LOKI_DB_PATH), + env.inMemoryPersistence(), + accountModels + ); + + await accountModelStore.init(); + const blobServerFactory = new BlobServerFactory(); + this.server = await blobServerFactory.createServer(env, accountModelStore); + + const config = this.server.config; Logger.default.strategy = config.enableDebugLog ? this.debuggerLoggerStrategy : new NoLoggerStrategy(); - this.server = new BlobServer(config); } public async startImpl(): Promise { @@ -63,38 +72,4 @@ export default class VSCServerManagerBlob extends VSCServerManagerBase { await this.createImpl(); await this.server!.clean(); } - - private async getConfiguration(): Promise { - const env = new VSCEnvironment(); - const location = await env.location(); - - DEFAULT_BLOB_PERSISTENCE_ARRAY[0].locationPath = join( - location, - DEFAULT_BLOB_PERSISTENCE_PATH - ); - AzuriteTelemetryClient.init(DEFAULT_BLOB_PERSISTENCE_ARRAY[0].locationPath, !env.disableTelemetry(), env.workspaceConfiguration, true); - - // Initialize server configuration - const config = new BlobConfiguration( - env.blobHost(), - env.blobPort(), - env.blobKeepAliveTimeout(), - join(location, DEFAULT_BLOB_LOKI_DB_PATH), - join(location, DEFAULT_BLOB_EXTENT_LOKI_DB_PATH), - DEFAULT_BLOB_PERSISTENCE_ARRAY, - !env.silent(), - this.accessChannelStream, - (await env.debug()) === true, - undefined, - env.loose(), - env.skipApiVersionCheck(), - env.cert(), - env.key(), - env.pwd(), - env.oauth(), - env.disableProductStyleUrl(), - env.inMemoryPersistence(), - ); - return config; - } } diff --git a/src/common/account/AccountModel.ts b/src/common/account/AccountModel.ts new file mode 100644 index 000000000..1d1692671 --- /dev/null +++ b/src/common/account/AccountModel.ts @@ -0,0 +1,8 @@ +export interface AccountModel { + key: string; + isBlobVersioningEnabled: boolean; +} + +export function normalizeAccountName(accountName: string): string { + return accountName.trim().toLowerCase(); +} diff --git a/src/common/account/IAccountModelStore.ts b/src/common/account/IAccountModelStore.ts new file mode 100644 index 000000000..fe71022b0 --- /dev/null +++ b/src/common/account/IAccountModelStore.ts @@ -0,0 +1,9 @@ +import ICleaner from "../ICleaner"; +import IDataStore from "../IDataStore"; +import { AccountModel } from "./AccountModel"; + +export default interface IAccountModelStore extends IDataStore, ICleaner { + getAccountModel(accountName: string): AccountModel | undefined; + isBlobVersioningEnabled(accountName: string): boolean; + hasBlobVersioningEnabled(): boolean; +} diff --git a/src/common/account/LokiAccountModelStore.ts b/src/common/account/LokiAccountModelStore.ts new file mode 100644 index 000000000..237577c07 --- /dev/null +++ b/src/common/account/LokiAccountModelStore.ts @@ -0,0 +1,275 @@ +import { stat } from "fs"; +import Loki from "lokijs"; +import { rimrafAsync } from "../utils/utils"; +import { AccountModel, normalizeAccountName } from "./AccountModel"; +import IAccountModelStore from "./IAccountModelStore"; + +/** + * LokiAccountModelStore manages account-level configuration using LokiJS. + * This store supports multiple accounts, with each account having its own configuration. + * + * The account name (key) is used as the unique identifier for each account. + * This allows different accounts to have different settings, such as blob versioning. + * + * @export + * @class LokiAccountModelStore + */ +export default class LokiAccountModelStore implements IAccountModelStore { + private readonly db: Loki; + private initialized: boolean = false; + private closed: boolean = true; + private readonly accountModelsFromArgs: Map | undefined; + + private readonly ACCOUNT_MODEL_COLLECTION = "$ACCOUNT_MODEL_COLLECTION$"; + + /** + * Creates an instance of LokiAccountDataStore. + * + * @param {string} lokiDBPath - Path to the LokiJS database file + * @param {boolean} inMemory - Whether to use in-memory persistence + * @param {Map} [accountModels] - Optional map of account configurations from environment + * @memberof LokiAccountDataStore + */ + public constructor( + public readonly lokiDBPath: string, + private readonly inMemory: boolean, + accountModels?: Map + ) { + this.accountModelsFromArgs = accountModels; + this.db = new Loki( + lokiDBPath, + inMemory + ? { + persistenceMethod: "memory" + } + : { + persistenceMethod: "fs", + autosave: true, + autosaveInterval: 5000 + } + ); + } + + /** + * Checks if the store is initialized. + * + * @returns {boolean} + * @memberof LokiAccountDataStore + */ + public isInitialized(): boolean { + return this.initialized; + } + + /** + * Checks if the store is closed. + * + * @returns {boolean} + * @memberof LokiAccountDataStore + */ + public isClosed(): boolean { + return this.closed; + } + + public async clean(): Promise { + if (this.isClosed()) { + if (!this.inMemory) { + await rimrafAsync(this.lokiDBPath); + } + + return; + } + throw new Error(`Cannot clean LokiAccountModelStore, it's not closed.`); + } + + /** + * Initializes the account data store. + * Creates the account model collection if it doesn't exist. + * Processes account models from environment arguments and merges them with existing DB configuration. + * + * Configuration Merge Logic: + * - For each account in accountModelsFromArgs: + * 1. Load existing account configuration from database + * 2. If account exists in DB, compare with new configuration + * 3. If no conflicts detected, merge and update with new configuration + * 4. If conflicts detected, throw error + * 5. If account doesn't exist in DB, insert new configuration + * + * Conflict Detection: + * - Currently, isBlobVersioningEnabled does NOT cause conflicts as it can be toggled on/off safely + * - Future account properties might require conflict detection if they cannot be changed after data exists + * - Example conflict scenario: If property "storageRedundancy" was added and changed from "LRS" to "GRS" + * after data was written, this would be a conflict requiring data migration + * + * @returns {Promise} + * @memberof LokiAccountDataStore + */ + public async init(): Promise { + await new Promise((resolve, reject) => { + stat(this.lokiDBPath, (statError, stats) => { + if (!statError) { + this.db.loadDatabase({}, (dbError) => { + if (dbError) { + reject(dbError); + } else { + resolve(); + } + }); + } else if (statError.code === "ENOENT") { + resolve(); + } else { + reject(statError); + } + }); + }); + + // Create account model collection if not exists + let accountModelCollection = this.db.getCollection( + this.ACCOUNT_MODEL_COLLECTION + ); + + if (accountModelCollection === null) { + accountModelCollection = this.db.addCollection( + this.ACCOUNT_MODEL_COLLECTION, + { + unique: ["key"] + } + ); + } + + const normalizedAccounts = new Map(); + const persistedAccounts = accountModelCollection.find(); + for (const accountModel of persistedAccounts) { + const key = normalizeAccountName(accountModel.key); + if (normalizedAccounts.has(key)) { + throw new Error( + `Account model configuration contains duplicate account '${key}'.` + ); + } + normalizedAccounts.set(key, accountModel); + } + for (const accountModel of persistedAccounts) { + const key = normalizeAccountName(accountModel.key); + if (accountModel.key !== key) { + accountModel.key = key; + accountModelCollection.update(accountModel); + } + } + + // Process account models from environment arguments + if (this.accountModelsFromArgs && this.accountModelsFromArgs.size > 0) { + for (const [accountName, newAccountModel] of this.accountModelsFromArgs) { + const normalizedAccountName = normalizeAccountName(accountName); + const existingAccount = accountModelCollection.by( + "key", + normalizedAccountName + ); + + if (existingAccount) { + // Account exists in DB - compare and merge configurations + + // TODO: Add conflict detection here for future account properties that cannot be changed + // Example: if (existingAccount.storageRedundancy !== newAccountModel.storageRedundancy) { + // throw new Error(`Conflict: Cannot change storageRedundancy for account '${accountName}'`); + // } + + // For now, isBlobVersioningEnabled can be toggled without conflicts + // Simply update the existing account with new configuration + existingAccount.isBlobVersioningEnabled = newAccountModel.isBlobVersioningEnabled; + accountModelCollection.update(existingAccount); + } else { + // Account doesn't exist in DB - insert new configuration + accountModelCollection.insert({ + key: normalizedAccountName, + isBlobVersioningEnabled: newAccountModel.isBlobVersioningEnabled + }); + } + } + } + + await new Promise((resolve, reject) => { + this.db.saveDatabase((err) => { + if (err) { + reject(err); + } else { + resolve(); + } + }); + }); + + this.initialized = true; + this.closed = false; + } + + /** + * Closes the LokiJS database. + * + * @returns {Promise} + * @memberof LokiAccountDataStore + */ + public async close(): Promise { + await new Promise((resolve, reject) => { + this.db.close((err) => { + if (err) { + reject(err); + } else { + this.closed = true; + resolve(); + } + }); + }); + } + + /** + * Gets the account model for a specific account. + * Returns undefined if the account doesn't exist. + * + * @param {string} accountName - The name of the account (used as the key) + * @returns {(AccountModel | undefined)} + * @memberof LokiAccountDataStore + */ + public getAccountModel(accountName: string): AccountModel | undefined { + const accountModelCollection = this.db.getCollection( + this.ACCOUNT_MODEL_COLLECTION + ); + + if (accountModelCollection === null) { + throw new Error("Account model collection is not initialized."); + } + + return accountModelCollection.by("key", normalizeAccountName(accountName)); + } + + /** + * Checks if blob versioning is enabled for a specific account. + * Returns false if the account doesn't exist. + * + * @param {string} accountName - The name of the account + * @returns {boolean} + * @memberof LokiAccountModelStore + */ + public isBlobVersioningEnabled(accountName: string): boolean { + const accountModel = this.getAccountModel(accountName); + return accountModel?.isBlobVersioningEnabled ?? false; + } + + public hasBlobVersioningEnabled(): boolean { + if (!this.initialized) { + return ( + this.accountModelsFromArgs !== undefined && + [...this.accountModelsFromArgs.values()].some( + (account) => account.isBlobVersioningEnabled + ) + ); + } + + const accountModelCollection = this.db.getCollection( + this.ACCOUNT_MODEL_COLLECTION + ); + return ( + accountModelCollection !== null && + accountModelCollection + .find() + .some((account) => account.isBlobVersioningEnabled) + ); + } +} diff --git a/src/common/account/index.ts b/src/common/account/index.ts new file mode 100644 index 000000000..c9c101060 --- /dev/null +++ b/src/common/account/index.ts @@ -0,0 +1,3 @@ +export * from "./AccountModel"; +export { default as IAccountModelStore } from "./IAccountModelStore"; +export { default as LokiAccountModelStore } from "./LokiAccountModelStore"; diff --git a/tests/BlobTestServerFactory.ts b/tests/BlobTestServerFactory.ts index 0867b07cb..5c9812625 100644 --- a/tests/BlobTestServerFactory.ts +++ b/tests/BlobTestServerFactory.ts @@ -5,6 +5,7 @@ import SqlBlobServer from "../src/blob/SqlBlobServer"; import { StoreDestinationArray } from "../src/common/persistence/IExtentStore"; import { DEFAULT_SQL_OPTIONS } from "../src/common/utils/constants"; import { DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT } from "../src/blob/utils/constants"; +import LokiAccountModelStore from "../src/common/account/LokiAccountModelStore"; import { LIVE_TEST_MODE } from "./testutils"; /** @@ -20,18 +21,26 @@ export class LiveModeStubServer { } export default class BlobTestServerFactory { + private createDefaultAccountModelStore(inMemory: boolean): LokiAccountModelStore { + // Create a default account model store with no account models (no specific configurations) + const accountDbPath = "__test_db_account_models_default__.json"; + return new LokiAccountModelStore(accountDbPath, inMemory, undefined); + } + public createServer( loose: boolean = false, skipApiVersionCheck: boolean = false, https: boolean = false, - oauth?: string + oauth?: string, + accountModelStore?: LokiAccountModelStore ): BlobServer | SqlBlobServer | LiveModeStubServer { if (LIVE_TEST_MODE) { return new LiveModeStubServer(); } const databaseConnectionString = process.env.AZURITE_TEST_DB; const isSQL = databaseConnectionString !== undefined; - const inMemoryPersistence = process.env.AZURITE_TEST_INMEMORYPERSISTENCE !== undefined; + const inMemoryPersistence = + process.env.AZURITE_TEST_INMEMORYPERSISTENCE !== undefined; const port = 11000; const host = "127.0.0.1"; @@ -47,7 +56,9 @@ export default class BlobTestServerFactory { if (isSQL) { if (inMemoryPersistence) { - throw new Error(`The in-memory persistence settings is not supported when using SQL-based metadata.`) + throw new Error( + `The in-memory persistence settings is not supported when using SQL-based metadata.` + ); } const config = new SqlBlobConfiguration( @@ -67,13 +78,17 @@ export default class BlobTestServerFactory { key, undefined, oauth, - undefined, + undefined ); return new SqlBlobServer(config); } else { const lokiMetadataDBPath = "__test_db_blob__.json"; const lokiExtentDBPath = "__test_db_blob_extent__.json"; + + // If no account model store is provided, create a default one + const finalAccountModelStore = accountModelStore || this.createDefaultAccountModelStore(inMemoryPersistence); + const config = new BlobConfiguration( host, port, @@ -92,7 +107,9 @@ export default class BlobTestServerFactory { undefined, oauth, undefined, - inMemoryPersistence + inMemoryPersistence, + undefined, + finalAccountModelStore ); return new BlobServer(config); } diff --git a/tests/blob/BlobServerFactory.unit.test.ts b/tests/blob/BlobServerFactory.unit.test.ts new file mode 100644 index 000000000..8e4323db5 --- /dev/null +++ b/tests/blob/BlobServerFactory.unit.test.ts @@ -0,0 +1,59 @@ +import * as assert from "assert"; + +import { BlobServerFactory } from "../../src/blob/BlobServerFactory"; +import IBlobEnvironment from "../../src/blob/IBlobEnvironment"; +import { AccountModel } from "../../src/common/account/AccountModel"; +import LokiAccountModelStore from "../../src/common/account/LokiAccountModelStore"; + +describe("BlobServerFactory", () => { + it("should reject versioning with SQL metadata", async () => { + const originalDatabase = process.env.AZURITE_DB; + process.env.AZURITE_DB = "mysql://unused"; + + const account: AccountModel = { + key: "devstoreaccount1", + isBlobVersioningEnabled: true + }; + const accountModelStore = new LokiAccountModelStore( + "", + true, + new Map([[account.key, account]]) + ); + const environment: IBlobEnvironment = { + blobHost: () => "127.0.0.1", + blobPort: () => 10000, + blobKeepAliveTimeout: () => 0, + location: async () => ".", + silent: () => true, + loose: () => false, + skipApiVersionCheck: () => false, + cert: () => undefined, + key: () => undefined, + pwd: () => undefined, + debug: async () => undefined, + oauth: () => undefined, + disableProductStyleUrl: () => false, + inMemoryPersistence: () => false, + extentMemoryLimit: () => undefined, + disableTelemetry: () => true, + getAccountModels: () => new Map([[account.key, account]]) + }; + + try { + await assert.rejects( + () => + new BlobServerFactory().createServer( + environment, + accountModelStore + ), + /Blob versioning is not supported when using SQL-based metadata storage/ + ); + } finally { + if (originalDatabase === undefined) { + delete process.env.AZURITE_DB; + } else { + process.env.AZURITE_DB = originalDatabase; + } + } + }); +}); diff --git a/tests/blob/apis/appendblob.test.ts b/tests/blob/apis/appendblob.test.ts index dbe72e78d..f431fb3b0 100644 --- a/tests/blob/apis/appendblob.test.ts +++ b/tests/blob/apis/appendblob.test.ts @@ -91,6 +91,11 @@ describe("AppendBlobAPIs", () => { assert.deepStrictEqual(properties.blobCommittedBlockCount, 0); }); + it("Create append blob should return versionId as undefined @loki", async () => { + const createResponse = await appendBlobClient.create(); + assert.strictEqual(createResponse.versionId, undefined); + }); + it("Create append blob with ifTags should work @loki", async () => { await appendBlobClient.create(); diff --git a/tests/blob/apis/appendblob.versioning.test.ts b/tests/blob/apis/appendblob.versioning.test.ts new file mode 100644 index 000000000..15ffbc3eb --- /dev/null +++ b/tests/blob/apis/appendblob.versioning.test.ts @@ -0,0 +1,618 @@ +import { + StorageSharedKeyCredential, + BlobServiceClient, + newPipeline, + Tags +} from "@azure/storage-blob"; +import assert = require("assert"); + +import { configLogger } from "../../../src/common/Logger"; +import BlobTestServerFactory from "../../BlobTestServerFactory"; +import { + bodyToString, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName, + sleep +} from "../../testutils"; +import { parseDateFromAssumedString } from "../../../src/blob/utils/utils"; +import { AccountModel } from "../../../src/common/account/AccountModel"; +import LokiAccountModelStore from "../../../src/common/account/LokiAccountModelStore"; + +// Set true to enable debug log +configLogger(false); + +const ACCOUNT_DB_FILE = "__test_db_account_models_appendblob_versioning__.json"; + +function createAccountModelStore(accountModel: AccountModel, inMemory: boolean = false): LokiAccountModelStore { + const accountModels = new Map(); + accountModels.set(accountModel.key || "devstoreaccount1", accountModel); + return new LokiAccountModelStore(ACCOUNT_DB_FILE, inMemory, accountModels); +} + +async function listBlobVersions(containerClient: any, blobName: string): Promise { + const listResponse = containerClient.listBlobsFlat({ + includeVersions: true + }); + const blobVersions = []; + for await (const blob of listResponse) { + if (blob.name === blobName) { + blobVersions.push(blob); + } + } + return blobVersions; +} + +describe("AppendBlobVersioningAPIs", () => { + const factory = new BlobTestServerFactory(); + const accountModel: AccountModel = + { + key: "devstoreaccount1", + isBlobVersioningEnabled: true + } + const accountModelStore = createAccountModelStore(accountModel, true); + const server = factory.createServer(false, false, false, undefined, accountModelStore); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + let containerName: string = getUniqueName("container"); + let containerClient = serviceClient.getContainerClient(containerName); + let blobName: string = getUniqueName("blob"); + let blobClient = containerClient.getBlobClient(blobName); + let appendBlobClient = blobClient.getAppendBlobClient(); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async () => { + containerName = getUniqueName("container"); + containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + blobName = getUniqueName("blob"); + blobClient = containerClient.getBlobClient(blobName); + appendBlobClient = blobClient.getAppendBlobClient(); + }); + + afterEach(async () => { + await containerClient.delete(); + }); + + // ===================== APPEND BLOB SPECIFIC TESTS ===================== + it("should return versionId when creating an append blob with versioning enabled", async () => { + const createResponse = await appendBlobClient.create(); + + // Verify versionId is returned and is a valid date + assert.ok( + createResponse.versionId, + "versionId should be present in create response" + ); + assert.ok( + parseDateFromAssumedString(createResponse.versionId), + "versionId should be a valid ISO date string" + ); + + // Verify other response properties + assert.strictEqual(createResponse._response.status, 201); + assert.ok(createResponse.etag); + assert.ok(createResponse.lastModified); + }); + + it("should create new versions when recreating append blob", async () => { + const metadata1 = { version: "1" }; + const metadata2 = { version: "2" }; + + // Create first version + const create1 = await appendBlobClient.create({ metadata: metadata1 }); + assert.ok(create1.versionId); + const version1Id = create1.versionId!; + + // Small delay to ensure different timestamps + await sleep(100); + + // Create second version (recreate the blob) + const create2 = await appendBlobClient.create({ metadata: metadata2 }); + assert.ok(create2.versionId); + const version2Id = create2.versionId!; + + // Verify different version IDs + assert.notStrictEqual(version1Id, version2Id); + + // Verify both are valid dates and version2 > version1 + const v1Date = parseDateFromAssumedString(version1Id)!; + const v2Date = parseDateFromAssumedString(version2Id)!; + assert.ok(v1Date instanceof Date); + assert.ok(v2Date instanceof Date); + assert.ok(v2Date > v1Date, "Second version should have later timestamp"); + + // List blobs with versions to verify both versions still exist + const blobVersions = await listBlobVersions(containerClient, blobName); + + // Should have 2 versions (both v1 and v2) + assert.strictEqual(blobVersions.length, 2, "Both versions should exist"); + const sortedVersions = blobVersions.sort( + (a, b) => new Date(a.versionId!).getTime() - new Date(b.versionId!).getTime() + ); + assert.strictEqual(sortedVersions[0].versionId, version1Id); + assert.strictEqual(sortedVersions[1].versionId, version2Id); + assert.strictEqual(sortedVersions[1].isCurrentVersion, true); + }); + + it("should NOT create new versions when appending blocks", async () => { + const content1 = "First append block"; + const content2 = "Second append block"; + + // Create append blob + const createResponse = await appendBlobClient.create(); + const originalVersionId = createResponse.versionId!; + + // Small delay to ensure different timestamps + await sleep(100); + + // Append first block (should NOT create new version) + await appendBlobClient.appendBlock(content1, content1.length); + // Note: appendBlock doesn't return versionId according to Azure docs + + await sleep(100); + + // Append second block (should NOT create new version) + await appendBlobClient.appendBlock(content2, content2.length); + + // Verify current blob properties - should still have same version + const properties = await appendBlobClient.getProperties(); + assert.strictEqual( + properties.versionId, + originalVersionId, + "Append operations should not create new versions" + ); + + // Verify content is concatenated + const download = await appendBlobClient.download(); + const content = await bodyToString(download, download.contentLength); + assert.strictEqual(content, content1 + content2); + + // List blobs with versions to verify only one version exists + const blobVersions = await listBlobVersions(containerClient, blobName); + + // Should still have 1 version + assert.strictEqual(blobVersions.length, 1, "Only one version should exist"); + assert.strictEqual(blobVersions[0].versionId, originalVersionId); + }); + + // ===================== GENERAL BLOB API TESTS ===================== + it("should return versionId when setting blob metadata with versioning enabled", async () => { + // First create an append blob + const createResponse = await appendBlobClient.create(); + const originalVersionId = createResponse.versionId!; + + await sleep(100); + + // Set metadata (this should create a new version) + const metadata = { key1: "value1", key2: "value2" }; + const setMetadataResponse = await appendBlobClient.setMetadata(metadata); + + // Verify versionId is returned and is different from original + assert.ok( + setMetadataResponse.versionId, + "versionId should be present in setMetadata response" + ); + assert.ok( + parseDateFromAssumedString(setMetadataResponse.versionId), + "versionId should be a valid ISO date string" + ); + assert.notStrictEqual( + setMetadataResponse.versionId, + originalVersionId, + "setMetadata should create new version" + ); + + // Verify the new version is later + const originalDate = parseDateFromAssumedString(originalVersionId)!; + const newDate = parseDateFromAssumedString(setMetadataResponse.versionId!)!; + assert.ok( + newDate > originalDate, + "New version should have later timestamp" + ); + + const listResponse = await listBlobVersions(containerClient, blobName); + assert.strictEqual(listResponse.length, 2, "There should be two versions now"); + assert.ok( + listResponse.some((b) => b.versionId === originalVersionId), + "Original version should still exist" + ); + assert.ok( + listResponse.some((b) => b.versionId === setMetadataResponse.versionId), + "New version should exist" + ); + }); + + it("should download specific blob version by versionId", async () => { + const content1 = "Version 1 content"; + const content2 = "Version 2 content"; + const metadata1 = { version: "1" }; + const metadata2 = { version: "2" }; + + // Create first version (append blob with content) + const create1 = await appendBlobClient.create({ metadata: metadata1 }); + await appendBlobClient.appendBlock(content1, content1.length); + const version1Id = create1.versionId!; + + await sleep(100); + + // Create second version (recreate append blob with different content) + const create2 = await appendBlobClient.create({ metadata: metadata2 }); + await appendBlobClient.appendBlock(content2, content2.length); + const version2Id = create2.versionId!; + + // Download current version (should be version 2) + const currentDownload = await appendBlobClient.download(); + const currentContent = await bodyToString( + currentDownload, + currentDownload.contentLength + ); + assert.strictEqual(currentContent, content2); + assert.strictEqual(currentDownload.metadata?.version, "2"); + + // Download specific version 1 + const version1Download = await appendBlobClient + .withVersion(version1Id) + .download(); + const version1Content = await bodyToString( + version1Download, + version1Download.contentLength + ); + assert.strictEqual(version1Content, content1); + assert.strictEqual(version1Download.metadata?.version, "1"); + assert.strictEqual(version1Download.versionId, version1Id); + + // Download specific version 2 + const version2Download = await appendBlobClient + .withVersion(version2Id) + .download(); + const version2Content = await bodyToString( + version2Download, + version2Download.contentLength + ); + assert.strictEqual(version2Content, content2); + assert.strictEqual(version2Download.metadata?.version, "2"); + assert.strictEqual(version2Download.versionId, version2Id); + }); + + it("should get properties for specific blob version by versionId", async () => { + const content = "Test content"; + const metadata1 = { version: "1", author: "user1" }; + const metadata2 = { version: "2", author: "user2" }; + + // Create first version (append blob with content) + const create1 = await appendBlobClient.create({ metadata: metadata1 }); + await appendBlobClient.appendBlock(content, content.length); + const version1Id = create1.versionId!; + + await sleep(100); + + // Create second version by setting metadata + const setMetadata = await appendBlobClient.setMetadata(metadata2); + const version2Id = setMetadata.versionId!; + + // Get properties for version 1 + const props1 = await appendBlobClient.withVersion(version1Id).getProperties(); + assert.strictEqual(props1.versionId, version1Id); + assert.strictEqual(props1.metadata?.version, "1"); + assert.strictEqual(props1.metadata?.author, "user1"); + + // Get properties for version 2 + const props2 = await appendBlobClient.withVersion(version2Id).getProperties(); + assert.strictEqual(props2.versionId, version2Id); + assert.strictEqual(props2.metadata?.version, "2"); + assert.strictEqual(props2.metadata?.author, "user2"); + + // Get properties for current version (should be version 2) + const currentProps = await appendBlobClient.getProperties(); + assert.strictEqual(currentProps.versionId, version2Id); + assert.strictEqual(currentProps.metadata?.version, "2"); + assert.strictEqual(currentProps.metadata?.author, "user2"); + + const listResponse = await listBlobVersions(containerClient, blobName); + assert.strictEqual(listResponse.length, 2, "There should be two versions now"); + assert.ok( + listResponse.some((b) => b.versionId === version1Id), + "Version 1 should still exist" + ); + assert.ok( + listResponse.some((b) => b.versionId === version2Id), + "Version 2 should still exist" + ); + }); + + it("should delete specific blob version by versionId", async () => { + const content1 = "Version 1 content"; + const content2 = "Version 2 content"; + const content3 = "Version 3 content"; + + // Create three versions (recreate append blob each time) + const create1 = await appendBlobClient.create(); + await appendBlobClient.appendBlock(content1, content1.length); + const version1Id = create1.versionId!; + + await sleep(100); + const create2 = await appendBlobClient.create(); + await appendBlobClient.appendBlock(content2, content2.length); + const version2Id = create2.versionId!; + + await sleep(100); + const create3 = await appendBlobClient.create(); + await appendBlobClient.appendBlock(content3, content3.length); + const version3Id = create3.versionId!; + + // Delete version 2 specifically + await appendBlobClient.withVersion(version2Id).delete(); + + // Verify current version (version 3) still exists + const currentDownload = await appendBlobClient.download(); + const currentContent = await bodyToString( + currentDownload, + currentDownload.contentLength + ); + assert.strictEqual(currentContent, content3); + assert.strictEqual(currentDownload.versionId, version3Id); + + // Verify version 1 still exists + const version1Download = await appendBlobClient + .withVersion(version1Id) + .download(); + const version1Content = await bodyToString( + version1Download, + version1Download.contentLength + ); + assert.strictEqual(version1Content, content1); + + // Verify version 2 is deleted + try { + await appendBlobClient.withVersion(version2Id).download(); + assert.fail("Should have thrown error for deleted version"); + } catch (error: any) { + assert.ok(error.statusCode === 404 || error.code === "BlobNotFound"); + } + }); + + it("should set and get tags for specific blob version", async () => { + const content = "Test content for tags"; + const tags1: Tags = { environment: "dev", version: "1.0" }; + const tags2: Tags = { environment: "prod", version: "2.0" }; + + // Create first version with tags (append blob) + const create1 = await appendBlobClient.create({ tags: tags1 }); + await appendBlobClient.appendBlock(content, content.length); + const version1Id = create1.versionId!; + + await sleep(100); + + // Create second version (recreate append blob with different tags) + const create2 = await appendBlobClient.create({ tags: tags2 }); + await appendBlobClient.appendBlock( + content + " updated", + (content + " updated").length + ); + const version2Id = create2.versionId!; + + // Get tags for version 1 + const version1Tags = await appendBlobClient.withVersion(version1Id).getTags(); + assert.deepStrictEqual(version1Tags.tags, tags1); + + // Get tags for version 2 + const version2Tags = await appendBlobClient.withVersion(version2Id).getTags(); + assert.deepStrictEqual(version2Tags.tags, tags2); + + // Get tags for current version (should be version 2) + const currentTags = await appendBlobClient.getTags(); + assert.deepStrictEqual(currentTags.tags, tags2); + }); + + it("should set tags on specific blob version", async () => { + const content = "Test content"; + const originalTags: Tags = { original: "true" }; + const newTags: Tags = { updated: "true", version: "modified" }; + + // Create append blob with original tags + const create = await appendBlobClient.create({ tags: originalTags }); + await appendBlobClient.appendBlock(content, content.length); + const versionId = create.versionId!; + + // Set new tags on the specific version + await appendBlobClient.withVersion(versionId).setTags(newTags); + + // Verify tags were updated on that version + const updatedTags = await appendBlobClient.withVersion(versionId).getTags(); + assert.deepStrictEqual(updatedTags.tags, newTags); + + // Verify current version also has the updated tags (since it's the same version) + const currentTags = await appendBlobClient.getTags(); + assert.deepStrictEqual(currentTags.tags, newTags); + }); + + it("should list blobs with version information", async () => { + const blobName1 = getUniqueName("blob1"); + const blobName2 = getUniqueName("blob2"); + const content1 = "Content for blob 1"; + const content2 = "Content for blob 2"; + + // Create append blobs with multiple versions + const blob1Client = containerClient.getAppendBlobClient(blobName1); + const blob2Client = containerClient.getAppendBlobClient(blobName2); + + const create1v1 = await blob1Client.create(); + await blob1Client.appendBlock(content1, content1.length); + await sleep(100); + const create1v2 = await blob1Client.create(); + await blob1Client.appendBlock(content1 + " v2", (content1 + " v2").length); + await sleep(100); + const create2v1 = await blob2Client.create(); + await blob2Client.appendBlock(content2, content2.length); + + // List blobs with versions + const listResponse = containerClient.listBlobsFlat({ + includeVersions: true + }); + const blobs = []; + for await (const blob of listResponse) { + blobs.push(blob); + } + + // Should have 3 versions total (2 for blob1, 1 for blob2) + assert.strictEqual(blobs.length, 3); + + // Find blob1 versions + const blob1Versions = blobs + .filter((b) => b.name === blobName1) + .sort( + (a, b) => + new Date(a.versionId!).getTime() - new Date(b.versionId!).getTime() + ); + assert.strictEqual(blob1Versions.length, 2); + assert.strictEqual(blob1Versions[0].versionId, create1v1.versionId); + assert.strictEqual(blob1Versions[1].versionId, create1v2.versionId); + assert.strictEqual(blob1Versions[0].isCurrentVersion, undefined); + assert.strictEqual(blob1Versions[1].isCurrentVersion, true); + + // Find blob2 version + const blob2Versions = blobs.filter((b) => b.name === blobName2); + assert.strictEqual(blob2Versions.length, 1); + assert.strictEqual(blob2Versions[0].versionId, create2v1.versionId); + assert.strictEqual(blob2Versions[0].isCurrentVersion, true); + }); + + it("should handle blob versioning with delete operations", async () => { + const content1 = "Version 1"; + const content2 = "Version 2"; + + // Create two versions (recreate append blob each time) + const create1 = await appendBlobClient.create(); + await appendBlobClient.appendBlock(content1, content1.length); + const version1Id = create1.versionId!; + + await sleep(100); + const create2 = await appendBlobClient.create(); + await appendBlobClient.appendBlock(content2, content2.length); + const version2Id = create2.versionId!; + + // Delete current version (without specifying version) + await appendBlobClient.delete(); + + // Current version should no longer exist + try { + await appendBlobClient.download(); + assert.fail("Should have thrown error for deleted current blob"); + } catch (error: any) { + assert.ok(error.statusCode === 404 || error.code === "BlobNotFound"); + } + + // But specific versions should still be accessible + const version1Download = await appendBlobClient + .withVersion(version1Id) + .download(); + const version1Content = await bodyToString( + version1Download, + version1Download.contentLength + ); + assert.strictEqual(version1Content, content1); + + const version2Download = await appendBlobClient + .withVersion(version2Id) + .download(); + const version2Content = await bodyToString( + version2Download, + version2Download.contentLength + ); + assert.strictEqual(version2Content, content2); + }); + + it("should validate versionId format in API calls", async () => { + const content = "Test content"; + await appendBlobClient.create(); + await appendBlobClient.appendBlock(content, content.length); + + // Test with invalid versionId format + const invalidVersionIds = [ + "invalid-date", + "2024-13-01T00:00:00.000Z", // Invalid month + "not-a-date-at-all", + "2024/01/01 00:00:00" // Wrong format + ]; + + for (const invalidVersionId of invalidVersionIds) { + try { + await appendBlobClient.withVersion(invalidVersionId).download(); + assert.fail( + `Should have thrown error for invalid versionId: ${invalidVersionId}` + ); + } catch (error: any) { + // Should throw an error for invalid versionId format + assert.ok( + error.statusCode === 400 || + error.code === "InvalidInput" || + error.statusCode === 404 + ); + } + } + }); + + it("should create snapshot and return versionId when versioning enabled", async () => { + const content = "Content for snapshot test"; + + // Create initial append blob + const create = await appendBlobClient.create(); + await appendBlobClient.appendBlock(content, content.length); + const originalVersionId = create.versionId!; + + await sleep(100); + + // Create snapshot (should also create new version) + const snapshotResponse = await appendBlobClient.createSnapshot(); + + // Verify snapshot properties + assert.ok( + snapshotResponse.snapshot, + "snapshot identifier should be present" + ); + assert.ok( + snapshotResponse.versionId, + "versionId should be present in snapshot response" + ); + assert.ok( + parseDateFromAssumedString(snapshotResponse.versionId), + "versionId should be valid date" + ); + + // New version should be different from original + assert.notStrictEqual(snapshotResponse.versionId, originalVersionId); + + // Verify chronological order + const originalDate = parseDateFromAssumedString(originalVersionId)!; + const snapshotDate = parseDateFromAssumedString( + snapshotResponse.versionId! + )!; + assert.ok( + snapshotDate > originalDate, + "Snapshot should create later version" + ); + }); +}); diff --git a/tests/blob/apis/blob.test.ts b/tests/blob/apis/blob.test.ts index 2dcaf37f4..c79a1582f 100644 --- a/tests/blob/apis/blob.test.ts +++ b/tests/blob/apis/blob.test.ts @@ -1603,6 +1603,126 @@ describe("BlobAPIs", () => { assert.fail(); }); + it("startCopyFromURL should fail with invalid snapshot date in source @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getBlockBlobClient(sourceBlob); + const destBlobClient = containerClient.getBlockBlobClient(destBlob); + + await sourceBlobClient.upload("hello", 5); + + // Try to copy with invalid snapshot date + const invalidSnapshotUrl = `${sourceBlobClient.url}?snapshot=invalid-date`; + try { + await destBlobClient.beginCopyFromURL(invalidSnapshotUrl); + assert.fail("Should have thrown error"); + } catch (error: any) { + assert.strictEqual(error.statusCode, 400); + assert.strictEqual(error.code, "InvalidQueryParameterValue"); + } + }); + + it("startCopyFromURL should fail with invalid versionId date in source @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getBlockBlobClient(sourceBlob); + const destBlobClient = containerClient.getBlockBlobClient(destBlob); + + await sourceBlobClient.upload("hello", 5); + + // Try to copy with invalid versionId date + const invalidVersionIdUrl = `${sourceBlobClient.url}?versionid=not-a-date`; + try { + await destBlobClient.beginCopyFromURL(invalidVersionIdUrl); + assert.fail("Should have thrown error"); + } catch (error: any) { + assert.strictEqual(error.statusCode, 400); + assert.strictEqual(error.code, "InvalidQueryParameterValue"); + } + }); + + it("startCopyFromURL should fail when both snapshot and versionId are present in source @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getBlockBlobClient(sourceBlob); + const destBlobClient = containerClient.getBlockBlobClient(destBlob); + + await sourceBlobClient.upload("hello", 5); + + // Try to copy with both snapshot and versionId + const bothParamsUrl = `${sourceBlobClient.url}?snapshot=2021-01-01T00:00:00.0000000Z&versionid=2021-01-02T00:00:00.0000000Z`; + try { + await destBlobClient.beginCopyFromURL(bothParamsUrl); + assert.fail("Should have thrown error"); + } catch (error: any) { + assert.strictEqual(error.statusCode, 400); + assert.strictEqual(error.code, "MutuallyExclusiveQueryParameters"); + } + }); + + it("syncCopyFromURL should fail with invalid snapshot date in source @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getBlockBlobClient(sourceBlob); + const destBlobClient = containerClient.getBlockBlobClient(destBlob); + + await sourceBlobClient.upload("hello", 5); + + // Try to copy with invalid snapshot date + const invalidSnapshotUrl = `${sourceBlobClient.url}?snapshot=bad-date-format`; + try { + await destBlobClient.syncCopyFromURL(invalidSnapshotUrl); + assert.fail("Should have thrown error"); + } catch (error: any) { + assert.strictEqual(error.statusCode, 400); + assert.strictEqual(error.code, "InvalidQueryParameterValue"); + } + }); + + it("syncCopyFromURL should fail with invalid versionId date in source @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getBlockBlobClient(sourceBlob); + const destBlobClient = containerClient.getBlockBlobClient(destBlob); + + await sourceBlobClient.upload("hello", 5); + + // Try to copy with invalid versionId date + const invalidVersionIdUrl = `${sourceBlobClient.url}?versionid=12345`; + try { + await destBlobClient.syncCopyFromURL(invalidVersionIdUrl); + assert.fail("Should have thrown error"); + } catch (error: any) { + assert.strictEqual(error.statusCode, 400); + assert.strictEqual(error.code, "InvalidQueryParameterValue"); + } + }); + + it("syncCopyFromURL should fail when both snapshot and versionId are present in source @loki @sql", async () => { + const sourceBlob = getUniqueName("blob"); + const destBlob = getUniqueName("blob"); + + const sourceBlobClient = containerClient.getBlockBlobClient(sourceBlob); + const destBlobClient = containerClient.getBlockBlobClient(destBlob); + + await sourceBlobClient.upload("hello", 5); + + // Try to copy with both snapshot and versionId + const bothParamsUrl = `${sourceBlobClient.url}?snapshot=2021-01-01T00:00:00.0000000Z&versionid=2021-01-02T00:00:00.0000000Z`; + try { + await destBlobClient.syncCopyFromURL(bothParamsUrl); + assert.fail("Should have thrown error"); + } catch (error: any) { + assert.strictEqual(error.statusCode, 400); + assert.strictEqual(error.code, "MutuallyExclusiveQueryParameters"); + } + }); + it("Synchronized copy blob should work @loki", async () => { const sourceBlob = getUniqueName("blob"); const destBlob = getUniqueName("blob"); @@ -2663,4 +2783,481 @@ describe("BlobAPIs", () => { it("UpdateSequenceNumber a Leased page blob, if input LeaseId matches, will success @loki @sql", async () => { // TODO: implement the case later }); -}); + it("download should fail with 400 when both snapshot and versionId are provided @loki @sql", async () => { + try { + // Try to download with both snapshot and versionId - should fail + await blobClient + .withVersion("randomString") + .withSnapshot("randomString") + .download(); + assert.fail( + "Should have thrown error when both snapshot and versionId provided" + ); + } catch (error: any) { + assert.strictEqual(error.statusCode, 400); + } + }); + + it("getProperties should fail with 400 when both snapshot and versionId are provided @loki @sql", async () => { + try { + // Try to get properties with both snapshot and versionId - should fail + await blobClient + .withVersion("randomString") + .withSnapshot("randomString") + .getProperties(); + assert.fail( + "Should have thrown error when both snapshot and versionId provided" + ); + } catch (error: any) { + assert.strictEqual(error.statusCode, 400); + } + }); + + it("delete should fail with 400 when both snapshot and versionId are provided @loki @sql", async () => { + try { + // Try to delete with both snapshot and versionId - should fail + await blobClient + .withVersion("randomString") + .withSnapshot("randomString") + .delete(); + assert.fail( + "Should have thrown error when both snapshot and versionId provided" + ); + } catch (error: any) { + assert.strictEqual(error.statusCode, 400); + } + }); + + it("setAccessTier should fail with 400 when both snapshot and versionId are provided @loki @sql", async () => { + try { + // Try to set access tier with both snapshot and versionId - should fail + await blobClient + .withVersion("randomString") + .withSnapshot("randomString") + .setAccessTier("Cool"); + assert.fail( + "Should have thrown error when both snapshot and versionId provided" + ); + } catch (error: any) { + assert.strictEqual(error.statusCode, 400); + } + }); + + it("getTags should fail with 400 when both snapshot and versionId are provided @loki @sql", async () => { + try { + // Try to get tags with both snapshot and versionId - should fail + await blobClient + .withVersion("randomString") + .withSnapshot("randomString") + .getTags(); + assert.fail( + "Should have thrown error when both snapshot and versionId provided" + ); + } catch (error: any) { + assert.strictEqual(error.statusCode, 400); + } + }); + + it("setTags should fail with 400 when both snapshot and versionId are provided @loki @sql", async () => { + const tags = { tag1: "value1", tag2: "value2" }; + try { + // Try to set tags with both snapshot and versionId - should fail + await blobClient + .withVersion("randomString") + .withSnapshot("randomString") + .setTags(tags); + assert.fail( + "Should have thrown error when both snapshot and versionId provided" + ); + } catch (error: any) { + assert.strictEqual(error.statusCode, 400); + } + }); + + // Tests for invalid versionId formats + it("download should fail with 400 when invalid versionId format is provided @loki @sql", async () => { + const invalidVersionIds = [ + "not-a-date", + "January 1, 2023", + "2023-01-01", + "1672531200", // epoch as string + "2023/01/01", + "01-01-2023", + "2023-13-40T25:70:70.000Z", // invalid date components + "2023-01-01T12:34:56.000Z", // version IDs require 7 fractional digits + "random-string-123", + "2023-01-01T12:34:56", // missing Z and fractional seconds + "abc123def456" + ]; + + for (const invalidVersionId of invalidVersionIds) { + try { + await blobClient.withVersion(invalidVersionId).download(); + assert.fail( + `Should have thrown error for invalid versionId: ${invalidVersionId}` + ); + } catch (error: any) { + assert.strictEqual( + error.statusCode, + 400, + `Failed for versionId: ${invalidVersionId}` + ); + } + } + }); + + it("getProperties should fail with 400 when invalid versionId format is provided @loki @sql", async () => { + const invalidVersionIds = [ + "not-a-date", + "January 1, 2023", + "2023-01-01", + "1672531200", // epoch as string + "2023/01/01", + "01-01-2023", + "2023-13-40T25:70:70.000Z", // invalid date components + "2023-01-01T12:34:56.000Z", // version IDs require 7 fractional digits + "random-string-123", + "2023-01-01T12:34:56", // missing Z and fractional seconds + "abc123def456" + ]; + + for (const invalidVersionId of invalidVersionIds) { + try { + await blobClient.withVersion(invalidVersionId).getProperties(); + assert.fail( + `Should have thrown error for invalid versionId: ${invalidVersionId}` + ); + } catch (error: any) { + assert.strictEqual( + error.statusCode, + 400, + `Failed for versionId: ${invalidVersionId}` + ); + } + } + }); + + it("delete should fail with 400 when invalid versionId format is provided @loki @sql", async () => { + const invalidVersionIds = [ + "not-a-date", + "January 1, 2023", + "2023-01-01", + "1672531200", // epoch as string + "2023/01/01", + "01-01-2023", + "2023-13-40T25:70:70.000Z", // invalid date components + "2023-01-01T12:34:56.000Z", // version IDs require 7 fractional digits + "random-string-123", + "2023-01-01T12:34:56", // missing Z and fractional seconds + "abc123def456" + ]; + + for (const invalidVersionId of invalidVersionIds) { + try { + await blobClient.withVersion(invalidVersionId).delete(); + assert.fail( + `Should have thrown error for invalid versionId: ${invalidVersionId}` + ); + } catch (error: any) { + assert.strictEqual( + error.statusCode, + 400, + `Failed for versionId: ${invalidVersionId}` + ); + } + } + }); + + it("setAccessTier should fail with 400 when invalid versionId format is provided @loki @sql", async () => { + const invalidVersionIds = [ + "not-a-date", + "January 1, 2023", + "2023-01-01", + "1672531200", // epoch as string + "2023/01/01", + "01-01-2023", + "2023-13-40T25:70:70.000Z", // invalid date components + "2023-01-01T12:34:56.000Z", // version IDs require 7 fractional digits + "random-string-123", + "2023-01-01T12:34:56", // missing Z and fractional seconds + "abc123def456" + ]; + + for (const invalidVersionId of invalidVersionIds) { + try { + await blobClient.withVersion(invalidVersionId).setAccessTier("Cool"); + assert.fail( + `Should have thrown error for invalid versionId: ${invalidVersionId}` + ); + } catch (error: any) { + assert.strictEqual( + error.statusCode, + 400, + `Failed for versionId: ${invalidVersionId}` + ); + } + } + }); + + it("getTags should fail with 400 when invalid versionId format is provided @loki @sql", async () => { + const invalidVersionIds = [ + "not-a-date", + "January 1, 2023", + "2023-01-01", + "1672531200", // epoch as string + "2023/01/01", + "01-01-2023", + "2023-13-40T25:70:70.000Z", // invalid date components + "2023-01-01T12:34:56.000Z", // version IDs require 7 fractional digits + "random-string-123", + "2023-01-01T12:34:56", // missing Z and fractional seconds + "abc123def456" + ]; + + for (const invalidVersionId of invalidVersionIds) { + try { + await blobClient.withVersion(invalidVersionId).getTags(); + assert.fail( + `Should have thrown error for invalid versionId: ${invalidVersionId}` + ); + } catch (error: any) { + assert.strictEqual( + error.statusCode, + 400, + `Failed for versionId: ${invalidVersionId}` + ); + } + } + }); + + it("setTags should fail with 400 when invalid versionId format is provided @loki @sql", async () => { + const tags = { tag1: "value1", tag2: "value2" }; + const invalidVersionIds = [ + "not-a-date", + "January 1, 2023", + "2023-01-01", + "1672531200", // epoch as string + "2023/01/01", + "01-01-2023", + "2023-13-40T25:70:70.000Z", // invalid date components + "2023-01-01T12:34:56.000Z", // version IDs require 7 fractional digits + "random-string-123", + "2023-01-01T12:34:56", // missing Z and fractional seconds + "abc123def456" + ]; + + for (const invalidVersionId of invalidVersionIds) { + try { + await blobClient.withVersion(invalidVersionId).setTags(tags); + assert.fail( + `Should have thrown error for invalid versionId: ${invalidVersionId}` + ); + } catch (error: any) { + assert.strictEqual( + error.statusCode, + 400, + `Failed for versionId: ${invalidVersionId}` + ); + } + } + }); + + // Tests for valid versionId formats + // These tests ensure that valid versionId formats do not result in 400 errors + // even if the version does not exist (should return 404 instead) + it("download return not found with valid versionId format @loki @sql", async () => { + const validVersionId = "2025-08-25T04:12:34.1195858Z"; + try { + await blobClient.withVersion(validVersionId).download(); + assert.fail(); + } catch (error: any) { + // Should not be a 400 error for format issues + assert.notStrictEqual( + error.statusCode, + 400, + "Should not fail with 400 for valid format" + ); + assert.strictEqual( + error.statusCode, + 404); + } + }); + + it("getProperties should work with valid versionId format @loki @sql", async () => { + const validVersionId = "2025-08-25T04:12:34.1195858Z"; + try { + await blobClient.withVersion(validVersionId).getProperties(); + assert.fail(); + } catch (error: any) { + // Should not be a 400 error for format issues + assert.notStrictEqual( + error.statusCode, + 400, + "Should not fail with 400 for valid format" + ); + assert.strictEqual( + error.statusCode, + 404); + } + }); + + it("delete should work with valid versionId format @loki @sql", async () => { + const validVersionId = "2025-08-25T04:12:34.1195858Z"; + try { + await blobClient.withVersion(validVersionId).delete(); + assert.fail(); + } catch (error: any) { + // Should not be a 400 error for format issues + assert.notStrictEqual( + error.statusCode, + 400, + "Should not fail with 400 for valid format" + ); + assert.strictEqual( + error.statusCode, + 404); + } + }); + + it("setAccessTier should work with valid versionId format @loki @sql", async () => { + const validVersionId = "2025-08-25T04:12:34.1195858Z"; + try { + await blobClient.withVersion(validVersionId).setAccessTier("Cool"); + assert.fail(); + } catch (error: any) { + // Should not be a 400 error for format issues + assert.notStrictEqual( + error.statusCode, + 400, + "Should not fail with 400 for valid format" + ); + assert.strictEqual( + error.statusCode, + 404); + } + }); + + it("getTags should work with valid versionId format @loki @sql", async () => { + const validVersionId = "2025-08-25T04:12:34.1195858Z"; + try { + await blobClient.withVersion(validVersionId).getTags(); + // If we reach here, the format was accepted (even if blob version doesn't exist) + assert.ok(true); + } catch (error: any) { + // Should not be a 400 error for format issues + assert.notStrictEqual( + error.statusCode, + 400, + "Should not fail with 400 for valid format" + ); + assert.strictEqual( + error.statusCode, + 404); + } + }); + + it("setTags should work with valid versionId format @loki @sql", async () => { + const tags = { tag1: "value1", tag2: "value2" }; + const validVersionId = "2025-08-25T04:12:34.1195858Z"; + try { + await blobClient.withVersion(validVersionId).setTags(tags); + assert.fail(); + } catch (error: any) { + // Should not be a 400 error for format issues + assert.notStrictEqual( + error.statusCode, + 400, + "Should not fail with 400 for valid format" + ); + assert.strictEqual( + error.statusCode, + 404); + } + }); + + // BlobHandler API Tests - Testing all APIs return versionId as undefined + describe("BlobHandler API versionId Tests", () => { + it("download should return versionId as undefined @loki @sql", async () => { + const deleteBlockBlobClient = blobClient.getBlockBlobClient(); + const uploadResponse = await deleteBlockBlobClient.upload( + "test content", + 12 + ); + assert.strictEqual(uploadResponse.versionId, undefined); + const result = await blobClient.download(); + assert.strictEqual(result.versionId, undefined); + }); + + it("download should return versionId as undefined when setting versionId to emptystring @loki @sql", async () => { + const deleteBlockBlobClient = blobClient.getBlockBlobClient(); + const uploadResponse = await deleteBlockBlobClient.upload( + "test content", + 12 + ); + assert.strictEqual(uploadResponse.versionId, undefined); + const result = await blobClient.withVersion("").download(); + assert.strictEqual(result.versionId, undefined); + }); + + it("getProperties should return versionId as undefined @loki @sql", async () => { + const deleteBlockBlobClient = blobClient.getBlockBlobClient(); + const uploadResponse = await deleteBlockBlobClient.upload( + "test content", + 12 + ); + assert.strictEqual(uploadResponse.versionId, undefined); + const result = await blobClient.getProperties(); + assert.strictEqual(result.versionId, undefined); + }); + + it("setMetadata should return versionId as undefined @loki @sql", async () => { + const deleteBlockBlobClient = blobClient.getBlockBlobClient(); + const uploadResponse = await deleteBlockBlobClient.upload( + "test content", + 12 + ); + assert.strictEqual(uploadResponse.versionId, undefined); + const metadata = { key1: "value1", key2: "value2" }; + const result = await blobClient.setMetadata(metadata); + assert.strictEqual(result.versionId, undefined); + }); + + it("createSnapshot should return versionId as undefined @loki @sql", async () => { + const deleteBlockBlobClient = blobClient.getBlockBlobClient(); + const uploadResponse = await deleteBlockBlobClient.upload( + "test content", + 12 + ); + assert.strictEqual(uploadResponse.versionId, undefined); + const result = await blobClient.createSnapshot(); + assert.strictEqual(result.versionId, undefined); + }); + + it("copyFromURL should return versionId as undefined @loki @sql", async () => { + const deleteBlockBlobClient = blobClient.getBlockBlobClient(); + const uploadResponse = await deleteBlockBlobClient.upload( + "test content", + 12 + ); + assert.strictEqual(uploadResponse.versionId, undefined); + const destBlobName = getUniqueName("destblob"); + const destBlobClient = containerClient.getBlobClient(destBlobName); + const result = await destBlobClient.syncCopyFromURL(blobClient.url); + assert.strictEqual(result.versionId, undefined); + }); + + it("beginCopyFromURL should return versionId as undefined @loki @sql", async () => { + const deleteBlockBlobClient = blobClient.getBlockBlobClient(); + const uploadResponse = await deleteBlockBlobClient.upload( + "test content", + 12 + ); + assert.strictEqual(uploadResponse.versionId, undefined); + const destBlobName = getUniqueName("destblob"); + const destBlobClient = containerClient.getBlobClient(destBlobName); + const result = await destBlobClient.beginCopyFromURL(blobClient.url); + const copyPoller = await result.pollUntilDone(); + assert.strictEqual(copyPoller.versionId, undefined); + }); + }); +}); \ No newline at end of file diff --git a/tests/blob/apis/blob.versioning.contract.test.ts b/tests/blob/apis/blob.versioning.contract.test.ts new file mode 100644 index 000000000..8d626b6fe --- /dev/null +++ b/tests/blob/apis/blob.versioning.contract.test.ts @@ -0,0 +1,582 @@ +import { + BlobServiceClient, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-blob"; +import * as assert from "assert"; + +import { AccountModel } from "../../../src/common/account/AccountModel"; +import { configLogger } from "../../../src/common/Logger"; +import LokiAccountModelStore from "../../../src/common/account/LokiAccountModelStore"; +import BlobTestServerFactory from "../../BlobTestServerFactory"; +import { + bodyToString, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getTestServerBaseURL, + getUniqueName +} from "../../testutils"; + +configLogger(false); + +const RUN_VERSIONING_CONTRACT_TESTS = + process.env.AZURITE_RUN_VERSIONING_CONTRACT_TESTS === "1"; +const contractDescribe = RUN_VERSIONING_CONTRACT_TESTS + ? describe + : describe.skip; + +const accountModel: AccountModel = { + key: EMULATOR_ACCOUNT_NAME, + isBlobVersioningEnabled: true +}; +const accountModels = new Map([ + [EMULATOR_ACCOUNT_NAME, accountModel] +]); + +contractDescribe("Blob Versioning Contract", () => { + const factory = new BlobTestServerFactory(); + const accountModelStore = new LokiAccountModelStore("", true, accountModels); + const server = factory.createServer( + false, + false, + false, + undefined, + accountModelStore + ); + const serviceClient = new BlobServiceClient( + getTestServerBaseURL(server), + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + keepAliveOptions: { enable: false } + } + ) + ); + + let containerName = getUniqueName("container"); + let containerClient = serviceClient.getContainerClient(containerName); + let blobName = getUniqueName("blob"); + let blockBlobClient = containerClient.getBlockBlobClient(blobName); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async () => { + containerName = getUniqueName("container"); + containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + blobName = getUniqueName("blob"); + blockBlobClient = containerClient.getBlockBlobClient(blobName); + }); + + afterEach(async () => { + await containerClient.delete(); + }); + + async function listVersions(name: string) { + const items = []; + for await (const item of containerClient.listBlobsFlat({ + includeVersions: true + })) { + if (item.name === name) { + items.push(item); + } + } + return items; + } + + async function pageThroughVersions(pageSize: number) { + const seen: string[] = []; + let continuationToken: string | undefined; + let pages = 0; + + do { + const result = await containerClient + .listBlobsFlat({ includeVersions: true }) + .byPage({ maxPageSize: pageSize, continuationToken }) + .next(); + if (result.done) { + break; + } + + for (const item of result.value.segment.blobItems) { + seen.push(`${item.name}@${item.versionId}`); + } + continuationToken = result.value.continuationToken; + pages++; + assert.ok(pages < 50, "Listing did not terminate"); + } while (continuationToken); + + return { seen, pages }; + } + + it("returns a timestamp version ID on upload @versioning-contract", async () => { + const upload = await blockBlobClient.upload("version1", 8); + + assert.notStrictEqual(upload.versionId, undefined); + assert.ok( + /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{7}Z$/.test(upload.versionId!) + ); + + const versions = await listVersions(blobName); + assert.strictEqual(versions.length, 1); + assert.strictEqual(versions[0].versionId, upload.versionId); + assert.strictEqual(versions[0].isCurrentVersion, true); + }); + + it("preserves overwritten content as a previous version @versioning-contract", async () => { + const first = await blockBlobClient.upload("version1", 8); + const second = await blockBlobClient.upload("version2", 8); + + assert.notStrictEqual(first.versionId, second.versionId); + assert.strictEqual( + await bodyToString(await blockBlobClient.download(), 8), + "version2" + ); + assert.strictEqual( + await bodyToString( + await blockBlobClient.withVersion(first.versionId!).download(), + 8 + ), + "version1" + ); + + const versions = await listVersions(blobName); + assert.deepStrictEqual( + versions.map((version) => version.versionId), + [first.versionId, second.versionId] + ); + assert.notStrictEqual(versions[0].isCurrentVersion, true); + assert.strictEqual(versions[1].isCurrentVersion, true); + }); + + it("hides previous versions unless requested @versioning-contract", async () => { + await blockBlobClient.upload("version1", 8); + await blockBlobClient.upload("version2", 8); + + const currentItems = []; + for await (const item of containerClient.listBlobsFlat()) { + currentItems.push(item); + } + + assert.strictEqual(currentItems.length, 1); + assert.strictEqual(currentItems[0].name, blobName); + assert.strictEqual((await listVersions(blobName)).length, 2); + }); + + it("reports current-version properties correctly @versioning-contract", async () => { + const first = await blockBlobClient.upload("version1", 8); + const second = await blockBlobClient.upload("version2", 8); + + const current = await blockBlobClient.getProperties(); + assert.strictEqual(current.versionId, second.versionId); + assert.strictEqual(current.isCurrentVersion, true); + + const currentByVersion = await blockBlobClient + .withVersion(second.versionId!) + .getProperties(); + assert.strictEqual(currentByVersion.versionId, second.versionId); + assert.strictEqual(currentByVersion.isCurrentVersion, true); + + const previous = await blockBlobClient + .withVersion(first.versionId!) + .getProperties(); + assert.strictEqual(previous.versionId, first.versionId); + assert.notStrictEqual(previous.isCurrentVersion, true); + }); + + it("returns 404 for an unknown version @versioning-contract", async () => { + await blockBlobClient.upload("version1", 8); + + let error; + try { + await blockBlobClient + .withVersion("2020-01-01T00:00:00.0000000Z") + .download(); + } catch (err) { + error = err; + } + + assert.strictEqual((error as any)?.statusCode, 404); + }); + + it("deletes one previous version without affecting others @versioning-contract", async () => { + const first = await blockBlobClient.upload("version1", 8); + const second = await blockBlobClient.upload("version2", 8); + + await blockBlobClient.withVersion(first.versionId!).delete(); + + const versions = await listVersions(blobName); + assert.strictEqual(versions.length, 1); + assert.strictEqual(versions[0].versionId, second.versionId); + assert.strictEqual( + await bodyToString(await blockBlobClient.download(), 8), + "version2" + ); + }); + + it("turns the current version into a previous version on delete @versioning-contract", async () => { + const first = await blockBlobClient.upload("version1", 8); + const second = await blockBlobClient.upload("version2", 8); + + await blockBlobClient.delete(); + + const versions = await listVersions(blobName); + assert.deepStrictEqual( + versions.map((version) => version.versionId), + [first.versionId, second.versionId] + ); + for (const version of versions) { + assert.notStrictEqual(version.isCurrentVersion, true); + assert.notStrictEqual(version.hasVersionsOnly, true); + } + assert.strictEqual(await blockBlobClient.exists(), false); + assert.strictEqual( + await bodyToString( + await blockBlobClient.withVersion(first.versionId!).download(), + 8 + ), + "version1" + ); + assert.strictEqual( + await bodyToString( + await blockBlobClient.withVersion(second.versionId!).download(), + 8 + ), + "version2" + ); + }); + + it("creates a new current version after delete @versioning-contract", async () => { + const first = await blockBlobClient.upload("version1", 8); + const second = await blockBlobClient.upload("version2", 8); + await blockBlobClient.delete(); + const third = await blockBlobClient.upload("version3", 8); + + const versions = await listVersions(blobName); + assert.deepStrictEqual( + versions.map((version) => version.versionId), + [first.versionId, second.versionId, third.versionId] + ); + assert.strictEqual(versions[2].isCurrentVersion, true); + }); + + it("retains versions when deleting snapshots with the blob @versioning-contract", async () => { + const first = await blockBlobClient.upload("version1", 8); + const second = await blockBlobClient.upload("version2", 8); + const snapshot = await blockBlobClient.createSnapshot(); + + await blockBlobClient.delete({ deleteSnapshots: "include" }); + + assert.deepStrictEqual( + (await listVersions(blobName)).map((version) => version.versionId), + [first.versionId, second.versionId, snapshot.versionId] + ); + + let snapshotCount = 0; + for await (const item of containerClient.listBlobsFlat({ + includeSnapshots: true + })) { + if (item.name === blobName && item.snapshot) { + snapshotCount++; + } + } + assert.strictEqual(snapshotCount, 0); + }); + + it("verifies current-version deletion by version ID @versioning-contract", async () => { + const only = await blockBlobClient.upload("version1", 8); + + let error; + try { + await blockBlobClient.withVersion(only.versionId!).delete(); + } catch (err) { + error = err; + } + + assert.strictEqual((error as any)?.statusCode, 403); + assert.strictEqual((error as any)?.code, "OperationNotAllowedOnRootBlob"); + }); + + it("restores by copying a previous version @versioning-contract", async () => { + const first = await blockBlobClient.upload("version1", 8); + await blockBlobClient.upload("version2", 8); + + const poller = await blockBlobClient.beginCopyFromURL( + blockBlobClient.withVersion(first.versionId!).url + ); + await poller.pollUntilDone(); + + assert.strictEqual( + await bodyToString(await blockBlobClient.download(), 8), + "version1" + ); + }); + + it("rejects snapshot and version ID together @versioning-contract", async () => { + const upload = await blockBlobClient.upload("version1", 8); + const snapshot = await blockBlobClient.createSnapshot(); + + let error; + try { + await blockBlobClient + .withSnapshot(snapshot.snapshot!) + .withVersion(upload.versionId!) + .download(); + } catch (err) { + error = err; + } + + assert.strictEqual((error as any)?.statusCode, 400); + assert.strictEqual( + (error as any)?.code, + "MutuallyExclusiveQueryParameters" + ); + }); + + it("creates a version when committing a block list @versioning-contract", async () => { + const first = await blockBlobClient.upload("version1", 8); + const blockId = Buffer.from("block-1").toString("base64"); + await blockBlobClient.stageBlock(blockId, "version2", 8); + const commit = await blockBlobClient.commitBlockList([blockId]); + + assert.notStrictEqual(commit.versionId, undefined); + assert.notStrictEqual(commit.versionId, first.versionId); + assert.deepStrictEqual( + (await listVersions(blobName)).map((version) => version.versionId), + [first.versionId, commit.versionId] + ); + }); + + it("paginates through all versions of one blob @versioning-contract", async () => { + const expected: string[] = []; + for (let i = 0; i < 5; i++) { + const upload = await blockBlobClient.upload(`v${i}`, 2); + expected.push(`${blobName}@${upload.versionId}`); + } + + const { seen, pages } = await pageThroughVersions(2); + assert.ok(pages > 1); + assert.deepStrictEqual(seen, expected); + }); + + it("paginates through versions across blobs @versioning-contract", async () => { + const expected: string[] = []; + for (const suffix of ["a", "b", "c"]) { + const name = `${blobName}-${suffix}`; + const client = containerClient.getBlockBlobClient(name); + for (let i = 0; i < 3; i++) { + const upload = await client.upload(`v${i}`, 2); + expected.push(`${name}@${upload.versionId}`); + } + } + + const { seen, pages } = await pageThroughVersions(2); + assert.ok(pages > 1); + assert.deepStrictEqual(seen, expected); + }); + + it("keeps non-version pagination unchanged @versioning-contract", async () => { + const names: string[] = []; + for (const suffix of ["a", "b", "c"]) { + const name = `${blobName}-${suffix}`; + const client = containerClient.getBlockBlobClient(name); + await client.upload("v0", 2); + await client.upload("v1", 2); + names.push(name); + } + + const seen: string[] = []; + let continuationToken: string | undefined; + do { + const result = await containerClient + .listBlobsFlat() + .byPage({ maxPageSize: 2, continuationToken }) + .next(); + if (result.done) { + break; + } + seen.push(...result.value.segment.blobItems.map((item) => item.name)); + continuationToken = result.value.continuationToken; + } while (continuationToken); + + assert.deepStrictEqual(seen, names); + }); + + it("creates a version when setting metadata @versioning-contract", async () => { + const first = await blockBlobClient.upload("version1", 8); + const result = await blockBlobClient.setMetadata({ k: "v2" }); + + assert.notStrictEqual(result.versionId, undefined); + assert.notStrictEqual(result.versionId, first.versionId); + assert.deepStrictEqual( + (await listVersions(blobName)).map((version) => version.versionId), + [first.versionId, result.versionId] + ); + assert.deepStrictEqual( + (await blockBlobClient.withVersion(first.versionId!).getProperties()) + .metadata ?? {}, + {} + ); + assert.deepStrictEqual((await blockBlobClient.getProperties()).metadata, { + k: "v2" + }); + }); + + it("does not create a version when setting properties @versioning-contract", async () => { + const first = await blockBlobClient.upload("version1", 8); + await blockBlobClient.setHTTPHeaders({ + blobContentType: "text/plain" + }); + + const versions = await listVersions(blobName); + assert.strictEqual(versions.length, 1); + assert.strictEqual(versions[0].versionId, first.versionId); + assert.strictEqual( + (await blockBlobClient.getProperties()).contentType, + "text/plain" + ); + + const pageName = getUniqueName("page"); + const pageClient = containerClient.getPageBlobClient(pageName); + await pageClient.create(512); + await pageClient.setHTTPHeaders({ blobContentType: "text/plain" }); + assert.strictEqual((await listVersions(pageName)).length, 1); + }); + + it("versions page and append blob creates only on supported writes @versioning-contract", async () => { + const pageName = getUniqueName("page"); + const pageClient = containerClient.getPageBlobClient(pageName); + assert.notStrictEqual((await pageClient.create(512)).versionId, undefined); + await pageClient.uploadPages("x".repeat(512), 0, 512); + assert.strictEqual((await listVersions(pageName)).length, 1); + + const appendName = getUniqueName("append"); + const appendClient = containerClient.getAppendBlobClient(appendName); + assert.notStrictEqual((await appendClient.create()).versionId, undefined); + await appendClient.appendBlock("y", 1); + assert.strictEqual((await listVersions(appendName)).length, 1); + + assert.notStrictEqual( + (await pageClient.setMetadata({ k: "v" })).versionId, + undefined + ); + assert.notStrictEqual( + (await appendClient.setMetadata({ k: "v" })).versionId, + undefined + ); + }); + + it("creates a version when taking a snapshot @versioning-contract", async () => { + const first = await blockBlobClient.upload("version1", 8); + const snapshot = await blockBlobClient.createSnapshot(); + + assert.notStrictEqual(snapshot.snapshot, undefined); + assert.notStrictEqual(snapshot.versionId, undefined); + assert.notStrictEqual(snapshot.versionId, first.versionId); + assert.deepStrictEqual( + (await listVersions(blobName)).map((version) => version.versionId), + [first.versionId, snapshot.versionId] + ); + }); + + it("returns the destination version ID from copy @versioning-contract", async () => { + const source = containerClient.getBlockBlobClient(getUniqueName("src")); + await source.upload("source12", 8); + const first = await blockBlobClient.upload("version1", 8); + + const poller = await blockBlobClient.beginCopyFromURL(source.url); + const copy = await poller.pollUntilDone(); + + assert.notStrictEqual(copy.versionId, undefined); + assert.notStrictEqual(copy.versionId, first.versionId); + assert.deepStrictEqual( + (await listVersions(blobName)).map((version) => version.versionId), + [first.versionId, copy.versionId] + ); + }); + + it("keeps tags isolated per version @versioning-contract", async () => { + const first = await blockBlobClient.upload("version1", 8); + await blockBlobClient.setTags({ tier: "old" }); + await blockBlobClient.upload("version2", 8); + await blockBlobClient.setTags({ tier: "new" }); + + assert.deepStrictEqual((await blockBlobClient.getTags()).tags, { + tier: "new" + }); + assert.deepStrictEqual( + (await blockBlobClient.withVersion(first.versionId!).getTags()).tags, + { tier: "old" } + ); + + await blockBlobClient + .withVersion(first.versionId!) + .setTags({ tier: "archived" }); + assert.deepStrictEqual( + (await blockBlobClient.withVersion(first.versionId!).getTags()).tags, + { tier: "archived" } + ); + }); + + it("sets access tier independently per version @versioning-contract", async () => { + const first = await blockBlobClient.upload("version1", 8); + await blockBlobClient.upload("version2", 8); + + await blockBlobClient.withVersion(first.versionId!).setAccessTier("Cool"); + + assert.strictEqual( + (await blockBlobClient.withVersion(first.versionId!).getProperties()) + .accessTier, + "Cool" + ); + assert.notStrictEqual( + (await blockBlobClient.getProperties()).accessTier, + "Cool" + ); + }); + + it("rejects malformed version IDs @versioning-contract", async () => { + await blockBlobClient.upload("version1", 8); + + for (const invalid of [ + "notatimestamp", + "2026-08-13", + "2026-08-13T10:00:00Z" + ]) { + let error; + try { + await blockBlobClient.withVersion(invalid).download(); + } catch (err) { + error = err; + } + + assert.strictEqual((error as any)?.statusCode, 400); + assert.strictEqual((error as any)?.code, "InvalidQueryParameterValue"); + } + }); + + it("keeps snapshots blocking base-blob deletion @versioning-contract", async () => { + await blockBlobClient.upload("version1", 8); + await blockBlobClient.createSnapshot(); + + let error; + try { + await blockBlobClient.delete(); + } catch (err) { + error = err; + } + + assert.strictEqual((error as any)?.statusCode, 409); + }); +}); diff --git a/tests/blob/apis/blob.versioning.hierarchy.contract.test.ts b/tests/blob/apis/blob.versioning.hierarchy.contract.test.ts new file mode 100644 index 000000000..59b720eea --- /dev/null +++ b/tests/blob/apis/blob.versioning.hierarchy.contract.test.ts @@ -0,0 +1,274 @@ +import { + BlobServiceClient, + newPipeline, + StorageSharedKeyCredential +} from "@azure/storage-blob"; +import * as assert from "assert"; + +import { AccountModel } from "../../../src/common/account/AccountModel"; +import { configLogger } from "../../../src/common/Logger"; +import LokiAccountModelStore from "../../../src/common/account/LokiAccountModelStore"; +import BlobTestServerFactory from "../../BlobTestServerFactory"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getTestServerBaseURL, + getUniqueName +} from "../../testutils"; + +configLogger(false); + +const RUN_VERSIONING_CONTRACT_TESTS = + process.env.AZURITE_RUN_VERSIONING_CONTRACT_TESTS === "1"; +const contractDescribe = RUN_VERSIONING_CONTRACT_TESTS + ? describe + : describe.skip; +const accountModel: AccountModel = { + key: EMULATOR_ACCOUNT_NAME, + isBlobVersioningEnabled: true +}; + +contractDescribe("Blob Versioning Hierarchy Contract", () => { + const factory = new BlobTestServerFactory(); + const accountModelStore = new LokiAccountModelStore( + "", + true, + new Map([[EMULATOR_ACCOUNT_NAME, accountModel]]) + ); + const server = factory.createServer( + false, + false, + false, + undefined, + accountModelStore + ); + const serviceClient = new BlobServiceClient( + getTestServerBaseURL(server), + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + keepAliveOptions: { enable: false } + } + ) + ); + + let containerName = getUniqueName("container"); + let containerClient = serviceClient.getContainerClient(containerName); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async () => { + containerName = getUniqueName("container"); + containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + }); + + afterEach(async () => { + await containerClient.delete(); + }); + + async function seed() { + const versions: { [name: string]: string[] } = {}; + const write = async (name: string, count: number) => { + versions[name] = []; + for (let i = 0; i < count; i++) { + const result = await containerClient + .getBlockBlobClient(name) + .upload(`v${i}`, 2); + versions[name].push(result.versionId!); + } + }; + + await write("p1/a", 2); + await write("p1/b", 2); + await write("p2/c", 2); + await write("root", 3); + return versions; + } + + async function seedInterleaved() { + const versions: { [name: string]: string[] } = {}; + const write = async (name: string, count: number) => { + versions[name] = []; + for (let i = 0; i < count; i++) { + const result = await containerClient + .getBlockBlobClient(name) + .upload(`v${i}`, 2); + versions[name].push(result.versionId!); + } + }; + + await write("a/1", 2); + await write("b", 3); + await write("c/1", 2); + await write("p", 2); + await write("p/x", 2); + return versions; + } + + async function pageHierarchy( + pageSize: number | undefined, + options: { includeVersions?: boolean; prefix?: string } = {} + ) { + const blobs: string[] = []; + const prefixes: string[] = []; + let continuationToken: string | undefined; + let pages = 0; + + do { + const result = await containerClient + .listBlobsByHierarchy("/", { + includeVersions: options.includeVersions, + prefix: options.prefix + }) + .byPage({ maxPageSize: pageSize, continuationToken }) + .next(); + if (result.done) { + break; + } + + for (const item of result.value.segment.blobItems ?? []) { + blobs.push(`${item.name}@${item.versionId ?? "-"}`); + } + for (const prefix of result.value.segment.blobPrefixes ?? []) { + prefixes.push(prefix.name); + } + continuationToken = result.value.continuationToken; + pages++; + assert.ok(pages < 60, "Listing did not terminate"); + } while (continuationToken); + + return { blobs, prefixes, pages }; + } + + it("returns each prefix once with versions enabled @versioning-contract", async () => { + const versions = await seed(); + const { blobs, prefixes } = await pageHierarchy(undefined, { + includeVersions: true + }); + + assert.deepStrictEqual(prefixes, ["p1/", "p2/"]); + assert.deepStrictEqual( + blobs, + versions.root.map((version) => `root@${version}`) + ); + }); + + it("keeps hierarchy listing unchanged without versions @versioning-contract", async () => { + await seed(); + const { blobs, prefixes } = await pageHierarchy(undefined); + + assert.deepStrictEqual(prefixes, ["p1/", "p2/"]); + assert.strictEqual(blobs.length, 1); + assert.ok(blobs[0].startsWith("root@")); + }); + + it("returns every version inside a prefix @versioning-contract", async () => { + const versions = await seed(); + const { blobs, prefixes } = await pageHierarchy(undefined, { + includeVersions: true, + prefix: "p1/" + }); + + assert.deepStrictEqual(prefixes, []); + assert.deepStrictEqual(blobs, [ + ...versions["p1/a"].map((version) => `p1/a@${version}`), + ...versions["p1/b"].map((version) => `p1/b@${version}`) + ]); + }); + + it("paginates hierarchy without duplicating prefixes @versioning-contract", async () => { + const versions = await seed(); + const { blobs, prefixes, pages } = await pageHierarchy(2, { + includeVersions: true + }); + + assert.ok(pages > 1); + assert.deepStrictEqual(prefixes, ["p1/", "p2/"]); + assert.deepStrictEqual( + blobs, + versions.root.map((version) => `root@${version}`) + ); + }); + + it("handles interleaved prefixes and blobs at page size one @versioning-contract", async () => { + const versions = await seedInterleaved(); + const { blobs, prefixes, pages } = await pageHierarchy(1, { + includeVersions: true + }); + + assert.ok(pages > 4); + assert.deepStrictEqual(prefixes, ["a/", "c/", "p/"]); + assert.deepStrictEqual(blobs, [ + ...versions.b.map((version) => `b@${version}`), + ...versions.p.map((version) => `p@${version}`) + ]); + }); + + it("lists a blob separately from a same-name prefix @versioning-contract", async () => { + const versions = await seedInterleaved(); + const { blobs, prefixes } = await pageHierarchy(undefined, { + includeVersions: true + }); + + assert.ok(prefixes.includes("p/")); + assert.deepStrictEqual( + blobs.filter((blob) => blob.startsWith("p@")), + versions.p.map((version) => `p@${version}`) + ); + }); + + it("returns every flat-list version at page size one @versioning-contract", async () => { + const versions = await seedInterleaved(); + const seen: string[] = []; + let continuationToken: string | undefined; + let pages = 0; + + do { + const result = await containerClient + .listBlobsFlat({ includeVersions: true }) + .byPage({ maxPageSize: 1, continuationToken }) + .next(); + if (result.done) { + break; + } + + for (const item of result.value.segment.blobItems) { + seen.push(`${item.name}@${item.versionId}`); + } + continuationToken = result.value.continuationToken; + pages++; + assert.ok(pages < 60, "Listing did not terminate"); + } while (continuationToken); + + const expected = ["a/1", "b", "c/1", "p", "p/x"].flatMap((name) => + versions[name].map((version) => `${name}@${version}`) + ); + assert.deepStrictEqual(seen, expected); + }); + + it("paginates every version inside a prefix @versioning-contract", async () => { + const versions = await seed(); + const { blobs, pages } = await pageHierarchy(2, { + includeVersions: true, + prefix: "p1/" + }); + + assert.ok(pages > 1); + assert.deepStrictEqual(blobs, [ + ...versions["p1/a"].map((version) => `p1/a@${version}`), + ...versions["p1/b"].map((version) => `p1/b@${version}`) + ]); + }); +}); diff --git a/tests/blob/apis/blockblob.test.ts b/tests/blob/apis/blockblob.test.ts index fd43e4a1e..761dda75e 100644 --- a/tests/blob/apis/blockblob.test.ts +++ b/tests/blob/apis/blockblob.test.ts @@ -682,6 +682,23 @@ describe("BlockBlobAPIs", () => { ); }); + it("commitBlockList should return versionId as undefined @loki @sql", async () => { + const body = "HelloWorld"; + await blockBlobClient.stageBlock(base64encode("1"), body, body.length); + await blockBlobClient.stageBlock(base64encode("2"), body, body.length); + const commitResponse = await blockBlobClient.commitBlockList([ + base64encode("1"), + base64encode("2") + ]); + assert.strictEqual(commitResponse.versionId, undefined); + + const properties = await blockBlobClient.getProperties(); + assert.strictEqual(properties.versionId, undefined); + + const downloadResponse = await blobClient.download(0); + assert.strictEqual(downloadResponse.versionId, undefined); + }); + it("commitBlockList with ifTags @loki @sql", async () => { const body = "HelloWorld"; await blockBlobClient.upload(body, 10); @@ -1109,5 +1126,4 @@ describe("BlockBlobAPIs", () => { assert.ok(resultWithPermission.copyId); assert.strictEqual(resultWithPermission.errorCode, undefined); }); - }); diff --git a/tests/blob/apis/blockblob.versioning.test.ts b/tests/blob/apis/blockblob.versioning.test.ts new file mode 100644 index 000000000..cf82dbeea --- /dev/null +++ b/tests/blob/apis/blockblob.versioning.test.ts @@ -0,0 +1,623 @@ +import { + StorageSharedKeyCredential, + BlobServiceClient, + newPipeline, + Tags +} from "@azure/storage-blob"; +import assert = require("assert"); + +import { configLogger } from "../../../src/common/Logger"; +import BlobTestServerFactory from "../../BlobTestServerFactory"; +import { + base64encode, + bodyToString, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName, + listBlobVersions, + sleep +} from "../../testutils"; +import { parseDateFromAssumedString } from "../../../src/blob/utils/utils"; +import { AccountModel } from "../../../src/common/account/AccountModel"; +import LokiAccountModelStore from "../../../src/common/account/LokiAccountModelStore"; + +// Set true to enable debug log +configLogger(false); + +const ACCOUNT_DB_FILE = "__test_db_account_models_blockblob_versioning__.json"; + +function createAccountModelStore(accountModel: AccountModel, inMemory: boolean = false): LokiAccountModelStore { + const accountModels = new Map(); + accountModels.set(accountModel.key || "devstoreaccount1", accountModel); + return new LokiAccountModelStore(ACCOUNT_DB_FILE, inMemory, accountModels); +} + +describe("BlockBlobVersioningAPIs", () => { + const factory = new BlobTestServerFactory(); + const accountModel: AccountModel = + { + key: "devstoreaccount1", + isBlobVersioningEnabled: true + } + const accountModelStore = createAccountModelStore(accountModel, true); + const server = factory.createServer(false, false, false, undefined, accountModelStore); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + let containerName: string = getUniqueName("container"); + let containerClient = serviceClient.getContainerClient(containerName); + let blobName: string = getUniqueName("blob"); + let blobClient = containerClient.getBlobClient(blobName); + let blockBlobClient = blobClient.getBlockBlobClient(); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async () => { + containerName = getUniqueName("container"); + containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + blobName = getUniqueName("blob"); + blobClient = containerClient.getBlobClient(blobName); + blockBlobClient = blobClient.getBlockBlobClient(); + }); + + afterEach(async () => { + await containerClient.delete(); + }); + + // ===================== BLOCK BLOB SPECIFIC TESTS ===================== + + it("should return versionId when uploading a block blob with versioning enabled", async () => { + const content = "Hello, Versioned World!"; + const uploadResponse = await blockBlobClient.upload( + content, + content.length + ); + + // Verify versionId is returned and is a valid date + assert.ok( + uploadResponse.versionId, + "versionId should be present in upload response" + ); + assert.ok( + parseDateFromAssumedString(uploadResponse.versionId), + "versionId should be a valid ISO date string" + ); + + // Verify other response properties + assert.strictEqual(uploadResponse._response.status, 201); + assert.ok(uploadResponse.etag); + assert.ok(uploadResponse.lastModified); + }); + + it("should create new versions when uploading to same blob multiple times", async () => { + const content1 = "Version 1 content"; + const content2 = "Version 2 content"; + + // Upload first version + const upload1 = await blockBlobClient.upload(content1, content1.length); + assert.ok(upload1.versionId); + const version1Id = upload1.versionId!; + + // Small delay to ensure different timestamps + await sleep(100); + + // Upload second version + const upload2 = await blockBlobClient.upload(content2, content2.length); + assert.ok(upload2.versionId); + const version2Id = upload2.versionId!; + + // Verify different version IDs + assert.notStrictEqual(version1Id, version2Id); + + // Verify both are valid dates and version2 > version1 + const v1Date = parseDateFromAssumedString(version1Id)!; + const v2Date = parseDateFromAssumedString(version2Id)!; + assert.ok(v1Date instanceof Date); + assert.ok(v2Date instanceof Date); + assert.ok(v2Date > v1Date, "Second version should have later timestamp"); + + const versions = await listBlobVersions(containerClient, blobName); + assert.strictEqual(versions.length, 2, "Should have two versions listed"); + assert.strictEqual(versions[0].versionId, version1Id); + assert.strictEqual(versions[1].versionId, version2Id); + }); + + it("should return versionId when committing block list with versioning enabled", async () => { + const blockIds = [ + base64encode("block1"), + base64encode("block2"), + base64encode("block3") + ]; + const blockContents = [ + "Block 1 content", + "Block 2 content", + "Block 3 content" + ]; + + // Stage blocks + for (let i = 0; i < blockIds.length; i++) { + await blockBlobClient.stageBlock( + blockIds[i], + blockContents[i], + blockContents[i].length + ); + } + + // Commit block list + const commitResponse = await blockBlobClient.commitBlockList(blockIds); + + // Verify versionId is returned + assert.ok( + commitResponse.versionId, + "versionId should be present in commit response" + ); + assert.ok( + parseDateFromAssumedString(commitResponse.versionId), + "versionId should be a valid ISO date string" + ); + + // Verify other response properties + assert.strictEqual(commitResponse._response.status, 201); + assert.ok(commitResponse.etag); + assert.ok(commitResponse.lastModified); + }); + + it("should create new versions when committing block lists multiple times", async () => { + const blockId1 = base64encode("block1"); + const blockId2 = base64encode("block2"); + const content1 = "First commit content"; + const content2 = "Second commit content"; + + // First commit + await blockBlobClient.stageBlock(blockId1, content1, content1.length); + const commit1 = await blockBlobClient.commitBlockList([blockId1]); + assert.ok(commit1.versionId); + const version1Id = commit1.versionId!; + + await sleep(100); + + // Second commit + await blockBlobClient.stageBlock(blockId2, content2, content2.length); + const commit2 = await blockBlobClient.commitBlockList([blockId2]); + assert.ok(commit2.versionId); + const version2Id = commit2.versionId!; + + // Verify different version IDs + assert.notStrictEqual(version1Id, version2Id); + + // Verify chronological order + const v1Date = parseDateFromAssumedString(version1Id)!; + const v2Date = parseDateFromAssumedString(version2Id)!; + assert.ok(v2Date > v1Date, "Second commit should have later timestamp"); + + const versions = await listBlobVersions(containerClient, blobName); + assert.strictEqual(versions.length, 2, "Should have two versions listed"); + assert.strictEqual(versions[0].versionId, version1Id); + assert.strictEqual(versions[1].versionId, version2Id); + }); + + // ===================== GENERAL BLOB API TESTS ===================== + it("should return versionId when setting blob metadata with versioning enabled", async () => { + // First create a blob + const content = "Test blob for metadata"; + const uploadResponse = await blockBlobClient.upload( + content, + content.length + ); + const originalVersionId = uploadResponse.versionId!; + + await sleep(100); + + // Set metadata (this should create a new version) + const metadata = { key1: "value1", key2: "value2" }; + const setMetadataResponse = await blobClient.setMetadata(metadata); + + // Verify versionId is returned and is different from original + assert.ok( + setMetadataResponse.versionId, + "versionId should be present in setMetadata response" + ); + assert.ok( + parseDateFromAssumedString(setMetadataResponse.versionId), + "versionId should be a valid ISO date string" + ); + assert.notStrictEqual( + setMetadataResponse.versionId, + originalVersionId, + "setMetadata should create new version" + ); + + // Verify the new version is later + const originalDate = parseDateFromAssumedString(originalVersionId)!; + const newDate = parseDateFromAssumedString(setMetadataResponse.versionId!)!; + assert.ok( + newDate > originalDate, + "New version should have later timestamp" + ); + + const versions = await listBlobVersions(containerClient, blobName); + assert.strictEqual(versions.length, 2, "Should have two versions listed"); + assert.strictEqual(versions[0].versionId, originalVersionId); + assert.strictEqual(versions[1].versionId, setMetadataResponse.versionId!); + }); + + it("should download specific blob version by versionId", async () => { + const content1 = "Version 1 content"; + const content2 = "Version 2 content"; + const metadata1 = { version: "1" }; + const metadata2 = { version: "2" }; + + // Create first version + const upload1 = await blockBlobClient.upload(content1, content1.length, { + metadata: metadata1 + }); + const version1Id = upload1.versionId!; + + await sleep(100); + + // Create second version + const upload2 = await blockBlobClient.upload(content2, content2.length, { + metadata: metadata2 + }); + const version2Id = upload2.versionId!; + + // Download current version (should be version 2) + const currentDownload = await blobClient.download(); + const currentContent = await bodyToString( + currentDownload, + currentDownload.contentLength + ); + assert.strictEqual(currentContent, content2); + assert.strictEqual(currentDownload.metadata?.version, "2"); + + // Download specific version 1 + const version1Download = await blobClient + .withVersion(version1Id) + .download(); + const version1Content = await bodyToString( + version1Download, + version1Download.contentLength + ); + assert.strictEqual(version1Content, content1); + assert.strictEqual(version1Download.metadata?.version, "1"); + assert.strictEqual(version1Download.versionId, version1Id); + + // Download specific version 2 + const version2Download = await blobClient + .withVersion(version2Id) + .download(); + const version2Content = await bodyToString( + version2Download, + version2Download.contentLength + ); + assert.strictEqual(version2Content, content2); + assert.strictEqual(version2Download.metadata?.version, "2"); + assert.strictEqual(version2Download.versionId, version2Id); + }); + + it("should get properties for specific blob version by versionId", async () => { + const content = "Test content"; + const metadata1 = { version: "1", author: "user1" }; + const metadata2 = { version: "2", author: "user2" }; + + // Create first version + const upload1 = await blockBlobClient.upload(content, content.length, { + metadata: metadata1 + }); + const version1Id = upload1.versionId!; + + await sleep(100); + + // Create second version by setting metadata + const setMetadata = await blobClient.setMetadata(metadata2); + const version2Id = setMetadata.versionId!; + + // Get properties for version 1 + const props1 = await blobClient.withVersion(version1Id).getProperties(); + assert.strictEqual(props1.versionId, version1Id); + assert.strictEqual(props1.metadata?.version, "1"); + assert.strictEqual(props1.metadata?.author, "user1"); + + // Get properties for version 2 + const props2 = await blobClient.withVersion(version2Id).getProperties(); + assert.strictEqual(props2.versionId, version2Id); + assert.strictEqual(props2.metadata?.version, "2"); + assert.strictEqual(props2.metadata?.author, "user2"); + + // Get properties for current version (should be version 2) + const currentProps = await blobClient.getProperties(); + assert.strictEqual(currentProps.versionId, version2Id); + assert.strictEqual(currentProps.metadata?.version, "2"); + assert.strictEqual(currentProps.metadata?.author, "user2"); + }); + + it("should delete specific blob version by versionId", async () => { + const content1 = "Version 1 content"; + const content2 = "Version 2 content"; + const content3 = "Version 3 content"; + + // Create three versions + const upload1 = await blockBlobClient.upload(content1, content1.length); + const version1Id = upload1.versionId!; + + await sleep(100); + const upload2 = await blockBlobClient.upload(content2, content2.length); + const version2Id = upload2.versionId!; + + await sleep(100); + const upload3 = await blockBlobClient.upload(content3, content3.length); + const version3Id = upload3.versionId!; + + // Delete version 2 specifically + await blobClient.withVersion(version2Id).delete(); + + // Verify current version (version 3) still exists + const currentDownload = await blobClient.download(); + const currentContent = await bodyToString( + currentDownload, + currentDownload.contentLength + ); + assert.strictEqual(currentContent, content3); + assert.strictEqual(currentDownload.versionId, version3Id); + + // Verify version 1 still exists + const version1Download = await blobClient + .withVersion(version1Id) + .download(); + const version1Content = await bodyToString( + version1Download, + version1Download.contentLength + ); + assert.strictEqual(version1Content, content1); + + // Verify version 2 is deleted + try { + await blobClient.withVersion(version2Id).download(); + assert.fail("Should have thrown error for deleted version"); + } catch (error: any) { + assert.ok(error.statusCode === 404 || error.code === "BlobNotFound"); + } + }); + + it("should set and get tags for specific blob version", async () => { + const content = "Test content for tags"; + const tags1: Tags = { environment: "dev", version: "1.0" }; + const tags2: Tags = { environment: "prod", version: "2.0" }; + + // Create first version with tags + const upload1 = await blockBlobClient.upload(content, content.length, { + tags: tags1 + }); + const version1Id = upload1.versionId!; + + await sleep(100); + + // Create second version (new blob content creates new version) + const upload2 = await blockBlobClient.upload( + content + " updated", + (content + " updated").length, + { tags: tags2 } + ); + const version2Id = upload2.versionId!; + + // Get tags for version 1 + const version1Tags = await blobClient.withVersion(version1Id).getTags(); + assert.deepStrictEqual(version1Tags.tags, tags1); + + // Get tags for version 2 + const version2Tags = await blobClient.withVersion(version2Id).getTags(); + assert.deepStrictEqual(version2Tags.tags, tags2); + + // Get tags for current version (should be version 2) + const currentTags = await blobClient.getTags(); + assert.deepStrictEqual(currentTags.tags, tags2); + }); + + it("should set tags on specific blob version", async () => { + const content = "Test content"; + const originalTags: Tags = { original: "true" }; + const newTags: Tags = { updated: "true", version: "modified" }; + + // Create blob with original tags + const upload = await blockBlobClient.upload(content, content.length, { + tags: originalTags + }); + const versionId = upload.versionId!; + + // Set new tags on the specific version + await blobClient.withVersion(versionId).setTags(newTags); + + // Verify tags were updated on that version + const updatedTags = await blobClient.withVersion(versionId).getTags(); + assert.deepStrictEqual(updatedTags.tags, newTags); + + // Verify current version also has the updated tags (since it's the same version) + const currentTags = await blobClient.getTags(); + assert.deepStrictEqual(currentTags.tags, newTags); + }); + + it("should list blobs with version information", async () => { + const blobName1 = getUniqueName("blob1"); + const blobName2 = getUniqueName("blob2"); + const content1 = "Content for blob 1"; + const content2 = "Content for blob 2"; + + // Create blobs with multiple versions + const blob1Client = containerClient.getBlockBlobClient(blobName1); + const blob2Client = containerClient.getBlockBlobClient(blobName2); + + const upload1v1 = await blob1Client.upload(content1, content1.length); + await sleep(100); + const upload1v2 = await blob1Client.upload( + content1 + " v2", + (content1 + " v2").length + ); + await sleep(100); + const upload2v1 = await blob2Client.upload(content2, content2.length); + + // List blobs with versions + const listResponse = containerClient.listBlobsFlat({ + includeVersions: true + }); + const blobs = []; + for await (const blob of listResponse) { + blobs.push(blob); + } + + // Should have 3 versions total (2 for blob1, 1 for blob2) + assert.strictEqual(blobs.length, 3); + + // Find blob1 versions + const blob1Versions = blobs + .filter((b) => b.name === blobName1) + .sort( + (a, b) => + new Date(a.versionId!).getTime() - new Date(b.versionId!).getTime() + ); + assert.strictEqual(blob1Versions.length, 2); + assert.strictEqual(blob1Versions[0].versionId, upload1v1.versionId); + assert.strictEqual(blob1Versions[1].versionId, upload1v2.versionId); + assert.strictEqual(blob1Versions[0].isCurrentVersion, undefined); + assert.strictEqual(blob1Versions[1].isCurrentVersion, true); + + // Find blob2 version + const blob2Versions = blobs.filter((b) => b.name === blobName2); + assert.strictEqual(blob2Versions.length, 1); + assert.strictEqual(blob2Versions[0].versionId, upload2v1.versionId); + assert.strictEqual(blob2Versions[0].isCurrentVersion, true); + }); + + it("should handle blob versioning with delete operations", async () => { + const content1 = "Version 1"; + const content2 = "Version 2"; + + // Create two versions + const upload1 = await blockBlobClient.upload(content1, content1.length); + const version1Id = upload1.versionId!; + + await sleep(100); + const upload2 = await blockBlobClient.upload(content2, content2.length); + const version2Id = upload2.versionId!; + + // Delete current version (without specifying version) + await blobClient.delete(); + + // Current version should no longer exist + try { + await blobClient.download(); + assert.fail("Should have thrown error for deleted current blob"); + } catch (error: any) { + assert.ok(error.statusCode === 404 || error.code === "BlobNotFound"); + } + + // But specific versions should still be accessible + const version1Download = await blobClient + .withVersion(version1Id) + .download(); + const version1Content = await bodyToString( + version1Download, + version1Download.contentLength + ); + assert.strictEqual(version1Content, content1); + + const version2Download = await blobClient + .withVersion(version2Id) + .download(); + const version2Content = await bodyToString( + version2Download, + version2Download.contentLength + ); + assert.strictEqual(version2Content, content2); + }); + + it("should validate versionId format in API calls", async () => { + const content = "Test content"; + await blockBlobClient.upload(content, content.length); + + // Test with invalid versionId format + const invalidVersionIds = [ + "invalid-date", + "2024-13-01T00:00:00.000Z", // Invalid month + "not-a-date-at-all", + "2024/01/01 00:00:00" // Wrong format + ]; + + for (const invalidVersionId of invalidVersionIds) { + try { + await blobClient.withVersion(invalidVersionId).download(); + assert.fail( + `Should have thrown error for invalid versionId: ${invalidVersionId}` + ); + } catch (error: any) { + // Should throw an error for invalid versionId format + assert.ok( + error.statusCode === 400 || + error.code === "InvalidInput" || + error.statusCode === 404 + ); + } + } + }); + + it("should create snapshot and return versionId when versioning enabled", async () => { + const content = "Content for snapshot test"; + + // Create initial blob + const upload = await blockBlobClient.upload(content, content.length); + const originalVersionId = upload.versionId!; + + await sleep(100); + + // Create snapshot (should also create new version) + const snapshotResponse = await blobClient.createSnapshot(); + + // Verify snapshot properties + assert.ok( + snapshotResponse.snapshot, + "snapshot identifier should be present" + ); + assert.ok( + snapshotResponse.versionId, + "versionId should be present in snapshot response" + ); + assert.ok( + parseDateFromAssumedString(snapshotResponse.versionId), + "versionId should be valid date" + ); + + // New version should be different from original + assert.notStrictEqual(snapshotResponse.versionId, originalVersionId); + + // Verify chronological order + const originalDate = parseDateFromAssumedString(originalVersionId)!; + const snapshotDate = parseDateFromAssumedString( + snapshotResponse.versionId! + )!; + assert.ok( + snapshotDate > originalDate, + "Snapshot should create later version" + ); + }); +}); diff --git a/tests/blob/apis/container.test.ts b/tests/blob/apis/container.test.ts index 505e20385..8c1f16052 100644 --- a/tests/blob/apis/container.test.ts +++ b/tests/blob/apis/container.test.ts @@ -828,7 +828,7 @@ describe("ContainerAPIs", () => { ).value; assert.ok(result.serviceEndpoint.length > 0); assert.ok(containerClient.url.indexOf(result.containerName)); - assert.equal(result.continuationToken, "blockblob/abc-003"); + assert.ok(result.continuationToken.startsWith("blockblob/abc-003")); assert.equal(result.segment.blobItems.length, 4); assert.equal( result._response.request.headers.get("x-ms-client-request-id"), @@ -852,7 +852,7 @@ describe("ContainerAPIs", () => { ).value; assert.ok(result.serviceEndpoint.length > 0); assert.ok(containerClient.url.indexOf(result.containerName)); - assert.equal(result.continuationToken, "blockblob/abc-007"); + assert.ok(result.continuationToken.startsWith("blockblob/abc-007")); assert.equal(result.segment.blobItems.length, 4); assert.equal( result._response.request.headers.get("x-ms-client-request-id"), @@ -934,7 +934,7 @@ describe("ContainerAPIs", () => { ).value; assert.ok(result.serviceEndpoint.length > 0); assert.ok(containerClient.url.indexOf(result.containerName)); - assert.equal(result.continuationToken, blobNames[9]); + assert.ok(result.continuationToken.startsWith(blobNames[9])); assert.equal(result.segment.blobItems.length, 10); assert.equal( result._response.request.headers.get("x-ms-client-request-id"), @@ -1239,6 +1239,140 @@ describe("ContainerAPIs", () => { assert.equal(result.segment.blobItems.length, 4); }); + it("list container should have isCurrentVersion and versionId as undefined @loki @sql", async () => { + // prepare blobs + const blobClients = []; + for (let i = 0; i < 3; i++) { + const blobClient = containerClient.getBlobClient( + getUniqueName(`blockblob${i}/${i}`) + ); + const blockBlobClient = blobClient.getBlockBlobClient(); + await blockBlobClient.upload("", 0); + blobClients.push(blobClient); + } + await blobClients[0].createSnapshot(); + + // create account sas + const storageSharedKeyCredential = new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ); + const tmr = new Date(); + tmr.setDate(tmr.getDate() + 1); + const sas = generateAccountSASQueryParameters( + { + expiresOn: tmr, + permissions: AccountSASPermissions.parse("rl"), + resourceTypes: AccountSASResourceTypes.parse("sco").toString(), + services: AccountSASServices.parse("b").toString(), + version: "2020-04-08" + }, + storageSharedKeyCredential as StorageSharedKeyCredential + ).toString(); + + // list with empty include + // create container client for + let pipeline = newPipeline( + new AnonymousCredential(), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new QueryRequestPolicyFactory("include=metadata", "include=") + ); + let serviceClientForOptions = new BlobServiceClient(`${baseURL}?${sas}`, pipeline); + + let ContainerClientForOptions = serviceClientForOptions.getContainerClient(containerName); + + // list blob with empty include + let result = ( + await ContainerClientForOptions + .listBlobsFlat({ + includeMetadata: true + }) + .byPage() + .next() + ).value; + assert.ok(result); + assert.strictEqual(result.segment.blobItems.length, 3); + + for (const blob of result.segment.blobItems) { + assert.strictEqual(blob.isCurrentVersion, undefined); + assert.strictEqual(blob.versionId, undefined); + } + + // list with include as upcase Snapshot + // create container client for + pipeline = newPipeline( + new AnonymousCredential(), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new QueryRequestPolicyFactory("include=metadata", "include=Snapshots") + ); + serviceClientForOptions = new BlobServiceClient(`${baseURL}?${sas}`, pipeline); + + ContainerClientForOptions = serviceClientForOptions.getContainerClient(containerName); + + // list blob with include as upcase Snapshot + result = ( + await ContainerClientForOptions + .listBlobsFlat({ + includeMetadata: true + }) + .byPage() + .next() + ).value; + assert.ok(result); + assert.strictEqual(result.segment.blobItems.length, 4); + + for (const blob of result.segment.blobItems) { + assert.strictEqual(blob.isCurrentVersion, undefined); + assert.strictEqual(blob.versionId, undefined); + } + + // list with multiple include + // create container client for + pipeline = newPipeline( + new AnonymousCredential(), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ); + pipeline.factories.unshift( + new QueryRequestPolicyFactory("include=metadata", "include=snapshots,metadata,uncommittedblobs,copy,deleted,tags,versions,deletedwithversions,immutabilitypolicy,legalhold,permissions") + ); + serviceClientForOptions = new BlobServiceClient(`${baseURL}?${sas}`, pipeline); + + ContainerClientForOptions = serviceClientForOptions.getContainerClient(containerName); + + // list blob with multiple include + result = ( + await ContainerClientForOptions + .listBlobsFlat({ + includeMetadata: true + }) + .byPage() + .next() + ).value; + assert.ok(result); + assert.strictEqual(result.segment.blobItems.length, 4); + + for (const blob of result.segment.blobItems) { + assert.strictEqual(blob.isCurrentVersion, undefined); + assert.strictEqual(blob.versionId, undefined); + } + }); + it("filter blob by tags should work on container @loki @sql", async () => { const blobName1 = getUniqueName("blobname1"); const appendBlobClient1 = containerClient.getAppendBlobClient(blobName1); diff --git a/tests/blob/apis/pageblob.test.ts b/tests/blob/apis/pageblob.test.ts index f1ebf0d4f..1e75a3b3e 100644 --- a/tests/blob/apis/pageblob.test.ts +++ b/tests/blob/apis/pageblob.test.ts @@ -88,6 +88,17 @@ describe("PageBlobAPIs", () => { ); }); + it("create page blob should return versionId as undefined @loki", async () => { + const createResponse = await pageBlobClient.create(512); + assert.strictEqual(createResponse.versionId, undefined); + + const properties = await pageBlobClient.getProperties(); + assert.strictEqual(properties.versionId, undefined); + + const downloadResponse = await blobClient.download(0); + assert.strictEqual(downloadResponse.versionId, undefined); + }); + it("create with all parameters set @loki", async () => { const options = { blobHTTPHeaders: { diff --git a/tests/blob/apis/pageblob.versioning.test.ts b/tests/blob/apis/pageblob.versioning.test.ts new file mode 100644 index 000000000..7e77042be --- /dev/null +++ b/tests/blob/apis/pageblob.versioning.test.ts @@ -0,0 +1,582 @@ +import { + newPipeline, + BlobServiceClient, + StorageSharedKeyCredential, + Tags +} from "@azure/storage-blob"; +import assert = require("assert"); + +import { configLogger } from "../../../src/common/Logger"; +import BlobTestServerFactory from "../../BlobTestServerFactory"; +import { + bodyToString, + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName, + listBlobVersions, + sleep +} from "../../testutils"; +import { parseDateFromAssumedString } from "../../../src/blob/utils/utils"; +import { AccountModel } from "../../../src/common/account/AccountModel"; +import LokiAccountModelStore from "../../../src/common/account/LokiAccountModelStore"; + +// Set true to enable debug log +configLogger(false); + +const ACCOUNT_DB_FILE = "__test_db_account_models_pageblob_versioning__.json"; + +function createAccountModelStore(accountModel: AccountModel, inMemory: boolean = false): LokiAccountModelStore { + const accountModels = new Map(); + accountModels.set(accountModel.key || "devstoreaccount1", accountModel); + return new LokiAccountModelStore(ACCOUNT_DB_FILE, inMemory, accountModels); +} + +describe("PageBlobVersioningAPIs", () => { + const factory = new BlobTestServerFactory(); + const accountModel: AccountModel = + { + key: "devstoreaccount1", + isBlobVersioningEnabled: true + } + const accountModelStore = createAccountModelStore(accountModel, true); + const server = factory.createServer(false, false, false, undefined, accountModelStore); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + const serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + // Make sure socket is closed once the operation is done. + keepAliveOptions: { enable: false } + } + ) + ); + + let containerName: string = getUniqueName("container"); + let containerClient = serviceClient.getContainerClient(containerName); + let blobName: string = getUniqueName("blob"); + let blobClient = containerClient.getBlobClient(blobName); + let pageBlobClient = blobClient.getPageBlobClient(); + + before(async () => { + await server.start(); + }); + + after(async () => { + await server.close(); + await server.clean(); + }); + + beforeEach(async () => { + containerName = getUniqueName("container"); + containerClient = serviceClient.getContainerClient(containerName); + await containerClient.create(); + blobName = getUniqueName("blob"); + blobClient = containerClient.getBlobClient(blobName); + pageBlobClient = blobClient.getPageBlobClient(); + }); + + afterEach(async () => { + await containerClient.delete(); + }); + + // ===================== PAGE BLOB SPECIFIC TESTS ===================== + it("should return versionId when creating a page blob with versioning enabled", async () => { + const createResponse = await pageBlobClient.create(512); + + // Verify versionId is returned and is a valid date + assert.ok( + createResponse.versionId, + "versionId should be present in create response" + ); + assert.ok( + parseDateFromAssumedString(createResponse.versionId), + "versionId should be a valid ISO date string" + ); + + // Verify other response properties + assert.strictEqual(createResponse._response.status, 201); + assert.ok(createResponse.etag); + assert.ok(createResponse.lastModified); + }); + + it("should create new versions when recreating page blob", async () => { + const metadata1 = { version: "1" }; + const metadata2 = { version: "2" }; + + // Create first version + const create1 = await pageBlobClient.create(512, { metadata: metadata1 }); + assert.ok(create1.versionId); + const version1Id = create1.versionId!; + + // Small delay to ensure different timestamps + await sleep(100); + + // Create second version (recreate the blob) + const create2 = await pageBlobClient.create(512, { metadata: metadata2 }); + assert.ok(create2.versionId); + const version2Id = create2.versionId!; + + // Verify different version IDs + assert.notStrictEqual(version1Id, version2Id); + + // Verify both are valid dates and version2 > version1 + const v1Date = parseDateFromAssumedString(version1Id)!; + const v2Date = parseDateFromAssumedString(version2Id)!; + assert.ok(v1Date instanceof Date); + assert.ok(v2Date instanceof Date); + assert.ok(v2Date > v1Date, "Second version should have later timestamp"); + }); + + it("should NOT create new versions when uploading pages", async () => { + const content1 = "A".repeat(512); // Page content must be 512-byte aligned + const content2 = "B".repeat(512); + + // Create page blob + const createResponse = await pageBlobClient.create(1024); // 2 pages + const originalVersionId = createResponse.versionId!; + + await sleep(100); + + // Upload first page (should NOT create new version) + await pageBlobClient.uploadPages(content1, 0, content1.length); + + await sleep(100); + + // Upload second page (should NOT create new version) + await pageBlobClient.uploadPages(content2, 512, content2.length); + + // Verify current blob properties - should still have same version + const properties = await pageBlobClient.getProperties(); + assert.strictEqual( + properties.versionId, + originalVersionId, + "Page upload operations should not create new versions" + ); + + // Verify content is written correctly + const download = await pageBlobClient.download(); + const content = await bodyToString(download, download.contentLength); + assert.strictEqual(content, content1 + content2); + }); + + // ===================== GENERAL BLOB API TESTS ===================== + it("should return versionId when setting blob metadata with versioning enabled", async () => { + // First create a page blob + const createResponse = await pageBlobClient.create(512); + const originalVersionId = createResponse.versionId!; + + await sleep(100); + + // Set metadata (this should create a new version) + const metadata = { key1: "value1", key2: "value2" }; + const setMetadataResponse = await pageBlobClient.setMetadata(metadata); + + // Verify versionId is returned and is different from original + assert.ok( + setMetadataResponse.versionId, + "versionId should be present in setMetadata response" + ); + assert.ok( + parseDateFromAssumedString(setMetadataResponse.versionId), + "versionId should be a valid ISO date string" + ); + assert.notStrictEqual( + setMetadataResponse.versionId, + originalVersionId, + "setMetadata should create new version" + ); + + // Verify the new version is later + const originalDate = parseDateFromAssumedString(originalVersionId)!; + const newDate = parseDateFromAssumedString(setMetadataResponse.versionId!)!; + assert.ok( + newDate > originalDate, + "New version should have later timestamp" + ); + + const versions = await listBlobVersions(containerClient, blobName); + assert.strictEqual(versions.length, 2, "Should have two versions listed"); + assert.strictEqual(versions[0].versionId, originalVersionId); + assert.strictEqual(versions[1].versionId, setMetadataResponse.versionId!); + }); + + it("should download specific blob version by versionId", async () => { + const content1 = "Version 1 content"; + const content2 = "Version 2 content"; + const metadata1 = { version: "1" }; + const metadata2 = { version: "2" }; + + // Create first version (page blob with content) + const content1Padded = content1.padEnd(512, "\0"); // Pad to 512 bytes + const create1 = await pageBlobClient.create(512, { metadata: metadata1 }); + await pageBlobClient.uploadPages(content1Padded, 0, 512); + const version1Id = create1.versionId!; + + await sleep(100); + + // Create second version (recreate page blob with different content) + const content2Padded = content2.padEnd(512, "\0"); // Pad to 512 bytes + const create2 = await pageBlobClient.create(512, { metadata: metadata2 }); + await pageBlobClient.uploadPages(content2Padded, 0, 512); + const version2Id = create2.versionId!; + + // Download current version (should be version 2) + const currentDownload = await pageBlobClient.download(); + const currentContent = await bodyToString( + currentDownload, + currentDownload.contentLength + ); + assert.strictEqual(currentContent, content2Padded); + assert.strictEqual(currentDownload.metadata?.version, "2"); + + // Download specific version 1 + const version1Download = await pageBlobClient + .withVersion(version1Id) + .download(); + const version1Content = await bodyToString( + version1Download, + version1Download.contentLength + ); + assert.strictEqual(version1Content, content1Padded); + assert.strictEqual(version1Download.metadata?.version, "1"); + assert.strictEqual(version1Download.versionId, version1Id); + + // Download specific version 2 + const version2Download = await pageBlobClient + .withVersion(version2Id) + .download(); + const version2Content = await bodyToString( + version2Download, + version2Download.contentLength + ); + assert.strictEqual(version2Content, content2Padded); + assert.strictEqual(version2Download.metadata?.version, "2"); + assert.strictEqual(version2Download.versionId, version2Id); + }); + + it("should get properties for specific blob version by versionId", async () => { + const content = "Test content"; + const metadata1 = { version: "1", author: "user1" }; + const metadata2 = { version: "2", author: "user2" }; + + // Create first version (page blob with content) + const contentPadded = content.padEnd(512, "\0"); // Pad to 512 bytes + const create1 = await pageBlobClient.create(512, { metadata: metadata1 }); + await pageBlobClient.uploadPages(contentPadded, 0, 512); + const version1Id = create1.versionId!; + + await sleep(100); + + // Create second version by setting metadata + const setMetadata = await pageBlobClient.setMetadata(metadata2); + const version2Id = setMetadata.versionId!; + + // Get properties for version 1 + const props1 = await pageBlobClient.withVersion(version1Id).getProperties(); + assert.strictEqual(props1.versionId, version1Id); + assert.strictEqual(props1.metadata?.version, "1"); + assert.strictEqual(props1.metadata?.author, "user1"); + + // Get properties for version 2 + const props2 = await pageBlobClient.withVersion(version2Id).getProperties(); + assert.strictEqual(props2.versionId, version2Id); + assert.strictEqual(props2.metadata?.version, "2"); + assert.strictEqual(props2.metadata?.author, "user2"); + + // Get properties for current version (should be version 2) + const currentProps = await pageBlobClient.getProperties(); + assert.strictEqual(currentProps.versionId, version2Id); + assert.strictEqual(currentProps.metadata?.version, "2"); + assert.strictEqual(currentProps.metadata?.author, "user2"); + }); + + it("should delete specific blob version by versionId", async () => { + const content1 = "Version 1 content"; + const content2 = "Version 2 content"; + const content3 = "Version 3 content"; + + // Create three versions (recreate page blob each time) + const content1Padded = content1.padEnd(512, "\0"); + const create1 = await pageBlobClient.create(512); + await pageBlobClient.uploadPages(content1Padded, 0, 512); + const version1Id = create1.versionId!; + + await sleep(100); + const content2Padded = content2.padEnd(512, "\0"); + const create2 = await pageBlobClient.create(512); + await pageBlobClient.uploadPages(content2Padded, 0, 512); + const version2Id = create2.versionId!; + + await sleep(100); + const content3Padded = content3.padEnd(512, "\0"); + const create3 = await pageBlobClient.create(512); + await pageBlobClient.uploadPages(content3Padded, 0, 512); + const version3Id = create3.versionId!; + + // Delete version 2 specifically + await pageBlobClient.withVersion(version2Id).delete(); + + // Verify current version (version 3) still exists + const currentDownload = await pageBlobClient.download(); + const currentContent = await bodyToString( + currentDownload, + currentDownload.contentLength + ); + assert.strictEqual(currentContent, content3Padded); + assert.strictEqual(currentDownload.versionId, version3Id); + + // Verify version 1 still exists + const version1Download = await pageBlobClient + .withVersion(version1Id) + .download(); + const version1Content = await bodyToString( + version1Download, + version1Download.contentLength + ); + assert.strictEqual(version1Content, content1Padded); + + // Verify version 2 is deleted + try { + await pageBlobClient.withVersion(version2Id).download(); + assert.fail("Should have thrown error for deleted version"); + } catch (error: any) { + assert.ok(error.statusCode === 404 || error.code === "BlobNotFound"); + } + }); + + it("should set and get tags for specific blob version", async () => { + const content = "Test content for tags"; + const tags1: Tags = { environment: "dev", version: "1.0" }; + const tags2: Tags = { environment: "prod", version: "2.0" }; + + // Create first version with tags (page blob) + const contentPadded = content.padEnd(512, "\0"); + const create1 = await pageBlobClient.create(512, { tags: tags1 }); + await pageBlobClient.uploadPages(contentPadded, 0, 512); + const version1Id = create1.versionId!; + + await sleep(100); + + // Create second version (recreate page blob with different tags) + const updatedContent = content + " updated"; + const updatedContentPadded = updatedContent.padEnd(512, "\0"); + const create2 = await pageBlobClient.create(512, { tags: tags2 }); + await pageBlobClient.uploadPages(updatedContentPadded, 0, 512); + const version2Id = create2.versionId!; + + // Get tags for version 1 + const version1Tags = await pageBlobClient.withVersion(version1Id).getTags(); + assert.deepStrictEqual(version1Tags.tags, tags1); + + // Get tags for version 2 + const version2Tags = await pageBlobClient.withVersion(version2Id).getTags(); + assert.deepStrictEqual(version2Tags.tags, tags2); + + // Get tags for current version (should be version 2) + const currentTags = await pageBlobClient.getTags(); + assert.deepStrictEqual(currentTags.tags, tags2); + }); + + it("should set tags on specific blob version", async () => { + const content = "Test content"; + const originalTags: Tags = { original: "true" }; + const newTags: Tags = { updated: "true", version: "modified" }; + + // Create page blob with original tags + const contentPadded = content.padEnd(512, "\0"); + const create = await pageBlobClient.create(512, { tags: originalTags }); + await pageBlobClient.uploadPages(contentPadded, 0, 512); + const versionId = create.versionId!; + + // Set new tags on the specific version + await pageBlobClient.withVersion(versionId).setTags(newTags); + + // Verify tags were updated on that version + const updatedTags = await pageBlobClient.withVersion(versionId).getTags(); + assert.deepStrictEqual(updatedTags.tags, newTags); + + // Verify current version also has the updated tags (since it's the same version) + const currentTags = await pageBlobClient.getTags(); + assert.deepStrictEqual(currentTags.tags, newTags); + }); + + it("should list blobs with version information", async () => { + const blobName1 = getUniqueName("blob1"); + const blobName2 = getUniqueName("blob2"); + const content1 = "Content for blob 1"; + const content2 = "Content for blob 2"; + + // Create page blobs with multiple versions + const blob1Client = containerClient.getPageBlobClient(blobName1); + const blob2Client = containerClient.getPageBlobClient(blobName2); + + const content1Padded = content1.padEnd(512, "\0"); + const create1v1 = await blob1Client.create(512); + await blob1Client.uploadPages(content1Padded, 0, 512); + await sleep(100); + const content1v2Padded = (content1 + " v2").padEnd(512, "\0"); + const create1v2 = await blob1Client.create(512); + await blob1Client.uploadPages(content1v2Padded, 0, 512); + await sleep(100); + const content2Padded = content2.padEnd(512, "\0"); + const create2v1 = await blob2Client.create(512); + await blob2Client.uploadPages(content2Padded, 0, 512); + + // List blobs with versions + const listResponse = containerClient.listBlobsFlat({ + includeVersions: true + }); + const blobs = []; + for await (const blob of listResponse) { + blobs.push(blob); + } + + // Should have 3 versions total (2 for blob1, 1 for blob2) + assert.strictEqual(blobs.length, 3); + + // Find blob1 versions + const blob1Versions = blobs + .filter((b) => b.name === blobName1) + .sort( + (a, b) => + new Date(a.versionId!).getTime() - new Date(b.versionId!).getTime() + ); + assert.strictEqual(blob1Versions.length, 2); + assert.strictEqual(blob1Versions[0].versionId, create1v1.versionId); + assert.strictEqual(blob1Versions[1].versionId, create1v2.versionId); + assert.strictEqual(blob1Versions[0].isCurrentVersion, undefined); + assert.strictEqual(blob1Versions[1].isCurrentVersion, true); + + // Find blob2 version + const blob2Versions = blobs.filter((b) => b.name === blobName2); + assert.strictEqual(blob2Versions.length, 1); + assert.strictEqual(blob2Versions[0].versionId, create2v1.versionId); + assert.strictEqual(blob2Versions[0].isCurrentVersion, true); + }); + + it("should handle blob versioning with delete operations", async () => { + const content1 = "Version 1"; + const content2 = "Version 2"; + + // Create two versions (recreate page blob each time) + const content1Padded = content1.padEnd(512, "\0"); + const create1 = await pageBlobClient.create(512); + await pageBlobClient.uploadPages(content1Padded, 0, 512); + const version1Id = create1.versionId!; + + await sleep(100); + const content2Padded = content2.padEnd(512, "\0"); + const create2 = await pageBlobClient.create(512); + await pageBlobClient.uploadPages(content2Padded, 0, 512); + const version2Id = create2.versionId!; + + // Delete current version (without specifying version) + await pageBlobClient.delete(); + + // Current version should no longer exist + try { + await pageBlobClient.download(); + assert.fail("Should have thrown error for deleted current blob"); + } catch (error: any) { + assert.ok(error.statusCode === 404 || error.code === "BlobNotFound"); + } + + // But specific versions should still be accessible + const version1Download = await pageBlobClient + .withVersion(version1Id) + .download(); + const version1Content = await bodyToString( + version1Download, + version1Download.contentLength + ); + assert.strictEqual(version1Content, content1Padded); + + const version2Download = await pageBlobClient + .withVersion(version2Id) + .download(); + const version2Content = await bodyToString( + version2Download, + version2Download.contentLength + ); + assert.strictEqual(version2Content, content2Padded); + }); + + it("should validate versionId format in API calls", async () => { + const content = "Test content"; + const contentPadded = content.padEnd(512, "\0"); + await pageBlobClient.create(512); + await pageBlobClient.uploadPages(contentPadded, 0, 512); + + // Test with invalid versionId format + const invalidVersionIds = [ + "invalid-date", + "2024-13-01T00:00:00.000Z", // Invalid month + "not-a-date-at-all", + "2024/01/01 00:00:00" // Wrong format + ]; + + for (const invalidVersionId of invalidVersionIds) { + try { + await pageBlobClient.withVersion(invalidVersionId).download(); + assert.fail( + `Should have thrown error for invalid versionId: ${invalidVersionId}` + ); + } catch (error: any) { + // Should throw an error for invalid versionId format + assert.ok( + error.statusCode === 400 || + error.code === "InvalidInput" || + error.statusCode === 404 + ); + } + } + }); + + it("should create snapshot and return versionId when versioning enabled", async () => { + const content = "Content for snapshot test"; + + // Create initial page blob + const contentPadded = content.padEnd(512, "\0"); + const create = await pageBlobClient.create(512); + await pageBlobClient.uploadPages(contentPadded, 0, 512); + const originalVersionId = create.versionId!; + + await sleep(100); + + // Create snapshot (should also create new version) + const snapshotResponse = await pageBlobClient.createSnapshot(); + + // Verify snapshot properties + assert.ok( + snapshotResponse.snapshot, + "snapshot identifier should be present" + ); + assert.ok( + snapshotResponse.versionId, + "versionId should be present in snapshot response" + ); + assert.ok( + parseDateFromAssumedString(snapshotResponse.versionId), + "versionId should be valid date" + ); + + // New version should be different from original + assert.notStrictEqual(snapshotResponse.versionId, originalVersionId); + + // Verify chronological order + const originalDate = parseDateFromAssumedString(originalVersionId)!; + const snapshotDate = parseDateFromAssumedString( + snapshotResponse.versionId! + )!; + assert.ok( + snapshotDate > originalDate, + "Snapshot should create later version" + ); + }); +}); diff --git a/tests/blob/apis/versioning.azurite.parity.test.ts b/tests/blob/apis/versioning.azurite.parity.test.ts new file mode 100644 index 000000000..4f4376834 --- /dev/null +++ b/tests/blob/apis/versioning.azurite.parity.test.ts @@ -0,0 +1,434 @@ +import * as assert from "assert"; +import { configLogger } from "../../../src/common/Logger"; +import BlobTestServerFactory from "../../BlobTestServerFactory"; +import { + EMULATOR_ACCOUNT_KEY, + EMULATOR_ACCOUNT_NAME, + getUniqueName +} from "../../testutils"; +import { isNullOrWhitespace } from "../../../src/blob/utils/utils"; +import { + StorageSharedKeyCredential, + newPipeline, + BlobServiceClient, + ContainerClient, + BlobItem +} from "@azure/storage-blob"; +import { AccountModel } from "../../../src/common/account/AccountModel"; +import LokiAccountModelStore from "../../../src/common/account/LokiAccountModelStore"; + +// Set to true when you want to debug the emulator +configLogger(false); + +const ACCOUNT_DB_FILE = "__test_db_account_models_versioning_parity__.json"; + +function createAccountModelStore(accountModel: AccountModel, inMemory: boolean = true): LokiAccountModelStore { + const accountModels = new Map(); + accountModels.set(accountModel.key || "devstoreaccount1", accountModel); + return new LokiAccountModelStore(ACCOUNT_DB_FILE, inMemory, accountModels); +} + +describe("Blob Versioning Parity Tests - Azurite", () => { + const factory = new BlobTestServerFactory(); + let server: any; + let serviceClient: BlobServiceClient; + let containerClient: ContainerClient; + let containerName: string; + + const createServerAndClient = async (versioningEnabled: boolean) => { + if (server) { + await server.close(); + } + + const accountModel: AccountModel = + { + key: "devstoreaccount1", + isBlobVersioningEnabled: versioningEnabled + } + + const accountModelStore = createAccountModelStore(accountModel, true); + server = factory.createServer(false, false, false, undefined, accountModelStore); + + await server.start(); + + const baseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; + serviceClient = new BlobServiceClient( + baseURL, + newPipeline( + new StorageSharedKeyCredential( + EMULATOR_ACCOUNT_NAME, + EMULATOR_ACCOUNT_KEY + ), + { + retryOptions: { maxTries: 1 }, + keepAliveOptions: { enable: false } + } + ) + ); + containerClient = serviceClient.getContainerClient(containerName); + await containerClient.createIfNotExists(); + }; + + after(async () => { + if (server) { + await server.close(); + await server.clean(); + } + }); + + beforeEach(async () => { + // Create unique container name for each test + containerName = getUniqueName("versioning-transition"); + }); + + afterEach(async () => { + if (containerClient) { + try { + await containerClient.delete(); + } catch { + // Ignore cleanup errors + } + } + }); + + it("should match versioning behaviour from production when setting metadata and downloading @azurite", async () => { + await createServerAndClient(true); + + const name = getUniqueName("blob"); + const blobClient = containerClient.getAppendBlobClient(name); + + // 1. Create blob with versioning ENABLED + const createdBlob = await blobClient.create(); + await blobClient.appendBlock("base", 4); + const createdBlobVersionId = createdBlob.versionId; + assert.ok(!isNullOrWhitespace(createdBlobVersionId)); + + // Set metadata to create new version (should create version when versioning enabled) + const modifiedMetadataResult = await blobClient.setMetadata({ + versionedmeta: "value1" + }); + const modifiedVersionId = modifiedMetadataResult.versionId; + assert.ok(!isNullOrWhitespace(modifiedVersionId)); + assert.notStrictEqual(modifiedVersionId, createdBlobVersionId); + + const versionedFetched = await blobClient.getProperties(); + assert.ok(!isNullOrWhitespace(versionedFetched.versionId)); + assert.deepStrictEqual(versionedFetched.metadata, { + versionedmeta: "value1" + }); + assert.strictEqual(versionedFetched.versionId, modifiedVersionId); + const enabledVersionId = versionedFetched.versionId; + + // 2. Switch to versioning DISABLED + await createServerAndClient(false); + + // Set metadata should NOT create new version (overwrite current) + const resp = await blobClient.setMetadata({ disabledmeta: "value2" }); + assert.strictEqual(resp.versionId, undefined); + assert.notStrictEqual(resp.versionId, enabledVersionId); + + const currentProps = await blobClient.getProperties(); + // With versioning disabled, behavior may vary but metadata should be updated + assert.deepStrictEqual(currentProps.metadata, { + disabledmeta: "value2" + }); + + // Should still be able to access the version created when versioning was enabled + const firstVersionClient = containerClient + .getBlobClient(name) + .withVersion(enabledVersionId!); + const firstVersionProps = await firstVersionClient.getProperties(); + assert.strictEqual(firstVersionProps.versionId, enabledVersionId); + // Original version should still have the original metadata + assert.deepStrictEqual(firstVersionProps.metadata, { + versionedmeta: "value1" + }); + + // 3. Re-enable versioning to verify behaviour + await createServerAndClient(true); + + const thirdModification = await blobClient.setMetadata({ + versionedmeta: "value3" + }); + const thirdModificationVersionId = thirdModification.versionId; + assert.ok(!isNullOrWhitespace(thirdModificationVersionId)); + assert.notStrictEqual(thirdModificationVersionId, createdBlobVersionId); + + const thirdModificationFetched = await blobClient.getProperties(); + assert.ok(!isNullOrWhitespace(thirdModificationFetched.versionId)); + assert.deepStrictEqual(thirdModificationFetched.metadata, { + versionedmeta: "value3" + }); + assert.strictEqual( + thirdModificationFetched.versionId, + thirdModificationVersionId + ); + + // 4. Switch to versioning DISABLED + // Verify downloading with versioning disabled returns the same version + // because no version modification operation was executed. + await createServerAndClient(false); + + const fetched = await blobClient.download(); + assert.ok(!isNullOrWhitespace(fetched.versionId)); + assert.deepStrictEqual(fetched.versionId, thirdModificationVersionId); + + await blobClient.appendBlock("bob", 3); + const downloadedAfterAppend = await blobClient.download(); + assert.ok(!isNullOrWhitespace(downloadedAfterAppend.versionId)); + assert.deepStrictEqual( + downloadedAfterAppend.versionId, + thirdModificationVersionId + ); + }); + + it("should match versioning behaviour from production when listing blobs after creation operations @production", async () => { + await createServerAndClient(true); + + // Ensure versioning is ENABLED first + const name = "blobA"; + const blobClient = containerClient.getAppendBlobClient(name); + + // 1. Create blob with versioning ENABLED + const createdBlob = await blobClient.create(); + await blobClient.appendBlock("base", 4); + const createdBlobVersionId = createdBlob.versionId; + assert.ok(!isNullOrWhitespace(createdBlobVersionId)); + + // Set metadata to create new version (should create version when versioning enabled) + const modifiedMetadataResult = await blobClient.setMetadata({ + versionedmeta: "value1" + }); + const modifiedVersionId = modifiedMetadataResult.versionId; + assert.ok(!isNullOrWhitespace(modifiedVersionId)); + assert.notStrictEqual(modifiedVersionId, createdBlobVersionId); + + const versionedFetched = await blobClient.getProperties(); + assert.ok(!isNullOrWhitespace(versionedFetched.versionId)); + assert.deepStrictEqual(versionedFetched.metadata, { + versionedmeta: "value1" + }); + assert.strictEqual(versionedFetched.versionId, modifiedVersionId); + const enabledVersionId = versionedFetched.versionId; + + const listingResult = containerClient.listBlobsFlat({ + includeVersions: true + }); + const pageable = await listingResult.byPage().next(); + assert.strictEqual(pageable.value.segment.blobItems.length, 2); + + for await (const item2 of pageable.value.segment.blobItems) { + assert.ok(!isNullOrWhitespace((item2 as BlobItem).versionId)); + } + + // 2. Switch to versioning DISABLED + await createServerAndClient(false); + + const listingResult2 = containerClient.listBlobsFlat({ + includeVersions: true + }); + const pageable2 = await listingResult2.byPage().next(); + assert.strictEqual(pageable2.value.segment.blobItems.length, 2); + + for await (const item2 of pageable2.value.segment.blobItems) { + assert.ok(!isNullOrWhitespace((item2 as BlobItem).versionId)); + } + + // Set metadata should NOT create new version (overwrite current) + const resp = await blobClient.setMetadata({ disabledmeta: "value2" }); + assert.strictEqual(resp.versionId, undefined); + assert.notStrictEqual(resp.versionId, enabledVersionId); + await blobClient.createSnapshot(); + + const listingResult3 = containerClient.listBlobsFlat({ + includeVersions: true, + includeSnapshots: true + }); + const pageable3 = await listingResult3.byPage().next(); + assert.strictEqual(pageable3.value.segment.blobItems.length, 4); + + for await (const item3 of pageable3.value.segment.blobItems) { + const asBlobModel = item3 as BlobItem; + assert.ok(!asBlobModel.isCurrentVersion); + } + + const currentProps = await blobClient.getProperties(); + // With versioning disabled, behavior may vary but metadata should be updated + assert.deepStrictEqual(currentProps.metadata, { + disabledmeta: "value2" + }); + + // Should still be able to access the version created when versioning was enabled + const firstVersionClient = containerClient + .getBlobClient(name) + .withVersion(enabledVersionId!); + const firstVersionProps = await firstVersionClient.getProperties(); + assert.strictEqual(firstVersionProps.versionId, enabledVersionId); + // Original version should still have the original metadata + assert.deepStrictEqual(firstVersionProps.metadata, { + versionedmeta: "value1" + }); + }); + + it("should throw when downloading with both versionId and snapshot @azurite", async () => { + await createServerAndClient(true); + + const name = getUniqueName("blob"); + const blobClient = containerClient.getBlockBlobClient(name); + + // Create blob + const created = await blobClient.upload("content", 7); + const versionId = created.versionId; + assert.ok(!isNullOrWhitespace(versionId)); + + // Create a snapshot + const snapshot = await blobClient.createSnapshot(); + + try { + // Try to download with both snapshot and versionId - should fail + await blobClient + .withVersion(versionId!) + .withSnapshot(snapshot.snapshot!) + .download(); + assert.fail( + "Should have thrown error when versionId provided with snapshot" + ); + } catch (error: any) { + // Azure Storage should return an error for this invalid combination + assert.ok(error.statusCode === 400); + // Note: Error message may vary between Azure Storage implementations + } + }); + + it("should throw error when versionId is provided with snapshot option only @azurite", async () => { + await createServerAndClient(true); + + const name = getUniqueName("blob"); + const blobClient = containerClient.getBlockBlobClient(name); + + // Create blob + const created = await blobClient.upload("content", 7); + + // Create a snapshot + await blobClient.createSnapshot(); + + try { + // Try to delete with both snapshot and versionId - should fail + await blobClient.withVersion(created.versionId!).delete({ + deleteSnapshots: "only" + }); + assert.fail( + "Should have thrown error when versionId provided with snapshot operations" + ); + } catch (error: any) { + // Azure Storage should return an error for this invalid combination + assert.ok( + error.statusCode === 400 || + error.code === "InvalidHeaderValue" || + error.code === "InvalidQueryParameterValue" + ); + // Note: Error message may vary between Azure Storage implementations + } + }); + + it("should throw error when versionId is provided with snapshot option include @azurite", async () => { + await createServerAndClient(true); + + const name = getUniqueName("blob"); + const blobClient = containerClient.getBlockBlobClient(name); + + // Create blob + const created = await blobClient.upload("content", 7); + + // Create a snapshot + await blobClient.createSnapshot(); + + try { + // Try to delete with both snapshot and versionId - should fail + await blobClient.withVersion(created.versionId!).delete({ + deleteSnapshots: "include" + }); + assert.fail( + "Should have thrown error when versionId provided with snapshot operations" + ); + } catch (error: any) { + // Azure Storage should return an error for this invalid combination + assert.ok(error.statusCode === 400); + // Note: Error message may vary between Azure Storage implementations + } + }); + + it("should throw error when versionId is provided with snapshot @azurite", async () => { + await createServerAndClient(true); + + const name = getUniqueName("blob"); + const blobClient = containerClient.getBlockBlobClient(name); + + // Create blob + const created = await blobClient.upload("content", 7); + + // Create a snapshot + const snapshot = await blobClient.createSnapshot(); + + try { + // Try to delete with both snapshot and versionId - should fail + await blobClient + .withVersion(created.versionId!) + .withSnapshot(snapshot.snapshot!) + .delete(); + assert.fail( + "Should have thrown error when versionId provided with snapshot operations" + ); + } catch (error: any) { + // Azure Storage should return an error for this invalid combination + assert.ok(error.statusCode === 400); + // Note: Error message may vary between Azure Storage implementations + } + }); + + it("should set the base blob as a previous version and delete the snapshots @azurite", async () => { + await createServerAndClient(true); + + const name = getUniqueName("blob"); + const blobClient = containerClient.getBlockBlobClient(name); + + // Create blob + const created = await blobClient.upload("content", 7); + const versionId = created.versionId; + assert.ok(!isNullOrWhitespace(versionId)); + + // Create a snapshot + await blobClient.createSnapshot(); + + await blobClient.delete({ + deleteSnapshots: "include" + }); + + const downloadDeleted = await blobClient.withVersion(versionId!).download(); + assert.ok(!isNullOrWhitespace(downloadDeleted.versionId)); + }); + + it("should fail to write with versioning enabled because IfNoneMatch was specified @azurite", async () => { + await createServerAndClient(true); + const name = getUniqueName("blob"); + const blobClient = containerClient.getBlockBlobClient(name); + + // Create blob + const created = await blobClient.upload("content", 7); + const versionId = created.versionId; + assert.ok(!isNullOrWhitespace(versionId)); + + try { + // Try to upload again with ifNoneMatch: "*" - should fail because blob exists, even with versioning + await blobClient.upload("new content", 11, { + conditions: { + ifNoneMatch: "*" + } + }); + assert.fail("Should have thrown error when uploading with ifNoneMatch to existing blob"); + } catch (error: any) { + // Should fail with 409 Conflict because blob already exists + assert.ok(error.statusCode === 409 || error.code === "BlobAlreadyExists"); + } + }); +}); diff --git a/tests/blob/apis/versioning.production.parity.test.ts b/tests/blob/apis/versioning.production.parity.test.ts new file mode 100644 index 000000000..198632101 --- /dev/null +++ b/tests/blob/apis/versioning.production.parity.test.ts @@ -0,0 +1,450 @@ +import * as assert from "assert"; +import { BlobItem, BlobServiceClient, ContainerClient } from "@azure/storage-blob"; +import { DefaultAzureCredential } from "@azure/identity"; +import { configLogger } from "../../../src/common/Logger"; +import { getUniqueName } from "../../testutils"; +import { isNullOrWhitespace } from "../../../src/blob/utils/utils"; + +// Set to true when you want to debug the emulator +configLogger(false); + +/** + * Helper function to wait for manual versioning configuration + */ +async function promptForVersioningStateChangeAndVerify( + realServiceClient: BlobServiceClient, + containerName: string, + requiredState: "enabled" | "disabled" +): Promise { + const stateMessage = + requiredState === "enabled" + ? "ENABLE blob versioning" + : "DISABLE blob versioning"; + + console.log("\n" + "=".repeat(80)); + console.log(`🔧 MANUAL ACTION REQUIRED`); + console.log("=".repeat(80)); + console.log(`Please ${stateMessage} for your Azure Storage account:`); + console.log(`⏱️ Waiting 10 seconds for you to configure versioning...`); + + // Wait 10 seconds instead of prompting + await new Promise((resolve) => setTimeout(resolve, 10000)); + + console.log(`✅ Proceeding with versioning ${requiredState}\n`); + await verifyVersioningState(realServiceClient, containerName, requiredState); +} + +/** + * Helper function to verify versioning state by attempting operations + */ +async function verifyVersioningState( + serviceClient: BlobServiceClient, + containerName: string, + expectedState: "enabled" | "disabled" +): Promise { + const testBlobName = getUniqueName("version-test"); + const testContent = "version test content"; + + const blockBlobClient = serviceClient + .getContainerClient(containerName) + .getBlockBlobClient(testBlobName); + + try { + const uploadResult = await blockBlobClient.upload( + testContent, + testContent.length + ); + + if (expectedState === "enabled") { + assert.ok( + uploadResult.versionId, + `Blob service should return version ID when versioning is enabled` + ); + assert.notStrictEqual( + uploadResult.versionId, + "", + `Blob service version ID should not be empty when versioning is enabled` + ); + } else { + // When versioning is disabled, some services might still return a version ID, so we'll be less strict + // The key difference is in behavior during multiple uploads and deletions + } + + // Clean up test blob + await blockBlobClient.delete(); + console.log(`✅ Blob service versioning state verified: ${expectedState}`); + } catch (error) { + console.error(`❌ Failed to verify Blob service versioning state:`, error); + throw error; + } +} + +// Skipping by default since these should be run manually +describe.skip("Blob Versioning Parity Tests - Production", () => { + let realServiceClient: BlobServiceClient; + let realContainerClient: ContainerClient; + let containerName: string; + + const realStorageAccountUrl = "YOUR_AZURE_STORAGE_ACCOUNT_URL"; + + before(async () => { + console.log("🚀 Setting up Blob Versioning Transition Parity Tests..."); + + // Initialize real Azure Storage client + realServiceClient = new BlobServiceClient( + realStorageAccountUrl, + new DefaultAzureCredential() + ); + }); + + beforeEach(async () => { + // Create unique container name for each test + containerName = getUniqueName("versioning-transition"); + realContainerClient = realServiceClient.getContainerClient(containerName); + await realContainerClient.create(); + }); + + it("should match versioning behaviour from lokidb when setting metadata and downloading @production", async () => { + // Ensure versioning is ENABLED first + const name = getUniqueName("blob"); + const blobClient = realContainerClient.getAppendBlobClient(name); + + // 1. Create blob with versioning ENABLED + const createdBlob = await blobClient.create(); + await blobClient.appendBlock("base", 4); + const createdBlobVersionId = createdBlob.versionId; + assert.ok(!isNullOrWhitespace(createdBlobVersionId)); + + // Set metadata to create new version (should create version when versioning enabled) + const modifiedMetadataResult = await blobClient.setMetadata({ + versionedmeta: "value1" + }); + const modifiedVersionId = modifiedMetadataResult.versionId; + assert.ok(!isNullOrWhitespace(modifiedVersionId)); + assert.notStrictEqual(modifiedVersionId, createdBlobVersionId); + + const versionedFetched = await blobClient.getProperties(); + assert.ok(!isNullOrWhitespace(versionedFetched.versionId)); + assert.deepStrictEqual(versionedFetched.metadata, { + versionedmeta: "value1" + }); + assert.strictEqual(versionedFetched.versionId, modifiedVersionId); + const enabledVersionId = versionedFetched.versionId; + + // 2. Switch to versioning DISABLED + await promptForVersioningStateChangeAndVerify( + realServiceClient, + containerName, + "disabled" + ); + + // Set metadata should NOT create new version (overwrite current) + const resp = await blobClient.setMetadata({ disabledmeta: "value2" }); + assert.strictEqual(resp.versionId, undefined); + assert.notStrictEqual(resp.versionId, enabledVersionId); + + const currentProps = await blobClient.getProperties(); + // With versioning disabled, behavior may vary but metadata should be updated + assert.deepStrictEqual(currentProps.metadata, { + disabledmeta: "value2" + }); + + // Should still be able to access the version created when versioning was enabled + const firstVersionClient = realContainerClient + .getBlobClient(name) + .withVersion(enabledVersionId!); + const firstVersionProps = await firstVersionClient.getProperties(); + assert.strictEqual(firstVersionProps.versionId, enabledVersionId); + // Original version should still have the original metadata + assert.deepStrictEqual(firstVersionProps.metadata, { + versionedmeta: "value1" + }); + + // 3. Re-enable versioning to verify behaviour + await promptForVersioningStateChangeAndVerify( + realServiceClient, + containerName, + "enabled" + ); + + const thirdModification = await blobClient.setMetadata({ + versionedmeta: "value3" + }); + const thirdModificationVersionId = thirdModification.versionId; + assert.ok(!isNullOrWhitespace(thirdModificationVersionId)); + assert.notStrictEqual(thirdModificationVersionId, createdBlobVersionId); + + const thirdModificationFetched = await blobClient.getProperties(); + assert.ok(!isNullOrWhitespace(thirdModificationFetched.versionId)); + assert.deepStrictEqual(thirdModificationFetched.metadata, { + versionedmeta: "value3" + }); + assert.strictEqual( + thirdModificationFetched.versionId, + thirdModificationVersionId + ); + + // 4. Switch to versioning DISABLED + // Verify downloading with versioning disabled returns the same version + // because no version modification operation was executed. + await promptForVersioningStateChangeAndVerify( + realServiceClient, + containerName, + "disabled" + ); + + const fetched = await blobClient.download(); + assert.ok(!isNullOrWhitespace(fetched.versionId)); + assert.deepStrictEqual(fetched.versionId, thirdModificationVersionId); + + await blobClient.appendBlock("bob", 3); + const downloadedAfterAppend = await blobClient.download(); + assert.ok(!isNullOrWhitespace(downloadedAfterAppend.versionId)); + assert.deepStrictEqual( + downloadedAfterAppend.versionId, + thirdModificationVersionId + ); + }); + + it("should match versioning behaviour from lokidb when listing blobs after creation operations @production", async () => { + // Ensure versioning is ENABLED first + const name = getUniqueName("blob"); + const blobClient = realContainerClient.getAppendBlobClient(name); + + // 1. Create blob with versioning ENABLED + const createdBlob = await blobClient.create(); + await blobClient.appendBlock("base", 4); + const createdBlobVersionId = createdBlob.versionId; + assert.ok(!isNullOrWhitespace(createdBlobVersionId)); + + // Set metadata to create new version (should create version when versioning enabled) + const modifiedMetadataResult = await blobClient.setMetadata({ + versionedmeta: "value1" + }); + const modifiedVersionId = modifiedMetadataResult.versionId; + assert.ok(!isNullOrWhitespace(modifiedVersionId)); + assert.notStrictEqual(modifiedVersionId, createdBlobVersionId); + + const versionedFetched = await blobClient.getProperties(); + assert.ok(!isNullOrWhitespace(versionedFetched.versionId)); + assert.deepStrictEqual(versionedFetched.metadata, { + versionedmeta: "value1" + }); + assert.strictEqual(versionedFetched.versionId, modifiedVersionId); + const enabledVersionId = versionedFetched.versionId; + + const listingResult = realContainerClient.listBlobsFlat({ + includeVersions: true + }); + const pageable = await listingResult.byPage().next(); + assert.strictEqual(pageable.value.segment.blobItems.length, 2); + + for await (const item2 of pageable.value.segment.blobItems) { + assert.ok(!isNullOrWhitespace((item2 as BlobItem).versionId)); + } + + // 2. Switch to versioning DISABLED + await promptForVersioningStateChangeAndVerify( + realServiceClient, + containerName, + "disabled" + ); + + const listingResult2 = realContainerClient.listBlobsFlat({ + includeVersions: true + }); + const pageable2 = await listingResult2.byPage().next(); + assert.strictEqual(pageable2.value.segment.blobItems.length, 2); + + for await (const item2 of pageable2.value.segment.blobItems) { + assert.ok(!isNullOrWhitespace((item2 as BlobItem).versionId)); + } + + // Set metadata should NOT create new version (overwrite current) + const resp = await blobClient.setMetadata({ disabledmeta: "value2" }); + assert.strictEqual(resp.versionId, undefined); + assert.notStrictEqual(resp.versionId, enabledVersionId); + await blobClient.createSnapshot(); + + const listingResult3 = realContainerClient.listBlobsFlat({ + includeVersions: true, + includeSnapshots: true + }); + const pageable3 = await listingResult3.byPage().next(); + assert.strictEqual(pageable3.value.segment.blobItems.length, 4); + + for await (const item3 of pageable3.value.segment.blobItems) { + const asBlobModel = item3 as BlobItem; + assert.ok(!asBlobModel.isCurrentVersion); + } + + const currentProps = await blobClient.getProperties(); + // With versioning disabled, behavior may vary but metadata should be updated + assert.deepStrictEqual(currentProps.metadata, { + disabledmeta: "value2" + }); + + // Should still be able to access the version created when versioning was enabled + const firstVersionClient = realContainerClient + .getBlobClient(name) + .withVersion(enabledVersionId!); + const firstVersionProps = await firstVersionClient.getProperties(); + assert.strictEqual(firstVersionProps.versionId, enabledVersionId); + // Original version should still have the original metadata + assert.deepStrictEqual(firstVersionProps.metadata, { + versionedmeta: "value1" + }); + }); + + it("should throw when downloading with both versionId and snapshot @production", async () => { + const name = getUniqueName("blob"); + const blobClient = realContainerClient.getBlockBlobClient(name); + + // Create blob + const created = await blobClient.upload("content", 7); + const versionId = created.versionId; + assert.ok(!isNullOrWhitespace(versionId)); + + // Create a snapshot + const snapshot = await blobClient.createSnapshot(); + + try { + // Try to download with both snapshot and versionId - should fail + await blobClient + .withVersion(versionId!) + .withSnapshot(snapshot.snapshot!) + .download(); + assert.fail( + "Should have thrown error when versionId provided with snapshot" + ); + } catch (error: any) { + // Azure Storage should return an error for this invalid combination + assert.ok(error.statusCode === 400); + // Note: Error message may vary between Azure Storage implementations + } + }); + + it("should throw error when versionId is provided with snapshot option only @production", async () => { + const name = getUniqueName("blob"); + const blobClient = realContainerClient.getBlockBlobClient(name); + + // Create blob + const created = await blobClient.upload("content", 7); + + // Create a snapshot + await blobClient.createSnapshot(); + + try { + // Try to delete with both snapshot and versionId - should fail + await blobClient.withVersion(created.versionId!).delete({ + deleteSnapshots: "only" + }); + assert.fail( + "Should have thrown error when versionId provided with snapshot operations" + ); + } catch (error: any) { + // Azure Storage should return an error for this invalid combination + assert.ok( + error.statusCode === 400 || + error.code === "InvalidHeaderValue" || + error.code === "InvalidQueryParameterValue" + ); + // Note: Error message may vary between Azure Storage implementations + } + }); + + it("should throw error when versionId is provided with snapshot option include @production", async () => { + const name = getUniqueName("blob"); + const blobClient = realContainerClient.getBlockBlobClient(name); + + // Create blob + const created = await blobClient.upload("content", 7); + + // Create a snapshot + await blobClient.createSnapshot(); + + try { + // Try to delete with both snapshot and versionId - should fail + await blobClient.withVersion(created.versionId!).delete({ + deleteSnapshots: "include" + }); + assert.fail( + "Should have thrown error when versionId provided with snapshot operations" + ); + } catch (error: any) { + // Azure Storage should return an error for this invalid combination + assert.ok(error.statusCode === 400); + // Note: Error message may vary between Azure Storage implementations + } + }); + + it("should throw error when versionId is provided with snapshot @production", async () => { + const name = getUniqueName("blob"); + const blobClient = realContainerClient.getBlockBlobClient(name); + + // Create blob + const created = await blobClient.upload("content", 7); + + // Create a snapshot + const snapshot = await blobClient.createSnapshot(); + + try { + // Try to delete with both snapshot and versionId - should fail + await blobClient + .withVersion(created.versionId!) + .withSnapshot(snapshot.snapshot!) + .delete(); + assert.fail( + "Should have thrown error when versionId provided with snapshot operations" + ); + } catch (error: any) { + // Azure Storage should return an error for this invalid combination + assert.ok(error.statusCode === 400); + // Note: Error message may vary between Azure Storage implementations + } + }); + + it("should set the base blob as a previous version and delete the snapshots @production", async () => { + const name = getUniqueName("blob"); + const blobClient = realContainerClient.getBlockBlobClient(name); + + // Create blob + const created = await blobClient.upload("content", 7); + const versionId = created.versionId; + assert.ok(!isNullOrWhitespace(versionId)); + + // Create a snapshot + await blobClient.createSnapshot(); + + // Try to delete with both snapshot and versionId - should fail + await blobClient.delete({ + deleteSnapshots: "include" + }); + + const downloadDeleted = await blobClient.withVersion(versionId!).download(); + assert.ok(!isNullOrWhitespace(downloadDeleted.versionId)); + }); + + it("should fail to write with versioning enabled because IfNoneMatch was specified @production", async () => { + const name = getUniqueName("blob"); + const blobClient = realContainerClient.getBlockBlobClient(name); + + // Create blob + const created = await blobClient.upload("content", 7); + const versionId = created.versionId; + assert.ok(!isNullOrWhitespace(versionId)); + + try { + // Try to upload again with ifNoneMatch: "*" - should fail because blob exists, even with versioning + await blobClient.upload("new content", 11, { + conditions: { + ifNoneMatch: "*" + } + }); + assert.fail("Should have thrown error when uploading with ifNoneMatch to existing blob"); + } catch (error: any) { + // Should fail with 409 Conflict because blob already exists + assert.ok(error.statusCode === 409 || error.code === "BlobAlreadyExists"); + } + }); +}); diff --git a/tests/blob/handlers/AppendBlobHandler.test.ts b/tests/blob/handlers/AppendBlobHandler.test.ts index a8ef5a57f..697ca7ade 100644 --- a/tests/blob/handlers/AppendBlobHandler.test.ts +++ b/tests/blob/handlers/AppendBlobHandler.test.ts @@ -61,6 +61,7 @@ describe("AppendBlobHandler", () => { blobCtx.account, blobCtx.container, blobCtx.blob, + undefined, undefined ) ).thenResolve({ @@ -87,6 +88,16 @@ describe("AppendBlobHandler", () => { undefined ) ).thenResolve(properties); + when( + metadataStore.createBlob(anything(), anything(), undefined, undefined) + ).thenResolve({ + name: blobCtx.blob, + accountName: blobCtx.account, + containerName: blobCtx.container, + isCommitted: true, + properties, + versionId: "" + }); const extentStore: IExtentStore = mock(); when( diff --git a/tests/blob/lokidb.test.ts b/tests/blob/lokidb.test.ts new file mode 100644 index 000000000..0adf412cd --- /dev/null +++ b/tests/blob/lokidb.test.ts @@ -0,0 +1,586 @@ +import assert = require("assert"); +import { randomUUID as uuid } from "crypto"; +import * as fs from "fs"; +import LokiBlobMetadataStore from "../../src/blob/persistence/LokiBlobMetadataStore"; +import * as Models from "../../src/blob/generated/artifacts/models"; +import Context from "../../src/blob/generated/Context"; +import { configLogger } from "../../src/common/Logger"; +import { isNullOrWhitespace } from "../../src/blob/utils/utils"; +import { + buildAppendBlob, + buildBlockBlob, + buildContainer, + buildPageBlob, + createContext +} from "../testutils"; +import { AccountModel } from "../../src/common/account/AccountModel"; +import LokiAccountModelStore from "../../src/common/account/LokiAccountModelStore"; +// Silence logs for tests +configLogger(false); + +const ACCOUNT = "devstoreaccount1"; +const ACCOUNT_DB_FILE = "__test_db_account_models_lokidb__.json"; + +function createAccountModelStore(accountModel: AccountModel, inMemory: boolean = false): LokiAccountModelStore { + const accountModels = new Map(); + accountModels.set(accountModel.key || ACCOUNT, accountModel); + return new LokiAccountModelStore(ACCOUNT_DB_FILE, inMemory, accountModels); +} + +describe("LokiBlobMetadataStore - Versioning Disabled", () => { + let accountModelStore: LokiAccountModelStore; + let store: LokiBlobMetadataStore; + let containerName: string; + let ctx: Context; + const DB_FILE = "__test_db_blob__.json"; // standard shared test db path + let originalDbContent: string | undefined; + let originalExists = false; + + before(() => { + if (fs.existsSync(DB_FILE)) { + originalExists = true; + originalDbContent = fs.readFileSync(DB_FILE, "utf8"); + } + }); + + beforeEach(async () => { + ctx = createContext(); + containerName = `container-${uuid()}`; + // Use in-memory for regular tests (fast); special test will override + const accountModel: AccountModel = + { + key: "account", + isBlobVersioningEnabled: false + }; + accountModelStore = createAccountModelStore(accountModel, true); + store = new LokiBlobMetadataStore(DB_FILE, true, accountModelStore); + await accountModelStore.init(); + await store.init(); + await store.createContainer(ctx, buildContainer(ACCOUNT, containerName)); + }); + + afterEach(async () => { + if (accountModelStore) { + await accountModelStore.close(); + await accountModelStore.clean(); + } + + if (store) { + await store.close(); + await store.clean(); + } + }); + + after(() => { + // Restore DB file to its original state + if (originalExists) { + fs.writeFileSync(DB_FILE, originalDbContent!); + } else if (fs.existsSync(DB_FILE)) { + try { + fs.unlinkSync(DB_FILE); + } catch { + /* ignore */ + } + } + }); + + it("creates base blob with versionId == '' and treats it as latest when no version specified @loki", async () => { + const name = `blob-${uuid()}`; + const contentV1 = "content-v1"; + const blobV1 = buildBlockBlob(ACCOUNT, containerName, name, contentV1); + await store.createBlob(ctx, blobV1); + + // Fetch without versionId (downloadBlob) => latest + const fetched = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual( + fetched.versionId, + "", + "Base version should have empty versionId" + ); + }); + + it("overwrites base blob and keeps latest (no version) when versioning disabled @loki", async () => { + const name = `blob-${uuid()}`; + const blobV1 = buildBlockBlob(ACCOUNT, containerName, name, "bob"); + const createdV1 = await store.createBlob(ctx, blobV1); + + const blobV2 = buildBlockBlob(ACCOUNT, containerName, name, "alice"); + const createdV2 = await store.createBlob(ctx, blobV2); + + assert.deepStrictEqual(createdV1.versionId, ""); + assert.deepStrictEqual(createdV2.versionId, ""); + assert.notDeepStrictEqual(createdV1.properties.contentLength, undefined); + assert.notDeepStrictEqual(createdV2.properties.contentLength, undefined); + assert.notDeepStrictEqual( + createdV1.properties.contentLength, + createdV2.properties.contentLength + ); + + // Download latest (no version) should return v2 (still versionId "") + const latest = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + assert.strictEqual( + latest.properties.contentLength, + blobV2.properties.contentLength + ); + assert.strictEqual(latest.versionId, "", "Still base version placeholder"); + }); + + it("can retrieve a version created while versioning was enabled after disabling versioning @loki", async () => { + // Close the in-memory disabled store from beforeEach; we need persistence for this scenario + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create persistent store with versioning enabled (inMemory=false) + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let persistent = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await persistent.init(); + await persistent.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const blobV = buildBlockBlob(ACCOUNT, containerName, name, "body"); + + const createdBlob = await persistent.createBlob(ctx, blobV); + const versionId = createdBlob.versionId; + assert.ok(!isNullOrWhitespace(versionId)); + await accountModelStore.close(); + await persistent.close(); // Do NOT clean so data persists + + // 2. Recreate store with versioning disabled using same DB file + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // 3. Attempt to fetch explicitly by the version id created earlier + const fetched = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + versionId + ); + assert.ok( + !isNullOrWhitespace(fetched.versionId), + "Fetched version should have a non-empty versionId" + ); + assert.deepStrictEqual(fetched.versionId, versionId); + }); + + it("should not create versions for subsequent blob modifications when versioning disabled @loki", async () => { + const name = `blob-${uuid()}`; + const blobV1 = buildBlockBlob(ACCOUNT, containerName, name, "version1"); + await store.createBlob(ctx, blobV1); + + // Multiple overwrites should all result in versionId "" + const blobV2 = buildBlockBlob(ACCOUNT, containerName, name, "version2"); + await store.createBlob(ctx, blobV2); + + const blobV3 = buildBlockBlob(ACCOUNT, containerName, name, "version3"); + await store.createBlob(ctx, blobV3); + + const latest = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + assert.strictEqual(latest.versionId, ""); + assert.strictEqual( + latest.properties.contentLength, + blobV3.properties.contentLength + ); + }); + + it("should handle delete operations without creating versions when versioning disabled @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + // Delete the blob + await store.deleteBlob(ctx, ACCOUNT, containerName, name, {}); + + // Verify blob is deleted + try { + await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.fail("Should have thrown error for deleted blob"); + } catch (error) { + // Expected behavior - blob should be deleted + } + }); + + it("should allow creation of new blob with same name after deletion when versioning disabled @loki", async () => { + const name = `blob-${uuid()}`; + const blob1 = buildBlockBlob(ACCOUNT, containerName, name, "content1"); + await store.createBlob(ctx, blob1); + + // Delete the blob + await store.deleteBlob(ctx, ACCOUNT, containerName, name, {}); + + // Create new blob with same name - should work and have versionId "" + const blob2 = buildBlockBlob(ACCOUNT, containerName, name, "content2"); + const created = await store.createBlob(ctx, blob2); + + assert.strictEqual(created.versionId, ""); + + const fetched = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + assert.strictEqual(fetched.versionId, ""); + assert.strictEqual( + fetched.properties.contentLength, + blob2.properties.contentLength + ); + }); + + // ================== SNAPSHOT TESTS WITH VERSIONING DISABLED ================== + it("should create snapshots without versions when versioning disabled @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + const beforeSnapshot = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Take snapshot should not create version when versioning disabled + ctx.startTime = new Date(Date.now() + 100); + const snapshotResponse = await store.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + + assert.ok(snapshotResponse.snapshot); + assert.strictEqual(snapshotResponse.versionId, ""); + + // Current blob should still exist and not have a version + const afterSnapshot = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + assert.strictEqual(afterSnapshot.versionId, ""); + assert.strictEqual(afterSnapshot.versionId, beforeSnapshot.versionId); + }); + + // ================== HTTP HEADERS TESTS WITH VERSIONING DISABLED ================== + it("should update HTTP headers in place without creating versions when versioning disabled @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + const beforeHeaders = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + ctx.startTime = new Date(Date.now() + 100); + await store.setBlobHTTPHeaders( + ctx, + ACCOUNT, + containerName, + name, + undefined, + { blobContentType: "text/plain" } + ); + + const afterHeaders = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Should update in place - same versionId (empty) but updated properties + assert.strictEqual(afterHeaders.versionId, beforeHeaders.versionId); + assert.strictEqual(afterHeaders.versionId, ""); + assert.strictEqual(afterHeaders.properties.contentType, "text/plain"); + }); + + // ================== BLOB TAGS TESTS WITH VERSIONING DISABLED ================== + it("should update blob tags in place without creating versions when versioning disabled @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + const beforeTags = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + ctx.startTime = new Date(Date.now() + 100); + await store.setBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined, + { blobTagSet: [{ key: "environment", value: "test" }] } + ); + + const afterTags = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Should update in place - same versionId (empty) + assert.strictEqual(afterTags.versionId, beforeTags.versionId); + assert.strictEqual(afterTags.versionId, ""); + + // Verify tags are set + const tags = await store.getBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined + ); + assert.deepStrictEqual(tags, { + blobTagSet: [{ key: "environment", value: "test" }] + }); + }); + + // ================== TIER MANAGEMENT TESTS WITH VERSIONING DISABLED ================== + it("should update tier in place without creating versions when versioning disabled @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + blob.properties.accessTier = Models.AccessTier.Hot; + await store.createBlob(ctx, blob); + + const beforeTier = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Set tier should update in place + await store.setTier( + ctx, + ACCOUNT, + containerName, + name, + undefined, + Models.AccessTier.Cool, + undefined + ); + + const afterTier = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Should update in place - same versionId (empty) but updated tier + assert.strictEqual(afterTier.versionId, beforeTier.versionId); + assert.strictEqual(afterTier.versionId, ""); + assert.strictEqual(afterTier.properties.accessTier, Models.AccessTier.Cool); + }); + + // ================== BLOB EXISTENCE AND PROPERTIES TESTS WITH VERSIONING DISABLED ================== + it("should check blob existence without version support when versioning disabled @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + // Check existence should work + await store.checkBlobExist(ctx, ACCOUNT, containerName, name); + + // Should throw for version-specific requests since versioning is disabled + try { + await store.checkBlobExist( + ctx, + ACCOUNT, + containerName, + name, + "", + "2099-01-01T00:00:00.0000000Z" + ); + assert.fail( + "Should have thrown for version-specific request when versioning disabled" + ); + } catch (error) { + // Expected - version requests not supported when versioning disabled + } + }); + + it("should get properties without version support when versioning disabled @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + // Set metadata + ctx.startTime = new Date(Date.now() + 100); + await store.setBlobMetadata(ctx, ACCOUNT, containerName, name, undefined, { + environment: "test" + }); + + // Get properties should work + const props = await store.getBlobProperties( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined + ); + + assert.deepStrictEqual(props.metadata, { environment: "test" }); + }); + + // ================== APPEND BLOB OPERATIONS TESTS WITH VERSIONING DISABLED ================== + it("should handle Append Block operations normally when versioning disabled @loki", async () => { + const name = `blob-${uuid()}`; + const appendBlob = buildAppendBlob(ACCOUNT, containerName, name); + await store.createBlob(ctx, appendBlob); + + const afterCreate = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Append block + const block = { + accountName: ACCOUNT, + containerName, + blobName: name, + name: "append1", + size: 10, + persistency: { id: uuid(), offset: 0, count: 10 } + } as any; + + ctx.startTime = new Date(Date.now() + 100); + await store.appendBlock(ctx, block); + + const afterAppend = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Should update in place - same versionId (empty) + assert.strictEqual(afterAppend.versionId, afterCreate.versionId); + assert.strictEqual(afterAppend.versionId, ""); + assert.strictEqual(afterAppend.properties.contentLength, 10); + }); + + // ================== PAGE BLOB OPERATIONS TESTS WITH VERSIONING DISABLED ================== + it("should handle Put Page operations normally when versioning disabled @loki", async () => { + const name = `blob-${uuid()}`; + const pageBlob = buildPageBlob(ACCOUNT, containerName, name, 512); + await store.createBlob(ctx, pageBlob); + + const afterCreate = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Put Page + const persistency = { id: uuid(), offset: 0, count: 512 }; + ctx.startTime = new Date(Date.now() + 100); + await store.uploadPages(ctx, pageBlob, 0, 511, persistency); + + const afterUpload = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Should update in place - same versionId (empty) + assert.strictEqual(afterUpload.versionId, afterCreate.versionId); + assert.strictEqual(afterUpload.versionId, ""); + }); +}); diff --git a/tests/blob/pagewithdelimiter.test.ts b/tests/blob/pagewithdelimiter.test.ts index c488ab8de..162278cfd 100644 --- a/tests/blob/pagewithdelimiter.test.ts +++ b/tests/blob/pagewithdelimiter.test.ts @@ -17,7 +17,7 @@ describe("PageWithDelimiter", () => { } // a namer is used by fill, just return the value for testing - const namer = (i: string) => { return i; }; + const namer = (i: string): [string, string] => { return [i, ""]; }; // return a reader for a list const createReader = (items: string[], maxResults: number): (o: number) => Promise => { @@ -45,13 +45,13 @@ describe("PageWithDelimiter", () => { it("fills 1 result properly @loki", async () => { const page = new PageWithDelimiter(1); const [items, prefixes, marker] = await page.fill(createReader(blobs, 1), namer); - checkResult(items, prefixes, marker, 1, 0, "a"); + checkResult(items, prefixes, marker, 1, 0, "a" + PageWithDelimiter.VERSIONING_MARKER); }); it("fills n results properly @loki", async () => { const page = new PageWithDelimiter(5); const [items, prefixes, marker] = await page.fill(createReader(blobs, 5), namer); - checkResult(items, prefixes, marker, 5, 0, "c/sub/1"); + checkResult(items, prefixes, marker, 5, 0, "c/sub/1" + PageWithDelimiter.VERSIONING_MARKER); }); it("fills exact count with no continuation @loki", async () => { @@ -65,6 +65,21 @@ describe("PageWithDelimiter", () => { const [items, prefixes, marker] = await page.fill(createReader(blobs, blobs.length + 1), namer); checkResult(items, prefixes, marker, blobs.length, 0, ""); }); + + it("supports legacy name-only markers with duplicate names @loki", async () => { + const page = new PageWithDelimiter( + 2, + undefined, + undefined, + "name" + ); + const [items, prefixes, marker] = await page.fill( + createReader(["a", "a", "b"], 2), + namer + ); + + checkResult(items, prefixes, marker, 2, 0, "a"); + }); }); describe("with '/' delimiter", () => { @@ -89,7 +104,7 @@ describe("PageWithDelimiter", () => { const blobs = ["a", "b"]; const page = new PageWithDelimiter(1, "/"); let [items, prefixes, marker] = await page.fill(createReader(blobs, 1), namer); - checkResult(items, prefixes, marker, 1, 0, "a"); + checkResult(items, prefixes, marker, 1, 0, "a" + PageWithDelimiter.VERSIONING_MARKER); // now cut off the end of the array and ensure no continuation is returned page.reset(); @@ -101,14 +116,14 @@ describe("PageWithDelimiter", () => { const blobs = ["a/1", "a/2", "a/3", "a/sub/1"]; const page = new PageWithDelimiter(1, "/", "a/"); const [items, prefixes, marker] = await page.fill(createReader(blobs, 1), namer); - checkResult(items, prefixes, marker, 1, 0, "a/1"); + checkResult(items, prefixes, marker, 1, 0, "a/1" + PageWithDelimiter.VERSIONING_MARKER); }); it("returns first prefix when blobs exist @loki", async () => { const blobs = ["a/s0/1", "a/s0/2", "a/s0/3", "a/s1/1", "a/s2/2", "a/z"]; const page = new PageWithDelimiter(1, "/", "a/"); const [items, prefixes, marker] = await page.fill(createReader(blobs, 1), namer); - checkResult(items, prefixes, marker, 0, 1, "a/s0/3"); + checkResult(items, prefixes, marker, 0, 1, "a/s0/3" + PageWithDelimiter.VERSIONING_MARKER); }); }); @@ -118,21 +133,21 @@ describe("PageWithDelimiter", () => { const blobs = ["a/s0/1", "a/s0/2", "a/s0/3", "a/s1/1", "a/s1/2", "a/s2/2", "a/z"]; const page = new PageWithDelimiter(2, "/", "a/"); const [items, prefixes, marker] = await page.fill(createReader(blobs, 2), namer); - checkResult(items, prefixes, marker, 0, 2, "a/s1/2"); + checkResult(items, prefixes, marker, 0, 2, "a/s1/2" + PageWithDelimiter.VERSIONING_MARKER); }); it("squashes a mix @loki", async () => { const blobs = ["a/a", "a/s0/1", "a/s0/2", "a/s1/1", "a/s1/2", "a/z"]; const page = new PageWithDelimiter(2, "/", "a/"); const [items, prefixes, marker] = await page.fill(createReader(blobs, 2), namer); - checkResult(items, prefixes, marker, 1, 1, "a/s0/2"); + checkResult(items, prefixes, marker, 1, 1, "a/s0/2" + PageWithDelimiter.VERSIONING_MARKER); }); it("follows squashed pages @loki", async () => { const blobs = ["a/a", "a/s0/1", "a/s0/2", "a/s1/1", "a/s1/2", "a/z"]; const page = new PageWithDelimiter(2, "/", "a/"); let [items, prefixes, marker] = await page.fill(createReader(blobs, 2), namer); - checkResult(items, prefixes, marker, 1, 1, "a/s0/2"); + checkResult(items, prefixes, marker, 1, 1, "a/s0/2" + PageWithDelimiter.VERSIONING_MARKER); // now cut off the end of the array and ensure no continuation is returned page.reset(); @@ -148,4 +163,141 @@ describe("PageWithDelimiter", () => { }); }); }); + + describe("with versioning scenarios", () => { + // Mock blob model for testing versioning logic + interface MockVersionedBlob { + name: string; + versionId?: string; + snapshot?: string; + lastModified?: string; + } + + // Namer that extracts name and timestamp tuple like the real implementation + const versioningNamer = (blob: MockVersionedBlob): [string, string] => { + // Snapshot: use snapshot timestamp + if (blob.snapshot && blob.snapshot.length > 0) { + return [blob.name, blob.snapshot]; + } + // Versioned blob: use versionId timestamp + if (blob.versionId && blob.versionId.length > 0) { + return [blob.name, blob.versionId]; + } + // Non-versioned blob: use lastModified timestamp + return [blob.name, blob.lastModified || "2023-01-01T00:00:00.000Z"]; + }; + + // Reader for versioned blobs + const createVersionedReader = (items: MockVersionedBlob[], maxResults: number): + (o: number) => Promise => { + return (o: number) => { return Promise.resolve(items.slice(o, o + maxResults)); } + }; + + // Helper to check versioned results + function checkVersionedResult( + items: MockVersionedBlob[], + prefixes: BlobPrefixModel[], + marker: string, + expected_items_count: number, + expected_prefixes_count: number, + expected_marker: string + ): void { + assert.equal(items.length, expected_items_count); + assert.equal(prefixes.length, expected_prefixes_count); + assert.equal(marker, expected_marker); + } + + it("handles blobs with versionIds @loki", async () => { + const blobs: MockVersionedBlob[] = [ + { name: "blob1", versionId: "2023-01-01T10:00:00.000Z" }, + { name: "blob1", versionId: "2023-01-01T11:00:00.000Z" }, + { name: "blob2", versionId: "2023-01-01T12:00:00.000Z" } + ]; + + const page = new PageWithDelimiter(2); + const [items, prefixes, marker] = await page.fill(createVersionedReader(blobs, 2), versioningNamer); + + checkVersionedResult(items, prefixes, marker, 2, 0, + "blob1" + PageWithDelimiter.VERSIONING_MARKER + "2023-01-01T11:00:00.000Z"); + }); + + it("handles blobs with snapshots @loki", async () => { + const blobs: MockVersionedBlob[] = [ + { name: "blob1", lastModified: "2023-01-01T10:00:00.000Z" }, + { name: "blob1", snapshot: "2023-01-01T10:30:00.0000000Z" }, + { name: "blob1", snapshot: "2023-01-01T11:00:00.0000000Z" } + ]; + + const page = new PageWithDelimiter(2); + const [items, prefixes, marker] = await page.fill(createVersionedReader(blobs, 2), versioningNamer); + + checkVersionedResult(items, prefixes, marker, 2, 0, + "blob1" + PageWithDelimiter.VERSIONING_MARKER + "2023-01-01T10:30:00.0000000Z"); + }); + + it("handles mixed versioning types with same name @loki", async () => { + const blobs: MockVersionedBlob[] = [ + { name: "blob1", lastModified: "2023-01-01T10:00:00.000Z" }, + { name: "blob1", versionId: "2023-01-01T11:00:00.000Z" }, + { name: "blob1", snapshot: "2023-01-01T12:00:00.0000000Z" }, + { name: "blob2", versionId: "2023-01-01T13:00:00.000Z" } + ]; + + const page = new PageWithDelimiter(3); + const [items, prefixes, marker] = await page.fill(createVersionedReader(blobs, 3), versioningNamer); + + checkVersionedResult(items, prefixes, marker, 3, 0, + "blob1" + PageWithDelimiter.VERSIONING_MARKER + "2023-01-01T12:00:00.0000000Z"); + }); + + it("handles different blob names with versions @loki", async () => { + const blobs: MockVersionedBlob[] = [ + { name: "apple", versionId: "2023-01-01T10:00:00.000Z" }, + { name: "banana", versionId: "2023-01-01T09:00:00.000Z" }, // Earlier timestamp + { name: "cherry", lastModified: "2023-01-01T11:00:00.000Z" } + ]; + + const page = new PageWithDelimiter(10); + const [items, prefixes, marker] = await page.fill(createVersionedReader(blobs, 10), versioningNamer); + + checkVersionedResult(items, prefixes, marker, 3, 0, ""); + }); + + it("handles pagination continuation with versioned blobs @loki", async () => { + const blobs: MockVersionedBlob[] = [ + { name: "blob1", versionId: "2023-01-01T10:00:00.000Z" }, + { name: "blob1", versionId: "2023-01-01T11:00:00.000Z" }, + { name: "blob2", snapshot: "2023-01-01T12:00:00.0000000Z" }, + { name: "blob3", lastModified: "2023-01-01T13:00:00.000Z" } + ]; + + // First page + const page = new PageWithDelimiter(2); + let [items, prefixes, marker] = await page.fill(createVersionedReader(blobs, 2), versioningNamer); + + checkVersionedResult(items, prefixes, marker, 2, 0, + "blob1" + PageWithDelimiter.VERSIONING_MARKER + "2023-01-01T11:00:00.000Z"); + + // Second page + page.reset(); + [items, prefixes, marker] = await page.fill(createVersionedReader(blobs.slice(2), 2), versioningNamer); + + checkVersionedResult(items, prefixes, marker, 2, 0, ""); + }); + + it("handles versioned blobs with delimiter @loki", async () => { + const blobs: MockVersionedBlob[] = [ + { name: "folder/blob1", versionId: "2023-01-01T10:00:00.000Z" }, + { name: "folder/blob1", versionId: "2023-01-01T11:00:00.000Z" }, + { name: "folder/sub/blob2", snapshot: "2023-01-01T12:00:00.000Z" }, + { name: "folder/blob3", lastModified: "2023-01-01T13:00:00.000Z" } + ]; + + const page = new PageWithDelimiter(2, "/", "folder/"); + const [items, prefixes, marker] = await page.fill(createVersionedReader(blobs, 2), versioningNamer); + + checkVersionedResult(items, prefixes, marker, 2, 0, + "folder/blob1" + PageWithDelimiter.VERSIONING_MARKER + "2023-01-01T11:00:00.000Z"); + }); + }); }); diff --git a/tests/blob/startupErrorRecovery.test.ts b/tests/blob/startupErrorRecovery.test.ts index d153ace82..ff3d8ee0a 100644 --- a/tests/blob/startupErrorRecovery.test.ts +++ b/tests/blob/startupErrorRecovery.test.ts @@ -4,8 +4,12 @@ import * as fs from "fs-extra"; import BlobConfiguration from "../../src/blob/BlobConfiguration"; import BlobServer from "../../src/blob/BlobServer"; import { configLogger } from "../../src/common/Logger"; -import { DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT } from "../../src/blob/utils/constants"; +import { + DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, + EMULATOR_ACCOUNT_NAME +} from "../../src/blob/utils/constants"; import { ServerStatus } from "../../src/common/ServerBase"; +import LokiAccountModelStore from "../../src/common/account/LokiAccountModelStore"; // Set true to enable debug log configLogger(false); @@ -15,6 +19,51 @@ describe("Blob Server Startup Error Recovery - Issue #2672 @loki", () => { const testDbExtentPath = "__test_startup_error_db_blob_extent__.json"; const blobStoragePath = "__test_startup_error_blobstorage__"; + function createConfiguration(): BlobConfiguration { + const accountModelStore = new LokiAccountModelStore( + "", + true, + new Map([ + [ + EMULATOR_ACCOUNT_NAME, + { + key: EMULATOR_ACCOUNT_NAME, + isBlobVersioningEnabled: false + } + ] + ]) + ); + + return new BlobConfiguration( + "127.0.0.1", + 0, + DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, + testDbPath, + testDbExtentPath, + [ + { + locationId: "test", + locationPath: blobStoragePath, + maxConcurrency: 10 + } + ], + false, + undefined, + false, + undefined, + false, + false, + "", + "", + "", + undefined, + false, + false, + undefined, + accountModelStore + ); + } + async function startWithTimeout(server: BlobServer): Promise { let timeout: ReturnType | undefined; @@ -60,21 +109,7 @@ describe("Blob Server Startup Error Recovery - Issue #2672 @loki", () => { } }); - const config = new BlobConfiguration( - "127.0.0.1", - 0, - DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, - testDbPath, - testDbExtentPath, - [ - { - locationId: "test", - locationPath: blobStoragePath, - maxConcurrency: 10 - } - ], - false - ); + const config = createConfiguration(); const server = new BlobServer(config); @@ -122,21 +157,7 @@ describe("Blob Server Startup Error Recovery - Issue #2672 @loki", () => { // Write corrupted metadata to simulate legacy data fs.writeFileSync(testDbPath, JSON.stringify(corruptedMetadata, null, 2)); - const config = new BlobConfiguration( - "127.0.0.1", - 0, - DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, - testDbPath, - testDbExtentPath, - [ - { - locationId: "test", - locationPath: blobStoragePath, - maxConcurrency: 10 - } - ], - false - ); + const config = createConfiguration(); const server = new BlobServer(config); try { @@ -184,21 +205,7 @@ describe("Blob Server Startup Error Recovery - Issue #2672 @loki", () => { } }); - const config = new BlobConfiguration( - "127.0.0.1", - 0, - DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, - testDbPath, - testDbExtentPath, - [ - { - locationId: "test", - locationPath: blobStoragePath, - maxConcurrency: 10 - } - ], - false - ); + const config = createConfiguration(); const server = new BlobServer(config); diff --git a/tests/blob/upgradeRegression.test.ts b/tests/blob/upgradeRegression.test.ts index 6ce4412c0..6a9f7d114 100644 --- a/tests/blob/upgradeRegression.test.ts +++ b/tests/blob/upgradeRegression.test.ts @@ -14,6 +14,7 @@ import { configLogger } from "../../src/common/Logger"; import { DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT } from "../../src/blob/utils/constants"; import { ServerStatus } from "../../src/common/ServerBase"; import { EMULATOR_ACCOUNT_KEY, EMULATOR_ACCOUNT_NAME } from "../testutils"; +import LokiAccountModelStore from "../../src/common/account/LokiAccountModelStore"; // Set true to enable debug log configLogger(false); @@ -47,6 +48,51 @@ describe("Azurite Upgrade Regression Tests @loki", () => { return new Promise((resolve) => setTimeout(resolve, ms)); } + function createConfiguration(): BlobConfiguration { + const accountModelStore = new LokiAccountModelStore( + "", + true, + new Map([ + [ + EMULATOR_ACCOUNT_NAME, + { + key: EMULATOR_ACCOUNT_NAME, + isBlobVersioningEnabled: false + } + ] + ]) + ); + + return new BlobConfiguration( + "127.0.0.1", + 0, + DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, + upgradeTestDbPath, + upgradeTestDbExtentPath, + [ + { + locationId: "test", + locationPath: upgradeBlobStoragePath, + maxConcurrency: 10 + } + ], + false, + undefined, + false, + undefined, + false, + false, + "", + "", + "", + undefined, + false, + false, + undefined, + accountModelStore + ); + } + async function removePathWithRetry(path: string): Promise { try { await nodefs.promises.rm(path, { @@ -168,21 +214,7 @@ describe("Azurite Upgrade Regression Tests @loki", () => { it("should upgrade without data loss", async () => { // PHASE 1: Simulate old version behavior - create initial data - const config1 = new BlobConfiguration( - "127.0.0.1", - 0, - DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, - upgradeTestDbPath, - upgradeTestDbExtentPath, - [ - { - locationId: "test", - locationPath: upgradeBlobStoragePath, - maxConcurrency: 10 - } - ], - false - ); + const config1 = createConfiguration(); const server1 = new BlobServer(config1); await server1.start(); @@ -228,21 +260,7 @@ describe("Azurite Upgrade Regression Tests @loki", () => { // PHASE 2: Simulate upgrade - load existing data - const config2 = new BlobConfiguration( - "127.0.0.1", - 0, - DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, - upgradeTestDbPath, - upgradeTestDbExtentPath, - [ - { - locationId: "test", - locationPath: upgradeBlobStoragePath, - maxConcurrency: 10 - } - ], - false - ); + const config2 = createConfiguration(); const server2 = new BlobServer(config2); @@ -315,21 +333,7 @@ describe("Azurite Upgrade Regression Tests @loki", () => { * Multiple accounts with existing persisted data */ it("should handle startup with multiple existing accounts and containers", async () => { - const config3 = new BlobConfiguration( - "127.0.0.1", - 0, - DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, - upgradeTestDbPath, - upgradeTestDbExtentPath, - [ - { - locationId: "test", - locationPath: upgradeBlobStoragePath, - maxConcurrency: 10 - } - ], - false - ); + const config3 = createConfiguration(); const server3 = new BlobServer(config3); await server3.start(); @@ -368,21 +372,7 @@ describe("Azurite Upgrade Regression Tests @loki", () => { } // Now restart and verify all data is still accessible - const config4 = new BlobConfiguration( - "127.0.0.1", - 0, - DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, - upgradeTestDbPath, - upgradeTestDbExtentPath, - [ - { - locationId: "test", - locationPath: upgradeBlobStoragePath, - maxConcurrency: 10 - } - ], - false - ); + const config4 = createConfiguration(); const server4 = new BlobServer(config4); @@ -444,21 +434,7 @@ describe("Azurite Upgrade Regression Tests @loki", () => { const compatibilityBlob = `blob-${shape}.txt`; const compatibilityContent = `compatibility data for ${shape}`; - const createConfig = new BlobConfiguration( - "127.0.0.1", - 0, - DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, - upgradeTestDbPath, - upgradeTestDbExtentPath, - [ - { - locationId: "test", - locationPath: upgradeBlobStoragePath, - maxConcurrency: 10 - } - ], - false - ); + const createConfig = createConfiguration(); const createServer = new BlobServer(createConfig); await createServer.start(); @@ -493,21 +469,7 @@ describe("Azurite Upgrade Regression Tests @loki", () => { rewritePersistedMd5Shape(shape); - const loadConfig = new BlobConfiguration( - "127.0.0.1", - 0, - DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, - upgradeTestDbPath, - upgradeTestDbExtentPath, - [ - { - locationId: "test", - locationPath: upgradeBlobStoragePath, - maxConcurrency: 10 - } - ], - false - ); + const loadConfig = createConfiguration(); const loadServer = new BlobServer(loadConfig); await loadServer.start(); @@ -554,21 +516,7 @@ describe("Azurite Upgrade Regression Tests @loki", () => { const compatibilityBlob = "blob-null-md5.txt"; const compatibilityContent = "compatibility data for null md5"; - const createConfig = new BlobConfiguration( - "127.0.0.1", - 0, - DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, - upgradeTestDbPath, - upgradeTestDbExtentPath, - [ - { - locationId: "test", - locationPath: upgradeBlobStoragePath, - maxConcurrency: 10 - } - ], - false - ); + const createConfig = createConfiguration(); const createServer = new BlobServer(createConfig); await createServer.start(); @@ -602,21 +550,7 @@ describe("Azurite Upgrade Regression Tests @loki", () => { rewritePersistedMd5AsNull(); - const loadConfig = new BlobConfiguration( - "127.0.0.1", - 0, - DEFAULT_BLOB_KEEP_ALIVE_TIMEOUT, - upgradeTestDbPath, - upgradeTestDbExtentPath, - [ - { - locationId: "test", - locationPath: upgradeBlobStoragePath, - maxConcurrency: 10 - } - ], - false - ); + const loadConfig = createConfiguration(); const loadServer = new BlobServer(loadConfig); await loadServer.start(); diff --git a/tests/blob/utils.test.ts b/tests/blob/utils.test.ts index 3efd12899..44a316aae 100644 --- a/tests/blob/utils.test.ts +++ b/tests/blob/utils.test.ts @@ -1,11 +1,15 @@ import * as assert from "assert"; import { PassThrough } from "stream"; -import { computeAndValidateTransactionalChecksums } from "../../src/blob/utils/utils"; +import { + computeAndValidateTransactionalChecksums, + isNullOrWhitespace, + parseDateFromAssumedString +} from "../../src/blob/utils/utils"; import { convertRawHeadersToMetadata, getCRC64FromStream, getCRC64FromString, - getMD5FromString + getMD5FromString, } from "../../src/common/utils/utils"; describe("Utils", () => { @@ -63,6 +67,75 @@ describe("Utils", () => { const metadata = convertRawHeadersToMetadata([]); assert.deepStrictEqual(metadata, undefined); }); + + describe("isNullOrWhitespace", () => { + it("returns true for undefined", () => { + assert.strictEqual(isNullOrWhitespace(undefined), true); + }); + it("returns true for null", () => { + assert.strictEqual(isNullOrWhitespace(null as any), true); + }); + it("returns true for empty string", () => { + assert.strictEqual(isNullOrWhitespace(""), true); + }); + it("returns true for whitespace-only string", () => { + assert.strictEqual(isNullOrWhitespace(" \t\n "), true); + }); + it("returns false for non-whitespace string", () => { + assert.strictEqual(isNullOrWhitespace("abc"), false); + }); + }); + + describe("parseDateFromAssumedString", () => { + it("returns undefined for undefined input", () => { + assert.strictEqual(parseDateFromAssumedString(undefined), undefined); + }); + it("returns same Date object when passed a Date", () => { + const d = new Date(); + assert.strictEqual(parseDateFromAssumedString(d), d); + }); + it("parses valid ISO string", () => { + const iso = "2024-12-31T23:59:59.123Z"; + const d = parseDateFromAssumedString(iso)!; + assert.ok(d instanceof Date); + assert.strictEqual(d.toISOString(), iso); + }); + it("returns undefined for whitespace string", () => { + assert.strictEqual(parseDateFromAssumedString(" "), undefined); + }); + it("returns undefined for non-date string", () => { + assert.strictEqual(parseDateFromAssumedString("not-a-date"), undefined); + }); + it("returns undefined for number input", () => { + assert.strictEqual(parseDateFromAssumedString(123), undefined); + assert.strictEqual(parseDateFromAssumedString(0), undefined); + assert.strictEqual(parseDateFromAssumedString(-1), undefined); + }); + it("returns undefined for boolean input", () => { + assert.strictEqual(parseDateFromAssumedString(true), undefined); + assert.strictEqual(parseDateFromAssumedString(false), undefined); + }); + it("returns undefined for object input", () => { + assert.strictEqual(parseDateFromAssumedString({}), undefined); + assert.strictEqual( + parseDateFromAssumedString({ date: "2024-01-01" }), + undefined + ); + }); + it("returns undefined for array input", () => { + assert.strictEqual(parseDateFromAssumedString([]), undefined); + assert.strictEqual(parseDateFromAssumedString(["2024-01-01"]), undefined); + }); + it("returns undefined for null input", () => { + assert.strictEqual(parseDateFromAssumedString(null), undefined); + }); + it("returns undefined for function input", () => { + const fn = function () { + return "test"; + }; + assert.strictEqual(parseDateFromAssumedString(fn), undefined); + }); + }); }); describe("CRC64", () => { diff --git a/tests/blob/versioning.lokidb.test.ts b/tests/blob/versioning.lokidb.test.ts new file mode 100644 index 000000000..c7d89b924 --- /dev/null +++ b/tests/blob/versioning.lokidb.test.ts @@ -0,0 +1,4871 @@ +import assert = require("assert"); +import { randomUUID as uuid } from "crypto"; +import * as fs from "fs"; +import Loki from "lokijs"; +import LokiBlobMetadataStore from "../../src/blob/persistence/LokiBlobMetadataStore"; +import { BlobModel } from "../../src/blob/persistence/IBlobMetadataStore"; +import LokiAccountModelStore from "../../src/common/account/LokiAccountModelStore"; +import { + buildAppendBlob, + buildBlockBlob, + buildContainer, + buildPageBlob, + createContext +} from "../testutils"; +import * as Models from "../../src/blob/generated/artifacts/models"; +import Context from "../../src/blob/generated/Context"; +import StorageError from "../../src/blob/errors/StorageError"; +import { configLogger } from "../../src/common/Logger"; +import { convertDateTimeStringMsTo7Digital } from "../../src/common/utils/utils"; +import { isNullOrWhitespace } from "../../src/blob/utils/utils"; +import { AccountModel } from "../../src/common/account/AccountModel"; + +// Silence logs for tests +configLogger(false); + +const ACCOUNT = "devstoreaccount1"; +const DEFAULT_LIST_BLOBS_MAX_RESULTS = 5000; +const DB_FILE = "__test_db_blob__.json"; // standard shared test db path +const ACCOUNT_DB_FILE = "__test_db_account_models__.json"; // account model DB + +// Helper function to create account model store with a given account model +function createAccountModelStore(accountModel: AccountModel, inMemory: boolean = false): LokiAccountModelStore { + const accountModels = new Map(); + accountModels.set(accountModel.key || ACCOUNT, accountModel); + return new LokiAccountModelStore(ACCOUNT_DB_FILE, inMemory, accountModels); +} + +describe("LokiBlobMetadataStore - Versioning Enabled", () => { + let store: LokiBlobMetadataStore; + let accountModelStore: LokiAccountModelStore; + let containerName: string; + let ctx: Context; + + beforeEach(async () => { + ctx = createContext(); + containerName = `container-${uuid()}`; + const accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: true + } + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + await store.createContainer(ctx, buildContainer(ACCOUNT, containerName)); + }); + + it("should reject deleting the current version by versionId @loki", async () => { + const name = `blob-${uuid()}`; + const created = await store.createBlob( + ctx, + buildBlockBlob(ACCOUNT, containerName, name, "current") + ); + + await assert.rejects( + () => + store.deleteBlob(ctx, ACCOUNT, containerName, name, { + versionId: created.versionId + }), + (error: unknown) => { + const storageError = error as StorageError; + return ( + storageError.statusCode === 403 && + storageError.storageErrorCode === "OperationNotAllowedOnRootBlob" + ); + } + ); + }); + + afterEach(async () => { + await accountModelStore.close(); + await accountModelStore.clean(); + await store.close(); + await store.clean(); + }); + + it("should add the versionId index when opening an existing workspace @loki", async () => { + await store.close(); + await store.clean(); + + const legacyDb = new Loki(DB_FILE); + legacyDb.addCollection("$BLOBS_COLLECTION$", { + indices: ["accountName", "containerName", "name", "snapshot"] + }); + await new Promise((resolve, reject) => { + legacyDb.saveDatabase((error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); + await new Promise((resolve, reject) => { + legacyDb.close((error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); + + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + const persistedDb = JSON.parse(fs.readFileSync(DB_FILE, "utf8")); + const blobsCollection = persistedDb.collections.find( + (collection: { name: string }) => + collection.name === "$BLOBS_COLLECTION$" + ); + assert.notStrictEqual(blobsCollection, undefined); + assert.notStrictEqual( + blobsCollection.binaryIndices.versionId, + undefined + ); + }); + + it("should treat blobs without versioning fields as current after upgrade @loki", async () => { + await store.close(); + await store.clean(); + + const legacyBlob = buildBlockBlob( + ACCOUNT, + containerName, + `legacy-${uuid()}`, + "legacy" + ); + legacyBlob.versionId = undefined; + legacyBlob.isCurrentVersion = undefined; + + const legacyDb = new Loki(DB_FILE); + legacyDb + .addCollection("$CONTAINERS_COLLECTION$", { + unique: ["accountName", "name"] + }) + .insert(buildContainer(ACCOUNT, containerName)); + legacyDb + .addCollection("$BLOBS_COLLECTION$", { + indices: ["accountName", "containerName", "name", "snapshot"] + }) + .insert(legacyBlob); + await new Promise((resolve, reject) => { + legacyDb.saveDatabase((error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); + await new Promise((resolve, reject) => { + legacyDb.close((error) => { + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); + + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + const found = await store.getBlob( + ctx, + ACCOUNT, + containerName, + legacyBlob.name + ); + assert.notStrictEqual(found, undefined); + + const [listed] = await store.listBlobs( + ctx, + ACCOUNT, + containerName + ); + assert.strictEqual(listed.length, 1); + assert.strictEqual(listed[0].name, legacyBlob.name); + }); + + // ================== VERSION MODE TRANSITION TESTS (ENABLED → DISABLED) ================== + it("should handle setBlobMetadata correctly when disabling versioning after creating versions @loki", async () => { + // Close the in-memory disabled store from beforeEach; we need persistence for this scenario + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning ENABLED and create versioned blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: true + } + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let enabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await enabledStore.init(); + await enabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + const createdBaseBlob = await enabledStore.createBlob(ctx, baseBlob); + + // Set metadata to create versions (should create version) + ctx.startTime = new Date(Date.now() + 100); + const modifiedMetadataBaseBlob = await enabledStore.setBlobMetadata( + ctx, + ACCOUNT, + containerName, + name, + undefined, + { versionedmeta: "value1" } + ); + assert.ok(!isNullOrWhitespace(createdBaseBlob.versionId)); + assert.ok(!isNullOrWhitespace(modifiedMetadataBaseBlob.versionId)); + assert.notStrictEqual( + modifiedMetadataBaseBlob.versionId, + createdBaseBlob.versionId + ); + + const versionedFetched = await enabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.ok(!isNullOrWhitespace(versionedFetched.versionId)); + assert.deepStrictEqual(versionedFetched.metadata, { + versionedmeta: "value1" + }); + assert.strictEqual( + versionedFetched.versionId, + modifiedMetadataBaseBlob.versionId + ); + const versionId = versionedFetched.versionId; + await accountModelStore.close(); + await enabledStore.close(); + + // 2. Re-open with versioning DISABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Set metadata should NOT create new version (overwrite current) + ctx.startTime = new Date(Date.now() + 200); + await store.setBlobMetadata(ctx, ACCOUNT, containerName, name, undefined, { + disabledmeta: "value2" + }); + + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Should be same version (no new version created) + assert.strictEqual(current.versionId, ""); + assert.notStrictEqual(current.versionId, versionId); + assert.deepStrictEqual(current.metadata, { disabledmeta: "value2" }); + + const firstVersion = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + versionId + ); + assert.strictEqual(firstVersion.versionId, versionId); + }); + + it("should handle setBlobHTTPHeaders correctly when disabling versioning after creating versions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning ENABLED and create versioned blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let enabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await enabledStore.init(); + await enabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + await enabledStore.createBlob(ctx, baseBlob); + + // Set HTTP headers (should NOT create version even when versioning enabled) + ctx.startTime = new Date(Date.now() + 100); + await enabledStore.setBlobHTTPHeaders( + ctx, + ACCOUNT, + containerName, + name, + undefined, + { blobContentType: "text/plain" } + ); + + const versionedFetched = await enabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.ok(!isNullOrWhitespace(versionedFetched.versionId)); + assert.strictEqual(versionedFetched.properties.contentType, "text/plain"); + const versionId = versionedFetched.versionId; + await accountModelStore.close(); + await enabledStore.close(); + + // 2. Re-open with versioning DISABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Set headers should continue to NOT create version and update in place + ctx.startTime = new Date(Date.now() + 200); + await store.setBlobHTTPHeaders( + ctx, + ACCOUNT, + containerName, + name, + undefined, + { blobContentType: "application/json" } + ); + + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + // Should be same version (headers don't create versions in either mode) + assert.strictEqual(current.versionId, versionId); + assert.strictEqual(current.properties.contentType, "application/json"); + }); + + it("should handle setBlobTag/getBlobTag correctly when disabling versioning after creating versions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning ENABLED and create versioned blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let enabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await enabledStore.init(); + await enabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + const creaatedBaseBlob = await enabledStore.createBlob(ctx, baseBlob); + + // Set tags (should NOT create version even when versioning enabled) + ctx.startTime = new Date(Date.now() + 100); + await enabledStore.setBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined, + { blobTagSet: [{ key: "env", value: "test" }] } + ); + + const versionedFetched = await enabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + const versionId = versionedFetched.versionId; + assert.ok(!isNullOrWhitespace(versionId)); + assert.strictEqual(creaatedBaseBlob.versionId, versionId); + const versionedTags = await enabledStore.getBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + versionId, + undefined + ); + assert.deepStrictEqual(versionedTags, { + blobTagSet: [{ key: "env", value: "test" }] + }); + await accountModelStore.close(); + await enabledStore.close(); + + // 2. Re-open with versioning DISABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Set tags should continue to NOT create version and update in place + ctx.startTime = new Date(Date.now() + 200); + await store.setBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined, + { blobTagSet: [{ key: "env", value: "prod" }] } + ); + + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + // Should be same version (tags don't create versions in either mode) + assert.strictEqual(current.versionId, versionId); + + const currentTags = await store.getBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + versionId, + undefined + ); + assert.deepStrictEqual(currentTags, { + blobTagSet: [{ key: "env", value: "prod" }] + }); + }); + + it("should handle setTier correctly when disabling versioning after creating versions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning ENABLED and create versioned blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let enabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await enabledStore.init(); + await enabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + baseBlob.properties.accessTier = Models.AccessTier.Hot; + const blobCreated = await enabledStore.createBlob(ctx, baseBlob); + + // Set tier (should NOT create version even when versioning enabled) + ctx.startTime = new Date(Date.now() + 100); + await enabledStore.setTier( + ctx, + ACCOUNT, + containerName, + name, + undefined, + Models.AccessTier.Cool, + undefined + ); + + const versionedFetched = await enabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + const versionId = versionedFetched.versionId; + assert.ok(!isNullOrWhitespace(versionId)); + assert.strictEqual(blobCreated.versionId, versionId); + assert.strictEqual( + versionedFetched.properties.accessTier, + Models.AccessTier.Cool + ); + await accountModelStore.close(); + await enabledStore.close(); + + // 2. Re-open with versioning DISABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Set tier should continue to work and update in place + ctx.startTime = new Date(Date.now() + 200); + await store.setTier( + ctx, + ACCOUNT, + containerName, + name, + undefined, + Models.AccessTier.Archive, + undefined + ); + + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + // Should be same version (tier operations don't create versions in either mode) + assert.strictEqual(current.versionId, versionId); + assert.strictEqual( + current.properties.accessTier, + Models.AccessTier.Archive + ); + }); + + it("should handle checkBlobExist correctly when disabling versioning after creating versions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning ENABLED and create versioned blobs + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let enabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await enabledStore.init(); + await enabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + const createdBaseBlob = await enabledStore.createBlob(ctx, baseBlob); + const firstVersionId = createdBaseBlob.versionId; + + // Create second version + ctx.startTime = new Date(Date.now() + 100); + const secondBlob = buildBlockBlob(ACCOUNT, containerName, name, "second"); + const createdSecondBlob = await enabledStore.createBlob(ctx, secondBlob); + + const current = await enabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + const currentVersionId = current.versionId; + assert.ok(!isNullOrWhitespace(currentVersionId)); + assert.strictEqual(currentVersionId, createdSecondBlob.versionId); + assert.notStrictEqual(currentVersionId, firstVersionId); + + // Get first version ID + await accountModelStore.close(); + await enabledStore.close(); + + // 2. Re-open with versioning DISABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Check existence should work for current blob + await store.checkBlobExist(ctx, ACCOUNT, containerName, name); + + // Should still be able to check existence by specific versionId + await store.checkBlobExist( + ctx, + ACCOUNT, + containerName, + name, + "", + currentVersionId + ); + + // Previous versions should still be accessible by versionId + await store.checkBlobExist( + ctx, + ACCOUNT, + containerName, + name, + "", + firstVersionId + ); + }); + + it("should handle getBlobProperties correctly when disabling versioning after creating versions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning ENABLED and create versioned blobs + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let enabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await enabledStore.init(); + await enabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + await enabledStore.createBlob(ctx, baseBlob); + + // Set metadata to create version + ctx.startTime = new Date(Date.now() + 100); + await enabledStore.setBlobMetadata( + ctx, + ACCOUNT, + containerName, + name, + undefined, + { env: "test" } + ); + + const secondVersion = await enabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + const secondVersionId = secondVersion.versionId; + + // Create second version + ctx.startTime = new Date(Date.now() + 200); + await enabledStore.setBlobMetadata( + ctx, + ACCOUNT, + containerName, + name, + undefined, + { env: "prod" } + ); + + const current = await enabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + const currentVersionId = current.versionId; + assert.ok(!isNullOrWhitespace(currentVersionId)); + assert.notStrictEqual(currentVersionId, secondVersionId); + await accountModelStore.close(); + await enabledStore.close(); + + // 2. Re-open with versioning DISABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Get properties should work for current version + const currentProps = await store.getBlobProperties( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined + ); + assert.deepStrictEqual(currentProps.metadata, { env: "prod" }); + + // Should still be able to get properties for specific versions by versionId + const currentPropsByVersion = await store.getBlobProperties( + ctx, + ACCOUNT, + containerName, + name, + undefined, + currentVersionId, + undefined + ); + assert.deepStrictEqual( + currentPropsByVersion.metadata, + currentProps.metadata + ); + + const secondVersionProps = await store.getBlobProperties( + ctx, + ACCOUNT, + containerName, + name, + undefined, + secondVersionId, + undefined + ); + assert.deepStrictEqual(secondVersionProps.metadata, { env: "test" }); + }); + + it("should handle createSnapshot correctly when disabling versioning after creating versions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning ENABLED and create versioned blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let enabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await enabledStore.init(); + await enabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + const createdBaseBlob = await enabledStore.createBlob(ctx, baseBlob); + + // Create snapshot (should create new version when versioning enabled) + ctx.startTime = new Date(Date.now() + 100); + const snapshotResponse1 = await enabledStore.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + assert.ok(snapshotResponse1.snapshot); + assert.ok(!isNullOrWhitespace(snapshotResponse1.versionId)); + assert.notStrictEqual( + snapshotResponse1.versionId, + createdBaseBlob.versionId + ); + + const versionedFetched = await enabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + const versionId = versionedFetched.versionId; + assert.ok(!isNullOrWhitespace(versionId)); + assert.strictEqual(versionId, snapshotResponse1.versionId); + await accountModelStore.close(); + await enabledStore.close(); + + // 2. Re-open with versioning DISABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Create snapshot should NOT create new version when versioning disabled + ctx.startTime = new Date(Date.now() + 200); + const snapshotResponse2 = await store.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + assert.ok(snapshotResponse2.snapshot); + assert.strictEqual(snapshotResponse2.versionId, ""); + + try { + // Snapshotting acts as a "write" + await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.fail("Expected error to be thrown"); + } catch (error) { + assert.ok(error); + } + }); + + it("should handle appendBlock correctly when disabling versioning after creating versions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning ENABLED and create versioned append blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let enabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await enabledStore.init(); + await enabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseAppendBlob = buildAppendBlob(ACCOUNT, containerName, name); + await enabledStore.createBlob(ctx, baseAppendBlob); + + // Append block (should NOT create version even when versioning enabled) + const block1 = { + accountName: ACCOUNT, + containerName, + blobName: name, + name: "append1", + size: 10, + persistency: { id: uuid(), offset: 0, count: 10 } + } as any; + + ctx.startTime = new Date(Date.now() + 100); + await enabledStore.appendBlock(ctx, block1); + + const versionedFetched = await enabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.ok(!isNullOrWhitespace(versionedFetched.versionId)); + assert.strictEqual(versionedFetched.properties.contentLength, 10); + const versionId = versionedFetched.versionId; + await accountModelStore.close(); + await enabledStore.close(); + + // 2. Re-open with versioning DISABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Append block should continue to NOT create version and update in place + const block2 = { + accountName: ACCOUNT, + containerName, + blobName: name, + name: "append2", + size: 15, + persistency: { id: uuid(), offset: 10, count: 15 } + } as any; + + ctx.startTime = new Date(Date.now() + 200); + await store.appendBlock(ctx, block2); + + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + // Should be same version (append operations don't create versions in either mode) + assert.strictEqual(current.versionId, versionId); + assert.strictEqual(current.properties.contentLength, 25); + }); + + it("should handle uploadPages correctly when disabling versioning after creating versions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning ENABLED and create versioned page blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let enabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await enabledStore.init(); + await enabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const basePageBlob = buildPageBlob(ACCOUNT, containerName, name, 512); + await enabledStore.createBlob(ctx, basePageBlob); + + // Upload pages (should NOT create version even when versioning enabled) + const persistency1 = { id: uuid(), offset: 0, count: 512 }; + ctx.startTime = new Date(Date.now() + 100); + await enabledStore.uploadPages(ctx, basePageBlob, 0, 511, persistency1); + + const versionedFetched = await enabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.ok(!isNullOrWhitespace(versionedFetched.versionId)); + const versionId = versionedFetched.versionId; + await accountModelStore.close(); + await enabledStore.close(); + + // 2. Re-open with versioning DISABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Upload pages should continue to NOT create version and update in place + const persistency2 = { id: uuid(), offset: 0, count: 512 }; + ctx.startTime = new Date(Date.now() + 200); + await store.uploadPages(ctx, basePageBlob, 0, 511, persistency2); + + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + // Should be same version (page operations don't create versions in either mode) + assert.strictEqual(current.versionId, versionId); + }); + + it("should handle deleteBlob correctly when disabling versioning after creating versions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning ENABLED and create versioned blobs + let accountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + + let accountModelStore = createAccountModelStore(accountModel, false); + + await accountModelStore.init(); + let enabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await enabledStore.init(); + await enabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + const createdBaseBlob = await enabledStore.createBlob(ctx, baseBlob); + + // Create second version + ctx.startTime = new Date(Date.now() + 100); + const secondBlob = buildBlockBlob(ACCOUNT, containerName, name, "second"); + await enabledStore.createBlob(ctx, secondBlob); + + const beforeDelete = await enabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + const currentVersionId = beforeDelete.versionId; + await accountModelStore.close(); + await enabledStore.close(); + + // 2. Re-open with versioning DISABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Delete current blob should completely remove it (not make it a previous version) + await store.deleteBlob(ctx, ACCOUNT, containerName, name, {}); + + // Current version should no longer exist + try { + await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.fail("Should have thrown error for deleted current blob"); + } catch (error) { + // Expected + } + + // But should still be able to access previous versions by specific versionId + const deletedVersion = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + createdBaseBlob.versionId + ); + assert.ok(!isNullOrWhitespace(deletedVersion.versionId)); + assert.notStrictEqual(deletedVersion.versionId, currentVersionId); + assert.strictEqual(deletedVersion.versionId, createdBaseBlob.versionId); + + // Should be able to delete specific version by versionId + await store.deleteBlob(ctx, ACCOUNT, containerName, name, { + versionId: createdBaseBlob.versionId + }); + + // That specific version should no longer exist + try { + await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + createdBaseBlob.versionId + ); + assert.fail("Should have thrown error for deleted specific version"); + } catch (error) { + // Expected + } + }); + + it("should preserve existing versions and allow operations on them when versioning is disabled @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning ENABLED and create multiple versions + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + + let accountModelStore = createAccountModelStore(accountModel, false); + + await accountModelStore.init(); + + let enabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await enabledStore.init(); + await enabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + + // Create first version + const blob1 = buildBlockBlob(ACCOUNT, containerName, name, "version1"); + await enabledStore.createBlob(ctx, blob1); + const version1 = await enabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + const version1Id = version1.versionId; + + // Create second version + ctx.startTime = new Date(Date.now() + 100); + const blob2 = buildBlockBlob(ACCOUNT, containerName, name, "version2"); + await enabledStore.createBlob(ctx, blob2); + const version2 = await enabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + const version2Id = version2.versionId; + + // Create third version + ctx.startTime = new Date(Date.now() + 200); + const blob3 = buildBlockBlob(ACCOUNT, containerName, name, "version3"); + await enabledStore.createBlob(ctx, blob3); + const version3 = await enabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + const version3Id = version3.versionId; + await accountModelStore.close(); + await enabledStore.close(); + + // 2. Re-open with versioning DISABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // All existing versions should remain accessible by versionId + const fetchedV1 = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + version1Id + ); + assert.strictEqual( + fetchedV1.properties.contentLength, + blob1.properties.contentLength + ); + assert.strictEqual(fetchedV1.versionId, version1Id); + + const fetchedV2 = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + version2Id + ); + assert.strictEqual( + fetchedV2.properties.contentLength, + blob2.properties.contentLength + ); + assert.strictEqual(fetchedV2.versionId, version2Id); + + const fetchedV3 = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + version3Id + ); + assert.strictEqual( + fetchedV3.properties.contentLength, + blob3.properties.contentLength + ); + assert.strictEqual(fetchedV3.versionId, version3Id); + + // Current version should be the latest (version3) + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual(current.versionId, version3Id); + assert.strictEqual( + current.properties.contentLength, + blob3.properties.contentLength + ); + + // Modifying current should create new version + ctx.startTime = new Date(Date.now() + 300); + const newBlob = buildBlockBlob( + ACCOUNT, + containerName, + name, + "modified_no_version" + ); + await store.createBlob(ctx, newBlob); + + const afterModify = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + // Should change to empty string + assert.strictEqual(afterModify.versionId, ""); + assert.notStrictEqual(afterModify.versionId, version3Id); + + // Previous versions should still exist and be unchanged + const stillV1 = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + version1Id + ); + assert.strictEqual( + stillV1.properties.contentLength, + blob1.properties.contentLength + ); + + const stillV2 = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + version2Id + ); + assert.strictEqual( + stillV2.properties.contentLength, + blob2.properties.contentLength + ); + }); + + it("creates a new version for each blob creation and marks previous current as not current @loki", async () => { + const name = `blob-${uuid()}`; + const v1 = buildBlockBlob(ACCOUNT, containerName, name, "v1"); + await store.createBlob(ctx, v1); + const afterV1 = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.ok( + afterV1.versionId || afterV1.versionId === "", + "First creation should have a version id (may be empty transitioning)" + ); + + // Second create -> new version id expected + ctx.startTime = new Date(Date.now() + 10); // ensure different timestamp base + const v2 = buildBlockBlob(ACCOUNT, containerName, name, "v2"); + await store.createBlob(ctx, v2); + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.notStrictEqual( + current.properties.etag, + afterV1.properties.etag, + "Etag should change for new version" + ); + assert.ok(current.isCurrentVersion, "Latest should be current version"); + }); + + it("promotes a non-versioned base blob (created with versioning disabled) to have a versionId equal to its original lastModified when versioning is later enabled @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning DISABLED (persistent) and create base blob (versionId will be ""). + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let disabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await disabledStore.init(); + await disabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + await disabledStore.createBlob(ctx, baseBlob); + const baseFetched = await disabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + const originalLastModifiedIso = convertDateTimeStringMsTo7Digital( + baseFetched.properties.lastModified.toISOString() + ); + await accountModelStore.close(); + await disabledStore.close(); + + // 2. Re-open SAME DB with versioning ENABLED. + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // 3. Create a new version (same name). This should assign a versionId to prior base blob + // using its lastModified timestamp, and mark it as not current. + ctx.startTime = new Date(Date.now() + 200); // ensure different timestamp for new current version + const secondBlob = buildBlockBlob(ACCOUNT, containerName, name, "second"); + const newBlobVer = await store.createBlob(ctx, secondBlob); + assert.ok( + !isNullOrWhitespace(newBlobVer.versionId), + "New blob version should have a versionId" + ); + + // 4. Fetch current (no version) and previous (by derived versionId) + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.ok( + current.isCurrentVersion, + "Latest blob should be current after enabling versioning" + ); + assert.ok( + !isNullOrWhitespace(current.versionId), + "Current blob should now have a non-empty versionId" + ); + + const previous = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + originalLastModifiedIso + ); + assert.strictEqual( + previous.versionId, + originalLastModifiedIso, + "Previous base blob should be promoted with versionId equal to its original lastModified ISO string" + ); + assert.strictEqual( + previous.isCurrentVersion, + false, + "Previous version should no longer be current" + ); + assert.notStrictEqual( + previous.versionId, + current.versionId, + "Current versionId should differ from promoted previous versionId" + ); + }); + + it("allows addressing previous version by its versionId @loki", async () => { + const name = `blob-${uuid()}`; + const v1 = buildBlockBlob(ACCOUNT, containerName, name, "v1"); + await store.createBlob(ctx, v1); + const first = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + ctx.startTime = new Date(Date.now() + 100); + const v2 = buildBlockBlob(ACCOUNT, containerName, name, "v2"); + await store.createBlob(ctx, v2); + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Try to fetch previous by versionId - should have non-empty version ID + assert.ok(!isNullOrWhitespace(first.versionId)); + const previousFetched = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + first.versionId + ); + assert.ok(previousFetched.versionId === first.versionId); + assert.ok(current.isCurrentVersion); + }); + + it("should assign unique version IDs based on timestamp when creating versions @loki", async () => { + const name = `blob-${uuid()}`; + ctx.startTime = new Date("2026-08-13T12:34:56.123Z"); + + // Create first version + const v1 = buildBlockBlob(ACCOUNT, containerName, name, "v1"); + const created1 = await store.createBlob(ctx, v1); + + // Create another version in the same JavaScript millisecond. + const v2 = buildBlockBlob(ACCOUNT, containerName, name, "v2"); + const created2 = await store.createBlob(ctx, v2); + + assert.strictEqual(created1.versionId, "2026-08-13T12:34:56.1230000Z"); + assert.strictEqual(created2.versionId, "2026-08-13T12:34:56.1230001Z"); + }); + + it("should maintain previous versions when creating new versions @loki", async () => { + const name = `blob-${uuid()}`; + + // Create first version + const v1 = buildBlockBlob(ACCOUNT, containerName, name, "ver1"); + const created1 = await store.createBlob(ctx, v1); + const version1Id = created1.versionId; + + // Create second version + ctx.startTime = new Date(Date.now() + 100); + const v2 = buildBlockBlob(ACCOUNT, containerName, name, "ver22"); + const created2 = await store.createBlob(ctx, v2); + const version2Id = created2.versionId; + + // Create third version + ctx.startTime = new Date(Date.now() + 200); + const v3 = buildBlockBlob(ACCOUNT, containerName, name, "ver333"); + await store.createBlob(ctx, v3); + + // Current should be v3 + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual( + current.properties.contentLength, + v3.properties.contentLength + ); + assert.ok(current.isCurrentVersion); + + // Previous versions should be accessible by versionId + assert.ok(!isNullOrWhitespace(version1Id)); + const prev1 = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + version1Id + ); + assert.strictEqual( + prev1.properties.contentLength, + v1.properties.contentLength + ); + assert.strictEqual(prev1.isCurrentVersion, false); + + assert.ok(!isNullOrWhitespace(version2Id)); + const prev2 = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + version2Id + ); + assert.strictEqual( + prev2.properties.contentLength, + v2.properties.contentLength + ); + assert.strictEqual(prev2.isCurrentVersion, false); + }); + + it("should handle delete operations by making current version a previous version @loki", async () => { + const name = `blob-${uuid()}`; + + // Create version + const v1 = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, v1); + + // Verify blob exists and is current + const beforeDelete = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.ok(beforeDelete.isCurrentVersion); + const versionIdBeforeDelete = beforeDelete.versionId; + + // Delete the blob (without version ID = delete current) + await store.deleteBlob(ctx, ACCOUNT, containerName, name, {}); + + // Current version should no longer exist + try { + await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.fail("Should have thrown error for deleted current blob"); + } catch (error) { + // Expected - no current version after delete + } + + // Previous version should still be accessible by version ID + assert.ok(!isNullOrWhitespace(versionIdBeforeDelete)); + const previousVersion = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + versionIdBeforeDelete + ); + assert.strictEqual(previousVersion.isCurrentVersion, false); + assert.strictEqual(previousVersion.versionId, versionIdBeforeDelete); + }); + + it("should allow creating new current version after deletion @loki", async () => { + const name = `blob-${uuid()}`; + + // Create and delete a version + const v1 = buildBlockBlob(ACCOUNT, containerName, name, "content1"); + await store.createBlob(ctx, v1); + const beforeDelete = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + const deletedVersionId = beforeDelete.versionId; + + await store.deleteBlob(ctx, ACCOUNT, containerName, name, {}); + + // Create new blob with same name + ctx.startTime = new Date(Date.now() + 100); + const v2 = buildBlockBlob(ACCOUNT, containerName, name, "content2"); + await store.createBlob(ctx, v2); + + // New blob should be current version + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.ok(current.isCurrentVersion); + assert.strictEqual( + current.properties.contentLength, + v2.properties.contentLength + ); + assert.notStrictEqual(current.versionId, deletedVersionId); + + // Previous version should still exist as non-current + assert.ok(!isNullOrWhitespace(deletedVersionId)); + const previous = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + deletedVersionId + ); + assert.strictEqual(previous.isCurrentVersion, false); + }); + + it("should allow deleting specific versions by versionId @loki", async () => { + const name = `blob-${uuid()}`; + + // Create multiple versions + const v1 = buildBlockBlob(ACCOUNT, containerName, name, "version1"); + const created1 = await store.createBlob(ctx, v1); + const version1Id = created1.versionId; + + ctx.startTime = new Date(Date.now() + 100); + const v2 = buildBlockBlob(ACCOUNT, containerName, name, "version2"); + await store.createBlob(ctx, v2); + + // Delete specific version (v1) by versionId + assert.ok(!isNullOrWhitespace(version1Id)); + await store.deleteBlob(ctx, ACCOUNT, containerName, name, { + versionId: version1Id + }); + + // Current version (v2) should still exist + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.ok(current.isCurrentVersion); + assert.strictEqual( + current.properties.contentLength, + v2.properties.contentLength + ); + + // Deleted version should no longer be accessible + try { + await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + version1Id + ); + assert.fail("Should have thrown error for deleted version"); + } catch (error) { + // Expected behavior + } + }); + + it("should create versions for write operations on existing blobs @loki", async () => { + const name = `blob-${uuid()}`; + + // Create initial blob + const initialBlob = buildBlockBlob(ACCOUNT, containerName, name, "initial"); + await store.createBlob(ctx, initialBlob); + const firstVersion = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Update metadata (write operation) should create new version + ctx.startTime = new Date(Date.now() + 100); + await store.setBlobMetadata(ctx, ACCOUNT, containerName, name, undefined, { + custommeta: "value" + }); + + const afterMetadataUpdate = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Should have new version ID and be current + assert.notStrictEqual( + afterMetadataUpdate.versionId, + firstVersion.versionId + ); + assert.ok(afterMetadataUpdate.isCurrentVersion); + + // Previous version should still exist as non-current + assert.ok(!isNullOrWhitespace(firstVersion.versionId)); + const previousVersion = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + firstVersion.versionId + ); + assert.strictEqual(previousVersion.isCurrentVersion, false); + assert.strictEqual(previousVersion.versionId, firstVersion.versionId); + }); + + it("should handle immutable versions correctly @loki", async () => { + const name = `blob-${uuid()}`; + + // Create version + const v1 = buildBlockBlob(ACCOUNT, containerName, name, "content"); + const created = await store.createBlob(ctx, v1); + const versionId = created.versionId; + + // Create new current version + ctx.startTime = new Date(Date.now() + 100); + const v2 = buildBlockBlob(ACCOUNT, containerName, name, "modified"); + await store.createBlob(ctx, v2); + + // Previous version should remain unchanged when accessed + assert.ok(!isNullOrWhitespace(versionId)); + const version1_read1 = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + versionId + ); + + // Wait and read again - should be identical + const version1_read2 = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + versionId + ); + + assert.strictEqual( + version1_read1.properties.contentLength, + version1_read2.properties.contentLength + ); + assert.strictEqual( + version1_read1.properties.etag, + version1_read2.properties.etag + ); + assert.strictEqual(version1_read1.versionId, version1_read2.versionId); + assert.strictEqual(version1_read1.isCurrentVersion, false); + assert.strictEqual(version1_read2.isCurrentVersion, false); + }); + + it("should return correct isCurrentVersion flag for different scenarios @loki", async () => { + const name = `blob-${uuid()}`; + + // Create first version + const v1 = buildBlockBlob(ACCOUNT, containerName, name, "v1"); + await store.createBlob(ctx, v1); + + // Should be current + const first = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual(first.isCurrentVersion, true); + + // Create second version + ctx.startTime = new Date(Date.now() + 100); + const v2 = buildBlockBlob(ACCOUNT, containerName, name, "v2"); + await store.createBlob(ctx, v2); + + // Second should be current, first should not be + const second = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual(second.isCurrentVersion, true); + + assert.ok(!isNullOrWhitespace(first.versionId)); + const firstAgain = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + first.versionId + ); + assert.strictEqual(firstAgain.isCurrentVersion, false); + }); + + // ================== SNAPSHOT TESTS ================== + it("should create a new version when taking a snapshot while versioning enabled @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + const beforeSnapshot = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Take snapshot should create new version according to Azure docs + ctx.startTime = new Date(Date.now() + 100); + const snapshotResponse = await store.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + + assert.ok(snapshotResponse.snapshot); + assert.ok(!isNullOrWhitespace(snapshotResponse.versionId)); + + // Current version should have changed after snapshot + const afterSnapshot = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + assert.notStrictEqual(afterSnapshot.versionId, beforeSnapshot.versionId); + assert.ok(afterSnapshot.isCurrentVersion); + }); + + // ================== HTTP HEADERS TESTS ================== + it("should NOT create new version when setting HTTP headers with versioning enabled @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + const beforeHeaders = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + ctx.startTime = new Date(Date.now() + 100); + await store.setBlobHTTPHeaders( + ctx, + ACCOUNT, + containerName, + name, + undefined, + { blobContentType: "text/plain" } + ); + + const afterHeaders = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Should NOT create new version - HTTP headers are metadata updates only + assert.strictEqual(afterHeaders.versionId, beforeHeaders.versionId); + assert.strictEqual(afterHeaders.properties.contentType, "text/plain"); + assert.ok(afterHeaders.isCurrentVersion); + + // Should be the same version with updated headers + assert.strictEqual(afterHeaders.versionId, beforeHeaders.versionId); + }); + + // ================== BLOB TAGS TESTS ================== + it("should NOT create new version when setting blob tags with versioning enabled @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + const beforeTags = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + ctx.startTime = new Date(Date.now() + 100); + await store.setBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined, + { blobTagSet: [{ key: "key1", value: "value1" }] } + ); + + const afterTags = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Should NOT create new version - tags are metadata updates only + assert.strictEqual(afterTags.versionId, beforeTags.versionId); + assert.ok(afterTags.isCurrentVersion); + + // Verify tags are set on current version (same version) + const tags = await store.getBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined + ); + assert.deepStrictEqual(tags, { + blobTagSet: [{ key: "key1", value: "value1" }] + }); + }); + + it("should access tags from specific versions @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + // Set tags on first version - this should NOT create a new version + await store.setBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined, + { blobTagSet: [{ key: "version", value: "1" }] } + ); + + const firstVersion = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Create second version using content change (Put Blob operation) + ctx.startTime = new Date(Date.now() + 100); + const blob2 = buildBlockBlob(ACCOUNT, containerName, name, "content2"); + await store.createBlob(ctx, blob2); + + // Set different tags on second version + await store.setBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined, + { blobTagSet: [{ key: "version", value: "2" }] } + ); + + // Verify each version has its own tags + const firstVersionTags = await store.getBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + firstVersion.versionId, + undefined + ); + + const currentTags = await store.getBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined + ); + + assert.deepStrictEqual(firstVersionTags, { + blobTagSet: [{ key: "version", value: "1" }] + }); + assert.deepStrictEqual(currentTags, { + blobTagSet: [{ key: "version", value: "2" }] + }); + }); + + // ================== TIER MANAGEMENT TESTS ================== + it("should set tier on specific blob versions independently @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + blob.properties.accessTier = Models.AccessTier.Hot; + await store.createBlob(ctx, blob); + + const firstVersion = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Create second version + ctx.startTime = new Date(Date.now() + 100); + const blob2 = buildBlockBlob(ACCOUNT, containerName, name, "content2"); + blob2.properties.accessTier = Models.AccessTier.Hot; + await store.createBlob(ctx, blob2); + + // Set tier on current version (without versionId) + await store.setTier( + ctx, + ACCOUNT, + containerName, + name, + "", + Models.AccessTier.Cool, + undefined + ); + + // Current version should have Cool tier + const currentAfterTier = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual( + currentAfterTier.properties.accessTier, + Models.AccessTier.Cool + ); + + // Previous version should still have Hot tier + assert.ok(!isNullOrWhitespace(firstVersion.versionId)); + const previousAfterTier = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + firstVersion.versionId + ); + assert.strictEqual( + previousAfterTier.properties.accessTier, + Models.AccessTier.Hot + ); + + // Now set tier on specific version (first version) by versionId + await store.setTier( + ctx, + ACCOUNT, + containerName, + name, + firstVersion.versionId, + Models.AccessTier.Archive, + undefined + ); + + // First version should now have Archive tier + const firstVersionAfterArchive = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + firstVersion.versionId + ); + assert.strictEqual( + firstVersionAfterArchive.properties.accessTier, + Models.AccessTier.Archive + ); + + // Current version should still have Cool tier (unchanged) + const currentStillCool = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual( + currentStillCool.properties.accessTier, + Models.AccessTier.Cool + ); + }); + + // ================== BLOB EXISTENCE AND PROPERTIES TESTS ================== + it("should check blob existence for specific versions @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + const firstVersion = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Create second version + ctx.startTime = new Date(Date.now() + 100); + const blob2 = buildBlockBlob(ACCOUNT, containerName, name, "content2"); + await store.createBlob(ctx, blob2); + + // Check existence of current version + await store.checkBlobExist(ctx, ACCOUNT, containerName, name); + + // Check existence of specific version + assert.ok(!isNullOrWhitespace(firstVersion.versionId)); + await store.checkBlobExist( + ctx, + ACCOUNT, + containerName, + name, + "", + firstVersion.versionId + ); + + // Should throw for non-existent version + try { + await store.checkBlobExist( + ctx, + ACCOUNT, + containerName, + name, + "", + "2099-01-01T00:00:00.0000000Z" + ); + assert.fail("Should have thrown for non-existent version"); + } catch (error) { + // Expected + } + }); + + it("should get properties for specific blob versions @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + // Set metadata to create version + ctx.startTime = new Date(Date.now() + 100); + await store.setBlobMetadata(ctx, ACCOUNT, containerName, name, undefined, { + version: "1" + }); + + const firstVersion = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Set different metadata to create second version + ctx.startTime = new Date(Date.now() + 200); + await store.setBlobMetadata(ctx, ACCOUNT, containerName, name, undefined, { + version: "2" + }); + + // Get properties of current version + const currentProps = await store.getBlobProperties( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined + ); + + assert.deepStrictEqual(currentProps.metadata, { version: "2" }); + + // Get properties of previous version + assert.ok(!isNullOrWhitespace(firstVersion.versionId)); + const prevProps = await store.getBlobProperties( + ctx, + ACCOUNT, + containerName, + name, + undefined, + firstVersion.versionId, + undefined + ); + + assert.deepStrictEqual(prevProps.metadata, { version: "1" }); + }); + + // ================== APPEND BLOB OPERATIONS TESTS ================== + it("should not create versions for Append Block operations @loki", async () => { + const name = `blob-${uuid()}`; + const appendBlob = buildAppendBlob(ACCOUNT, containerName, name); + await store.createBlob(ctx, appendBlob); + + const afterCreate = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Append block should not create version + const block = { + accountName: ACCOUNT, + containerName, + blobName: name, + name: "append1", + size: 10, + persistency: { id: uuid(), offset: 0, count: 10 } + } as any; + + ctx.startTime = new Date(Date.now() + 100); + await store.appendBlock(ctx, block); + + const afterAppend = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Should be same version, just updated properties + assert.strictEqual(afterAppend.versionId, afterCreate.versionId); + assert.ok(afterAppend.isCurrentVersion); + assert.strictEqual(afterAppend.properties.contentLength, 10); + }); + + it("should create versions for Put Blob operations on append blobs @loki", async () => { + const name = `blob-${uuid()}`; + const appendBlob1 = buildAppendBlob(ACCOUNT, containerName, name); + await store.createBlob(ctx, appendBlob1); + + const firstVersion = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Put Blob (replace) should create new version + ctx.startTime = new Date(Date.now() + 100); + const appendBlob2 = buildAppendBlob(ACCOUNT, containerName, name); + appendBlob2.properties.contentLength = 20; + await store.createBlob(ctx, appendBlob2); + + const secondVersion = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + assert.notStrictEqual(secondVersion.versionId, firstVersion.versionId); + assert.ok(secondVersion.isCurrentVersion); + assert.strictEqual(secondVersion.properties.contentLength, 20); + }); + + // ================== PAGE BLOB OPERATIONS TESTS ================== + it("should not create versions for Put Page operations @loki", async () => { + const name = `blob-${uuid()}`; + const pageBlob = buildPageBlob(ACCOUNT, containerName, name, 512); + await store.createBlob(ctx, pageBlob); + + const afterCreate = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Put Page should not create version according to Azure docs + const persistency = { id: uuid(), offset: 0, count: 512 }; + ctx.startTime = new Date(Date.now() + 100); + await store.uploadPages(ctx, pageBlob, 0, 511, persistency); + + const afterUpload = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Should be same version + assert.strictEqual(afterUpload.versionId, afterCreate.versionId); + assert.ok(afterUpload.isCurrentVersion); + }); + + it("should create versions for Put Blob operations on page blobs @loki", async () => { + const name = `blob-${uuid()}`; + const pageBlob1 = buildPageBlob(ACCOUNT, containerName, name, 512); + await store.createBlob(ctx, pageBlob1); + + const firstVersion = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Put Blob (replace) should create new version + ctx.startTime = new Date(Date.now() + 100); + const pageBlob2 = buildPageBlob(ACCOUNT, containerName, name, 1024); + await store.createBlob(ctx, pageBlob2); + + const secondVersion = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + assert.notStrictEqual(secondVersion.versionId, firstVersion.versionId); + assert.ok(secondVersion.isCurrentVersion); + assert.strictEqual(secondVersion.properties.contentLength, 1024); + }); + + // ================== LIST BLOBS VERSIONING TESTS ================== + it("should list preserved versions after deleting the current blob @loki", async () => { + const blob1Name = `blob1-${uuid()}`; + const blob2Name = `blob2-${uuid()}`; + + // Create first blob with multiple versions + const blob1v1 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v1"); + await store.createBlob(ctx, blob1v1); + const blob1v1Downloaded = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + blob1Name, + undefined, + undefined + ); + const blob1v1Id = blob1v1Downloaded.versionId; + + ctx.startTime = new Date(Date.now() + 100); + const blob1v2 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v2"); + await store.createBlob(ctx, blob1v2); + const blob1v2Downloaded = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + blob1Name, + undefined, + undefined + ); + const blob1v2Id = blob1v2Downloaded.versionId; + + // Create second blob + ctx.startTime = new Date(Date.now() + 200); + const blob2v1 = buildBlockBlob( + ACCOUNT, + containerName, + blob2Name, + "content" + ); + await store.createBlob(ctx, blob2v1); + + // Create third blob with multiple versions, then delete it + const blob3Name = `blob3-${uuid()}`; + ctx.startTime = new Date(Date.now() + 300); + const blob3v1 = buildBlockBlob(ACCOUNT, containerName, blob3Name, "v1"); + await store.createBlob(ctx, blob3v1); + + ctx.startTime = new Date(Date.now() + 400); + const blob3v2 = buildBlockBlob(ACCOUNT, containerName, blob3Name, "v2"); + await store.createBlob(ctx, blob3v2); + + // Delete blob3. Its current version becomes a previous version. + await store.deleteBlob(ctx, ACCOUNT, containerName, blob3Name, {}); + + // includeVersions continues to return all preserved versions. + const [blobs, ,] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, + undefined, + "", + DEFAULT_LIST_BLOBS_MAX_RESULTS, + "", + undefined, + undefined, + true, + undefined + ); + + assert.strictEqual(blobs.length, 5); + + // Verify blob1 versions are sorted chronologically (earliest first) + const blob1Versions = blobs.filter((b) => b.name === blob1Name); + assert.strictEqual(blob1Versions.length, 2); + assert.strictEqual(blob1Versions[0].versionId, blob1v1Id); // Earlier version first + assert.strictEqual(blob1Versions[1].versionId, blob1v2Id); // Current version last + assert.strictEqual(blob1Versions[0].isCurrentVersion, false); + assert.strictEqual(blob1Versions[1].isCurrentVersion, true); + + // Verify blob2 is present + const blob2Versions = blobs.filter((b) => b.name === blob2Name); + assert.strictEqual(blob2Versions.length, 1); + assert.strictEqual(blob2Versions[0].isCurrentVersion, true); + + // Both blob3 versions remain visible, with no current version. + const blob3Versions = blobs.filter((b) => b.name === blob3Name); + assert.strictEqual(blob3Versions.length, 2); + assert.ok( + blob3Versions.every((version) => version.isCurrentVersion !== true) + ); + }); + + it("should list blobs with includeVersions=false showing only current versions @loki", async () => { + const blob1Name = `blob1-${uuid()}`; + const blob2Name = `blob2-${uuid()}`; + + // Create first blob with multiple versions + const blob1v1 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v1"); + await store.createBlob(ctx, blob1v1); + + ctx.startTime = new Date(Date.now() + 100); + const blob1v2 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v2"); + await store.createBlob(ctx, blob1v2); + + // Create second blob + ctx.startTime = new Date(Date.now() + 200); + const blob2v1 = buildBlockBlob( + ACCOUNT, + containerName, + blob2Name, + "content" + ); + await store.createBlob(ctx, blob2v1); + + // List with includeVersions=false (default) should show only current versions + const [blobs, ,] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, + undefined, + "", + DEFAULT_LIST_BLOBS_MAX_RESULTS, + "", + undefined, + undefined, + false, + undefined + ); + + assert.strictEqual(blobs.length, 2); // Only current versions + + const blob1Result = blobs.find((b) => b.name === blob1Name); + const blob2Result = blobs.find((b) => b.name === blob2Name); + + assert.ok(blob1Result); + assert.ok(blob2Result); + assert.strictEqual(blob1Result.isCurrentVersion, true); + assert.strictEqual(blob2Result.isCurrentVersion, true); + }); + + it("should list blobs with includeDeletedWithVersions=true showing all versions including deleted @loki", async () => { + const blob1Name = `blob1-${uuid()}`; + const blob2Name = `blob2-${uuid()}`; + + // Create first blob with multiple versions + const blob1v1 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v1"); + await store.createBlob(ctx, blob1v1); + const blob1v1Downloaded = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + blob1Name, + undefined, + undefined + ); + const blob1v1Id = blob1v1Downloaded.versionId; + + ctx.startTime = new Date(Date.now() + 100); + const blob1v2 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v2"); + await store.createBlob(ctx, blob1v2); + const blob1v2Downloaded = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + blob1Name, + undefined, + undefined + ); + const blob1v2Id = blob1v2Downloaded.versionId; + + // Create second blob + ctx.startTime = new Date(Date.now() + 200); + const blob2v1 = buildBlockBlob( + ACCOUNT, + containerName, + blob2Name, + "content" + ); + await store.createBlob(ctx, blob2v1); + + // Delete first blob (current version becomes previous version) + await store.deleteBlob(ctx, ACCOUNT, containerName, blob1Name, {}); + + // List with includeDeletedWithVersions=true should show all versions including deleted + const [blobs, ,] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, + undefined, + "", + DEFAULT_LIST_BLOBS_MAX_RESULTS, + "", + undefined, + undefined, + undefined, + true + ); + + assert.strictEqual(blobs.length, 3); // blob1v1, blob1v2 (both now non-current), blob2v1 + + // Verify blob1 versions are present but marked as non-current (deleted) + const blob1Versions = blobs.filter((b) => b.name === blob1Name); + assert.strictEqual(blob1Versions.length, 2); + assert.strictEqual(blob1Versions[0].versionId, blob1v1Id); // Earlier version first + assert.strictEqual(blob1Versions[1].versionId, blob1v2Id); // Later version + assert.strictEqual(blob1Versions[0].isCurrentVersion, false); + assert.strictEqual(blob1Versions[1].isCurrentVersion, false); // Deleted, no longer current + + // Verify blob2 is still current + const blob2Versions = blobs.filter((b) => b.name === blob2Name); + assert.strictEqual(blob2Versions.length, 1); + assert.strictEqual(blob2Versions[0].isCurrentVersion, true); + }); + + it("should properly sort versions chronologically with current version last @loki", async () => { + const blobName = `blob-${uuid()}`; + + // Create multiple versions with specific timestamps + const timestamps = [ + new Date(Date.now() + 100), + new Date(Date.now() + 200), + new Date(Date.now() + 300), + new Date(Date.now() + 400) + ]; + + const versionIds = []; + for (let i = 0; i < timestamps.length; i++) { + ctx.startTime = timestamps[i]; + const blob = buildBlockBlob( + ACCOUNT, + containerName, + blobName, + `v${i + 1}` + ); + await store.createBlob(ctx, blob); + const downloaded = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + blobName, + undefined, + undefined + ); + versionIds.push(downloaded.versionId); + } + + // List with includeVersions=true + const [blobs, ,] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, + undefined, + "", + DEFAULT_LIST_BLOBS_MAX_RESULTS, + "", + undefined, + undefined, + true, + undefined + ); + + const blobVersions = blobs.filter((b) => b.name === blobName); + assert.strictEqual(blobVersions.length, 4); + + // Verify chronological order (earliest first, current last) + for (let i = 0; i < blobVersions.length; i++) { + assert.strictEqual(blobVersions[i].versionId, versionIds[i]); + if (i === blobVersions.length - 1) { + assert.strictEqual(blobVersions[i].isCurrentVersion, true); // Last one is current + } else { + assert.strictEqual(blobVersions[i].isCurrentVersion, false); // Others are previous + } + } + }); + + it("should handle snapshots correctly with includeSnapshots option @loki", async () => { + const blobName = `blob-${uuid()}`; + + // Create blob + const blob = buildBlockBlob(ACCOUNT, containerName, blobName, "content"); + await store.createBlob(ctx, blob); + + // Create snapshot + ctx.startTime = new Date(Date.now() + 100); + const snapshotResponse = await store.createSnapshot( + ctx, + ACCOUNT, + containerName, + blobName + ); + const snapshotId = snapshotResponse.snapshot; + + // Create another version + ctx.startTime = new Date(Date.now() + 200); + const blob2 = buildBlockBlob(ACCOUNT, containerName, blobName, "content2"); + await store.createBlob(ctx, blob2); + + // List with includeSnapshots=true and includeVersions=true + const [blobs, ,] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, + undefined, + "", + DEFAULT_LIST_BLOBS_MAX_RESULTS, + "", + true, + undefined, + true, + undefined + ); + + const blobItems = blobs.filter((b) => b.name === blobName); + + // Should have: original version, snapshot, current version + assert.ok(blobItems.length >= 2); // At least the versions, snapshot handling may vary + + // Find the snapshot entry + const snapshotEntry = blobItems.find((b) => b.snapshot === snapshotId); + if (snapshotEntry) { + assert.strictEqual(snapshotEntry.name, blobName); + assert.strictEqual(snapshotEntry.snapshot, snapshotId); + } + + // Verify current version is marked correctly + const currentVersionEntry = blobItems.find( + (b) => b.isCurrentVersion === true + ); + assert.ok(currentVersionEntry); + assert.strictEqual(currentVersionEntry.name, blobName); + }); + + it("should list snapshots without versions when includeSnapshots=true and includeVersions=false @loki", async () => { + const blobName = `blob-${uuid()}`; + + // Create blob + const blob = buildBlockBlob(ACCOUNT, containerName, blobName, "content"); + await store.createBlob(ctx, blob); + + // Create snapshot + ctx.startTime = new Date(Date.now() + 100); + const snapshotResponse = await store.createSnapshot( + ctx, + ACCOUNT, + containerName, + blobName + ); + const snapshotId = snapshotResponse.snapshot; + + // Create another version + ctx.startTime = new Date(Date.now() + 200); + const blob2 = buildBlockBlob(ACCOUNT, containerName, blobName, "content2"); + await store.createBlob(ctx, blob2); + + // List with includeSnapshots=true but includeVersions=false + const [blobs, ,] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, + undefined, + "", + DEFAULT_LIST_BLOBS_MAX_RESULTS, + "", + true, + undefined, + false, + undefined + ); + + assert.strictEqual(blobs.length, 2); + + const blobItems = blobs.filter((b) => b.name === blobName); + + // Should have: current version + snapshot + const currentVersionEntry = blobItems.find( + (b) => b.isCurrentVersion === true && !b.snapshot + ); + const snapshotEntry = blobItems.find((b) => b.snapshot === snapshotId); + + assert.ok(currentVersionEntry); + assert.strictEqual(currentVersionEntry.name, blobName); + + if (snapshotEntry) { + assert.strictEqual(snapshotEntry.name, blobName); + assert.strictEqual(snapshotEntry.snapshot, snapshotId); + } + }); + + it("should handle complex scenario with multiple blobs, versions, snapshots, and deletions @loki", async () => { + const blob1Name = `blob1-${uuid()}`; + const blob2Name = `blob2-${uuid()}`; + const blob3Name = `blob3-${uuid()}`; + + // Create blob1 with multiple versions + const blob1v1 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v1"); + await store.createBlob(ctx, blob1v1); + + ctx.startTime = new Date(Date.now() + 100); + const blob1v2 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v2"); + await store.createBlob(ctx, blob1v2); + + // Create snapshot of blob1 + ctx.startTime = new Date(Date.now() + 150); + await store.createSnapshot(ctx, ACCOUNT, containerName, blob1Name); + + // Create blob2 + ctx.startTime = new Date(Date.now() + 200); + const blob2v1 = buildBlockBlob( + ACCOUNT, + containerName, + blob2Name, + "content" + ); + await store.createBlob(ctx, blob2v1); + + // Create blob3 and then delete it + ctx.startTime = new Date(Date.now() + 300); + const blob3v1 = buildBlockBlob( + ACCOUNT, + containerName, + blob3Name, + "content" + ); + await store.createBlob(ctx, blob3v1); + await store.deleteBlob(ctx, ACCOUNT, containerName, blob3Name, {}); + + // Test 1: includeVersions=true, includeSnapshots=true, includeDeletedWithVersions=false + const [blobs1, ,] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, + undefined, + "", + DEFAULT_LIST_BLOBS_MAX_RESULTS, + "", + true, + undefined, + true, + false + ); + + // include=versions returns preserved versions even when no current blob exists. + const blob1Items = blobs1.filter((b) => b.name === blob1Name); + const blob2Items = blobs1.filter((b) => b.name === blob2Name); + const blob3Items = blobs1.filter((b) => b.name === blob3Name); + + assert.strictEqual(blob1Items.length, 4); // versions + assert.strictEqual(blob2Items.length, 1); // current version only + assert.strictEqual(blob3Items.length, 1); + assert.strictEqual(blob3Items[0].isCurrentVersion, false); + + // Test 2: includeDeletedWithVersions=true + const [blobs2, ,] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, + undefined, + "", + DEFAULT_LIST_BLOBS_MAX_RESULTS, + "", + true, + undefined, + true, + true + ); + + const blob3ItemsWithDeleted = blobs2.filter((b) => b.name === blob3Name); + assert.ok(blob3ItemsWithDeleted.length > 0); // Should include deleted blob3 versions + assert.strictEqual(blob3ItemsWithDeleted[0].isCurrentVersion, false); // Should be marked as non-current + }); + + // ================== VERSION MODE TRANSITION TESTS ================== + it("should handle setBlobMetadata correctly across versioning mode transitions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning DISABLED and create base blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let disabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await disabledStore.init(); + await disabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + await disabledStore.createBlob(ctx, baseBlob); + + // Set metadata on base blob (should not create version) + ctx.startTime = new Date(Date.now() + 100); + await disabledStore.setBlobMetadata( + ctx, + ACCOUNT, + containerName, + name, + undefined, + { basemeta: "value1" } + ); + + const baseFetched = await disabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual(baseFetched.versionId, ""); + assert.deepStrictEqual(baseFetched.metadata, { basemeta: "value1" }); + await accountModelStore.close(); + await disabledStore.close(); + + // 2. Re-open with versioning ENABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Set metadata should create new version and promote previous + ctx.startTime = new Date(Date.now() + 200); + await store.setBlobMetadata(ctx, ACCOUNT, containerName, name, undefined, { + versionedmeta: "value2" + }); + + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.ok(!isNullOrWhitespace(current.versionId)); + assert.ok(current.isCurrentVersion); + assert.deepStrictEqual(current.metadata, { versionedmeta: "value2" }); + + // Previous version should be accessible with original metadata + const originalLastModifiedIso = convertDateTimeStringMsTo7Digital( + baseFetched.properties.lastModified.toISOString() + ); + const previous = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + originalLastModifiedIso + ); + assert.strictEqual(previous.isCurrentVersion, false); + assert.deepStrictEqual(previous.metadata, { basemeta: "value1" }); + }); + + it("should handle setBlobHTTPHeaders correctly across versioning mode transitions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning DISABLED and create base blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let disabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await disabledStore.init(); + await disabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + await disabledStore.createBlob(ctx, baseBlob); + + // Set HTTP headers on base blob (should not create version) + ctx.startTime = new Date(Date.now() + 100); + await disabledStore.setBlobHTTPHeaders( + ctx, + ACCOUNT, + containerName, + name, + undefined, + { blobContentType: "text/plain" } + ); + + const baseFetched = await disabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual(baseFetched.versionId, ""); + assert.strictEqual(baseFetched.properties.contentType, "text/plain"); + await accountModelStore.close(); + await disabledStore.close(); + + // 2. Re-open with versioning ENABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Set headers should NOT create new version (metadata operation) + ctx.startTime = new Date(Date.now() + 200); + await store.setBlobHTTPHeaders( + ctx, + ACCOUNT, + containerName, + name, + undefined, + { blobContentType: "application/json" } + ); + + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + // Should be promoted version but same version (no new version for headers) + assert.strictEqual(current.versionId, ""); + assert.ok(!current.isCurrentVersion); + assert.strictEqual(current.properties.contentType, "application/json"); + }); + + it("should handle setBlobTag/getBlobTag correctly across versioning mode transitions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning DISABLED and create base blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let disabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await disabledStore.init(); + await disabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + await disabledStore.createBlob(ctx, baseBlob); + + // Set tags on base blob (should not create version) + ctx.startTime = new Date(Date.now() + 100); + await disabledStore.setBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined, + { blobTagSet: [{ key: "env", value: "test" }] } + ); + + const baseFetched = await disabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual(baseFetched.versionId, ""); + const baseTags = await disabledStore.getBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined + ); + assert.deepStrictEqual(baseTags, { + blobTagSet: [{ key: "env", value: "test" }] + }); + await accountModelStore.close(); + await disabledStore.close(); + + // 2. Re-open with versioning ENABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Set tags should NOT create new version (metadata operation) + ctx.startTime = new Date(Date.now() + 200); + await store.setBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined, + { blobTagSet: [{ key: "env", value: "prod" }] } + ); + + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + // Should be promoted version but same version (no new version for tags) + assert.strictEqual(current.versionId, ""); + assert.ok(!current.isCurrentVersion); + + const currentTags = await store.getBlobTag( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined + ); + assert.deepStrictEqual(currentTags, { + blobTagSet: [{ key: "env", value: "prod" }] + }); + }); + + it("should handle setTier correctly across versioning mode transitions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning DISABLED and create base blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let disabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await disabledStore.init(); + await disabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + baseBlob.properties.accessTier = Models.AccessTier.Hot; + await disabledStore.createBlob(ctx, baseBlob); + + // Set tier on base blob (should not create version) + ctx.startTime = new Date(Date.now() + 100); + await disabledStore.setTier( + ctx, + ACCOUNT, + containerName, + name, + undefined, + Models.AccessTier.Cool, + undefined + ); + + const baseFetched = await disabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual(baseFetched.versionId, ""); + assert.strictEqual( + baseFetched.properties.accessTier, + Models.AccessTier.Cool + ); + await accountModelStore.close(); + await disabledStore.close(); + + // 2. Re-open with versioning ENABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Set tier should work on promoted version + ctx.startTime = new Date(Date.now() + 200); + await store.setTier( + ctx, + ACCOUNT, + containerName, + name, + undefined, + Models.AccessTier.Archive, + undefined + ); + + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + // Should be promoted version + assert.strictEqual(current.versionId, ""); + assert.ok(!current.isCurrentVersion); + assert.strictEqual( + current.properties.accessTier, + Models.AccessTier.Archive + ); + }); + + it("should handle checkBlobExist correctly across versioning mode transitions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning DISABLED and create base blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let disabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await disabledStore.init(); + await disabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + await disabledStore.createBlob(ctx, baseBlob); + + const baseFetched = await disabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual(baseFetched.versionId, ""); + const originalLastModifiedIso = convertDateTimeStringMsTo7Digital( + baseFetched.properties.lastModified.toISOString() + ); + + // Check existence should work + await disabledStore.checkBlobExist(ctx, ACCOUNT, containerName, name); + await accountModelStore.close(); + await disabledStore.close(); + + // 2. Re-open with versioning ENABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Check existence should work for promoted base blob + await store.checkBlobExist(ctx, ACCOUNT, containerName, name); + + // No writes yet, so versionId should be empty + await store.checkBlobExist(ctx, ACCOUNT, containerName, name, "", ""); + + // Create new version to ensure previous works + ctx.startTime = new Date(Date.now() + 200); + const newBlob = buildBlockBlob(ACCOUNT, containerName, name, "new"); + await store.createBlob(ctx, newBlob); + + // Both current and previous should exist + await store.checkBlobExist(ctx, ACCOUNT, containerName, name); + await store.checkBlobExist( + ctx, + ACCOUNT, + containerName, + name, + "", + originalLastModifiedIso + ); + }); + + it("should handle getBlobProperties correctly across versioning mode transitions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning DISABLED and create base blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let disabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await disabledStore.init(); + await disabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + await disabledStore.createBlob(ctx, baseBlob); + + // Set metadata + ctx.startTime = new Date(Date.now() + 100); + await disabledStore.setBlobMetadata( + ctx, + ACCOUNT, + containerName, + name, + undefined, + { env: "test" } + ); + + const baseFetched = await disabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual(baseFetched.versionId, ""); + const originalLastModifiedIso = convertDateTimeStringMsTo7Digital( + baseFetched.properties.lastModified.toISOString() + ); + + // Get properties should work + const baseProps = await disabledStore.getBlobProperties( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined + ); + assert.deepStrictEqual(baseProps.metadata, { env: "test" }); + await accountModelStore.close(); + await disabledStore.close(); + + // 2. Re-open with versioning ENABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Get properties should work for promoted base blob + const promotedProps = await store.getBlobProperties( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined + ); + assert.deepStrictEqual(promotedProps.metadata, { env: "test" }); + + // Create new version + ctx.startTime = new Date(Date.now() + 200); + await store.setBlobMetadata(ctx, ACCOUNT, containerName, name, undefined, { + env: "prod" + }); + + // Get current properties + const currentProps = await store.getBlobProperties( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined, + undefined + ); + assert.deepStrictEqual(currentProps.metadata, { env: "prod" }); + + // Get previous version properties + const prevProps = await store.getBlobProperties( + ctx, + ACCOUNT, + containerName, + name, + undefined, + originalLastModifiedIso, + undefined + ); + assert.deepStrictEqual(prevProps.metadata, { env: "test" }); + }); + + it("should handle createSnapshot correctly across versioning mode transitions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning DISABLED and create base blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let disabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await disabledStore.init(); + await disabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + await disabledStore.createBlob(ctx, baseBlob); + + // Create snapshot (should not create version) + ctx.startTime = new Date(Date.now() + 100); + const snapshotResponse1 = await disabledStore.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + assert.ok(snapshotResponse1.snapshot); + assert.strictEqual(snapshotResponse1.versionId, ""); + + const baseFetched = await disabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual(baseFetched.versionId, ""); + const originalLastModifiedIso = convertDateTimeStringMsTo7Digital( + baseFetched.properties.lastModified.toISOString() + ); + await accountModelStore.close(); + await disabledStore.close(); + + // 2. Re-open with versioning ENABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Create snapshot should create new version and promote previous + ctx.startTime = new Date(Date.now() + 200); + const snapshotResponse2 = await store.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + assert.ok(snapshotResponse2.snapshot); + assert.ok(!isNullOrWhitespace(snapshotResponse2.versionId)); + + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.ok(!isNullOrWhitespace(current.versionId)); + assert.ok(current.isCurrentVersion); + + // Previous version should be accessible + const previous = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + originalLastModifiedIso + ); + assert.strictEqual(previous.isCurrentVersion, false); + assert.strictEqual(previous.versionId, originalLastModifiedIso); + }); + + it("should handle appendBlock correctly across versioning mode transitions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning DISABLED and create append blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let disabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await disabledStore.init(); + await disabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseAppendBlob = buildAppendBlob(ACCOUNT, containerName, name); + await disabledStore.createBlob(ctx, baseAppendBlob); + + // Append block (should not create version) + const block1 = { + accountName: ACCOUNT, + containerName, + blobName: name, + name: "append1", + size: 10, + persistency: { id: uuid(), offset: 0, count: 10 } + } as any; + + ctx.startTime = new Date(Date.now() + 100); + await disabledStore.appendBlock(ctx, block1); + + const baseFetched = await disabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual(baseFetched.versionId, ""); + assert.strictEqual(baseFetched.properties.contentLength, 10); + await accountModelStore.close(); + await disabledStore.close(); + + // 2. Re-open with versioning ENABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Append block should NOT create new version (per Azure spec) + const block2 = { + accountName: ACCOUNT, + containerName, + blobName: name, + name: "append2", + size: 15, + persistency: { id: uuid(), offset: 10, count: 15 } + } as any; + + ctx.startTime = new Date(Date.now() + 200); + await store.appendBlock(ctx, block2); + + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + // Should not be promoted version (no new version for page upload). + // It does not count as a versioning "write." + assert.strictEqual(current.versionId, ""); + assert.ok(!current.isCurrentVersion); + assert.strictEqual(current.properties.contentLength, 25); + }); + + it("should handle uploadPages correctly across versioning mode transitions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning DISABLED and create page blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let disabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await disabledStore.init(); + await disabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const basePageBlob = buildPageBlob(ACCOUNT, containerName, name, 512); + await disabledStore.createBlob(ctx, basePageBlob); + + // Upload pages (should not create version) + const persistency1 = { id: uuid(), offset: 0, count: 512 }; + ctx.startTime = new Date(Date.now() + 100); + await disabledStore.uploadPages(ctx, basePageBlob, 0, 511, persistency1); + + const baseFetched = await disabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual(baseFetched.versionId, ""); + await accountModelStore.close(); + await disabledStore.close(); + + // 2. Re-open with versioning ENABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Upload pages should NOT create new version (per Azure spec) + const persistency2 = { id: uuid(), offset: 0, count: 512 }; + ctx.startTime = new Date(Date.now() + 200); + await store.uploadPages(ctx, basePageBlob, 0, 511, persistency2); + + const current = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + // Should not be promoted version (no new version for page upload). + // It does not count as a versioning "write." + assert.strictEqual(current.versionId, ""); + assert.ok(!current.isCurrentVersion); + }); + + it("should handle deleteBlob correctly across versioning mode transitions @loki", async () => { + await store.close(); + await store.clean(); + + const name = `blob-${uuid()}`; + + // 1. Create store with versioning DISABLED and create base blob + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let disabledStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await disabledStore.init(); + await disabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + const baseBlob = buildBlockBlob(ACCOUNT, containerName, name, "base"); + await disabledStore.createBlob(ctx, baseBlob); + + const baseFetched = await disabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.strictEqual(baseFetched.versionId, ""); + const originalLastModifiedIso = convertDateTimeStringMsTo7Digital( + baseFetched.properties.lastModified.toISOString() + ); + await accountModelStore.close(); + await disabledStore.close(); + + // 2. Re-open with versioning ENABLED + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // Create new version first so we have something to delete + ctx.startTime = new Date(Date.now() + 200); + const newBlob = buildBlockBlob(ACCOUNT, containerName, name, "new"); + await store.createBlob(ctx, newBlob); + + const currentBeforeDelete = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + + // Delete current blob should make it non-current + await store.deleteBlob(ctx, ACCOUNT, containerName, name, {}); + + // Current version should no longer exist + try { + await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + undefined + ); + assert.fail("Should have thrown error for deleted current blob"); + } catch (error) { + // Expected + } + + // Previous versions should still exist as non-current + const originalVersion = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + originalLastModifiedIso + ); + assert.strictEqual(originalVersion.isCurrentVersion, false); + + const deletedVersion = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + currentBeforeDelete.versionId + ); + assert.strictEqual(deletedVersion.isCurrentVersion, false); + + // Should be able to delete specific version by versionId + await store.deleteBlob(ctx, ACCOUNT, containerName, name, { + versionId: originalLastModifiedIso + }); + + // That specific version should no longer exist + try { + await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + undefined, + originalLastModifiedIso + ); + assert.fail("Should have thrown error for deleted specific version"); + } catch (error) { + // Expected + } + }); +}); + +describe("LokiBlobMetadataStore - Versioning Enabled - deleteBlob comprehensive code path coverage @loki", () => { + let store: LokiBlobMetadataStore; + let disabledStore: LokiBlobMetadataStore; + let accountModelStore: LokiAccountModelStore; + let disabledAccountModelStore: LokiAccountModelStore; + let ctx: Context; + const containerName = "test-container"; + + beforeEach(async () => { + ctx = createContext(); + // Versioning enabled + let accountModel: AccountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore("__test_db_blob__.json", false, accountModelStore); + await store.init(); + await store.createContainer(ctx, buildContainer(ACCOUNT, containerName)); + + // Versioning disabled + accountModel = + { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + disabledAccountModelStore = createAccountModelStore(accountModel, false); + await disabledAccountModelStore.init(); + disabledStore = new LokiBlobMetadataStore( + "__test_db_blob_disabled__.json", + false, + disabledAccountModelStore + ); + await disabledStore.init(); + await disabledStore.createContainer( + ctx, + buildContainer(ACCOUNT, containerName) + ); + }); + + afterEach(async () => { + await accountModelStore.close(); + await store.close(); + await store.clean(); + await disabledAccountModelStore.close(); + await disabledStore.close(); + await disabledStore.clean(); + }); + + it("should throw error when versionId is provided with snapshot option @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + const created = await store.createBlob(ctx, blob); + + // Create a snapshot + const snapshot = await store.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + + try { + await store.deleteBlob(ctx, ACCOUNT, containerName, name, { + snapshot: snapshot.snapshot, + versionId: created.versionId + }); + assert.fail( + "Should have thrown error when versionId provided with snapshot" + ); + } catch (error) { + assert.strictEqual(error.statusCode, 400); + assert.ok( + error.message.includes( + "When deleting a blob version, you cannot specify a snapshot" + ) + ); + } + }); + + it("should throw error when versionId is provided with deleteSnapshots option @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + const created = await store.createBlob(ctx, blob); + + try { + await store.deleteBlob(ctx, ACCOUNT, containerName, name, { + deleteSnapshots: Models.DeleteSnapshotsOptionType.Include, + versionId: created.versionId + }); + assert.fail( + "Should have thrown error when versionId provided with deleteSnapshots" + ); + } catch (error) { + assert.strictEqual(error.statusCode, 400); + assert.ok( + error.message.includes( + "When deleting a blob version, you cannot specify a snapshot" + ) + ); + } + }); + + it("should throw BlobNotFound when deleting non-existent blob @loki", async () => { + try { + await store.deleteBlob( + ctx, + ACCOUNT, + containerName, + "non-existent-blob", + {} + ); + assert.fail("Should have thrown BlobNotFound error"); + } catch (error) { + assert.strictEqual(error.statusCode, 404); + } + }); + + it("should throw error when trying to use deleteSnapshots against a snapshot @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + // Create a snapshot + const snapshot = await store.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + + try { + await store.deleteBlob(ctx, ACCOUNT, containerName, name, { + snapshot: snapshot.snapshot, + deleteSnapshots: Models.DeleteSnapshotsOptionType.Include + }); + assert.fail( + "Should have thrown error when using deleteSnapshots against snapshot" + ); + } catch (error) { + assert.strictEqual(error.statusCode, 400); + assert.ok( + error.message.includes("Invalid operation against a blob snapshot") + ); + } + }); + + it("should delete specific version when versionId is provided @loki", async () => { + const name = `blob-${uuid()}`; + + // Create version 1 + const v1 = buildBlockBlob(ACCOUNT, containerName, name, "version1"); + const created1 = await store.createBlob(ctx, v1); + + // Create version 2 + ctx.startTime = new Date(Date.now() + 100); + const v2 = buildBlockBlob(ACCOUNT, containerName, name, "version2"); + await store.createBlob(ctx, v2); + + // Delete version 1 specifically + await store.deleteBlob(ctx, ACCOUNT, containerName, name, { + versionId: created1.versionId + }); + + // Version 2 should still exist as current + const current = await store.downloadBlob(ctx, ACCOUNT, containerName, name); + assert.strictEqual( + current.properties.contentLength, + Buffer.byteLength("version2") + ); + + // Version 1 should be gone + try { + await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + "", + created1.versionId + ); + assert.fail("Should have thrown error for deleted version"); + } catch (error) { + assert.strictEqual(error.statusCode, 404); + } + }); + + it("should throw SnapshotsPresent when deleting base blob with snapshots (versioning enabled) @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + // Create snapshots + await store.createSnapshot(ctx, ACCOUNT, containerName, name); + await store.createSnapshot(ctx, ACCOUNT, containerName, name); + + try { + await store.deleteBlob(ctx, ACCOUNT, containerName, name, {}); + assert.fail("Should have thrown SnapshotsPresent error"); + } catch (error) { + assert.strictEqual(error.statusCode, 409); + assert.ok(error.message.includes("has snapshots")); + } + }); + + it("should throw SnapshotsPresent when deleting base blob with snapshots (versioning disabled) @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await disabledStore.createBlob(ctx, blob); + + // Create snapshots + await disabledStore.createSnapshot(ctx, ACCOUNT, containerName, name); + await disabledStore.createSnapshot(ctx, ACCOUNT, containerName, name); + + try { + await disabledStore.deleteBlob(ctx, ACCOUNT, containerName, name, {}); + assert.fail("Should have thrown SnapshotsPresent error"); + } catch (error) { + assert.strictEqual(error.statusCode, 409); + assert.ok(error.message.includes("has snapshots")); + } + }); + + it("should mark blob as non-current when deleting base blob without snapshots (versioning enabled) @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + const created = await store.createBlob(ctx, blob); + + // Delete the blob (no snapshots exist) + await store.deleteBlob(ctx, ACCOUNT, containerName, name, {}); + + // Blob should be marked as non-current, not physically deleted + try { + await store.downloadBlob(ctx, ACCOUNT, containerName, name); + assert.fail("Should have thrown error for non-current blob"); + } catch (error) { + assert.strictEqual(error.statusCode, 404); + } + + // But should still be accessible by version ID + const versionedBlob = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + "", + created.versionId + ); + assert.strictEqual( + versionedBlob.properties.contentLength, + Buffer.byteLength("content") + ); + }); + + it("should physically delete blob when deleting base blob without snapshots (versioning disabled) @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + const created = await disabledStore.createBlob(ctx, blob); + + // Delete the blob (no snapshots exist) + await disabledStore.deleteBlob(ctx, ACCOUNT, containerName, name, {}); + + // Blob should be completely gone + try { + await disabledStore.downloadBlob(ctx, ACCOUNT, containerName, name); + assert.fail("Should have thrown error for deleted blob"); + } catch (error) { + assert.strictEqual(error.statusCode, 404); + } + + // Should not be accessible by version ID either (versioning disabled) + try { + await disabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + "", + created.versionId + ); + assert.fail("Should have thrown error for deleted blob by version"); + } catch (error) { + assert.strictEqual(error.statusCode, 404); + } + }); + + it("should delete individual snapshot when targeting specific snapshot @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + // Create multiple snapshots + const snapshot1 = await store.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + ctx.startTime = new Date(Date.now() + 100); + const snapshot2 = await store.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + + // Delete first snapshot specifically + await store.deleteBlob(ctx, ACCOUNT, containerName, name, { + snapshot: snapshot1.snapshot + }); + + // Base blob should still exist + const baseBlob = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name + ); + assert.strictEqual( + baseBlob.properties.contentLength, + Buffer.byteLength("content") + ); + + // Second snapshot should still exist + const snap2 = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + snapshot2.snapshot + ); + assert.strictEqual( + snap2.properties.contentLength, + Buffer.byteLength("content") + ); + + // First snapshot should be gone + try { + await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + snapshot1.snapshot + ); + assert.fail("Should have thrown error for deleted snapshot"); + } catch (error) { + assert.strictEqual(error.statusCode, 404); + } + }); + + it("should delete base blob and all snapshots when deleteSnapshots=include (versioning disabled) @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await disabledStore.createBlob(ctx, blob); + + // Create snapshots + const snapshot1 = await disabledStore.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + const snapshot2 = await disabledStore.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + + // Delete blob and all snapshots + await disabledStore.deleteBlob(ctx, ACCOUNT, containerName, name, { + deleteSnapshots: Models.DeleteSnapshotsOptionType.Include + }); + + // Base blob should be gone + try { + await disabledStore.downloadBlob(ctx, ACCOUNT, containerName, name); + assert.fail("Should have thrown error for deleted blob"); + } catch (error) { + assert.strictEqual(error.statusCode, 404); + } + + // All snapshots should be gone + try { + await disabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + snapshot1.snapshot + ); + assert.fail("Should have thrown error for deleted snapshot1"); + } catch (error) { + assert.strictEqual(error.statusCode, 404); + } + + try { + await disabledStore.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + snapshot2.snapshot + ); + assert.fail("Should have thrown error for deleted snapshot2"); + } catch (error) { + assert.strictEqual(error.statusCode, 404); + } + }); + + it("should delete snapshots only and mark base blob as non-current when deleteSnapshots=include (versioning enabled) @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + const created = await store.createBlob(ctx, blob); + + // Create snapshots + const snapshot1 = await store.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + const snapshot2 = await store.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + + // Delete blob and all snapshots + await store.deleteBlob(ctx, ACCOUNT, containerName, name, { + deleteSnapshots: Models.DeleteSnapshotsOptionType.Include + }); + + // Base blob should be marked as non-current + try { + await store.downloadBlob(ctx, ACCOUNT, containerName, name); + assert.fail("Should have thrown error for non-current blob"); + } catch (error) { + assert.strictEqual(error.statusCode, 404); + } + + // All snapshots should be gone + try { + await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + snapshot1.snapshot + ); + assert.fail("Should have thrown error for deleted snapshot1"); + } catch (error) { + assert.strictEqual(error.statusCode, 404); + } + + try { + await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + snapshot2.snapshot + ); + assert.fail("Should have thrown error for deleted snapshot2"); + } catch (error) { + assert.strictEqual(error.statusCode, 404); + } + + // But base blob should still be accessible by version ID + const versionedBlob = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + "", + created.versionId + ); + assert.strictEqual( + versionedBlob.properties.contentLength, + Buffer.byteLength("content") + ); + }); + + it("should delete only snapshots when deleteSnapshots=only @loki", async () => { + const name = `blob-${uuid()}`; + const blob = buildBlockBlob(ACCOUNT, containerName, name, "content"); + await store.createBlob(ctx, blob); + + // Create snapshots + const snapshot1 = await store.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + const snapshot2 = await store.createSnapshot( + ctx, + ACCOUNT, + containerName, + name + ); + + // Delete only snapshots + await store.deleteBlob(ctx, ACCOUNT, containerName, name, { + deleteSnapshots: Models.DeleteSnapshotsOptionType.Only + }); + + // Base blob should still exist and be current + const baseBlob = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name + ); + assert.strictEqual( + baseBlob.properties.contentLength, + Buffer.byteLength("content") + ); + + // All snapshots should be gone + try { + await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + snapshot1.snapshot + ); + assert.fail("Should have thrown error for deleted snapshot1"); + } catch (error) { + assert.strictEqual(error.statusCode, 404); + } + + try { + await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + name, + snapshot2.snapshot + ); + assert.fail("Should have thrown error for deleted snapshot2"); + } catch (error) { + assert.strictEqual(error.statusCode, 404); + } + }); +}); + +describe("LokiBlobMetadataStore - Versioning Enabled - listBlobs and filterBlobs pagination tests @loki", () => { + let store: LokiBlobMetadataStore; + let accountModelStore: LokiAccountModelStore; + let containerName: string; + let ctx: Context; + const DB_FILE = "__test_db_blob__.json"; + + beforeEach(async () => { + ctx = createContext(); + containerName = `container-${uuid()}`; + const accountModel: AccountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + await store.createContainer(ctx, buildContainer(ACCOUNT, containerName)); + }); + + afterEach(async () => { + await accountModelStore.close(); + await accountModelStore.clean(); + await store.close(); + await store.clean(); + }); + + it("should paginate listBlobs correctly with includeVersions=true using name+versionId marker @loki", async () => { + // Create multiple versions of different blobs to test pagination + const blob1Name = `blob-a`; + const blob2Name = `blob-b`; + + // Create first blob with multiple versions + const blob1v1 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v1"); + await store.createBlob(ctx, blob1v1); + + ctx.startTime = new Date(Date.now() + 100); + const blob1v2 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v2"); + await store.createBlob(ctx, blob1v2); + + ctx.startTime = new Date(Date.now() + 200); + const blob1v3 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v3"); + await store.createBlob(ctx, blob1v3); + + // Create second blob with multiple versions + ctx.startTime = new Date(Date.now() + 300); + const blob2v1 = buildBlockBlob(ACCOUNT, containerName, blob2Name, "v1"); + await store.createBlob(ctx, blob2v1); + + ctx.startTime = new Date(Date.now() + 400); + const blob2v2 = buildBlockBlob(ACCOUNT, containerName, blob2Name, "v2"); + await store.createBlob(ctx, blob2v2); + + // Test pagination with small maxResults to trigger marker logic + const [firstPage, , firstMarker] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, // delimiter + undefined, // blob + "", // prefix + 3, // maxResults - should get 3 versions + "", // marker + false, // includeSnapshots + false, // includeUncommittedBlobs + true, // includeVersions - this triggers the changed code path + false // includeDeletedWithVersions + ); + + assert.strictEqual(firstPage.length, 3, "First page should have 3 versions"); + assert.ok(firstMarker, "Should have a marker for next page"); + + // Continue pagination with marker + const [secondPage, , secondMarker] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, + undefined, + "", + 3, + firstMarker!, // Use marker from first page + false, + false, + true, // includeVersions=true triggers name+versionId comparison + false + ); + + assert.strictEqual(secondPage.length, 2, "Second page should have remaining 2 versions"); + assert.strictEqual(secondMarker, "", "Should not have marker when all results returned"); + + // Verify all versions are accounted for + const totalVersions = firstPage.length + secondPage.length; + assert.strictEqual(totalVersions, 5, "Should have 5 total versions across both pages"); + + // Verify versions are properly ordered by name+versionId + const allVersions = [...firstPage, ...secondPage]; + for (let i = 1; i < allVersions.length; i++) { + const prev = allVersions[i - 1]; + const curr = allVersions[i]; + const prevKey = prev.name + prev.versionId; + const currKey = curr.name + curr.versionId; + assert.ok(prevKey <= currKey, `Versions should be ordered: ${prevKey} <= ${currKey}`); + } + }); + + it("should paginate listBlobs correctly with includeVersions=false using name-only marker @loki", async () => { + // Create multiple blobs (current versions only) + const blobNames = [`blob-a`, `blob-b`, `blob-c`, `blob-d`]; + + for (let i = 0; i < blobNames.length; i++) { + ctx.startTime = new Date(Date.now() + i * 100); + const blob = buildBlockBlob(ACCOUNT, containerName, blobNames[i], `content${i}`); + await store.createBlob(ctx, blob); + + // Create additional versions for some blobs + if (i % 2 === 0) { + ctx.startTime = new Date(Date.now() + i * 100 + 50); + const blob2 = buildBlockBlob(ACCOUNT, containerName, blobNames[i], `content${i}v2`); + await store.createBlob(ctx, blob2); + } + } + + // Test pagination with includeVersions=false (should use name-only marker) + const [firstPage, , firstMarker] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, + undefined, + "", + 2, // maxResults + "", + false, + false, + false, // includeVersions=false - uses original name-only logic + false + ); + + assert.strictEqual(firstPage.length, 2, "First page should have 2 current versions"); + assert.ok(firstMarker, "Should have marker for next page"); + + // Continue pagination + const [secondPage, , secondMarker] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, + undefined, + "", + 2, + firstMarker!, + false, + false, + false, // includeVersions=false + false + ); + + assert.strictEqual(secondPage.length, 2, "Second page should have 2 more current versions"); + assert.strictEqual(secondMarker, "", "Should not have marker when all results returned"); + + // Verify only current versions are returned + const allBlobs = [...firstPage, ...secondPage]; + assert.strictEqual(allBlobs.length, 4, "Should have 4 current versions total"); + allBlobs.forEach(blob => { + assert.ok(blob.isCurrentVersion, `Blob ${blob.name} should be current version`); + }); + }); + + it("should handle filterBlobs with versioning enabled returning only current versions @loki", async () => { + // Create multiple blobs with tags that change across versions + const blob1Name = `tagged-blob-a`; + const blob2Name = `tagged-blob-b`; + + // Create first blob with multiple versions, each with different tags + const blob1v1 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v1"); + blob1v1.blobTags = { blobTagSet: [{ key: "env", value: "dev" }] }; + await store.createBlob(ctx, blob1v1); + + ctx.startTime = new Date(Date.now() + 100); + const blob1v2 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v2"); + blob1v2.blobTags = { blobTagSet: [{ key: "env", value: "test" }] }; + await store.createBlob(ctx, blob1v2); + + ctx.startTime = new Date(Date.now() + 200); + const blob1v3 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v3"); + blob1v3.blobTags = { blobTagSet: [{ key: "env", value: "prod" }] }; + await store.createBlob(ctx, blob1v3); + + // Create second blob with different tags across versions + ctx.startTime = new Date(Date.now() + 300); + const blob2v1 = buildBlockBlob(ACCOUNT, containerName, blob2Name, "v1"); + blob2v1.blobTags = { blobTagSet: [{ key: "env", value: "dev" }] }; + await store.createBlob(ctx, blob2v1); + + ctx.startTime = new Date(Date.now() + 400); + const blob2v2 = buildBlockBlob(ACCOUNT, containerName, blob2Name, "v2"); + blob2v2.blobTags = { blobTagSet: [{ key: "env", value: "prod" }] }; + await store.createBlob(ctx, blob2v2); + + // Search for blobs with env=prod (current versions of both blobs) + const [prodResults,] = await store.filterBlobs( + ctx, + ACCOUNT, + containerName, + `"env" = 'prod'`, + 10, + "" + ); + + assert.strictEqual(prodResults.length, 2, "Should find only 2 current versions with env=prod"); + assert.strictEqual(prodResults[0].name, blob1Name, "First result should be blob1"); + assert.strictEqual(prodResults[1].name, blob2Name, "Second result should be blob2"); + + // Verify that filtered results include versionId and isCurrentVersion + prodResults.forEach(result => { + assert.ok(result.versionId, `Result ${result.name} should have versionId`); + assert.ok(!isNullOrWhitespace(result.versionId), `Result ${result.name} versionId should not be empty`); + assert.strictEqual(result.isCurrentVersion, true, `Result ${result.name} should be current version`); + }); + + // Search for blobs with env=dev (previous versions only) + const [devResults,] = await store.filterBlobs( + ctx, + ACCOUNT, + containerName, + `"env" = 'dev'`, + 10, + "" + ); + + assert.strictEqual(devResults.length, 0, "Should NOT find previous versions with env=dev"); + + // Search for blobs with env=test (previous version only) + const [testResults,] = await store.filterBlobs( + ctx, + ACCOUNT, + containerName, + `"env" = 'test'`, + 10, + "" + ); + + assert.strictEqual(testResults.length, 0, "Should NOT find previous version with env=test"); + + // Verify that previous version tags are preserved (even though not searchable) + assert.ok(blob1v1.versionId, "blob1v1 should have versionId"); + const blob1v1Retrieved = await store.downloadBlob( + ctx, + ACCOUNT, + containerName, + blob1Name, + "", + blob1v1.versionId + ); + assert.ok(blob1v1Retrieved.blobTags, "Previous version should have tags"); + assert.strictEqual( + blob1v1Retrieved.blobTags.blobTagSet.find(t => t.key === "env")?.value, + "dev", + "Previous version tags should be preserved" + ); + }); + + it("should handle mixed scenario with multiple blobs, versions, and pagination boundaries @loki", async () => { + const blobNames = [`blob-001-${uuid()}`, `blob-002-${uuid()}`, `blob-003-${uuid()}`]; + const versionIds: string[] = []; + + // Create multiple versions for each blob + for (let blobIndex = 0; blobIndex < blobNames.length; blobIndex++) { + for (let version = 1; version <= 3; version++) { + ctx.startTime = new Date(Date.now() + (blobIndex * 1000) + (version * 100)); + const blob = buildBlockBlob(ACCOUNT, containerName, blobNames[blobIndex], `content-${version}`); + const created = await store.createBlob(ctx, blob); + versionIds.push(created.versionId!); + } + } + + // Test with maxResults that doesn't align with blob boundaries + const [page1, , marker1] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, undefined, "", 4, "", false, false, true, false + ); + + assert.strictEqual(page1.length, 4, "Page 1 should have 4 versions"); + assert.ok(marker1, "Should have marker after page 1"); + + const [page2, , marker2] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, undefined, "", 4, marker1!, false, false, true, false + ); + + assert.strictEqual(page2.length, 4, "Page 2 should have 4 versions"); + assert.ok(marker2, "Should have marker after page 2"); + + const [page3, , marker3] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, undefined, "", 4, marker2!, false, false, true, false + ); + + assert.strictEqual(page3.length, 1, "Page 3 should have 1 remaining version"); + assert.strictEqual(marker3, "", "Should not have marker after final page"); + + // Verify all versions accounted for + const allPages = [...page1, ...page2, ...page3]; + assert.strictEqual(allPages.length, 9, "Should have 9 total versions (3 blobs × 3 versions)"); + + // Verify version IDs are properly distributed + const returnedVersionIds = allPages.map(b => b.versionId!); + versionIds.forEach(vid => { + assert.ok(returnedVersionIds.includes(vid), `Version ID ${vid} should be in results`); + }); + }); + + it("should handle edge cases with empty results and boundary conditions @loki", async () => { + // Test empty container + const [emptyPage, , emptyMarker] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, undefined, "", 10, "", false, false, true, false + ); + + assert.strictEqual(emptyPage.length, 0, "Empty container should return no results"); + assert.strictEqual(emptyMarker, "", "Empty container should not return marker"); + + // Create single blob with single version + const singleBlobName = `single-${uuid()}`; + const singleBlob = buildBlockBlob(ACCOUNT, containerName, singleBlobName, "content"); + await store.createBlob(ctx, singleBlob); + + // Test with maxResults larger than available + const [singlePage, , singleMarker] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, undefined, "", 100, "", false, false, true, false + ); + + assert.strictEqual(singlePage.length, 1, "Should return single version"); + assert.strictEqual(singleMarker, "", "Should not return marker when all results fit"); + + // Test with maxResults of 1 + const [onePage, , oneMarker] = await store.listBlobs( + ctx, + ACCOUNT, + containerName, + undefined, undefined, "", 1, "", false, false, true, false + ); + + assert.strictEqual(onePage.length, 1, "Should return exactly 1 result"); + assert.strictEqual(oneMarker, "", "Should not have marker when no more results"); + }); + + it("should correctly handle listBlobs versioning transitions @loki", async () => { + await store.close(); + await store.clean(); + + // Start with versioning enabled + let accountModel: AccountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: true + }; + let accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + let versioningStore = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await versioningStore.init(); + await versioningStore.createContainer(ctx, buildContainer(ACCOUNT, containerName)); + + const blobName = `transition-${uuid()}`; + + // Create multiple versions + const v1 = buildBlockBlob(ACCOUNT, containerName, blobName, "v1"); + await versioningStore.createBlob(ctx, v1); + + ctx.startTime = new Date(Date.now() + 100); + const v2 = buildBlockBlob(ACCOUNT, containerName, blobName, "v2"); + await versioningStore.createBlob(ctx, v2); + + // Test with versioning enabled - includeVersions=true should show both + const [enabledVersions, ,] = await versioningStore.listBlobs( + ctx, ACCOUNT, containerName, undefined, undefined, "", 10, "", false, false, true, false + ); + assert.strictEqual(enabledVersions.length, 2, "Should show 2 versions when versioning enabled"); + + // Test with versioning enabled - includeVersions=false should show only current + const [enabledCurrent, ,] = await versioningStore.listBlobs( + ctx, ACCOUNT, containerName, undefined, undefined, "", 10, "", false, false, false, false + ); + assert.strictEqual(enabledCurrent.length, 1, "Should show 1 current version"); + assert.ok(enabledCurrent[0].isCurrentVersion, "Result should be current version"); + + await versioningStore.close(); + await accountModelStore.close(); + + // Switch to versioning disabled + accountModel = { + key: ACCOUNT, + isBlobVersioningEnabled: false + }; + accountModelStore = createAccountModelStore(accountModel, false); + await accountModelStore.init(); + store = new LokiBlobMetadataStore(DB_FILE, false, accountModelStore); + await store.init(); + + // With versioning disabled, includeVersions should still work but use different logic + const [disabledVersions, ,] = await store.listBlobs( + ctx, ACCOUNT, containerName, undefined, undefined, "", 10, "", false, false, true, false + ); + assert.strictEqual(disabledVersions.length, 2, "Should still show existing versions even when versioning disabled"); + + const [disabledCurrent, ,] = await store.listBlobs( + ctx, ACCOUNT, containerName, undefined, undefined, "", 10, "", false, false, false, false + ); + assert.strictEqual(disabledCurrent.length, 1, "Should show 1 current version when versioning disabled"); + }); + + it("should paginate listBlobs correctly with snapshots, versions and includeSnapshots=true @loki", async () => { + // Create multiple blobs with versions and snapshots + const blob1Name = `snap-blob-a`; + const blob2Name = `snap-blob-b`; + + // Create first blob with versions + const blob1v1 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v1"); + await store.createBlob(ctx, blob1v1); + + // Create snapshot of first blob current version + ctx.startTime = new Date(Date.now() + 200); + const snapshot1 = await store.createSnapshot(ctx, ACCOUNT, containerName, blob1Name); + + ctx.startTime = new Date(Date.now() + 300); + const blob1v2 = buildBlockBlob(ACCOUNT, containerName, blob1Name, "v2"); + await store.createBlob(ctx, blob1v2); + + // Create second blob with versions + ctx.startTime = new Date(Date.now() + 400); + const blob2v1 = buildBlockBlob(ACCOUNT, containerName, blob2Name, "v1"); + await store.createBlob(ctx, blob2v1); + + // Create snapshot of second blob + ctx.startTime = new Date(Date.now() + 500); + const snapshot2 = await store.createSnapshot(ctx, ACCOUNT, containerName, blob2Name); + + ctx.startTime = new Date(Date.now() + 600); + const blob2v2 = buildBlockBlob(ACCOUNT, containerName, blob2Name, "v2"); + await store.createBlob(ctx, blob2v2); + + // Test pagination with includeVersions=true and includeSnapshots=true + const [firstPage, , firstMarker] = await store.listBlobs( + ctx, ACCOUNT, containerName, undefined, undefined, "", 4, "", true, false, true, false + ); + + assert.strictEqual(firstPage.length, 4, "First page should have 4 items (versions only, snapshots at end)"); + assert.ok(firstMarker, "Should have marker for next page"); + + // Continue pagination + const [secondPage, , secondMarker] = await store.listBlobs( + ctx, ACCOUNT, containerName, undefined, undefined, "", 4, firstMarker, true, false, true, false + ); + + assert.strictEqual(secondPage.length, 4, "Second page should have remaining items"); + assert.strictEqual(secondMarker, "", "Should not have marker when all results returned"); + + // Verify snapshots are included and ordered correctly (snapshots come after versions) + const allItems = [...firstPage, ...secondPage]; + + // Should have: blob1v1, blob1v2, blob1v3(from snapshot), blob2v1, blob2v2, blob2v3(from snapshot), blob1-snapshot, blob2-snapshot + assert.ok(allItems.length >= 6, "Should have at least 6 items including versions and snapshots"); + + // Check that snapshots are present + const snapshots = allItems.filter(item => item.snapshot && item.snapshot.length > 0); + assert.strictEqual(snapshots.length, 2, "Should have 2 snapshots"); + assert.ok(snapshots.some(s => s.snapshot === snapshot1.snapshot), "Should include first snapshot"); + assert.ok(snapshots.some(s => s.snapshot === snapshot2.snapshot), "Should include second snapshot"); + }); +}); diff --git a/tests/common/EnvironmentFunctions.test.ts b/tests/common/EnvironmentFunctions.test.ts new file mode 100644 index 000000000..6f6be4c27 --- /dev/null +++ b/tests/common/EnvironmentFunctions.test.ts @@ -0,0 +1,717 @@ +import * as assert from "assert"; +import { writeFileSync, mkdirSync, rmSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; + +import { parseAccountModelFlags } from "../../src/common/EnvironmentFunctions"; +import { AccountModel } from "../../src/common/account/AccountModel"; + +describe("EnvironmentFunctions", () => { + describe("parseAccountModelFlags", () => { + let tempDir: string; + let configFilePath1: string; + let configFilePath2: string; + + beforeEach(() => { + // Create a temporary directory for test files + tempDir = join(tmpdir(), `azurite-test-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`); + mkdirSync(tempDir, { recursive: true }); + configFilePath1 = join(tempDir, "account-config1.json"); + configFilePath2 = join(tempDir, "account-config2.json"); + }); + + afterEach(() => { + // Clean up temporary files + try { + rmSync(tempDir, { recursive: true, force: true }); + } catch (error) { + // Ignore cleanup errors + } + }); + + // ===================== SUCCESS CASES - Single Account ===================== + + it("should return undefined when neither configFilePath nor configAsJson is provided", () => { + const result = parseAccountModelFlags({}); + assert.strictEqual(result, undefined); + }); + + it("should return undefined when flags object is empty", () => { + const result = parseAccountModelFlags({}); + assert.strictEqual(result, undefined); + }); + + it("should parse single account JSON string with versioning enabled", () => { + const flags = { + accountConfigAsJson: 'devstoreaccount1:{"isBlobVersioningEnabled": true}' + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.ok(result instanceof Map); + assert.strictEqual(result.size, 1); + + const account = result.get("devstoreaccount1"); + assert.ok(account); + assert.strictEqual(account.key, "devstoreaccount1"); + assert.strictEqual(account.isBlobVersioningEnabled, true); + }); + + it("should parse single account JSON string with versioning disabled", () => { + const flags = { + accountConfigAsJson: 'myaccount:{"isBlobVersioningEnabled": false}' + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + + const account = result.get("myaccount"); + assert.ok(account); + assert.strictEqual(account.key, "myaccount"); + assert.strictEqual(account.isBlobVersioningEnabled, false); + }); + + it("should read and parse single account config file with versioning enabled", () => { + const config = { isBlobVersioningEnabled: true }; + writeFileSync(configFilePath1, JSON.stringify(config)); + + const flags = { + accountConfigFilePath: `account1:${configFilePath1}` + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + + const account = result.get("account1"); + assert.ok(account); + assert.strictEqual(account.key, "account1"); + assert.strictEqual(account.isBlobVersioningEnabled, true); + }); + + it("should read and parse single account config file with versioning disabled", () => { + const config = { isBlobVersioningEnabled: false }; + writeFileSync(configFilePath1, JSON.stringify(config)); + + const flags = { + accountConfigFilePath: `testaccount:${configFilePath1}` + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + + const account = result.get("testaccount"); + assert.ok(account); + assert.strictEqual(account.key, "testaccount"); + assert.strictEqual(account.isBlobVersioningEnabled, false); + }); + + it("should parse JSON with additional properties (should ignore them)", () => { + const flags = { + accountConfigAsJson: 'account1:{"isBlobVersioningEnabled": true, "extraProperty": "ignored", "anotherProp": 123}' + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + + const account = result.get("account1"); + assert.ok(account); + assert.strictEqual(account.key, "account1"); + assert.strictEqual(account.isBlobVersioningEnabled, true); + // Should only have the two expected properties + assert.strictEqual(Object.keys(account).length, 2); + }); + + // ===================== SUCCESS CASES - Multiple Accounts ===================== + + it("should parse multiple accounts from JSON string", () => { + const flags = { + accountConfigAsJson: 'account1:{"isBlobVersioningEnabled": true},account2:{"isBlobVersioningEnabled": false}' + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 2); + + const account1 = result.get("account1"); + assert.ok(account1); + assert.strictEqual(account1.key, "account1"); + assert.strictEqual(account1.isBlobVersioningEnabled, true); + + const account2 = result.get("account2"); + assert.ok(account2); + assert.strictEqual(account2.key, "account2"); + assert.strictEqual(account2.isBlobVersioningEnabled, false); + }); + + it("should parse multiple accounts from config files", () => { + writeFileSync(configFilePath1, JSON.stringify({ isBlobVersioningEnabled: true })); + writeFileSync(configFilePath2, JSON.stringify({ isBlobVersioningEnabled: false })); + + const flags = { + accountConfigFilePath: `account1:${configFilePath1},account2:${configFilePath2}` + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 2); + + const account1 = result.get("account1"); + assert.ok(account1); + assert.strictEqual(account1.key, "account1"); + assert.strictEqual(account1.isBlobVersioningEnabled, true); + + const account2 = result.get("account2"); + assert.ok(account2); + assert.strictEqual(account2.key, "account2"); + assert.strictEqual(account2.isBlobVersioningEnabled, false); + }); + + it("should parse three or more accounts", () => { + const flags = { + accountConfigAsJson: 'dev:{"isBlobVersioningEnabled": true},staging:{"isBlobVersioningEnabled": false},prod:{"isBlobVersioningEnabled": true}' + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 3); + assert.ok(result.get("dev")); + assert.ok(result.get("staging")); + assert.ok(result.get("prod")); + }); + + it("should handle JSON with commas inside values", () => { + const flags = { + accountConfigAsJson: 'account1:{"isBlobVersioningEnabled": true}' + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + assert.ok(result.get("account1")); + }); + + // ===================== ERROR CASES ===================== + + it("should throw error when both configFilePath and configAsJson are provided", () => { + const config = { isBlobVersioningEnabled: true }; + writeFileSync(configFilePath1, JSON.stringify(config)); + + const flags = { + accountConfigFilePath: `account1:${configFilePath1}`, + accountConfigAsJson: 'account1:{"isBlobVersioningEnabled": false}' + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Specify either accountConfigFilePath or accountConfigAsJson, not both\./ + ); + }); + + it("should throw error when config file does not exist", () => { + const flags = { + accountConfigFilePath: `account1:${join(tempDir, "nonexistent-file.json")}` + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Account configuration file not found for account 'account1'/ + ); + }); + + it("should throw error when config file is empty", () => { + writeFileSync(configFilePath1, ""); + + const flags = { + accountConfigFilePath: `account1:${configFilePath1}` + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Account configuration file is empty for account 'account1'/ + ); + }); + + it("should return undefined when configAsJson is empty string", () => { + const flags = { + accountConfigAsJson: "" + }; + + const result = parseAccountModelFlags(flags); + assert.strictEqual(result, undefined); + }); + + it("should throw error when JSON is invalid", () => { + const flags = { + accountConfigAsJson: 'account1:{"isBlobVersioningEnabled": true' // Missing closing brace + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Invalid JSON in account configuration for account 'account1'/ + ); + }); + + it("should throw error when JSON file contains invalid JSON", () => { + writeFileSync(configFilePath1, '{"invalid": json}'); + + const flags = { + accountConfigFilePath: `account1:${configFilePath1}` + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Invalid JSON in account configuration for account 'account1'/ + ); + }); + + it("should throw error when parsed JSON is null", () => { + const flags = { + accountConfigAsJson: "account1:null" + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Account configuration must be a JSON object for account 'account1'/ + ); + }); + + it("should default isBlobVersioningEnabled to false when omitted", () => { + const flags = { + accountConfigAsJson: 'account1:{"someOtherProperty": true}' + }; + + const result = parseAccountModelFlags(flags); + assert.strictEqual( + result?.get("account1")?.isBlobVersioningEnabled, + false + ); + }); + + it("should throw error when isBlobVersioningEnabled is null", () => { + const flags = { + accountConfigAsJson: 'account1:{"isBlobVersioningEnabled": null}' + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Account configuration value 'isBlobVersioningEnabled' must be a boolean for account 'account1'/ + ); + }); + + it("should throw error when isBlobVersioningEnabled is a string", () => { + const flags = { + accountConfigAsJson: 'account1:{"isBlobVersioningEnabled": "true"}' + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Account configuration value 'isBlobVersioningEnabled' must be a boolean for account 'account1'/ + ); + }); + + it("should throw error when isBlobVersioningEnabled is a number", () => { + const flags = { + accountConfigAsJson: 'account1:{"isBlobVersioningEnabled": 1}' + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Account configuration value 'isBlobVersioningEnabled' must be a boolean for account 'account1'/ + ); + }); + + it("should throw error when isBlobVersioningEnabled is an object", () => { + const flags = { + accountConfigAsJson: 'account1:{"isBlobVersioningEnabled": {"enabled": true}}' + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Account configuration value 'isBlobVersioningEnabled' must be a boolean for account 'account1'/ + ); + }); + + it("should throw error when isBlobVersioningEnabled is an array", () => { + const flags = { + accountConfigAsJson: 'account1:{"isBlobVersioningEnabled": [true]}' + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Account configuration value 'isBlobVersioningEnabled' must be a boolean for account 'account1'/ + ); + }); + + it("should throw error when account name is missing", () => { + const flags = { + accountConfigAsJson: ':{"isBlobVersioningEnabled": true}' + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Account name is missing in configuration entry/ + ); + }); + + it("should throw error when configuration value is missing", () => { + const flags = { + accountConfigAsJson: 'account1:' + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Configuration value is missing for account 'account1'/ + ); + }); + + it("should throw error when no valid accounts are found", () => { + const flags = { + accountConfigAsJson: ' ' + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Account configuration was specified but no valid accounts were found/ + ); + }); + + // ===================== EDGE CASES ===================== + + it("should handle flags object with undefined values", () => { + const flags = { + accountConfigFilePath: undefined, + accountConfigAsJson: undefined, + someOtherFlag: "value" + }; + + const result = parseAccountModelFlags(flags); + assert.strictEqual(result, undefined); + }); + + it("should handle config file with whitespace-only content", () => { + writeFileSync(configFilePath1, " \n\t \r\n "); + + const flags = { + accountConfigFilePath: `account1:${configFilePath1}` + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Account configuration file is empty for account 'account1'/ + ); + }); + + it("should handle JSON string with extra whitespace", () => { + const flags = { + accountConfigAsJson: ' account1:{"isBlobVersioningEnabled": true} ' + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + + const account = result.get("account1"); + assert.ok(account); + assert.strictEqual(account.key, "account1"); + assert.strictEqual(account.isBlobVersioningEnabled, true); + }); + + it("should handle account names with whitespace around them", () => { + const flags = { + accountConfigAsJson: ' account1 : {"isBlobVersioningEnabled": true} ' + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + assert.ok(result.get("account1")); + }); + + it("should normalize account names and reject case-insensitive duplicates", () => { + const flags = { + accountConfigAsJson: + 'Account1:{"isBlobVersioningEnabled": true},account1:{"isBlobVersioningEnabled": false}' + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /duplicate account 'account1'/ + ); + }); + + it("should return correct AccountModel structure for each account", () => { + const flags = { + accountConfigAsJson: 'account1:{"isBlobVersioningEnabled": true},account2:{"isBlobVersioningEnabled": false}' + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 2); + + // Verify first account + const account1 = result.get("account1"); + assert.ok(account1); + assert.strictEqual(account1.key, "account1"); + assert.strictEqual(typeof account1.isBlobVersioningEnabled, "boolean"); + + // Verify only expected properties exist + const expectedKeys = ["key", "isBlobVersioningEnabled"]; + const actualKeys1 = Object.keys(account1); + assert.deepStrictEqual(actualKeys1.sort(), expectedKeys.sort()); + + // Verify second account + const account2 = result.get("account2"); + assert.ok(account2); + assert.strictEqual(account2.key, "account2"); + const actualKeys2 = Object.keys(account2); + assert.deepStrictEqual(actualKeys2.sort(), expectedKeys.sort()); + }); + + it("should handle nested objects in config (should ignore extra properties)", () => { + const complexConfig = { + isBlobVersioningEnabled: false, + database: { + host: "localhost", + port: 5432 + }, + features: ["versioning", "encryption"], + metadata: { + version: "1.0.0", + author: "test" + } + }; + + const flags = { + accountConfigAsJson: `account1:${JSON.stringify(complexConfig)}` + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + + const account = result.get("account1"); + assert.ok(account); + assert.strictEqual(account.key, "account1"); + assert.strictEqual(account.isBlobVersioningEnabled, false); + assert.strictEqual(Object.keys(account).length, 2); + }); + + it("should handle multiple accounts with mixed configurations", () => { + writeFileSync(configFilePath1, JSON.stringify({ isBlobVersioningEnabled: true })); + + const flags = { + accountConfigFilePath: `fileAccount:${configFilePath1}`, + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + assert.ok(result.get("fileaccount")); + }); + + it("should handle special characters in account names", () => { + const flags = { + accountConfigAsJson: 'account-test_123:{"isBlobVersioningEnabled": true}' + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + + const account = result.get("account-test_123"); + assert.ok(account); + assert.strictEqual(account.key, "account-test_123"); + }); + + it("should correctly parse when JSON contains nested braces", () => { + const flags = { + accountConfigAsJson: 'account1:{"isBlobVersioningEnabled": true}' + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + assert.ok(result.get("account1")); + }); + + it("should handle Windows-style file paths", () => { + if (process.platform === "win32") { + const winPath = join(tempDir, "config.json"); + writeFileSync(winPath, JSON.stringify({ isBlobVersioningEnabled: true })); + + const flags = { + accountConfigFilePath: `account1:${winPath}` + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + assert.ok(result.get("account1")); + } + }); + + it("should throw error when one account in multi-account config is invalid", () => { + const flags = { + accountConfigAsJson: 'account1:{"isBlobVersioningEnabled": true},account2:{"isBlobVersioningEnabled": "invalid"}' + }; + + assert.throws( + () => parseAccountModelFlags(flags), + /Account configuration value 'isBlobVersioningEnabled' must be a boolean for account 'account2'/ + ); + }); + + it("should handle large number of accounts", () => { + const accounts: string[] = []; + for (let i = 0; i < 100; i++) { + accounts.push(`account${i}:{"isBlobVersioningEnabled": ${i % 2 === 0}}`); + } + + const flags = { + accountConfigAsJson: accounts.join(',') + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 100); + + for (let i = 0; i < 100; i++) { + const account: AccountModel | undefined = result.get(`account${i}`); + assert.ok(account); + assert.strictEqual(account.isBlobVersioningEnabled, i % 2 === 0); + } + }); + + // ===================== Single Account Without Prefix ===================== + + it("should parse single account JSON without account name prefix (no prefix)", () => { + const flags = { + accountConfigAsJson: '{"isBlobVersioningEnabled": true}' + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + + const account: AccountModel | undefined = result.get('devstoreaccount1'); + assert.ok(account); + assert.strictEqual(account.key, 'devstoreaccount1'); + assert.strictEqual(account.isBlobVersioningEnabled, true); + }); + + it("should parse single account JSON without prefix with versioning disabled (no prefix)", () => { + const flags = { + accountConfigAsJson: '{"isBlobVersioningEnabled": false}' + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + + const account: AccountModel | undefined = result.get('devstoreaccount1'); + assert.ok(account); + assert.strictEqual(account.key, 'devstoreaccount1'); + assert.strictEqual(account.isBlobVersioningEnabled, false); + }); + + it("should parse single account file path without account name prefix (no prefix)", () => { + const configContent = JSON.stringify({ isBlobVersioningEnabled: true }); + writeFileSync(configFilePath1, configContent, "utf-8"); + + const flags = { + accountConfigFilePath: configFilePath1 + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + + const account: AccountModel | undefined = result.get('devstoreaccount1'); + assert.ok(account); + assert.strictEqual(account.key, 'devstoreaccount1'); + assert.strictEqual(account.isBlobVersioningEnabled, true); + }); + + it("should parse single account file path without prefix with versioning disabled (no prefix)", () => { + const configContent = JSON.stringify({ isBlobVersioningEnabled: false }); + writeFileSync(configFilePath1, configContent, "utf-8"); + + const flags = { + accountConfigFilePath: configFilePath1 + }; + + const result = parseAccountModelFlags(flags); + + assert.ok(result); + assert.strictEqual(result.size, 1); + + const account: AccountModel | undefined = result.get('devstoreaccount1'); + assert.ok(account); + assert.strictEqual(account.key, 'devstoreaccount1'); + assert.strictEqual(account.isBlobVersioningEnabled, false); + }); + + it("should throw error for non-existent file in no prefix mode", () => { + const flags = { + accountConfigFilePath: '/non/existent/path.json' + }; + + assert.throws( + () => parseAccountModelFlags(flags), + (err: Error) => { + return err.message.includes('Account configuration file not found'); + } + ); + }); + + it("should throw error for invalid JSON in no prefix mode", () => { + const flags = { + accountConfigAsJson: '{invalid json}' + }; + + assert.throws( + () => parseAccountModelFlags(flags), + (err: Error) => { + return err.message.includes('Invalid JSON'); + } + ); + }); + + it("should default versioning to false in no prefix mode", () => { + const flags = { + accountConfigAsJson: '{"someOtherField": true}' + }; + + const result = parseAccountModelFlags(flags); + assert.strictEqual( + result?.get("devstoreaccount1")?.isBlobVersioningEnabled, + false + ); + }); + }); +}); diff --git a/tests/common/LokiAccountModelStore.test.ts b/tests/common/LokiAccountModelStore.test.ts new file mode 100644 index 000000000..a2b031c0d --- /dev/null +++ b/tests/common/LokiAccountModelStore.test.ts @@ -0,0 +1,265 @@ +import * as assert from "assert"; +import { existsSync, unlinkSync } from "fs"; +import { join } from "path"; +import { tmpdir } from "os"; + +import LokiAccountModelStore from "../../src/common/account/LokiAccountModelStore"; +import { AccountModel } from "../../src/common/account/AccountModel"; + +describe("LokiAccountModelStore", () => { + let store: LokiAccountModelStore; + let dbPath: string; + + beforeEach(() => { + // Create a unique temporary database file for each test + dbPath = join(tmpdir(), `test-account-model-${Date.now()}-${Math.random().toString(36).slice(2, 11)}.json`); + }); + + afterEach(async () => { + // Clean up + if (store && !store.isClosed()) { + await store.close(); + } + try { + unlinkSync(dbPath); + } catch (error) { + // Ignore cleanup errors + } + }); + + describe("constructor", () => { + it("should create store with file persistence", () => { + store = new LokiAccountModelStore(dbPath, false); + assert.ok(store); + assert.strictEqual(store.lokiDBPath, dbPath); + }); + + it("should create store with in-memory persistence", () => { + store = new LokiAccountModelStore(dbPath, true); + assert.ok(store); + assert.strictEqual(store.lokiDBPath, dbPath); + }); + + it("should create store with account models from args", () => { + const accountModels = new Map(); + accountModels.set("account1", { key: "account1", isBlobVersioningEnabled: true }); + + store = new LokiAccountModelStore(dbPath, false, accountModels); + assert.ok(store); + }); + }); + + describe("init", () => { + it("should initialize empty store successfully", async () => { + store = new LokiAccountModelStore(dbPath, false); + + assert.strictEqual(store.isInitialized(), false); + assert.strictEqual(store.isClosed(), true); + + await store.init(); + + assert.strictEqual(store.isInitialized(), true); + assert.strictEqual(store.isClosed(), false); + }); + + it("should initialize store with account models from args", async () => { + const accountModels = new Map(); + accountModels.set("account1", { key: "account1", isBlobVersioningEnabled: true }); + accountModels.set("account2", { key: "account2", isBlobVersioningEnabled: false }); + + store = new LokiAccountModelStore(dbPath, false, accountModels); + await store.init(); + + assert.strictEqual(store.isInitialized(), true); + + const account1 = store.getAccountModel("account1"); + assert.ok(account1); + assert.strictEqual(account1.key, "account1"); + assert.strictEqual(account1.isBlobVersioningEnabled, true); + + const account2 = store.getAccountModel("account2"); + assert.ok(account2); + assert.strictEqual(account2.key, "account2"); + assert.strictEqual(account2.isBlobVersioningEnabled, false); + }); + + it("should merge account models from args with existing DB", async () => { + // First initialization with one account + const accountModels1 = new Map(); + accountModels1.set("account1", { key: "account1", isBlobVersioningEnabled: true }); + + store = new LokiAccountModelStore(dbPath, false, accountModels1); + await store.init(); + await store.close(); + + // Second initialization with updated configuration + const accountModels2 = new Map(); + accountModels2.set("account1", { key: "account1", isBlobVersioningEnabled: false }); + + store = new LokiAccountModelStore(dbPath, false, accountModels2); + await store.init(); + + const account1 = store.getAccountModel("account1"); + assert.ok(account1); + assert.strictEqual(account1.isBlobVersioningEnabled, false); + }); + + it("should preserve existing accounts when adding new ones", async () => { + // First initialization with one account + const accountModels1 = new Map(); + accountModels1.set("account1", { key: "account1", isBlobVersioningEnabled: true }); + + store = new LokiAccountModelStore(dbPath, false, accountModels1); + await store.init(); + await store.close(); + + // Second initialization with a different account + const accountModels2 = new Map(); + accountModels2.set("account2", { key: "account2", isBlobVersioningEnabled: false }); + + store = new LokiAccountModelStore(dbPath, false, accountModels2); + await store.init(); + + // Both accounts should exist + const account1 = store.getAccountModel("account1"); + assert.ok(account1); + assert.strictEqual(account1.isBlobVersioningEnabled, true); + + const account2 = store.getAccountModel("account2"); + assert.ok(account2); + assert.strictEqual(account2.isBlobVersioningEnabled, false); + }); + }); + + describe("getAccountModel", () => { + beforeEach(async () => { + const accountModels = new Map(); + accountModels.set("account1", { key: "account1", isBlobVersioningEnabled: true }); + + store = new LokiAccountModelStore(dbPath, false, accountModels); + await store.init(); + }); + + it("should return account model for existing account", () => { + const account = store.getAccountModel("account1"); + + assert.ok(account); + assert.strictEqual(account.key, "account1"); + assert.strictEqual(account.isBlobVersioningEnabled, true); + }); + + it("should resolve account names case-insensitively", () => { + const account = store.getAccountModel("ACCOUNT1"); + + assert.ok(account); + assert.strictEqual(account.key, "account1"); + }); + + it("should return undefined for non-existent account", () => { + const account = store.getAccountModel("nonexistent"); + + assert.strictEqual(account, undefined); + }); + + it("should throw error if store is not initialized", () => { + store = new LokiAccountModelStore(dbPath, false); + + assert.throws( + () => store.getAccountModel("account1"), + /Account model collection is not initialized/ + ); + }); + }); + + describe("isBlobVersioningEnabled", () => { + beforeEach(async () => { + const accountModels = new Map(); + accountModels.set("versioned", { key: "versioned", isBlobVersioningEnabled: true }); + accountModels.set("notversioned", { key: "notversioned", isBlobVersioningEnabled: false }); + + store = new LokiAccountModelStore(dbPath, false, accountModels); + await store.init(); + }); + + it("should return true for account with versioning enabled", () => { + assert.strictEqual(store.isBlobVersioningEnabled("versioned"), true); + }); + + it("should return false for account with versioning disabled", () => { + assert.strictEqual(store.isBlobVersioningEnabled("notversioned"), false); + }); + + it("should return false for non-existent account", () => { + assert.strictEqual(store.isBlobVersioningEnabled("nonexistent"), false); + }); + }); + + describe("close", () => { + it("should close successfully", async () => { + store = new LokiAccountModelStore(dbPath, false); + await store.init(); + + assert.strictEqual(store.isClosed(), false); + + await store.close(); + + assert.strictEqual(store.isClosed(), true); + }); + + it("should remove its persisted database when cleaned", async () => { + store = new LokiAccountModelStore(dbPath, false); + await store.init(); + await store.close(); + + assert.strictEqual(existsSync(dbPath), true); + await store.clean(); + assert.strictEqual(existsSync(dbPath), false); + }); + }); + + describe("multiple accounts scenario", () => { + it("should handle multiple accounts with different configurations", async () => { + const accountModels = new Map(); + accountModels.set("devstoreaccount1", { key: "devstoreaccount1", isBlobVersioningEnabled: false }); + accountModels.set("testaccount1", { key: "testaccount1", isBlobVersioningEnabled: true }); + accountModels.set("prodaccount1", { key: "prodaccount1", isBlobVersioningEnabled: true }); + + store = new LokiAccountModelStore(dbPath, false, accountModels); + await store.init(); + + assert.strictEqual(store.isBlobVersioningEnabled("devstoreaccount1"), false); + assert.strictEqual(store.isBlobVersioningEnabled("testaccount1"), true); + assert.strictEqual(store.isBlobVersioningEnabled("prodaccount1"), true); + + // Verify persistence + await store.close(); + + store = new LokiAccountModelStore(dbPath, false); + await store.init(); + + assert.strictEqual(store.isBlobVersioningEnabled("devstoreaccount1"), false); + assert.strictEqual(store.isBlobVersioningEnabled("testaccount1"), true); + assert.strictEqual(store.isBlobVersioningEnabled("prodaccount1"), true); + }); + }); + + describe("in-memory persistence", () => { + it("should not persist data with in-memory mode", async () => { + const accountModels = new Map(); + accountModels.set("account1", { key: "account1", isBlobVersioningEnabled: true }); + + store = new LokiAccountModelStore(dbPath, true, accountModels); + await store.init(); + + assert.strictEqual(store.isBlobVersioningEnabled("account1"), true); + + await store.close(); + + // Reopen - should not have the data + store = new LokiAccountModelStore(dbPath, true); + await store.init(); + + assert.strictEqual(store.isBlobVersioningEnabled("account1"), false); + }); + }); +}); diff --git a/tests/testutils.ts b/tests/testutils.ts index 8faaeb762..bfe69af0b 100644 --- a/tests/testutils.ts +++ b/tests/testutils.ts @@ -1,13 +1,221 @@ -import { randomBytes } from "crypto"; +import { randomBytes, randomUUID as uuid } from "crypto"; import { createWriteStream, readFileSync, promises as fsPromises } from "fs"; import { sign } from "jsonwebtoken"; import { join } from "path"; import { URL } from "url"; +import { + BlobModel, + ContainerModel +} from "../src/blob/persistence/IBlobMetadataStore"; +import * as Models from "../src/blob/generated/artifacts/models"; +import Context from "../src/blob/generated/Context"; import { EMULATOR_ACCOUNT_KEY_STR as DEFAULT_EMULATOR_ACCOUNT_KEY_STR, EMULATOR_ACCOUNT_NAME as DEFAULT_EMULATOR_ACCOUNT_NAME } from "../src/blob/utils/constants"; + +/** + * Helper to list all versions of a blob. + * + * @export + * @param {any} containerClient Container client + * @param {string} blobName Blob name + * @returns {Promise} List of blob versions + */ +export + async function listBlobVersions(containerClient: any, blobName: string): Promise { + const listResponse = containerClient.listBlobsFlat({ + includeVersions: true + }); + const blobVersions = []; + for await (const blob of listResponse) { + if (blob.name === blobName) { + blobVersions.push(blob); + } + } + return blobVersions; +} + +/** + * Helper to create a minimal Context object. + */ +export function createContext(): Context { + return { + contextId: uuid(), + startTime: new Date() + } as any as Context; // Cast to simplify test construction +} + +/** + * Helper to build a minimal ContainerModel for tests. + */ +export function buildContainer(account: string, name: string): ContainerModel { + const now = new Date(); + return { + accountName: account, + name, + properties: { + lastModified: now, + etag: '"test-etag"', + leaseStatus: Models.LeaseStatusType.Unlocked, + leaseState: Models.LeaseStateType.Available, + defaultEncryptionScope: undefined, + denyEncryptionScopeOverride: undefined, + hasImmutabilityPolicy: undefined, + hasLegalHold: undefined, + publicAccess: undefined, + leaseDuration: undefined + } + } as any as ContainerModel; +} + +/** + * Helper to build a minimal Block Blob BlobModel for tests. + */ +export function buildBlockBlob( + account: string, + container: string, + name: string, + content: string +): BlobModel { + const now = new Date(); + return { + accountName: account, + containerName: container, + name, + properties: { + creationTime: now, + lastModified: now, + etag: `\"etag-${uuid()}\"`, + blobType: Models.BlobType.BlockBlob, + contentLength: Buffer.byteLength(content), + serverEncrypted: false, + accessTier: Models.AccessTier.Hot, + accessTierInferred: true, + cacheControl: undefined, + contentType: undefined, + contentMD5: undefined, + contentEncoding: undefined, + contentLanguage: undefined, + contentDisposition: undefined, + leaseDuration: undefined, + leaseState: Models.LeaseStateType.Available, + leaseStatus: Models.LeaseStatusType.Unlocked, + tagCount: undefined, + archiveStatus: undefined, + accessTierChangeTime: undefined, + deletedTime: undefined, + remainingRetentionDays: undefined, + deleted: false, + rehydratePriority: undefined, + lastAccessedOn: undefined, + snapshot: undefined + }, + isCommitted: true, + committedBlocksInOrder: [], + // Versioning top-level fields (duplicated when persisted in Loki) + snapshot: "" + } as any as BlobModel; +} + +/** + * Helper to build a minimal Page Blob BlobModel for tests. + */ +export function buildPageBlob( + account: string, + container: string, + name: string, + contentLength: number +): BlobModel { + const now = new Date(); + return { + accountName: account, + containerName: container, + name, + properties: { + creationTime: now, + lastModified: now, + etag: `\"etag-${uuid()}\"`, + blobType: Models.BlobType.PageBlob, + contentLength, + serverEncrypted: false, + accessTier: undefined, + accessTierInferred: undefined, + cacheControl: undefined, + contentType: undefined, + contentMD5: undefined, + contentEncoding: undefined, + contentLanguage: undefined, + contentDisposition: undefined, + leaseDuration: undefined, + leaseState: Models.LeaseStateType.Available, + leaseStatus: Models.LeaseStatusType.Unlocked, + tagCount: undefined, + archiveStatus: undefined, + accessTierChangeTime: undefined, + deletedTime: undefined, + remainingRetentionDays: undefined, + deleted: false, + rehydratePriority: undefined, + lastAccessedOn: undefined, + snapshot: undefined, + blobSequenceNumber: 0 + }, + isCommitted: true, + pageRangesInOrder: [], + snapshot: "" + } as any as BlobModel; +} + +/** + * Helper to build a minimal Append Blob BlobModel for tests. + */ +export function buildAppendBlob( + account: string, + container: string, + name: string +): BlobModel { + const now = new Date(); + return { + accountName: account, + containerName: container, + name, + properties: { + creationTime: now, + lastModified: now, + etag: `\"etag-${uuid()}\"`, + blobType: Models.BlobType.AppendBlob, + contentLength: 0, + serverEncrypted: false, + accessTier: undefined, + accessTierInferred: undefined, + cacheControl: undefined, + contentType: undefined, + contentMD5: undefined, + contentEncoding: undefined, + contentLanguage: undefined, + contentDisposition: undefined, + leaseDuration: undefined, + leaseState: Models.LeaseStateType.Available, + leaseStatus: Models.LeaseStatusType.Unlocked, + tagCount: undefined, + archiveStatus: undefined, + accessTierChangeTime: undefined, + deletedTime: undefined, + remainingRetentionDays: undefined, + deleted: false, + rehydratePriority: undefined, + lastAccessedOn: undefined, + snapshot: undefined, + isSealed: false + }, + isCommitted: true, + committedBlocksInOrder: [], + snapshot: "" + } as any as BlobModel; +} + // ---- Live Azure mode ------------------------------------------------------- // // Set AZURITE_LIVE_TEST_CONNECTION_STRING to a full storage account connection @@ -226,7 +434,7 @@ export async function createRandomLocalFile( ws.on("open", () => { // tslint:disable-next-line:no-empty - while (offsetInMB++ < blockNumber && ws.write(randomValueHex())) {} + while (offsetInMB++ < blockNumber && ws.write(randomValueHex())) { } if (offsetInMB >= blockNumber) { ws.end(); } @@ -234,7 +442,7 @@ export async function createRandomLocalFile( ws.on("drain", () => { // tslint:disable-next-line:no-empty - while (offsetInMB++ < blockNumber && ws.write(randomValueHex())) {} + while (offsetInMB++ < blockNumber && ws.write(randomValueHex())) { } if (offsetInMB >= blockNumber) { ws.end(); }