diff --git a/.gitignore b/.gitignore index 3a8821c..230baae 100644 --- a/.gitignore +++ b/.gitignore @@ -50,4 +50,7 @@ temp/ UI.html -response.json \ No newline at end of file +response.json + +# Local commit orchestration helpers +/commit-benchmark-changes.sh diff --git a/apps/api/package.json b/apps/api/package.json index f14fe22..0dce9cc 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -11,6 +11,8 @@ "jobs": "tsx src/jobs/worker.ts", "migrate": "tsx src/db/migrate.ts", "seed:demo": "node src/scripts/seed-demo.js", + "seed:benchmark": "tsx src/scripts/seed-benchmark.ts", + "backfill:cost-rollups": "tsx src/scripts/backfill-cost-rollups.ts", "build": "tsc -p tsconfig.json && node scripts/copy-build-assets.mjs", "build:lambda": "npm run build && node scripts/build-lambda-package.mjs", "start": "node dist/server.js", diff --git a/apps/api/src/db/migrations/003_add_cost_snapshots_workspace_date_index.sql b/apps/api/src/db/migrations/003_add_cost_snapshots_workspace_date_index.sql new file mode 100644 index 0000000..9e09324 --- /dev/null +++ b/apps/api/src/db/migrations/003_add_cost_snapshots_workspace_date_index.sql @@ -0,0 +1,8 @@ +-- The benchmark summary and reporting queries filter by workspace and date +-- without always filtering by AWS account. The original unique index orders +-- aws_account_id before usage_date, so it cannot efficiently serve that path. +CREATE INDEX IF NOT EXISTS idx_cost_snapshots_workspace_date + ON cost_snapshots (workspace_id, usage_date) + INCLUDE (amount, currency); + +ANALYZE cost_snapshots; diff --git a/apps/api/src/db/migrations/004_cover_cost_reporting_queries.sql b/apps/api/src/db/migrations/004_cover_cost_reporting_queries.sql new file mode 100644 index 0000000..aec7289 --- /dev/null +++ b/apps/api/src/db/migrations/004_cover_cost_reporting_queries.sql @@ -0,0 +1,12 @@ +-- The by-service report needs service_name in addition to the values covered +-- by the workspace/date index. Including it allows all three cost-reporting +-- queries to avoid fetching the matching rows from the heap. +CREATE INDEX IF NOT EXISTS idx_cost_snapshots_workspace_date_reporting + ON cost_snapshots (workspace_id, usage_date) + INCLUDE (service_name, amount, currency); + +-- The reporting index is a strict covering replacement for the narrower index +-- introduced by migration 003. +DROP INDEX IF EXISTS idx_cost_snapshots_workspace_date; + +ANALYZE cost_snapshots; diff --git a/apps/api/src/db/migrations/005_add_workspace_cost_daily_rollups.sql b/apps/api/src/db/migrations/005_add_workspace_cost_daily_rollups.sql new file mode 100644 index 0000000..e331778 --- /dev/null +++ b/apps/api/src/db/migrations/005_add_workspace_cost_daily_rollups.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS workspace_cost_daily_rollups ( + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + usage_date DATE NOT NULL, + service_name VARCHAR(255) NOT NULL, + total_amount DECIMAL(18, 6) NOT NULL, + currency VARCHAR(10) NOT NULL DEFAULT 'USD', + updated_at TIMESTAMP NOT NULL DEFAULT NOW(), + PRIMARY KEY (workspace_id, usage_date, service_name) +); + +CREATE INDEX IF NOT EXISTS idx_workspace_cost_daily_rollups_workspace_service_date + ON workspace_cost_daily_rollups (workspace_id, service_name, usage_date) + INCLUDE (total_amount, currency); diff --git a/apps/api/src/repositories/cost.repository.ts b/apps/api/src/repositories/cost.repository.ts index 26e9ae4..00d40bf 100644 --- a/apps/api/src/repositories/cost.repository.ts +++ b/apps/api/src/repositories/cost.repository.ts @@ -1,3 +1,5 @@ +import type { PoolClient } from "pg"; + import { pool } from "../config/db.js"; import type { CostQueryInput, @@ -8,6 +10,31 @@ import type { TimeseriesCostPoint, } from "../types/aws-account.types.js"; +const rebuildWorkspaceRollups = async ( + client: PoolClient, + workspaceId: string, + from: string, + to: string, +): Promise => { + await client.query( + `DELETE FROM workspace_cost_daily_rollups + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3`, + [workspaceId, from, to], + ); + await client.query( + `INSERT INTO workspace_cost_daily_rollups ( + workspace_id, usage_date, service_name, total_amount, currency, updated_at + ) + SELECT workspace_id, usage_date, service_name, SUM(amount), MAX(currency), NOW() + FROM cost_snapshots + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3 + GROUP BY workspace_id, usage_date, service_name`, + [workspaceId, from, to], + ); +}; + export const costRepository = { async createSyncRun(awsAccountId: string): Promise { const result = await pool.query( @@ -45,6 +72,8 @@ export const costRepository = { async replaceSnapshots(input: { workspaceId: string; awsAccountId: string; + from: string; + to: string; entries: Array<{ usageDate: string; serviceName: string; @@ -52,35 +81,69 @@ export const costRepository = { currency: string; }>; }): Promise { - for (const entry of input.entries) { - await pool.query( - `INSERT INTO cost_snapshots (workspace_id, aws_account_id, usage_date, service_name, amount, currency) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (workspace_id, aws_account_id, usage_date, service_name) - DO UPDATE SET amount = EXCLUDED.amount, currency = EXCLUDED.currency`, - [ - input.workspaceId, - input.awsAccountId, - entry.usageDate, - entry.serviceName, - entry.amount, - entry.currency, - ], + const client = await pool.connect(); + + try { + await client.query("BEGIN"); + // Account syncs are independently locked, so serialize mutations and + // rollup refreshes for the same workspace. + await client.query( + "SELECT pg_advisory_xact_lock(hashtext('cost-rollup:' || $1::text))", + [input.workspaceId], ); - } + await client.query( + `DELETE FROM cost_snapshots + WHERE workspace_id = $1 + AND aws_account_id = $2 + AND usage_date BETWEEN $3 AND $4`, + [input.workspaceId, input.awsAccountId, input.from, input.to], + ); + + for (const entry of input.entries) { + await client.query( + `INSERT INTO cost_snapshots (workspace_id, aws_account_id, usage_date, service_name, amount, currency) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (workspace_id, aws_account_id, usage_date, service_name) + DO UPDATE SET amount = EXCLUDED.amount, currency = EXCLUDED.currency`, + [ + input.workspaceId, + input.awsAccountId, + entry.usageDate, + entry.serviceName, + entry.amount, + entry.currency, + ], + ); + } - return input.entries.length; + await rebuildWorkspaceRollups(client, input.workspaceId, input.from, input.to); + await client.query("COMMIT"); + return input.entries.length; + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } finally { + client.release(); + } }, async getSummary(workspaceId: string, input: CostQueryInput): Promise { - const result = await pool.query( - `SELECT COALESCE(SUM(amount), 0) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency - FROM cost_snapshots - WHERE workspace_id = $1 - AND usage_date BETWEEN $2 AND $3 - AND ($4::uuid IS NULL OR aws_account_id = $4::uuid)`, - [workspaceId, input.from, input.to, input.awsAccountId ?? null], - ); + const result = input.awsAccountId + ? await pool.query( + `SELECT COALESCE(SUM(amount), 0) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency + FROM cost_snapshots + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3 + AND aws_account_id = $4::uuid`, + [workspaceId, input.from, input.to, input.awsAccountId], + ) + : await pool.query( + `SELECT COALESCE(SUM(total_amount), 0) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency + FROM workspace_cost_daily_rollups + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3`, + [workspaceId, input.from, input.to], + ); return { totalAmount: Number(result.rows[0]?.total_amount ?? 0), @@ -94,16 +157,26 @@ export const costRepository = { workspaceId: string, input: CostQueryInput, ): Promise { - const result = await pool.query( - `SELECT service_name, SUM(amount) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency - FROM cost_snapshots - WHERE workspace_id = $1 - AND usage_date BETWEEN $2 AND $3 - AND ($4::uuid IS NULL OR aws_account_id = $4::uuid) - GROUP BY service_name - ORDER BY total_amount DESC`, - [workspaceId, input.from, input.to, input.awsAccountId ?? null], - ); + const result = input.awsAccountId + ? await pool.query( + `SELECT service_name, SUM(amount) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency + FROM cost_snapshots + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3 + AND aws_account_id = $4::uuid + GROUP BY service_name + ORDER BY total_amount DESC`, + [workspaceId, input.from, input.to, input.awsAccountId], + ) + : await pool.query( + `SELECT service_name, SUM(total_amount) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency + FROM workspace_cost_daily_rollups + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3 + GROUP BY service_name + ORDER BY total_amount DESC`, + [workspaceId, input.from, input.to], + ); return result.rows.map((row) => ({ serviceName: String(row.service_name), @@ -116,16 +189,26 @@ export const costRepository = { workspaceId: string, input: CostQueryInput, ): Promise { - const result = await pool.query( - `SELECT usage_date, SUM(amount) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency - FROM cost_snapshots - WHERE workspace_id = $1 - AND usage_date BETWEEN $2 AND $3 - AND ($4::uuid IS NULL OR aws_account_id = $4::uuid) - GROUP BY usage_date - ORDER BY usage_date ASC`, - [workspaceId, input.from, input.to, input.awsAccountId ?? null], - ); + const result = input.awsAccountId + ? await pool.query( + `SELECT usage_date, SUM(amount) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency + FROM cost_snapshots + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3 + AND aws_account_id = $4::uuid + GROUP BY usage_date + ORDER BY usage_date ASC`, + [workspaceId, input.from, input.to, input.awsAccountId], + ) + : await pool.query( + `SELECT usage_date, SUM(total_amount) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency + FROM workspace_cost_daily_rollups + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3 + GROUP BY usage_date + ORDER BY usage_date ASC`, + [workspaceId, input.from, input.to], + ); return result.rows.map((row) => ({ usageDate: String(row.usage_date).slice(0, 10), diff --git a/apps/api/src/scripts/backfill-cost-rollups.ts b/apps/api/src/scripts/backfill-cost-rollups.ts new file mode 100644 index 0000000..6d9f53c --- /dev/null +++ b/apps/api/src/scripts/backfill-cost-rollups.ts @@ -0,0 +1,56 @@ +if (process.env.ALLOW_COST_ROLLUP_BACKFILL !== "true") { + throw new Error("Refusing to backfill cost rollups without ALLOW_COST_ROLLUP_BACKFILL=true"); +} + +const { pool } = await import("../config/db.js"); + +const run = async (): Promise => { + const client = await pool.connect(); + + try { + await client.query("BEGIN"); + await client.query( + "SELECT pg_advisory_xact_lock(hashtext('cost-rollup-backfill'))", + ); + await client.query("LOCK TABLE cost_snapshots IN SHARE MODE"); + await client.query("LOCK TABLE workspace_cost_daily_rollups IN EXCLUSIVE MODE"); + await client.query("DELETE FROM workspace_cost_daily_rollups"); + const result = await client.query<{ count: string }>( + `WITH inserted AS ( + INSERT INTO workspace_cost_daily_rollups ( + workspace_id, usage_date, service_name, total_amount, currency, updated_at + ) + SELECT workspace_id, usage_date, service_name, SUM(amount), MAX(currency), NOW() + FROM cost_snapshots + GROUP BY workspace_id, usage_date, service_name + RETURNING 1 + ) + SELECT COUNT(*)::text AS count FROM inserted`, + ); + await client.query("ANALYZE workspace_cost_daily_rollups"); + await client.query("COMMIT"); + + console.log( + JSON.stringify({ + costRollupBackfill: true, + rows: Number(result.rows[0]?.count ?? 0), + }), + ); + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } finally { + client.release(); + await pool.end(); + } +}; + +run().catch((error: unknown) => { + console.error( + JSON.stringify({ + message: "Cost rollup backfill failed", + error: error instanceof Error ? error.message : "Unknown error", + }), + ); + process.exitCode = 1; +}); diff --git a/apps/api/src/scripts/seed-benchmark.ts b/apps/api/src/scripts/seed-benchmark.ts new file mode 100644 index 0000000..fe26e29 --- /dev/null +++ b/apps/api/src/scripts/seed-benchmark.ts @@ -0,0 +1,243 @@ +const BENCHMARK_GUARD = "ALLOW_BENCHMARK_SEED"; +const EXPECTED_COST_SNAPSHOTS = 3_650_000; +const EXPECTED_COST_ROLLUPS = 182_500; + +const assertBenchmarkGuard = (): void => { + if (process.env[BENCHMARK_GUARD] !== "true") { + const environment = process.env.NODE_ENV ?? "development"; + throw new Error( + `Refusing to seed benchmark data in ${environment}: set ${BENCHMARK_GUARD}=true explicitly`, + ); + } +}; + +assertBenchmarkGuard(); + +const benchmarkPassword = process.env.BENCHMARK_PASSWORD; + +if (!benchmarkPassword || benchmarkPassword.length < 12) { + throw new Error("BENCHMARK_PASSWORD must contain at least 12 characters"); +} + +const { pool } = await import("../config/db.js"); +const { hashPassword } = await import("../utils/password.js"); + +const benchmarkUuid = (kind: number, sequence: number): string => + `${kind}0000000-0000-4000-8000-${String(sequence).padStart(12, "0")}`; + +const userIds = Array.from({ length: 10 }, (_, index) => benchmarkUuid(1, index + 1)); +const workspaceIds = Array.from( + { length: 10 }, + (_, index) => benchmarkUuid(2, index + 1), +); + +const run = async (): Promise => { + const passwordHash = await hashPassword(benchmarkPassword); + const client = await pool.connect(); + + try { + await client.query("BEGIN"); + + // The fixed UUID ranges and reserved example.invalid identities are the complete + // ownership boundary for this disposable dataset. + await client.query("DELETE FROM workspaces WHERE id = ANY($1::uuid[])", [workspaceIds]); + await client.query( + `DELETE FROM users + WHERE id = ANY($1::uuid[]) + OR email = ANY( + SELECT format('benchmark+%s@example.invalid', lpad(i::text, 2, '0')) + FROM generate_series(1, 10) AS i + )`, + [userIds], + ); + + await client.query( + `INSERT INTO users ( + id, email, password_hash, first_name, last_name, role, + is_active, is_email_verified, password_changed_at, session_version + ) + SELECT + format('10000000-0000-4000-8000-%s', lpad(i::text, 12, '0'))::uuid, + format('benchmark+%s@example.invalid', lpad(i::text, 2, '0')), + $1, + 'Benchmark', + format('User %s', lpad(i::text, 2, '0')), + 'customer', TRUE, TRUE, TIMESTAMP '2025-01-01 00:00:00', 1 + FROM generate_series(1, 10) AS i`, + [passwordHash], + ); + + await client.query( + `INSERT INTO workspaces (id, name, slug, owner_user_id, created_at, updated_at) + SELECT + format('20000000-0000-4000-8000-%s', lpad(i::text, 12, '0'))::uuid, + format('Benchmark Workspace %s', lpad(i::text, 2, '0')), + format('benchmark-workspace-%s', lpad(i::text, 2, '0')), + format('10000000-0000-4000-8000-%s', lpad(i::text, 12, '0'))::uuid, + TIMESTAMP '2025-01-01 00:00:00', TIMESTAMP '2025-01-01 00:00:00' + FROM generate_series(1, 10) AS i`, + ); + + await client.query( + `INSERT INTO workspace_members (id, workspace_id, user_id, role, created_at) + SELECT + format('21000000-0000-4000-8000-%s', lpad(i::text, 12, '0'))::uuid, + format('20000000-0000-4000-8000-%s', lpad(i::text, 12, '0'))::uuid, + format('10000000-0000-4000-8000-%s', lpad(i::text, 12, '0'))::uuid, + 'owner', TIMESTAMP '2025-01-01 00:00:00' + FROM generate_series(1, 10) AS i`, + ); + + await client.query( + `INSERT INTO aws_accounts ( + id, workspace_id, name, aws_account_id, role_arn, external_id, + status, last_verified_at, last_sync_at, created_at, updated_at + ) + SELECT + format( + '30000000-0000-4000-8000-%s', + lpad((((workspace_number - 1) * 20) + account_number)::text, 12, '0') + )::uuid, + format( + '20000000-0000-4000-8000-%s', + lpad(workspace_number::text, 12, '0') + )::uuid, + format('Synthetic Account %s-%s', workspace_number, lpad(account_number::text, 2, '0')), + lpad((workspace_number * 1000 + account_number)::text, 12, '0'), + format( + 'arn:aws:iam::%s:role/UnderflowSyntheticBenchmark', + lpad((workspace_number * 1000 + account_number)::text, 12, '0') + ), + format('benchmark-external-%s-%s', workspace_number, account_number), + 'verified', TIMESTAMP '2025-01-01 00:00:00', + TIMESTAMP '2025-12-31 23:59:00', TIMESTAMP '2025-01-01 00:00:00', + TIMESTAMP '2025-12-31 23:59:00' + FROM generate_series(1, 10) AS workspace_number + CROSS JOIN generate_series(1, 20) AS account_number`, + ); + + await client.query( + `INSERT INTO cost_sync_runs ( + id, aws_account_id, status, started_at, finished_at, error_message + ) + SELECT + format('40000000-0000-4000-8000-%s', lpad(sequence_number::text, 12, '0'))::uuid, + format('30000000-0000-4000-8000-%s', lpad(sequence_number::text, 12, '0'))::uuid, + 'completed', TIMESTAMP '2025-12-31 23:55:00', + TIMESTAMP '2025-12-31 23:59:00', NULL + FROM generate_series(1, 200) AS sequence_number`, + ); + + await client.query( + `INSERT INTO cost_snapshots ( + workspace_id, aws_account_id, usage_date, service_name, amount, currency, created_at + ) + SELECT + format( + '20000000-0000-4000-8000-%s', + lpad(workspace_number::text, 12, '0') + )::uuid, + format( + '30000000-0000-4000-8000-%s', + lpad((((workspace_number - 1) * 20) + account_number)::text, 12, '0') + )::uuid, + DATE '2025-01-01' + day_offset, + format('Synthetic Service %s', lpad(service_number::text, 2, '0')), + ( + ((workspace_number * 100000 + account_number * 1000 + day_offset * 10 + service_number) + % 500000) + 1 + )::numeric / 100, + 'USD', TIMESTAMP '2025-01-01 00:00:00' + FROM generate_series(1, 10) AS workspace_number + CROSS JOIN generate_series(1, 20) AS account_number + CROSS JOIN generate_series(0, 364) AS day_offset + CROSS JOIN generate_series(1, 50) AS service_number`, + ); + + await client.query( + `INSERT INTO workspace_cost_daily_rollups ( + workspace_id, usage_date, service_name, total_amount, currency, updated_at + ) + SELECT workspace_id, usage_date, service_name, SUM(amount), MAX(currency), NOW() + FROM cost_snapshots + WHERE workspace_id = ANY($1::uuid[]) + GROUP BY workspace_id, usage_date, service_name`, + [workspaceIds], + ); + + const countsResult = await client.query<{ + users: string; + workspaces: string; + workspace_members: string; + aws_accounts: string; + cost_sync_runs: string; + cost_snapshots: string; + cost_rollups: string; + }>( + `SELECT + (SELECT COUNT(*) FROM users WHERE id = ANY($1::uuid[])) AS users, + (SELECT COUNT(*) FROM workspaces WHERE id = ANY($2::uuid[])) AS workspaces, + (SELECT COUNT(*) FROM workspace_members WHERE workspace_id = ANY($2::uuid[])) AS workspace_members, + (SELECT COUNT(*) FROM aws_accounts WHERE workspace_id = ANY($2::uuid[])) AS aws_accounts, + (SELECT COUNT(*) FROM cost_sync_runs csr + JOIN aws_accounts aa ON aa.id = csr.aws_account_id + WHERE aa.workspace_id = ANY($2::uuid[])) AS cost_sync_runs, + (SELECT COUNT(*) FROM cost_snapshots WHERE workspace_id = ANY($2::uuid[])) AS cost_snapshots, + (SELECT COUNT(*) FROM workspace_cost_daily_rollups WHERE workspace_id = ANY($2::uuid[])) AS cost_rollups`, + [userIds, workspaceIds], + ); + + const row = countsResult.rows[0]; + const counts = { + users: Number(row?.users ?? 0), + workspaces: Number(row?.workspaces ?? 0), + workspaceMembers: Number(row?.workspace_members ?? 0), + awsAccounts: Number(row?.aws_accounts ?? 0), + costSyncRuns: Number(row?.cost_sync_runs ?? 0), + costSnapshots: Number(row?.cost_snapshots ?? 0), + costRollups: Number(row?.cost_rollups ?? 0), + }; + + if ( + counts.users !== 10 || + counts.workspaces !== 10 || + counts.workspaceMembers !== 10 || + counts.awsAccounts !== 200 || + counts.costSyncRuns !== 200 || + counts.costSnapshots !== EXPECTED_COST_SNAPSHOTS || + counts.costRollups !== EXPECTED_COST_ROLLUPS + ) { + throw new Error(`Unexpected benchmark row counts: ${JSON.stringify(counts)}`); + } + + await client.query( + "ANALYZE users, workspaces, workspace_members, aws_accounts, cost_sync_runs, cost_snapshots, workspace_cost_daily_rollups", + ); + await client.query("COMMIT"); + + console.log( + JSON.stringify({ + dataset: "underflow-api-benchmark-v1", + dateRange: { from: "2025-01-01", to: "2025-12-31" }, + servicesPerAccountPerDay: 50, + ...counts, + }), + ); + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } finally { + client.release(); + await pool.end(); + } +}; + +run().catch((error: unknown) => { + console.error( + JSON.stringify({ + message: "Benchmark seed failed", + error: error instanceof Error ? error.message : "Unknown error", + }), + ); + process.exitCode = 1; +}); diff --git a/apps/api/src/services/cost.service.ts b/apps/api/src/services/cost.service.ts index 13cf67b..3d33e28 100644 --- a/apps/api/src/services/cost.service.ts +++ b/apps/api/src/services/cost.service.ts @@ -66,6 +66,8 @@ export const costService = { const recordsSynced = await costRepository.replaceSnapshots({ workspaceId: awsAccount.workspaceId, awsAccountId: awsAccount.id, + from: syncRange.from, + to: syncRange.to, entries, }); @@ -164,6 +166,8 @@ export const costService = { await costRepository.replaceSnapshots({ workspaceId: account.workspaceId, awsAccountId: account.id, + from: range.from, + to: range.to, entries, }); diff --git a/apps/api/src/test/cost-monitor.integration.spec.ts b/apps/api/src/test/cost-monitor.integration.spec.ts index 8d07a2c..87b0ba8 100644 --- a/apps/api/src/test/cost-monitor.integration.spec.ts +++ b/apps/api/src/test/cost-monitor.integration.spec.ts @@ -77,6 +77,7 @@ const truncateTables = async (): Promise => { notification_deliveries, alert_events, budget_alerts, + workspace_cost_daily_rollups, cost_snapshots, cost_sync_runs, aws_accounts, @@ -171,6 +172,24 @@ test("workspace, aws account, sync, reporting, and alerts work together against assert.equal(summaryResponse.status, 200); assert.equal(summaryResponse.body.summary.totalAmount, 120); + const rollupResult = await pool.query( + `SELECT COUNT(*) AS count, SUM(total_amount) AS total_amount + FROM workspace_cost_daily_rollups + WHERE workspace_id = $1`, + [workspaceId], + ); + assert.equal(Number(rollupResult.rows[0]?.count ?? 0), 3); + assert.equal(Number(rollupResult.rows[0]?.total_amount ?? 0), 120); + + const accountSummaryResponse = await request(app) + .get( + `/api/v1/workspaces/${workspaceId}/costs/summary?from=${dayOne}&to=${dayThree}&awsAccountId=${awsAccountId}`, + ) + .set("Authorization", `Bearer ${accessToken}`); + + assert.equal(accountSummaryResponse.status, 200); + assert.equal(accountSummaryResponse.body.summary.totalAmount, 120); + const byServiceResponse = await request(app) .get(`/api/v1/workspaces/${workspaceId}/costs/by-service?from=${dayOne}&to=${dayThree}`) .set("Authorization", `Bearer ${accessToken}`); @@ -199,6 +218,24 @@ test("workspace, aws account, sync, reporting, and alerts work together against assert.equal(Number(alertEventsResult.rows[0]?.count ?? 0), 1); assert.equal(Number(deliveriesResult.rows[0]?.count ?? 0), 1); + + costExplorerService.fetchCostData = async () => []; + const emptyReplacementResponse = await request(app) + .post(`/api/v1/aws-accounts/${awsAccountId}/sync`) + .set("Authorization", `Bearer ${accessToken}`) + .send({ from: dayOne, to: dayThree }); + + assert.equal(emptyReplacementResponse.status, 200); + assert.equal(emptyReplacementResponse.body.recordsSynced, 0); + + const replacedCounts = await pool.query( + `SELECT + (SELECT COUNT(*) FROM cost_snapshots WHERE workspace_id = $1) AS snapshots, + (SELECT COUNT(*) FROM workspace_cost_daily_rollups WHERE workspace_id = $1) AS rollups`, + [workspaceId], + ); + assert.equal(Number(replacedCounts.rows[0]?.snapshots ?? 0), 0); + assert.equal(Number(replacedCounts.rows[0]?.rollups ?? 0), 0); } finally { costExplorerService.fetchCostData = originalFetch; } diff --git a/benchmarks/underflow-api/README.md b/benchmarks/underflow-api/README.md new file mode 100644 index 0000000..d9d7698 --- /dev/null +++ b/benchmarks/underflow-api/README.md @@ -0,0 +1,371 @@ +# Underflow authenticated API benchmark + +This benchmark measures the real bearer-authenticated, PostgreSQL-backed cost-reporting paths against a deterministic synthetic dataset. It deliberately excludes login latency from the measured metric while retaining bearer authentication—and therefore the database-backed user lookup—on every measured request. + +No result in this directory should be treated as measured evidence until all commands complete and a timestamped result directory contains the required JSON files. A failed k6 threshold is evidence and must be preserved as-is. + +## Final measured result + +The completed benchmark sustained **170.90 requests/second for 15 minutes at 45 continuously active VUs** with a 0.0325% functional failure rate. Across every one-minute ALB server-side datapoint, the worst p50 was 15.22 ms, p95 was 89.89 ms, and p99 was 226.96 ms. The 50-VU soak was the first sustained failing level because its p95 and p99 targets were breached. See [`results/2026-09-10-c0cf1f68/RESULTS.md`](results/2026-09-10-c0cf1f68/RESULTS.md) for the generated report and limitations; VUs are continuously active workers, not registered-user counts. + +The disposable stack was destroyed on 2026-09-11. Terraform state and independent AWS inventory checks both reported zero remaining benchmark resources; the sanitized verification record is stored with the result evidence. + +## Safety boundary + +The only authorized Terraform root is `infra/terraform/envs/api-benchmark`. It has local, isolated state and does not reference production state or modules. Every named resource begins with `underflow-api-bench-` and AWS provider default tags apply: + +- `project = underflow` +- `environment = api-benchmark` +- `purpose = disposable-load-test` +- `managed = terraform` + +Never run these commands from `infra/terraform/envs/production`. Never use production data or real AWS account integrations. The benchmark creates one VPC, two public subnets, an internet gateway and route table, three narrowly scoped security groups, ECR, one ECS cluster/service/task definition, one ALB/listener/target group, one log group, one non-public single-AZ RDS instance, IAM execution/runtime roles, and one zero-recovery Secrets Manager secret. It creates no NAT gateway, private subnet, DNS, certificate, HTTPS listener, SES, Lambda, EventBridge, worker, autoscaling, Stripe, S3, or CloudFront resource. + +## Prerequisites and run identity + +Commands below assume Bash plus AWS CLI v2, Docker with BuildKit, Terraform, jq, Python 3, and a native k6 installation available in the same shell that runs the benchmark. Docker is used to build the API image, but not to run k6. Keep shell history disabled while handling the generated benchmark password. + +```bash +export REPO_ROOT="$(git rev-parse --show-toplevel)" +export TF_ROOT="$REPO_ROOT/infra/terraform/envs/api-benchmark" +export GIT_SHA="$(git -C "$REPO_ROOT" rev-parse HEAD)" +export SHORT_SHA="$(printf '%s' "$GIT_SHA" | cut -c1-8)" +export BENCHMARK_ID="$SHORT_SHA" +export AWS_REGION="us-east-1" +export LOAD_TEST_IP="$(curl --fail --silent https://checkip.amazonaws.com | tr -d '\r\n')" +export LOAD_TEST_CIDR="$LOAD_TEST_IP/32" # public /32 of this host, where native k6 runs +export RUN_DATE="$(date -u +%F)" +export RESULTS_DIR="$REPO_ROOT/benchmarks/underflow-api/results/${RUN_DATE}-${SHORT_SHA}" +mkdir -p "$RESULTS_DIR" +command -v k6 >/dev/null +k6 version +``` + +Record the clean baseline before editing or deployment: + +```bash +git -C "$REPO_ROOT" status --short +git -C "$REPO_ROOT" rev-parse HEAD +cd "$REPO_ROOT/apps/api" +npm ci +npm run build +npm test +# Only when a disposable PostgreSQL DATABASE_URL is configured: +npm run test:db +``` + +## Configure and validate isolated Terraform + +Create the ignored `terraform.tfvars`; do not commit it: + +```bash +cat > "$TF_ROOT/terraform.tfvars" </dev/null && tfsec "$TF_ROOT" + +terraform -chdir="$TF_ROOT" plan -out=tfplan +terraform -chdir="$TF_ROOT" show -no-color tfplan > "$RESULTS_DIR/terraform-plan.txt.raw" +"$REPO_ROOT/benchmarks/underflow-api/scripts/sanitize-results.sh" \ + "$RESULTS_DIR/terraform-plan.txt.raw" "$RESULTS_DIR/terraform-plan.txt" +rm "$RESULTS_DIR/terraform-plan.txt.raw" +``` + +Before applying, read the plan and confirm: + +```bash +export PREFIX="underflow-api-bench-$BENCHMARK_ID" +terraform -chdir="$TF_ROOT" show -no-color tfplan | less +terraform -chdir="$TF_ROOT" show -json tfplan | jq -r \ + '.resource_changes[] | select(.mode == "managed") | [.type, .name, (.change.actions | join(","))] | @tsv' +``` + +The review must show only this root's benchmark resources, the benchmark prefix on every name-capable AWS resource, the four tags on every tag-capable resource, non-public RDS, ECS port 3080 ingress only from the ALB security group, ALB port 80 ingress only from `load_test_cidr`, and none of the excluded service types. Terraform state must be local to this root. + +If the native load generator's public IP changes after deployment, update only the ALB ingress rule and verify health with the guarded helper. It refuses to apply a plan that changes any managed resource other than `aws_security_group.alb`: + +```bash +bash "$REPO_ROOT/benchmarks/underflow-api/scripts/update-load-test-ip.sh" +``` + +## Bootstrap ECR, build, and deploy + +The reviewed configuration is applied first to only its disposable ECR resource so the immutable image exists before ECS starts. This target is in the isolated benchmark root; it does not address any existing Underflow resource. + +```bash +terraform -chdir="$TF_ROOT" apply -target=aws_ecr_repository.api +export ECR_REPOSITORY="$(terraform -chdir="$TF_ROOT" output -raw ecr_repository_url)" +aws ecr get-login-password --region "$AWS_REGION" | \ + docker login --username AWS --password-stdin "${ECR_REPOSITORY%%/*}" +docker buildx build --platform linux/amd64 \ + --tag "$ECR_REPOSITORY:$GIT_SHA" --push "$REPO_ROOT/apps/api" +export IMAGE_DIGEST="$(aws ecr describe-images --region "$AWS_REGION" \ + --repository-name "${ECR_REPOSITORY##*/}" --image-ids imageTag="$GIT_SHA" \ + --query 'imageDetails[0].imageDigest' --output text)" +test -n "$IMAGE_DIGEST" && test "$IMAGE_DIGEST" != "None" + +# Refresh and review the complete plan after the immutable image exists. +terraform -chdir="$TF_ROOT" plan -out=tfplan +terraform -chdir="$TF_ROOT" show -no-color tfplan | less +terraform -chdir="$TF_ROOT" apply tfplan +aws ecs wait services-stable --region "$AWS_REGION" \ + --cluster "$(terraform -chdir="$TF_ROOT" output -raw ecs_cluster_name)" \ + --services "$(terraform -chdir="$TF_ROOT" output -raw ecs_service_name)" +export BASE_URL="$(terraform -chdir="$TF_ROOT" output -raw api_base_url)" +test "$(curl --silent --output /dev/null --write-out '%{http_code}' "$BASE_URL/api/v1/health")" = "200" +``` + +Record only sanitized, non-secret environment metadata. Do not save the full `terraform output -json`, because it includes a sensitive secret ARN. The helper script creates and validates the metadata file: + +```bash +"$REPO_ROOT/benchmarks/underflow-api/scripts/collect-environment.sh" +jq . "$RESULTS_DIR/environment.json" +``` + +## Migrate and seed + +Both commands launch one-off Fargate tasks from the deployed API task definition with identical subnets, security group, image, environment, and Secrets Manager injection. The runner waits for task completion and requires container exit code zero. + +```bash +export BENCHMARK_RESULTS_DIR="$RESULTS_DIR" +"$REPO_ROOT/benchmarks/underflow-api/scripts/run-one-off-task.sh" migrate +"$REPO_ROOT/benchmarks/underflow-api/scripts/run-one-off-task.sh" seed +jq -e '.users == 10 and .workspaces == 10 and .awsAccounts == 200 and .costSnapshots == 3650000 and .costRollups == 182500' \ + "$RESULTS_DIR/dataset.json" +``` + +The seeder requires `ALLOW_BENCHMARK_SEED=true`, uses only reserved UUIDs and `example.invalid` identities, bulk-loads with PostgreSQL set operations, replaces only its fixed dataset, verifies exact counts, builds 182,500 workspace/date/service rollups from the 3,650,000 raw snapshots, and runs `ANALYZE`. The generated password is never logged. The first identity is `benchmark+01@example.invalid`, and its workspace is `20000000-0000-4000-8000-000000000001`. + +After deploying the rollup migration over an already-seeded benchmark database, backfill without reseeding the raw dataset: + +```bash +export BENCHMARK_RESULTS_DIR="$RESULTS_DIR" +"$REPO_ROOT/benchmarks/underflow-api/scripts/run-one-off-task.sh" rollup +``` + +The guarded backfill rebuilds all rollups transactionally and records its sanitized output in `rollup-task.log`. Normal synchronization refreshes only the affected workspace/date range under a workspace-scoped transaction lock. Workspace-wide summary, timeseries, and by-service reads use rollups; account-filtered reads continue to use raw snapshots. + +To capture a baseline PostgreSQL execution plan for the full-year cost-summary query without exposing RDS publicly, run the diagnostic one-off task and preserve its sanitized output: + +```bash +export BENCHMARK_RESULTS_DIR="$RESULTS_DIR" +"$REPO_ROOT/benchmarks/underflow-api/scripts/run-one-off-task.sh" explain +``` + +The resulting `explain-task.log` contains `EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT JSON)` for summary, timeseries, and by-service queries, plus sanitized `cost_snapshots` table/index size and scan counters. Capture this baseline before adding an index or changing a query. + +## Preflight and k6 execution + +Run native k6 from this host, outside ECS and outside Docker. The ALB permits only `load_test_cidr`, so it must be this host's current public `/32`; if the address changes, update `terraform.tfvars`, review a new plan, and apply it before continuing. Fetch the generated synthetic password into an environment variable without echoing it, then disable shell history and verify one healthy task, the seed counts, and health before testing. + +```bash +set +o history +export TEST_EMAIL="benchmark+01@example.invalid" +export WORKSPACE_ID="20000000-0000-4000-8000-000000000001" +export TEST_PASSWORD="$(aws secretsmanager get-secret-value --region "$AWS_REGION" \ + --secret-id "$PREFIX-runtime" --query SecretString --output text | jq -r '.BENCHMARK_PASSWORD')" +test -n "$TEST_PASSWORD" + +aws ecs describe-services --region "$AWS_REGION" \ + --cluster "$(terraform -chdir="$TF_ROOT" output -raw ecs_cluster_name)" \ + --services "$(terraform -chdir="$TF_ROOT" output -raw ecs_service_name)" \ + --query 'services[0].{desired:desiredCount,running:runningCount,pending:pendingCount}' +curl --fail --silent "$BASE_URL/api/v1/health" + +export RESULTS_DIR +k6 run "$REPO_ROOT/benchmarks/underflow-api/k6/smoke.js" + +# Unmeasured 60-second warm-up; preserve separately from smoke and normal load. +SMOKE_DURATION=60s SUMMARY_NAME=warmup-summary.json \ + k6 run "$REPO_ROOT/benchmarks/underflow-api/k6/smoke.js" +sleep 30 + +export LOAD_START="$(date -u +%FT%TZ)" +k6 run "$REPO_ROOT/benchmarks/underflow-api/k6/load.js" +export LOAD_END="$(date -u +%FT%TZ)" +``` + +The scripts use deterministic iteration buckets for the 30/30/25/10/5 distribution, validate status, JSON, response shape, and authorization/server errors, and save compact summaries through `handleSummary`. k6 enforces the client-observed failure rate `<1%` and checks `>99%`; its end-to-end latency and per-endpoint values are retained as diagnostic evidence but are not used for backend acceptance. Because a developer-machine run includes location-dependent Internet transit, backend latency targets are evaluated from the ALB `TargetResponseTime` p50/p90/p95/p99 collected from CloudWatch. The required server-side targets are p50 `<30 ms`, p95 `<100 ms`, and p99 `<250 ms` in every one-minute datapoint. Client-observed k6 latency must be labeled with the load-generator location rather than relabeled or discarded. Login is tagged as setup and excluded from `measured_*` metrics. + +### Capacity discovery on the baseline database class + +When the normal profile saturates the documented `db.t4g.micro`, preserve that failed run and determine the configuration's actual capacity with independent constant-load levels. Test increasing VU levels, stopping once the first failing level above the highest passing level is established. A passing level has a measured failure rate below 1%, a checks pass rate above 99%, and every one-minute ALB `TargetResponseTime` p50 below 30 ms, p95 below 100 ms, and p99 below 250 ms. Refresh the ALB ingress CIDR before each level; never update it during a measured interval. + +The all-in-one runner performs the IP refresh, health and credential preflight, k6 run, CloudWatch wait and collection, and final capacity check. It defaults to 7 VUs for 3 minutes; the optional arguments are VUs and duration: + +```bash +bash "$REPO_ROOT/benchmarks/underflow-api/scripts/run-capacity-test.sh" 7 3m +``` + +It refuses to overwrite existing evidence. Set a distinct label when intentionally repeating a level, for example `CAPACITY_LABEL=post-rollup-repeat`. + +The equivalent manual sequence follows for troubleshooting or inspecting each step: + +```bash +bash "$REPO_ROOT/benchmarks/underflow-api/scripts/update-load-test-ip.sh" +export CAPACITY_VUS=5 +export CAPACITY_START="$(date -u +%FT%TZ)" +CAPACITY_VUS="$CAPACITY_VUS" CAPACITY_DURATION=3m \ + k6 run "$REPO_ROOT/benchmarks/underflow-api/k6/capacity.js" +export CAPACITY_END="$(date -u +%FT%TZ)" + +# Allow one-minute CloudWatch statistics to become available. +sleep 120 + +AWS_REGION="$AWS_REGION" \ +ECS_CLUSTER="$(terraform -chdir="$TF_ROOT" output -raw ecs_cluster_name | tr -d '\r')" \ +ECS_SERVICE="$(terraform -chdir="$TF_ROOT" output -raw ecs_service_name | tr -d '\r')" \ +RDS_IDENTIFIER="$(terraform -chdir="$TF_ROOT" output -raw rds_identifier | tr -d '\r')" \ +ALB_ARN_SUFFIX="$(terraform -chdir="$TF_ROOT" output -raw load_balancer_arn_suffix | tr -d '\r')" \ +TARGET_GROUP_ARN_SUFFIX="$(terraform -chdir="$TF_ROOT" output -raw target_group_arn_suffix | tr -d '\r')" \ +START_TIME="$CAPACITY_START" END_TIME="$CAPACITY_END" \ +OUTPUT_FILE="$RESULTS_DIR/capacity-${CAPACITY_VUS}vus-cloudwatch.json" \ + "$REPO_ROOT/benchmarks/underflow-api/scripts/collect-aws-metadata.sh" + +"$REPO_ROOT/benchmarks/underflow-api/scripts/check-capacity-result.sh" \ + "$RESULTS_DIR/capacity-${CAPACITY_VUS}vus-summary.json" \ + "$RESULTS_DIR/capacity-${CAPACITY_VUS}vus-cloudwatch.json" +``` + +The k6 summary is saved as `capacity-vus-summary.json`. The checker requires functional success plus p50 `<30 ms`, p95 `<100 ms`, and p99 `<250 ms` in every collected ALB minute. A run aborted for network failures is invalid, must be preserved as such, and cannot establish capacity. The supported level is the highest fully valid level whose server-side threshold passes; do not interpolate or claim the next failing level. + +Run stress only when the normal profile completes, thresholds are reviewed, the service remains healthy, and the load generator is not saturated: + +```bash +export STRESS_START="$(date -u +%FT%TZ)" +k6 run "$REPO_ROOT/benchmarks/underflow-api/k6/stress.js" +export STRESS_END="$(date -u +%FT%TZ)" +``` + +Stress aborts after a sustained measured failure rate above 5%. Treat its breaking point as configuration-specific, never as a fabricated supported-user claim. Record load-generator CPU/network separately if it approaches saturation. + +## Collect CloudWatch evidence and generate results + +Collect metrics immediately after each measured interval (use distinct output names if collecting smoke and stress separately). The required final `cloudwatch-summary.json` should cover normal load. ALB target-response datapoints contain server-side p50, p90, p95, and p99 extended statistics for each 60-second period; do not describe these as end-to-end client latency. + +```bash +AWS_REGION="$AWS_REGION" \ +ECS_CLUSTER="$(terraform -chdir="$TF_ROOT" output -raw ecs_cluster_name)" \ +ECS_SERVICE="$(terraform -chdir="$TF_ROOT" output -raw ecs_service_name)" \ +RDS_IDENTIFIER="$(terraform -chdir="$TF_ROOT" output -raw rds_identifier)" \ +ALB_ARN_SUFFIX="$(terraform -chdir="$TF_ROOT" output -raw load_balancer_arn_suffix)" \ +TARGET_GROUP_ARN_SUFFIX="$(terraform -chdir="$TF_ROOT" output -raw target_group_arn_suffix)" \ +START_TIME="$LOAD_START" END_TIME="$LOAD_END" \ +OUTPUT_FILE="$RESULTS_DIR/cloudwatch-summary.json" \ + "$REPO_ROOT/benchmarks/underflow-api/scripts/collect-aws-metadata.sh" + +"$REPO_ROOT/benchmarks/underflow-api/scripts/generate-results.sh" \ + "$RESULTS_DIR" "$RESULTS_DIR/RESULTS.md" +``` + +The generator refuses missing evidence and mechanically derives requests/second, p95, concurrency, dataset size, and the résumé bullet from preserved JSON. Review `RESULTS.md` for supported bottleneck observations; do not replace failed thresholds or missing stress evidence with estimates. + +Required final evidence for this capacity-discovery run is: + +```text +environment.json +dataset.json +smoke-summary.json +load-summary.json # retained invalid run, excluded from capacity claims +capacity-45vus-post-rollup-soak-summary.json +capacity-45vus-post-rollup-soak-cloudwatch.json +capacity-50vus-post-rollup-soak-summary.json +capacity-50vus-post-rollup-soak-cloudwatch.json +explain-after-rollup.json +rollup.json +RESULTS.md +``` + +## Repository validation and secret scan + +Before teardown, verify the repository and evidence while the environment is still available for diagnosis: + +```bash +cd "$REPO_ROOT/apps/api" +npm run build +npm test +terraform -chdir="$TF_ROOT" fmt -check -recursive +terraform -chdir="$TF_ROOT" validate + +node --check "$REPO_ROOT/benchmarks/underflow-api/k6/common.js" +node --check "$REPO_ROOT/benchmarks/underflow-api/k6/smoke.js" +node --check "$REPO_ROOT/benchmarks/underflow-api/k6/load.js" +node --check "$REPO_ROOT/benchmarks/underflow-api/k6/stress.js" +bash -n "$REPO_ROOT/benchmarks/underflow-api/scripts/"*.sh + +git -C "$REPO_ROOT" diff --check +git -C "$REPO_ROOT" status --short +git -C "$REPO_ROOT" diff --stat +git -C "$REPO_ROOT" diff -- . ':!infra/terraform/envs/production' + +# These scans must print nothing. Review any match before proceeding. +git -C "$REPO_ROOT" ls-files --others --exclude-standard --cached | \ + grep -E '(\.tfstate|\.tfvars$|\.tfplan$)' || true +rg -n --hidden --glob '!**/.terraform/**' --glob '!**/node_modules/**' \ + '(AKIA[0-9A-Z]{16}|ASIA[0-9A-Z]{16}|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.|[0-9]{12}\.dkr\.ecr\.|\.rds\.amazonaws\.com)' \ + "$REPO_ROOT/infra/terraform/envs/api-benchmark" "$REPO_ROOT/benchmarks/underflow-api" || true +``` + +The synthetic 12-digit account-number generator in the seeder is expected source code; no real account number may appear in code or evidence. + +## Mandatory Terraform-only destruction + +Do not begin until the timestamped evidence exists outside `.terraform`, is sanitized, and `RESULTS.md` has been generated. Destroy only from the exact isolated root. + +```bash +test "$(cd "$TF_ROOT" && pwd)" = "$TF_ROOT" +for file in environment.json dataset.json smoke-summary.json load-summary.json cloudwatch-summary.json RESULTS.md; do + test -s "$RESULTS_DIR/$file" +done + +# Infrastructure characteristics and the tested image digest are already +# recorded in environment.json. Do not save live resource IDs or endpoints. +terraform -chdir="$TF_ROOT" state list | tee "$RESULTS_DIR/terraform-state-list.txt" +terraform -chdir="$TF_ROOT" state list | grep -E 'envs.production|platform_stack' && exit 1 || true + +# Stop only active one-off tasks in this isolated benchmark cluster. Service +# tasks have a `service:` group and are left for Terraform to remove. +export CLUSTER="$(terraform -chdir="$TF_ROOT" output -raw ecs_cluster_name)" +export TASK_ARNS="$(aws ecs list-tasks --region "$AWS_REGION" --cluster "$CLUSTER" \ + --desired-status RUNNING --family "$PREFIX-api" --query 'taskArns[]' --output text)" +if [[ -n "$TASK_ARNS" && "$TASK_ARNS" != "None" ]]; then + for task in $(aws ecs describe-tasks --region "$AWS_REGION" --cluster "$CLUSTER" \ + --tasks $TASK_ARNS --output json | \ + jq -r '.tasks[] | select(.group | startswith("family:")) | .taskArn'); do + aws ecs stop-task --region "$AWS_REGION" --cluster "$CLUSTER" --task "$task" \ + --reason 'benchmark teardown' >/dev/null + done +fi + +terraform -chdir="$TF_ROOT" plan -destroy -out=destroy.tfplan +terraform -chdir="$TF_ROOT" show -no-color destroy.tfplan | less +terraform -chdir="$TF_ROOT" apply destroy.tfplan +test -z "$(terraform -chdir="$TF_ROOT" state list)" +``` + +Finally verify by benchmark prefix that ECS, ELBv2, RDS, ECR, Secrets Manager, CloudWatch Logs, and EC2/VPC resources are absent. Queries must use `$PREFIX`; never delete unrelated matches manually. + +```bash +aws ecs list-clusters --region "$AWS_REGION" --query "clusterArns[?contains(@, '$PREFIX')]" +aws elbv2 describe-load-balancers --region "$AWS_REGION" --query "LoadBalancers[?contains(LoadBalancerName, '$PREFIX')]" +aws rds describe-db-instances --region "$AWS_REGION" --query "DBInstances[?contains(DBInstanceIdentifier, '$PREFIX')]" +aws ecr describe-repositories --region "$AWS_REGION" --query "repositories[?contains(repositoryName, '$PREFIX')]" +aws secretsmanager list-secrets --region "$AWS_REGION" --include-planned-deletion \ + --query "SecretList[?contains(Name, '$PREFIX')]" +aws logs describe-log-groups --region "$AWS_REGION" --log-group-name-prefix "/ecs/$PREFIX" +aws ec2 describe-vpcs --region "$AWS_REGION" \ + --filters "Name=tag:Name,Values=$PREFIX*" --query 'Vpcs[].VpcId' +aws ecs list-tasks --region "$AWS_REGION" --cluster "$PREFIX-cluster" --desired-status RUNNING 2>/dev/null || true +``` + +All result arrays must be empty. Remove ignored `terraform.tfvars`, plan files, local state, and any temporary raw logs only after successful destruction; retain sanitized timestamped results and repository changes. Do not commit or push without explicit authorization. diff --git a/benchmarks/underflow-api/RESULTS_TEMPLATE.md b/benchmarks/underflow-api/RESULTS_TEMPLATE.md new file mode 100644 index 0000000..bb5f1a0 --- /dev/null +++ b/benchmarks/underflow-api/RESULTS_TEMPLATE.md @@ -0,0 +1,55 @@ +# Underflow API benchmark results + +> This template intentionally contains no performance claims. Generate the completed file from captured summaries with `scripts/generate-results.sh`. + +## Objective + +Measure authenticated, PostgreSQL-backed cost-reporting reads on the documented disposable AWS configuration. + +## Tested commit + +See `environment.json`. + +## Infrastructure + +See `environment.json`. Results apply only to the documented one-task ECS and single-AZ RDS configuration. + +## Dataset + +See `dataset.json` for exact verified counts. + +## Workload + +30% summary, 30% timeseries, 25% by service, 10% AWS account list, and 5% sync history. Authentication occurs before measurement; every measured request still uses bearer authentication and performs the application's PostgreSQL user lookup. + +## Thresholds + +Measured failure rate `< 1%` and checks pass rate `> 99%`. For every one-minute CloudWatch datapoint, ALB server-side `TargetResponseTime` p50 `< 30 ms`, p95 `< 100 ms`, and p99 `< 250 ms`. Client-observed latency is retained as diagnostic evidence and is not used for backend acceptance. + +## Results + +Generated from `smoke-summary.json`, `load-summary.json`, and, when safely run, `stress-summary.json`. + +## Per-endpoint results + +Generated from named endpoint trend metrics. + +## AWS resource metrics + +See `cloudwatch-summary.json`. + +## Observed bottlenecks + +Record only observations supported by k6 and CloudWatch evidence. + +## Limitations + +Synthetic data, one region, one Fargate task, one load-generator location, HTTP restricted by CIDR, and a single-AZ burstable RDS instance. + +## Reproduction commands + +See [`README.md`](README.md). + +## Evidence-backed résumé bullet + +Generated mechanically only after a completed normal-load run. diff --git a/benchmarks/underflow-api/k6/capacity.js b/benchmarks/underflow-api/k6/capacity.js new file mode 100644 index 0000000..4a8695a --- /dev/null +++ b/benchmarks/underflow-api/k6/capacity.js @@ -0,0 +1,29 @@ +import { authenticate, runMeasuredRequest, standardThresholds, writeSummary } from "./common.js"; + +const capacityVus = Number(__ENV.CAPACITY_VUS ?? "5"); +if (!Number.isInteger(capacityVus) || capacityVus < 1 || capacityVus > 100) { + throw new Error("CAPACITY_VUS must be an integer from 1 through 100"); +} + +const capacityDuration = __ENV.CAPACITY_DURATION ?? "3m"; + +export const options = { + scenarios: { + capacity: { + executor: "constant-vus", + vus: capacityVus, + duration: capacityDuration, + gracefulStop: "30s", + }, + }, + // Abort sustained functional failures so a changed public IP does not turn + // the remainder of a capacity run into misleading fast failures. + thresholds: standardThresholds("rate<0.01", true), +}; + +export const setup = authenticate; + +export default runMeasuredRequest; + +export const handleSummary = (data) => + writeSummary(data, `capacity-${capacityVus}vus-summary.json`); diff --git a/benchmarks/underflow-api/k6/common.js b/benchmarks/underflow-api/k6/common.js new file mode 100644 index 0000000..1ae48a4 --- /dev/null +++ b/benchmarks/underflow-api/k6/common.js @@ -0,0 +1,157 @@ +import http from "k6/http"; +import { check, sleep } from "k6"; +import exec from "k6/execution"; +import { Counter, Rate, Trend } from "k6/metrics"; + +const measuredRequests = new Counter("measured_requests"); +const measuredFailures = new Rate("measured_failures"); +const measuredDuration = new Trend("measured_duration", true); + +const endpointTrends = { + cost_summary: new Trend("endpoint_cost_summary_duration", true), + cost_timeseries: new Trend("endpoint_cost_timeseries_duration", true), + cost_by_service: new Trend("endpoint_cost_by_service_duration", true), + aws_account_list: new Trend("endpoint_aws_account_list_duration", true), + sync_history: new Trend("endpoint_sync_history_duration", true), +}; + +const requiredEnvironment = ["BASE_URL", "TEST_EMAIL", "TEST_PASSWORD", "WORKSPACE_ID"]; + +export const assertEnvironment = () => { + for (const name of requiredEnvironment) { + if (!__ENV[name]) { + throw new Error(`${name} is required`); + } + } + if (!/^https?:\/\//.test(__ENV.BASE_URL)) { + throw new Error("BASE_URL must start with http:// or https://"); + } +}; + +export const standardThresholds = (failureThreshold = "rate<0.01", abortOnFailure = false) => ({ + measured_failures: abortOnFailure + ? [{ threshold: failureThreshold, abortOnFail: true, delayAbortEval: "30s" }] + : [failureThreshold], + checks: ["rate>0.99"], +}); + +export const authenticate = () => { + assertEnvironment(); + const response = http.post( + `${__ENV.BASE_URL}/api/v1/auth/mobile/login`, + JSON.stringify({ email: __ENV.TEST_EMAIL, password: __ENV.TEST_PASSWORD }), + { + headers: { "Content-Type": "application/json" }, + tags: { name: "auth_setup", scope: "setup" }, + }, + ); + + if (response.status !== 200) { + throw new Error(`Authentication failed with HTTP ${response.status}`); + } + + let payload; + try { + payload = response.json(); + } catch (_error) { + throw new Error("Authentication returned invalid JSON"); + } + + const accessToken = payload?.tokens?.accessToken; + if (typeof accessToken !== "string" || accessToken.length === 0) { + throw new Error("Authentication response did not contain an access token"); + } + + return { accessToken }; +}; + +const requestDefinitions = [ + { + upperBound: 30, + name: "cost_summary", + path: `/api/v1/workspaces/${__ENV.WORKSPACE_ID}/costs/summary?from=2025-01-01&to=2025-12-31`, + shape: (body) => typeof body?.summary?.totalAmount === "number" && body.summary.currency === "USD", + }, + { + upperBound: 60, + name: "cost_timeseries", + path: `/api/v1/workspaces/${__ENV.WORKSPACE_ID}/costs/timeseries?from=2025-01-01&to=2025-12-31`, + shape: (body) => Array.isArray(body?.points) && body.points.length === 365, + }, + { + upperBound: 85, + name: "cost_by_service", + path: `/api/v1/workspaces/${__ENV.WORKSPACE_ID}/costs/by-service?from=2025-01-01&to=2025-12-31`, + shape: (body) => Array.isArray(body?.services) && body.services.length === 50, + }, + { + upperBound: 95, + name: "aws_account_list", + path: `/api/v1/workspaces/${__ENV.WORKSPACE_ID}/aws-accounts`, + shape: (body) => Array.isArray(body?.awsAccounts) && body.awsAccounts.length === 20, + }, + { + upperBound: 100, + name: "sync_history", + path: `/api/v1/workspaces/${__ENV.WORKSPACE_ID}/sync-history?limit=25`, + shape: (body) => Array.isArray(body?.syncRuns) && body.syncRuns.length > 0, + }, +]; + +const chooseRequest = () => { + // Multiplication by a value coprime to 100 deterministically spreads short + // runs across the complete weighted distribution instead of exhausting each + // endpoint's contiguous bucket in order. + const bucket = (exec.scenario.iterationInTest * 37) % 100; + return requestDefinitions.find((definition) => bucket < definition.upperBound); +}; + +export const runMeasuredRequest = (data) => { + const definition = chooseRequest(); + if (!definition) { + throw new Error("No request definition selected"); + } + + const response = http.get(`${__ENV.BASE_URL}${definition.path}`, { + headers: { + Authorization: `Bearer ${data.accessToken}`, + Accept: "application/json", + }, + tags: { name: definition.name, endpoint: definition.name, scope: "measured" }, + }); + + let body = null; + let validJson = true; + try { + body = response.json(); + } catch (_error) { + validJson = false; + } + + const validStatus = response.status === 200; + const validShape = validJson && definition.shape(body); + const passed = check(response, { + [`${definition.name}: status is 200`]: () => validStatus, + [`${definition.name}: response is JSON`]: () => validJson, + [`${definition.name}: response shape is valid`]: () => validShape, + [`${definition.name}: no auth or server error`]: () => + response.status !== 401 && response.status !== 403 && response.status < 500, + }); + + measuredRequests.add(1); + measuredFailures.add(!passed || !validStatus || !validJson || !validShape); + measuredDuration.add(response.timings.duration); + endpointTrends[definition.name].add(response.timings.duration); + sleep(Number(__ENV.THINK_TIME_SECONDS ?? "0.1")); +}; + +export const writeSummary = (data, defaultName) => { + const resultsDirectory = (__ENV.RESULTS_DIR ?? "results").replace(/[\\/]$/, ""); + const summaryName = __ENV.SUMMARY_NAME ?? defaultName; + const sanitizedData = { ...data }; + delete sanitizedData.setup_data; + return { + [`${resultsDirectory}/${summaryName}`]: JSON.stringify(sanitizedData, null, 2), + stdout: `Saved compact k6 summary to ${resultsDirectory}/${summaryName}\n`, + }; +}; diff --git a/benchmarks/underflow-api/k6/load.js b/benchmarks/underflow-api/k6/load.js new file mode 100644 index 0000000..9466287 --- /dev/null +++ b/benchmarks/underflow-api/k6/load.js @@ -0,0 +1,27 @@ +import { authenticate, runMeasuredRequest, standardThresholds, writeSummary } from "./common.js"; + +export const options = { + scenarios: { + normal_load: { + executor: "ramping-vus", + startVUs: 0, + gracefulRampDown: "30s", + stages: [ + { duration: "1m", target: 25 }, + { duration: "3m", target: 25 }, + { duration: "1m", target: 50 }, + { duration: "5m", target: 50 }, + { duration: "2m", target: 100 }, + { duration: "5m", target: 100 }, + { duration: "1m", target: 0 }, + ], + }, + }, + thresholds: standardThresholds(), +}; + +export const setup = authenticate; + +export default runMeasuredRequest; + +export const handleSummary = (data) => writeSummary(data, "load-summary.json"); diff --git a/benchmarks/underflow-api/k6/smoke.js b/benchmarks/underflow-api/k6/smoke.js new file mode 100644 index 0000000..7c36b00 --- /dev/null +++ b/benchmarks/underflow-api/k6/smoke.js @@ -0,0 +1,18 @@ +import { authenticate, runMeasuredRequest, standardThresholds, writeSummary } from "./common.js"; + +export const options = { + scenarios: { + smoke: { + executor: "constant-vus", + vus: Number(__ENV.SMOKE_VUS ?? "2"), + duration: __ENV.SMOKE_DURATION ?? "30s", + }, + }, + thresholds: standardThresholds(), +}; + +export const setup = authenticate; + +export default runMeasuredRequest; + +export const handleSummary = (data) => writeSummary(data, "smoke-summary.json"); diff --git a/benchmarks/underflow-api/k6/stress.js b/benchmarks/underflow-api/k6/stress.js new file mode 100644 index 0000000..3ec4645 --- /dev/null +++ b/benchmarks/underflow-api/k6/stress.js @@ -0,0 +1,27 @@ +import { authenticate, runMeasuredRequest, standardThresholds, writeSummary } from "./common.js"; + +export const options = { + scenarios: { + stress: { + executor: "ramping-vus", + startVUs: 0, + gracefulRampDown: "15s", + stages: [ + { duration: "2m", target: 100 }, + { duration: "3m", target: 100 }, + { duration: "1m", target: 150 }, + { duration: "3m", target: 150 }, + { duration: "1m", target: 200 }, + { duration: "3m", target: 200 }, + { duration: "1m", target: 0 }, + ], + }, + }, + thresholds: standardThresholds("rate<0.05", true), +}; + +export const setup = authenticate; + +export default runMeasuredRequest; + +export const handleSummary = (data) => writeSummary(data, "stress-summary.json"); diff --git a/benchmarks/underflow-api/results/.gitkeep b/benchmarks/underflow-api/results/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/benchmarks/underflow-api/results/.gitkeep @@ -0,0 +1 @@ + diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/RESULTS.md b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/RESULTS.md new file mode 100644 index 0000000..d19abba --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/RESULTS.md @@ -0,0 +1,85 @@ +# Underflow API benchmark results + +## Objective + +Measure authenticated, PostgreSQL-backed cost-reporting reads on an isolated, disposable AWS configuration and establish a repeatable server-side capacity boundary. + +## Tested commit + +The deployed image was built from local commit `2624a6ccafc427521745b1180764fcde963f90bc` using image digest `sha256:b1eec7b5c7652dfab0fe8259ab66f9b0682e228930da769c8d1688961680e644`. After an evidence-only history rewrite removed a bearer token from an earlier commit, the code-equivalent repository commit is `f6c356d5e38409b5dfcd44908af68cc968335631`. + +## Infrastructure + +One ECS Fargate API task with 0.5 vCPU and 1,024 MiB memory, backed by a private, single-AZ `db.t4g.micro` PostgreSQL 16.13 instance with 20 GiB encrypted storage in `us-east-1`. + +## Dataset + +3,650,000 deterministic synthetic cost snapshots across 10 workspaces and 200 synthetic AWS accounts. Workspace/date/service reads use 182,500 daily rollup rows, a 20x row-count reduction. + +## Workload + +Bearer-authenticated read traffic distributed as 30% cost summary, 30% timeseries, 25% by service, 10% AWS account list, and 5% sync history. Login occurs once during setup and is excluded from measured requests. The supported and failing boundary runs each lasted 15 minutes. + +## Acceptance targets + +Measured failure rate `<1%`, checks pass rate `>99%`, and every one-minute ALB `TargetResponseTime` datapoint p50 `<30 ms`, p95 `<100 ms`, and p99 `<250 ms`. ALB target response time excludes load-generator-to-ALB Internet transit. + +## Capacity results + +| Run | Duration | Requests | Requests/s | Failure rate | Checks | Worst p50 | Worst p95 | Worst p99 | Result | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| 45 VUs | 15m | 153,992 | 170.90 | 0.0325% | 99.9752% | 15.22 ms | 89.89 ms | 226.96 ms | Pass | +| 50 VUs | 15m | 169,377 | 187.92 | 0.0549% | 99.9588% | 16.54 ms | 127.47 ms | 265.47 ms | Fail | + +The documented configuration therefore sustained at least **170.90 requests/second at 45 continuously active VUs**. The first sustained failing level tested was 50 VUs; this establishes a tested boundary, not a claim that 45 VUs equals 45 registered users or that 45 is the absolute maximum. + +## Initial smoke and normal-load outcomes + +The initial 2-VU smoke run completed 66 measured requests with 0.000% functional failures and 100.000% checks passed. It exposed the pre-rollup cost-query latency bottleneck. + +The original ramped normal-load attempt recorded 79.88% failures after the load generator's public IP changed and was therefore invalid for capacity claims. It is retained as failure evidence but excluded from the supported result. A separate stress profile was not run after the fixed-load capacity boundary was established. + +## Database optimization evidence + +The raw `cost_snapshots` relation occupied 1423 MB; the rollup relation occupied 52 MB. Post-rollup full-year `EXPLAIN (ANALYZE, BUFFERS)` execution times were 63.087 ms for summary, 11.025 ms for timeseries, and 8.155 ms for by-service. All used index-only scans. + +## Client-observed endpoint diagnostics + +These values include Internet transit from the load-generator location and are retained for diagnosis, not backend acceptance. + +| Endpoint | Average | Median | p90 | p95 | Maximum | +| --- | ---: | ---: | ---: | ---: | ---: | +| Cost summary | 140.52 ms | 130.19 ms | 162.82 ms | 195.43 ms | 1631.54 ms | +| Cost timeseries | 208.21 ms | 141.46 ms | 276.01 ms | 373.12 ms | 6610.41 ms | +| Cost by service | 144.52 ms | 131.19 ms | 167.54 ms | 202.12 ms | 1823.98 ms | +| AWS account list | 148.64 ms | 125.90 ms | 165.70 ms | 221.99 ms | 3462.79 ms | +| Sync history | 139.22 ms | 123.70 ms | 152.18 ms | 191.61 ms | 1638.74 ms | + +## AWS resource metrics at the supported level + +Peak one-minute ECS CPU was 76.21% and ECS memory was 6.93%. Peak RDS CPU was 66.63%, the minimum average freeable memory was 79.84 MiB, and database connections peaked at 10. The ALB recorded 3 load-balancer-generated 5xx responses and 0 target-generated 5xx responses during 153,992 measured requests. + +## Observed bottleneck + +At 50 VUs, p95 exceeded 100 ms in 5 of 15 one-minute periods and p99 exceeded 250 ms in 1 period. Peak one-minute ECS CPU reached 83.85% and RDS CPU reached 71.19%. Functional reliability still met its target, so server-side tail latency—not widespread request failure—defined the tested capacity boundary. + +## Limitations + +Synthetic data; one AWS region; one Fargate task; HTTP restricted to one changing public CIDR; one load-generator location; a single-AZ burstable database; 15-minute confirmation windows; no production AWS integrations; and no post-boundary stress run. Results apply only to the documented configuration and workload. + +## Exact reproduction + +See `../../README.md`. The final confirmation command was: + +```bash +CAPACITY_LABEL=post-rollup-soak \ + bash benchmarks/underflow-api/scripts/run-capacity-test.sh 45 15m +``` + +## Evidence-backed résumé bullet + +Built and benchmarked a multi-tenant AWS cost-monitoring API against 3.65M cost records, sustaining 170.90 requests/second with 89.89 ms worst-minute p95 server-side latency under 45 concurrent virtual users. + +```latex +\item Built and benchmarked a multi-tenant AWS cost-monitoring API against \textbf{3.65M cost records}, sustaining \textbf{170.90 requests/second} with \textbf{89.89 ms worst-minute p95 server-side latency} under \textbf{45 concurrent virtual users}. +``` diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-10vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-10vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..9dd9028 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-10vus-post-rollup-cloudwatch.json @@ -0,0 +1,233 @@ +{ + "startTime": "2026-09-10T18:18:20Z", + "endTime": "2026-09-10T18:21:22Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.07967098018474315, + "p90": 0.01480722002149371, + "p50": 0.011078576614027212, + "p95": 0.01778017725998972 + } + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.05131022634561018, + "p90": 0.014367543387150064, + "p50": 0.010959019176451491, + "p95": 0.016009033974398738 + } + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.055445969415233325, + "p90": 0.014988910955111562, + "p50": 0.011117015799361602, + "p95": 0.02148186070452684 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 15.721466579901366, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 22.787106196085613, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 22.53944714864095, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 6.396484375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 6.4453125, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 6.510416666666667, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 5.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 3.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 5.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 4.9305393610287505, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 18.993249437453123, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 19.44902754862257, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 135.18083483333334, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 76001280.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 90918912.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 83263488.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 0.002413793103448276, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 0.0012627986348122866, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 0.00034129692832764505, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 0.0004347826086956522, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-10vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-10vus-post-rollup-summary.json new file mode 100644 index 0000000..ed1a64b --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-10vus-post-rollup-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 390, + "fails": 0 + }, + { + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 390, + "fails": 0 + }, + { + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 390, + "fails": 0, + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid" + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 390, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 2340 + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 2340, + "fails": 0 + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 2340, + "fails": 0 + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 2340, + "fails": 0 + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 780, + "fails": 0 + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 780, + "fails": 0 + }, + { + "passes": 780, + "fails": 0, + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2" + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 780, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 1950 + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 1950, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 1950 + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 1950, + "fails": 0 + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 2340, + "fails": 0 + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 2340, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 2340 + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 2340, + "fails": 0 + } + ], + "name": "", + "path": "" + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "testRunDurationMs": 180996.5683, + "isStdOutTTY": true, + "isStdErrTTY": true + }, + "metrics": { + "http_req_tls_handshaking": { + "contains": "time", + "values": { + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0 + }, + "type": "trend" + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 129.28671610256404, + "min": 114.2435, + "med": 126.31815, + "max": 492.9923, + "p(90)": 132.0206, + "p(95)": 137.0889 + } + }, + "http_req_duration": { + "values": { + "avg": 130.4328539546217, + "min": 108.3892, + "med": 126.6433, + "max": 1177.3111, + "p(90)": 133.2004, + "p(95)": 137.9782 + }, + "type": "trend", + "contains": "time" + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 31200, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "http_req_connecting": { + "contains": "time", + "values": { + "avg": 0.16524775028842456, + "min": 0, + "med": 0, + "max": 122.3256, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend" + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "max": 0.5032, + "p(90)": 0, + "p(95)": 0, + "avg": 6.450455069862837E-05, + "min": 0, + "med": 0 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 124.03695423076923, + "min": 108.3892, + "med": 119.81190000000001, + "max": 803.2904, + "p(90)": 125.4834, + "p(95)": 129.17446499999997 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 145.19138999999996, + "avg": 137.0559506837606, + "min": 116.6628, + "med": 128.9137, + "max": 1177.3111, + "p(90)": 135.39408 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 7801 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 7800 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_reqs": { + "values": { + "count": 7801, + "rate": 43.10026468054311 + }, + "type": "counter", + "contains": "default" + }, + "measured_requests": { + "contains": "default", + "values": { + "count": 7800, + "rate": 43.09473971391313 + }, + "type": "counter" + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 109.4097, + "med": 119.7698, + "max": 688.5023, + "p(90)": 124.95237, + "p(95)": 126.39524, + "avg": 122.81241153846159 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 7800, + "rate": 43.09473971391313 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 10, + "min": 10, + "max": 10 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 230.9306075128205, + "min": 208.6462, + "med": 227.03505, + "max": 1277.5961, + "p(90)": 233.67709, + "p(95)": 238.31685 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 5003592, + "rate": 27644.678830079232 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 124.82565, + "max": 493.9183, + "p(90)": 129.97223, + "p(95)": 133.4991549999998, + "avg": 127.93604846153845, + "min": 113.0858 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "med": 125.6587, + "max": 715.1738, + "p(90)": 132.5007, + "p(95)": 135.7257, + "avg": 128.1199522881682, + "min": 108.3892 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 72206634, + "rate": 398939.2433138192 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 130.4328539546217, + "min": 108.3892, + "med": 126.6433, + "max": 1177.3111, + "p(90)": 133.2004, + "p(95)": 137.9782 + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0.16633584155877448, + "min": 0, + "med": 0, + "max": 122.3256 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 10, + "min": 10, + "max": 10 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 130.36359476923124, + "min": 108.3892, + "med": 126.63905, + "max": 1177.3111, + "p(90)": 133.19968, + "p(95)": 137.96328499999998 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 2.3128371619023183, + "min": 0, + "med": 0.3316, + "max": 848.2044, + "p(90)": 3.271, + "p(95)": 4.8441 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-2vus-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-2vus-cloudwatch.json new file mode 100644 index 0000000..bbbf6c6 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-2vus-cloudwatch.json @@ -0,0 +1,233 @@ +{ + "startTime": "2026-09-10T14:20:37Z", + "endTime": "2026-09-10T14:23:38Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.4469220790572148, + "p90": 0.13038611785583315, + "p50": 0.07453238033386977, + "p95": 0.15222797425470158 + } + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.2615802315178325, + "p90": 0.12640921432900043, + "p50": 0.07695024109803843, + "p95": 0.14644058085164235 + } + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.20681019823844912, + "p90": 0.11525497158473341, + "p50": 0.07536445397883569, + "p95": 0.13858705730247767 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 2.8502402305603027, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 4.901556015014648, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 4.135225772857666, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 0.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 2.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 2.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 3.968584815994397, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 36.016666666666666, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 43.94396228615215, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 117.7651533, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 101449728.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 92250112.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 85606400.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 0.0008333333333333334, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 0.01, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 0.0005660377358490566, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 0.0007142857142857143, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-2vus-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-2vus-summary.json new file mode 100644 index 0000000..30e676a --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-2vus-summary.json @@ -0,0 +1,492 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 331, + "fails": 0 + }, + { + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 331, + "fails": 0, + "name": "cost_summary: response is JSON" + }, + { + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 331, + "fails": 0, + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid" + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 331, + "fails": 0 + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 331, + "fails": 0 + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 331, + "fails": 0 + }, + { + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 331, + "fails": 0, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid" + }, + { + "fails": 0, + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 331 + }, + { + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 275, + "fails": 0 + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 275, + "fails": 0 + }, + { + "passes": 275, + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937" + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 275, + "fails": 0 + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 110, + "fails": 0 + }, + { + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 110, + "fails": 0, + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON" + }, + { + "fails": 0, + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 110 + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 110, + "fails": 0 + }, + { + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 55, + "fails": 0, + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200" + }, + { + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 55, + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON" + }, + { + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 55, + "fails": 0 + }, + { + "passes": 55, + "fails": 0, + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d" + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 181055.0222 + }, + "metrics": { + "measured_failures": { + "thresholds": { + "rate<0.01": { + "ok": true + } + }, + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 1102 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 707168, + "rate": 3905.818194972998 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 1103 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 1, + "min": 1, + "max": 2 + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 1102, + "rate": 6.08654754013225 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 1103, + "rate": 6.092070723018033 + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 111.1337, + "med": 192.2361, + "max": 3316.7455, + "p(90)": 258.65104, + "p(95)": 362.6146499999999, + "avg": 226.65930571169542 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0 + } + }, + "http_req_receiving": { + "contains": "time", + "values": { + "p(95)": 8.661789999999984, + "avg": 21.79715593834995, + "min": 0, + "med": 0.5157, + "max": 3024.6642, + "p(90)": 5.43742 + }, + "type": "trend" + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.3155878513145966, + "min": 0, + "med": 0, + "max": 122.0562, + "p(90)": 0, + "p(95)": 0 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 4408, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 1444.6597, + "p(90)": 130.72054, + "p(95)": 167.45368999999974, + "avg": 159.6348818181818, + "min": 117.3265, + "med": 119.4087 + }, + "thresholds": { + "p(99)<500": { + "ok": false + }, + "p(95)<200": { + "ok": true + } + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 169.62410636363634, + "min": 111.1337, + "med": 120.11015, + "max": 3291.5441, + "p(90)": 134.77812000000003, + "p(95)": 175.40819 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": false + } + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 292.69355, + "max": 3416.9556, + "p(90)": 358.76344, + "p(95)": 470.00490000000036, + "avg": 326.8671863883846, + "min": 211.4726 + } + }, + "endpoint_cost_by_service_duration": { + "contains": "time", + "values": { + "avg": 248.13789818181814, + "min": 197.4729, + "med": 214.6382, + "max": 1479.5777, + "p(90)": 269.5189, + "p(95)": 362.23875 + }, + "thresholds": { + "p(99)<500": { + "ok": false + }, + "p(95)<200": { + "ok": false + } + }, + "type": "trend" + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 10207627, + "rate": 56378.590750850766 + } + }, + "measured_duration": { + "thresholds": { + "p(99)<500": { + "ok": false + }, + "p(95)<200": { + "ok": false + } + }, + "type": "trend", + "contains": "time", + "values": { + "p(90)": 258.24661999999995, + "p(95)": 362.03845, + "avg": 226.20558348457357, + "min": 111.1337, + "med": 192.21555, + "max": 3316.7455 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "max": 1477.6435, + "p(90)": 246.01572000000007, + "p(95)": 278.9438899999998, + "avg": 204.86214977334555, + "min": 109.1615, + "med": 189.6664 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 1102, + "rate": 6.08654754013225 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 226.65930571169542, + "min": 111.1337, + "med": 192.2361, + "max": 3316.7455, + "p(90)": 258.65104, + "p(95)": 362.6146499999999 + } + }, + "http_req_blocked": { + "contains": "time", + "values": { + "max": 128.3212, + "p(90)": 0, + "p(95)": 0, + "avg": 0.321267815049864, + "min": 0, + "med": 0 + }, + "type": "trend" + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 170.0405, + "med": 183.3613, + "max": 931.3722, + "p(90)": 227.3473, + "p(95)": 244.928, + "avg": 201.63510211480371 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 262.4194622356497, + "min": 178.3248, + "med": 191.0472, + "max": 3316.7455, + "p(90)": 322.0324, + "p(95)": 454.4524 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-3vus-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-3vus-cloudwatch.json new file mode 100644 index 0000000..c7cab4c --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-3vus-cloudwatch.json @@ -0,0 +1,227 @@ +{ + "startTime": "2026-09-10T14:52:04Z", + "endTime": "2026-09-10T14:55:06Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.3216822121114268, + "p90": 0.15344186149861608, + "p50": 0.08822177870264457, + "p95": 0.178307124267672 + } + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.275133227621956, + "p90": 0.14913721967881835, + "p50": 0.08592872289167662, + "p95": 0.1697540344935103 + } + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.262079332238483, + "p90": 0.14766303987496063, + "p50": 0.0901778833758862, + "p95": 0.17626883014707334 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 4.248351162920396, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 5.800196806589763, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 4.955207506815593, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 3.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 3.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 3.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 22.277434093145022, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 57.641666666666666, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 55.647176411794106, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 89739264.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 90087424.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 87236608.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 0.000273972602739726, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 0.002631578947368421, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 0.0005263157894736842, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 0.008511705685618728, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 0.000625, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-3vus-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-3vus-summary.json new file mode 100644 index 0000000..4e9acf9 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-3vus-summary.json @@ -0,0 +1,492 @@ +{ + "root_group": { + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 482, + "fails": 0 + }, + { + "passes": 482, + "fails": 0, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c" + }, + { + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 482, + "fails": 0, + "name": "cost_summary: response shape is valid" + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 482, + "fails": 0 + }, + { + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 401, + "fails": 0 + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 401, + "fails": 0 + }, + { + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 401, + "fails": 0 + }, + { + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 401, + "fails": 0, + "name": "cost_by_service: no auth or server error" + }, + { + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 481, + "fails": 0, + "name": "cost_timeseries: status is 200" + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 481, + "fails": 0 + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 481, + "fails": 0 + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 481, + "fails": 0 + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 160, + "fails": 0 + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 160, + "fails": 0 + }, + { + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 160, + "fails": 0, + "name": "aws_account_list: response shape is valid" + }, + { + "fails": 0, + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 160 + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 80, + "fails": 0 + }, + { + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 80, + "fails": 0 + }, + { + "passes": 80, + "fails": 0, + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc" + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 80, + "fails": 0 + } + ], + "name": "" + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdErrTTY": true, + "testRunDurationMs": 180977.8361, + "isStdOutTTY": true + }, + "metrics": { + "measured_duration": { + "thresholds": { + "p(99)<500": { + "ok": false + }, + "p(95)<200": { + "ok": false + } + }, + "type": "trend", + "contains": "time", + "values": { + "max": 1223.4112, + "p(90)": 314.64201, + "p(95)": 376.4216349999997, + "avg": 236.23136197007494, + "min": 111.767, + "med": 213.53379999999999 + } + }, + "measured_failures": { + "thresholds": { + "rate<0.01": { + "ok": true + } + }, + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 1604 + } + }, + "endpoint_sync_history_duration": { + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time", + "values": { + "avg": 131.49302375, + "min": 115.1849, + "med": 122.44595, + "max": 583.3741, + "p(90)": 135.3736, + "p(95)": 148.8091 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 236.5069182554519, + "min": 111.767, + "med": 213.5847, + "max": 1223.4112, + "p(90)": 314.74686, + "p(95)": 377.31175999999994 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.28470255451713394, + "min": 0, + "med": 0, + "max": 114.5268, + "p(90)": 0, + "p(95)": 0 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 113.69002, + "p(95)": 115.72668, + "avg": 25.1699171339564, + "min": 0, + "med": 0.5669, + "max": 1015.2661 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "min": 109.9895, + "med": 204.6345, + "max": 874.686, + "p(90)": 268.07598, + "p(95)": 297.1818399999999, + "avg": 211.33674423676 + } + }, + "data_sent": { + "values": { + "rate": 5686.795809799165, + "count": 1029184 + }, + "type": "counter", + "contains": "data" + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 1605, + "rate": 8.86848928347862 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "rate": 8.86296374498424, + "count": 1604 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.00025688473520249223, + "min": 0, + "med": 0, + "max": 0.4123, + "p(90)": 0, + "p(95)": 0 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "passes": 6416, + "fails": 0, + "rate": 1 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "max": 3, + "value": 3, + "min": 3 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 3, + "min": 3, + "max": 3 + } + }, + "http_req_duration": { + "contains": "time", + "values": { + "p(90)": 314.74686, + "p(95)": 377.31175999999994, + "avg": 236.5069182554519, + "min": 111.767, + "med": 213.5847, + "max": 1223.4112 + }, + "type": "trend" + }, + "http_req_failed": { + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 1605 + }, + "type": "rate" + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 14840765, + "rate": 82003.21829353557 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 291.92862993762975, + "min": 179.9106, + "med": 240.5608, + "max": 1223.4112, + "p(90)": 405.7211, + "p(95)": 590.2395 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 336.9036885286786, + "min": 212.1592, + "med": 313.86969999999997, + "max": 1323.9538, + "p(90)": 415.08569, + "p(95)": 477.61089 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 172.7006, + "med": 205.39325, + "max": 554.7547, + "p(90)": 255.6989, + "p(95)": 276.23161, + "avg": 216.18656639004152 + }, + "thresholds": { + "p(99)<500": { + "ok": true + }, + "p(95)<200": { + "ok": false + } + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0 + } + }, + "endpoint_aws_account_list_duration": { + "values": { + "max": 786.5083, + "p(90)": 139.15415000000002, + "p(95)": 197.9365599999998, + "avg": 141.850551875, + "min": 111.767, + "med": 120.71719999999999 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": false + } + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 237.589, + "max": 1180.7206, + "p(90)": 291.9862, + "p(95)": 333.3501, + "avg": 252.0697715710724, + "min": 203.0169 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "measured_requests": { + "values": { + "count": 1604, + "rate": 8.86296374498424 + }, + "type": "counter", + "contains": "default" + }, + "http_req_blocked": { + "values": { + "avg": 0.29025925233644856, + "min": 0, + "med": 0, + "max": 123.2969, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend", + "contains": "time" + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-45vus-post-rollup-soak-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-45vus-post-rollup-soak-cloudwatch.json new file mode 100644 index 0000000..d48d3e1 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-45vus-post-rollup-soak-cloudwatch.json @@ -0,0 +1,859 @@ +{ + "startTime": "2026-09-10T21:07:35Z", + "endTime": "2026-09-10T21:22:36Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Sum": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Sum": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Sum": 1.0, + "Unit": "Count" + } + ] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.22696422849622452, + "p90": 0.03841808723311325, + "p50": 0.012767480580308825, + "p95": 0.08663686471205145 + } + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.12127545537418484, + "p90": 0.034861958298139366, + "p50": 0.01334601622259427, + "p95": 0.066006978014974 + } + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.15553184370961143, + "p90": 0.046290560453668486, + "p50": 0.014499876365229116, + "p95": 0.08189163656668233 + } + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.14957885435136659, + "p90": 0.055556492335232244, + "p50": 0.014669070692200284, + "p95": 0.0892821045873521 + } + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.13140495969438273, + "p90": 0.0327728163515166, + "p50": 0.013030088230694301, + "p95": 0.06352527315394052 + } + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1394391257218174, + "p90": 0.05100416210263806, + "p50": 0.013983387667723654, + "p95": 0.08133548737421174 + } + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1304723344686426, + "p90": 0.03894339707974348, + "p50": 0.014039776647280941, + "p95": 0.0682588712314812 + } + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1292886727605574, + "p90": 0.0355498927717731, + "p50": 0.01330494667270981, + "p95": 0.06552079636554493 + } + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.12057463938746758, + "p90": 0.05183884785424821, + "p50": 0.014719794287451161, + "p95": 0.08010717484458531 + } + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.12272407084884956, + "p90": 0.02965406917127382, + "p50": 0.01293879627331918, + "p95": 0.05889423773359602 + } + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.11532587931874197, + "p90": 0.04470771790836263, + "p50": 0.01461153690611638, + "p95": 0.07023591080244401 + } + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.16658662201219457, + "p90": 0.05131801693381506, + "p50": 0.015001231002625093, + "p95": 0.08203423009262112 + } + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1429754692803315, + "p90": 0.05179071890944367, + "p50": 0.015222384966804284, + "p95": 0.08382716786850573 + } + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.16332998520837083, + "p90": 0.05782130915813708, + "p50": 0.014950578453640933, + "p95": 0.08989301919448041 + } + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.14521047708988097, + "p90": 0.03558024893885173, + "p50": 0.01367234586144698, + "p95": 0.07367894105208644 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 19.006610927482445, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 71.08097585042317, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 75.06921895345052, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 73.82374827067058, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 66.42612202962239, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 72.2046381632487, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 72.75210571289062, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 67.82080332438152, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 74.06728871663411, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 63.4721425374349, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 75.20867665608723, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 74.73727671305339, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 76.2130126953125, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 75.22903442382812, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 64.43630854288737, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 6.901041666666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 6.884765625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 6.901041666666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 6.884765625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 10.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 3.7185259296314825, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 48.449999999999996, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 60.64164709409825, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 66.40318842029782, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 58.09166666666666, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 58.19166666666666, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 63.633333333333326, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 57.08074430691831, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 64.2393823681446, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 55.93333333333334, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 62.0054967935371, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 64.52311897856451, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 65.89851761684814, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 66.38848633251712, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 66.62944196279751, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 116.5389658, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 111.45983285, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 106.11632803333333, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 100311040.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 94199808.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 91463680.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 87752704.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 83718144.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 94822400.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 93990912.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 91389952.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 86355968.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 84824064.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 102236160.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 95895552.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 94117888.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 88563712.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 87482368.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 0.001, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 0.0025, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 0.0036075949367088606, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 0.0007142857142857143, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 8.064516129032258e-05, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 0.002, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 0.0008333333333333334, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 0.0063686131386861316, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 0.00020833333333333335, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 0.009750889679715304, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 0.0005, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 0.000425531914893617, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 0.00036363636363636367, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 0.00020833333333333335, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 0.007356115107913669, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 0.0004347826086956522, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 0.00017857142857142857, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-45vus-post-rollup-soak-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-45vus-post-rollup-soak-summary.json new file mode 100644 index 0000000..1346578 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-45vus-post-rollup-soak-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 38485, + "fails": 13, + "name": "cost_by_service: status is 200" + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 38498, + "fails": 0 + }, + { + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 38485, + "fails": 13 + }, + { + "fails": 13, + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 38485 + }, + { + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 46183, + "fails": 14, + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200" + }, + { + "fails": 1, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 46196 + }, + { + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 46183, + "fails": 14, + "name": "cost_summary: response shape is valid" + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 46183, + "fails": 14 + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 15393, + "fails": 6 + }, + { + "passes": 15398, + "fails": 1, + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516" + }, + { + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 15393, + "fails": 6, + "name": "aws_account_list: response shape is valid" + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 15393, + "fails": 6 + }, + { + "fails": 15, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 46183 + }, + { + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 46197, + "fails": 1, + "name": "cost_timeseries: response is JSON" + }, + { + "passes": 46183, + "fails": 15, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131" + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 46183, + "fails": 15 + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 7698, + "fails": 2 + }, + { + "passes": 7700, + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe" + }, + { + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 7698, + "fails": 2 + }, + { + "passes": 7698, + "fails": 2, + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d" + } + ], + "name": "" + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 901061.1801 + }, + "metrics": { + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 6710.6686, + "p(90)": 344.09765999999996, + "p(95)": 360.70271999999994, + "avg": 263.03108682269135, + "min": 205.1959, + "med": 232.58125 + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.037130415668244675, + "min": 0, + "med": 0, + "max": 137.8727, + "p(90)": 0, + "p(95)": 0 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "passes": 50, + "fails": 153943, + "rate": 0.0003246900833154754 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 195.43402, + "avg": 140.52127147433748, + "min": 110.9487, + "med": 130.1859, + "max": 1631.5384, + "p(90)": 162.82008000000002 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "avg": 9.886228594806257E-05, + "min": 0, + "med": 0, + "max": 0.6337, + "p(90)": 0, + "p(95)": 0 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 98778449, + "rate": 109624.57509160203 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 260.1895499999999, + "avg": 162.57455058379756, + "min": 105.1924, + "med": 132.15019999999998, + "max": 6610.4124, + "p(90)": 243.62148 + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 162.57810323131667, + "min": 105.1924, + "med": 132.1503, + "max": 6610.4124, + "p(90)": 243.62272000000002, + "p(95)": 260.2007199999999 + } + }, + "http_req_waiting": { + "contains": "time", + "values": { + "p(90)": 164.22142000000002, + "p(95)": 198.62101999999993, + "avg": 142.03721965803447, + "min": 105.1924, + "med": 130.9389, + "max": 1742.544 + }, + "type": "trend" + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 141.46480000000003, + "max": 6610.4124, + "p(90)": 276.0128, + "p(95)": 373.121465, + "avg": 208.20745710853464, + "min": 113.3273 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 153993, + "rate": 170.90182487154738 + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "rate": 170.90071506899224, + "count": 153992 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "med": 132.1538, + "max": 6610.4124, + "p(90)": 243.63308000000004, + "p(95)": 260.22270999999995, + "avg": 162.59244396172707, + "min": 105.1924 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 1425079623, + "rate": 1581557.0068636674 + } + }, + "endpoint_aws_account_list_duration": { + "contains": "time", + "values": { + "avg": 148.64191929995468, + "min": 105.1924, + "med": 125.9046, + "max": 3462.7897, + "p(90)": 165.69952000000004, + "p(95)": 221.98865999999987 + }, + "type": "trend" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0.9997516104732713, + "passes": 615815, + "fails": 153 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "iterations": { + "contains": "default", + "values": { + "count": 153992, + "rate": 170.90071506899224 + }, + "type": "counter" + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 110.8367, + "med": 123.70125, + "max": 1638.7445, + "p(90)": 152.17794, + "p(95)": 191.6138749999998, + "avg": 139.21970505194807 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.03706307299682454, + "min": 0, + "med": 0, + "max": 131.9592, + "p(90)": 0, + "p(95)": 0 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0.00032469219180217154, + "passes": 50, + "fails": 153942 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 6469.2012, + "p(90)": 112.06322000000002, + "p(95)": 118.33564, + "avg": 20.5407847109933 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "max": 45, + "value": 16, + "min": 16 + } + }, + "vus_max": { + "values": { + "max": 45, + "value": 45, + "min": 45 + }, + "type": "gauge", + "contains": "default" + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 144.5223717465835, + "min": 112.9334, + "med": 131.18675, + "max": 1823.9798, + "p(90)": 167.54245, + "p(95)": 202.11836000000005 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-cloudwatch.json new file mode 100644 index 0000000..5074f01 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-cloudwatch.json @@ -0,0 +1,233 @@ +{ + "startTime": "2026-09-10T15:05:09Z", + "endTime": "2026-09-10T15:08:11Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.5330995033881941, + "p90": 0.20571109858608008, + "p50": 0.11230603773633399, + "p95": 0.24711965236213634 + } + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.33188158154488734, + "p90": 0.1878762817176436, + "p50": 0.10822247509484144, + "p95": 0.21563325793715646 + } + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.3072971678957348, + "p90": 0.19351794759431754, + "p50": 0.1084316882746557, + "p95": 0.2143750914823625 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 4.707579294840495, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 6.98672358194987, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 6.7127617200215655, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 6.673177083333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 6.689453125, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 6.73828125, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 4.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 3.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 4.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 21.3753562559376, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 64.99808240649648, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 65.79100797921073, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 117.83615691666667, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 94597120.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 95064064.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 77701120.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 0.002, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 0.00013793103448275863, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 0.002, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 0.00025316455696202533, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..2aa10a0 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-post-rollup-cloudwatch.json @@ -0,0 +1,233 @@ +{ + "startTime": "2026-09-10T16:59:43Z", + "endTime": "2026-09-10T17:02:45Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.10853015813738111, + "p90": 0.014967646517032092, + "p50": 0.011690917322607587, + "p95": 0.017136104420685527 + } + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.15150425628671077, + "p90": 0.016295435084666084, + "p50": 0.011824906477696875, + "p95": 0.024563172947103407 + } + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.04178329584805352, + "p90": 0.015012652418217379, + "p50": 0.011604221704336868, + "p95": 0.01809424324589384 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 5.039510101079941, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 10.869722684224447, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 10.522063573201498, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 4.182942708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 5.696614583333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 6.070963541666667, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 0.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 2.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 2.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 3.4934133733533437, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 8.95, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 9.625, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 127.73084505, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 100794368.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 88588288.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 80076800.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 0.00022988505747126436, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 0.0002786377708978328, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 0.00145985401459854, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 0.0002, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 0.005596026490066225, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-post-rollup-summary.json new file mode 100644 index 0000000..7960265 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-post-rollup-summary.json @@ -0,0 +1,492 @@ +{ + "root_group": { + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 715, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 715 + }, + { + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 715, + "fails": 0 + }, + { + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 715, + "fails": 0, + "name": "cost_by_service: no auth or server error" + }, + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 859, + "fails": 0 + }, + { + "passes": 859, + "fails": 0, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c" + }, + { + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 859, + "fails": 0, + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid" + }, + { + "passes": 859, + "fails": 0, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f" + }, + { + "passes": 860, + "fails": 0, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa" + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 860, + "fails": 0 + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 860, + "fails": 0 + }, + { + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 860, + "fails": 0, + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error" + }, + { + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 286, + "fails": 0, + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200" + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 286, + "fails": 0 + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 286, + "fails": 0 + }, + { + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 286, + "fails": 0, + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error" + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 144, + "fails": 0 + }, + { + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 144, + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON" + }, + { + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 144, + "fails": 0 + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 144, + "fails": 0 + } + ], + "name": "", + "path": "" + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 181058.8887 + }, + "metrics": { + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 2864, + "rate": 15.818057984136958 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 2864, + "rate": 15.818057984136958 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 248.50261999999992, + "avg": 151.18177898778413, + "min": 109.0623, + "med": 127.4804, + "max": 2717.0338, + "p(90)": 240.85442 + } + }, + "http_req_failed": { + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 2865 + }, + "type": "rate" + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 240.85442, + "p(95)": 248.50261999999992, + "avg": 151.18177898778413, + "min": 109.0623, + "med": 127.4804, + "max": 2717.0338 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 119.0387, + "p(90)": 0, + "p(95)": 0, + "avg": 0.2013464572425829 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "fails": 0, + "rate": 1, + "passes": 11456 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 132.18870181818156, + "min": 114.6623, + "med": 126.7956, + "max": 830.597, + "p(90)": 135.99886, + "p(95)": 142.25488 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "endpoint_cost_summary_duration": { + "contains": "time", + "values": { + "max": 836.0096, + "p(90)": 134.83686, + "p(95)": 143.32411, + "avg": 132.18542409778811, + "min": 113.6895, + "med": 125.277 + }, + "thresholds": { + "p(99)<500": { + "ok": true + }, + "p(95)<200": { + "ok": true + } + }, + "type": "trend" + }, + "http_req_receiving": { + "contains": "time", + "values": { + "avg": 20.356390541012203, + "min": 0, + "med": 0.5418, + "max": 2062.5126, + "p(90)": 112.49145999999999, + "p(95)": 116.43809999999996 + }, + "type": "trend" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 1837389, + "rate": 10148.018764460692 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 350.005825, + "avg": 251.59314270251315, + "min": 209.3998, + "med": 227.97475, + "max": 2817.3962, + "p(90)": 341.48843000000005 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 4, + "min": 4, + "max": 4 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 760.5899, + "p(90)": 126.71690000000001, + "p(95)": 135.55020000000002, + "avg": 131.2874493006994, + "min": 109.0623, + "med": 119.9335 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "avg": 130.82538844677123, + "min": 109.0623, + "med": 126.1781, + "max": 830.0697, + "p(90)": 134.30367999999999, + "p(95)": 141.82551999999998 + } + }, + "endpoint_sync_history_duration": { + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time", + "values": { + "max": 615.5881, + "p(90)": 127.61873000000001, + "p(95)": 134.280035, + "avg": 126.67868611111112, + "min": 115.5742, + "med": 119.7502 + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "max": 125.5202, + "p(90)": 0, + "p(95)": 0, + "avg": 0.2049466317626527, + "min": 0, + "med": 0 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 26529049, + "rate": 146521.66038617687 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 2864 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 150.98240593575477, + "min": 109.0623, + "med": 127.4751, + "max": 2717.0338, + "p(90)": 240.84785, + "p(95)": 248.24059499999998 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 118.2723, + "med": 133.357, + "max": 2717.0338, + "p(90)": 252.21209000000002, + "p(95)": 346.2328099999998, + "avg": 196.00170720930205 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 2865, + "rate": 15.823581049075553 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 1, + "min": 1, + "max": 4 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-summary.json new file mode 100644 index 0000000..3192246 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-summary.json @@ -0,0 +1,492 @@ +{ + "root_group": { + "groups": [], + "checks": [ + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 567, + "fails": 0 + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 567, + "fails": 0 + }, + { + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 567, + "fails": 0, + "name": "cost_summary: response shape is valid" + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 567, + "fails": 0 + }, + { + "passes": 473, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b" + }, + { + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 473, + "fails": 0, + "name": "cost_by_service: response is JSON" + }, + { + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 473, + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid" + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 473, + "fails": 0 + }, + { + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 568, + "fails": 0, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200" + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 568, + "fails": 0 + }, + { + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 568, + "fails": 0, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid" + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 568, + "fails": 0 + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 189, + "fails": 0 + }, + { + "passes": 189, + "fails": 0, + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516" + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 189, + "fails": 0 + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 189, + "fails": 0 + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 95, + "fails": 0 + }, + { + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 95, + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON" + }, + { + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 95, + "fails": 0 + }, + { + "fails": 0, + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 95 + } + ], + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e" + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 181152.8998 + }, + "metrics": { + "checks": { + "contains": "default", + "values": { + "rate": 1, + "passes": 7568, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate" + }, + "http_reqs": { + "values": { + "count": 1893, + "rate": 10.449736118438883 + }, + "type": "counter", + "contains": "default" + }, + "http_req_duration{expected_response:true}": { + "values": { + "min": 109.7353, + "med": 253.0464, + "max": 1848.1695, + "p(90)": 392.87459999999993, + "p(95)": 498.0584399999997, + "avg": 280.45375324881155 + }, + "type": "trend", + "contains": "time" + }, + "http_req_waiting": { + "contains": "time", + "values": { + "p(90)": 317.15079999999995, + "p(95)": 352.3152999999997, + "avg": 239.33004352879004, + "min": 109.7353, + "med": 228.5054, + "max": 1016.7855 + }, + "type": "trend" + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 1213904, + "rate": 6700.991269475665 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 1892 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 17525377, + "rate": 96743.56314112947 + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 997.2257, + "p(90)": 336.24538, + "p(95)": 396.10353999999984, + "avg": 281.8976847780127, + "min": 202.4459, + "med": 260.235 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 1892, + "rate": 10.444215919749798 + } + }, + "endpoint_cost_summary_duration": { + "values": { + "avg": 242.40121763668418, + "min": 176.2123, + "med": 230.3304, + "max": 748.1477, + "p(90)": 298.06056, + "p(95)": 333.2647599999999 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + }, + "type": "trend", + "contains": "time" + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0.311429265715795, + "min": 0, + "med": 0, + "max": 121.1994, + "p(90)": 0 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 379.31989630281686, + "min": 185.7742, + "med": 333.6333, + "max": 1848.1695, + "p(90)": 496.58567, + "p(95)": 732.1242349999997 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 4, + "min": 4, + "max": 4 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 41.12370972002117, + "min": 0, + "med": 0.8435, + "max": 1542.8107, + "p(90)": 118.36088, + "p(95)": 125.69785999999998 + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 1892, + "rate": 10.444215919749798 + } + }, + "http_req_connecting": { + "contains": "time", + "values": { + "avg": 0.30842921288959324, + "min": 0, + "med": 0, + "max": 119.2845, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend" + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "fails": 1893, + "rate": 0, + "passes": 0 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 214.3145, + "med": 353.30425, + "max": 1948.608, + "p(90)": 493.588, + "p(95)": 598.6581749999995, + "avg": 380.90038319238965 + } + }, + "endpoint_aws_account_list_duration": { + "values": { + "avg": 158.1309185185185, + "min": 109.7353, + "med": 126.4681, + "max": 812.4448, + "p(90)": 147.97592000000003, + "p(95)": 211.26947999999962 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + }, + "type": "trend", + "contains": "time" + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 280.45375324881155, + "min": 109.7353, + "med": 253.0464, + "max": 1848.1695, + "p(90)": 392.87459999999993, + "p(95)": 498.0584399999997 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 4 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 253.0396, + "max": 1848.1695, + "p(90)": 392.0442500000001, + "p(95)": 496.69095499999986, + "avg": 280.2357935517972, + "min": 109.7353 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "endpoint_sync_history_duration": { + "contains": "time", + "values": { + "max": 762.6455, + "p(90)": 144.69596, + "p(95)": 155.57877, + "avg": 148.2798042105263, + "min": 115.0667, + "med": 126.1415 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": false + } + }, + "type": "trend" + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..2b3414e --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-cloudwatch.json @@ -0,0 +1,239 @@ +{ + "startTime": "2026-09-10T18:38:03Z", + "endTime": "2026-09-10T18:41:05Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Sum": 1.0, + "Unit": "Count" + } + ] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.16124677999101966, + "p90": 0.04288549457706691, + "p50": 0.01335367132537653, + "p95": 0.08024803408355827 + } + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.13391357113303803, + "p90": 0.059012790514521074, + "p50": 0.014505140989362704, + "p95": 0.08803598019997966 + } + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1463262010119467, + "p90": 0.049828100975514794, + "p50": 0.013342769202778854, + "p95": 0.08662856869215846 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 53.25306828816732, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 76.05701700846355, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 68.35044860839844, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 6.673177083333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 6.73828125, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 6.803385416666667, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 10.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 22.12736155808641, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 60.85, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 62.600989488764135, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 134.26899898333335, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 71598080.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 73609216.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 86646784.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 0.0010695187165775401, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 0.00033018867924528304, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 0.0001282051282051282, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 0.00014705882352941178, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 0.004074074074074074, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 0.000425531914893617, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-soak-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-soak-cloudwatch.json new file mode 100644 index 0000000..cd06ee8 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-soak-cloudwatch.json @@ -0,0 +1,843 @@ +{ + "startTime": "2026-09-10T20:40:37Z", + "endTime": "2026-09-10T20:55:39Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.26547326186521625, + "p90": 0.09294158986409308, + "p50": 0.016536228178670235, + "p95": 0.12747345451562245 + } + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.17346124005650101, + "p90": 0.07507393920092932, + "p50": 0.016285825444213125, + "p95": 0.10550231332342269 + } + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.14881264126091515, + "p90": 0.05625411187867282, + "p50": 0.01394742836887384, + "p95": 0.08522614579148616 + } + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1619191384153045, + "p90": 0.06312529991127364, + "p50": 0.014618052565306072, + "p95": 0.09654172274626481 + } + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.13082859473711783, + "p90": 0.05421661827612759, + "p50": 0.014845335099832178, + "p95": 0.08558981252585243 + } + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.17898169761226565, + "p90": 0.07728137813055257, + "p50": 0.016322467216005537, + "p95": 0.10429778298826516 + } + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.13536103384583714, + "p90": 0.051090133310175305, + "p50": 0.013822467752679679, + "p95": 0.08097465215572476 + } + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1444790046329604, + "p90": 0.05438658564937058, + "p50": 0.014962453538007492, + "p95": 0.08505956109290651 + } + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1717701376998449, + "p90": 0.07624222907770911, + "p50": 0.015347009431461618, + "p95": 0.10705705545556463 + } + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.19530985855272281, + "p90": 0.06968408185596575, + "p50": 0.015741647860514525, + "p95": 0.09830719195689931 + } + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.17184858624088614, + "p90": 0.06435065183355768, + "p50": 0.01423167976933858, + "p95": 0.09664364288086877 + } + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.16999014648970473, + "p90": 0.07183276565896418, + "p50": 0.015850197658450565, + "p95": 0.10381483079363371 + } + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1350000009605071, + "p90": 0.05838512388694818, + "p50": 0.01518167924987536, + "p95": 0.08763211211987929 + } + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.15774214322854335, + "p90": 0.05221038945771639, + "p50": 0.014080250164527998, + "p95": 0.09186672384761485 + } + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.12448582699691704, + "p90": 0.03866099536963388, + "p50": 0.013520679615344857, + "p95": 0.07188973995583374 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 27.78227763498823, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 82.95702616373698, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 72.49641672770183, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 78.04808044433594, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 75.48256174723308, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 82.08821360270183, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 67.92955780029297, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 74.08062489827473, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 82.41480000813802, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 81.00458526611328, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 76.92803700764973, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 83.85089111328125, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 79.1140874226888, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 79.1009521484375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 68.52549235026042, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 6.868489583333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 6.884765625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 6.901041666666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 0.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 10.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 3.991666666666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 58.475, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 66.0, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 64.25953689821756, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 68.87610471902617, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 71.18940529735131, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 65.90833333333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 66.54994163748542, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 70.18040678982526, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 65.49832419002517, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 70.44401480049335, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 67.43333333333334, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 67.6, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 69.98167582875227, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 64.27392876785387, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 135.46236903333335, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 129.5748806, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 123.84171361666667, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 106991616.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 81727488.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 85204992.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 85090304.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 93011968.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 88014848.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 85815296.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 84688896.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 100077568.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 94621696.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 91594752.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 89759744.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 88977408.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 92823552.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 90112000.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 0.0002054794520547945, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 0.0029896907216494842, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 0.0009090909090909091, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 0.0006666666666666666, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 0.0002, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 0.0005660377358490566, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 0.008998211091234347, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 0.0023076923076923075, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 0.0004166666666666667, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 0.006395759717314488, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 0.0005555555555555556, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 0.00028169014084507044, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 0.0004347826086956522, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 0.0035106382978723405, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-soak-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-soak-summary.json new file mode 100644 index 0000000..36ca1c1 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-soak-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "groups": [], + "checks": [ + { + "passes": 50785, + "fails": 28, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa" + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 50813, + "fails": 0 + }, + { + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 50785, + "fails": 28, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid" + }, + { + "passes": 50785, + "fails": 28, + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7" + }, + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 50785, + "fails": 29 + }, + { + "fails": 0, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 50814 + }, + { + "fails": 29, + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 50785 + }, + { + "passes": 50785, + "fails": 29, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f" + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 8465, + "fails": 4 + }, + { + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 8469, + "fails": 0, + "name": "sync_history: response is JSON" + }, + { + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 8465, + "fails": 4, + "name": "sync_history: response shape is valid" + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 8465, + "fails": 4 + }, + { + "fails": 9, + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 16928 + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 16937, + "fails": 0 + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 16928, + "fails": 9 + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 16928, + "fails": 9 + }, + { + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 42321, + "fails": 23 + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 42344, + "fails": 0 + }, + { + "passes": 42321, + "fails": 23, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937" + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 42321, + "fails": 23 + } + ], + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e" + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 901347.467 + }, + "metrics": { + "iterations": { + "values": { + "count": 169377, + "rate": 187.91532256006218 + }, + "type": "counter", + "contains": "default" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + } + }, + "endpoint_aws_account_list_duration": { + "contains": "time", + "values": { + "avg": 150.59239438507305, + "min": 95.295, + "med": 122.2539, + "max": 2913.1937, + "p(90)": 185.20208, + "p(95)": 237.59875999999994 + }, + "type": "trend" + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "fails": 169284, + "rate": 0.0005490710072796188, + "passes": 93 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_req_connecting": { + "values": { + "avg": 0.03584236264449928, + "min": 0, + "med": 0, + "max": 123.5881, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend", + "contains": "time" + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 3838.8797, + "p(90)": 242.02646000000001, + "p(95)": 280.06849499999987, + "avg": 165.27676774374635, + "min": 95.295, + "med": 129.62754999999999 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 280.03352, + "avg": 165.28775312402323, + "min": 95.295, + "med": 129.6361, + "max": 3838.8797, + "p(90)": 242.03010000000003 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "rate": 1738643.3704816746, + "count": 1567121798 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 195.3291, + "med": 230.0238, + "max": 3938.9685, + "p(90)": 342.47162, + "p(95)": 380.4692599999996, + "avg": 265.7277010024966 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "fails": 169285, + "rate": 0.0005490677655893918, + "passes": 93 + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 169377, + "rate": 187.91532256006218 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "max": 0.5196, + "p(90)": 0, + "p(95)": 0, + "avg": 5.200025977399663E-05, + "min": 0, + "med": 0 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 217.95698016846194, + "min": 106.7566, + "med": 163.8585, + "max": 3838.8797, + "p(90)": 314.913, + "p(95)": 411.22083999999944 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "med": 127.5148, + "max": 1771.7581, + "p(90)": 177.84153000000003, + "p(95)": 212.58007499999994, + "avg": 141.04151085796562, + "min": 95.295 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 165.2739052244418, + "min": 95.295, + "med": 129.6275, + "max": 3838.8797, + "p(90)": 242.02622, + "p(95)": 280.06283999999994 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0.9995881967445402, + "passes": 677229, + "fails": 279 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 108647176, + "rate": 120538.6157700269 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 8, + "min": 8, + "max": 50 + } + }, + "http_req_blocked": { + "values": { + "med": 0, + "max": 123.5881, + "p(90)": 0, + "p(95)": 0, + "avg": 0.03594038245817049, + "min": 0 + }, + "type": "trend", + "contains": "time" + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 169378, + "rate": 187.91643201012067 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 98.3583, + "med": 121.1113, + "max": 1715.5437, + "p(90)": 170.04915999999997, + "p(95)": 210.82805999999988, + "avg": 141.09344468059976 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 24.235204885522197, + "min": 0, + "med": 0, + "max": 3618.1492, + "p(90)": 106.37881000000002, + "p(95)": 117.50994999999999 + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 215.883825, + "avg": 143.8517163258072, + "min": 104.0605, + "med": 128.3699, + "max": 1772.5991, + "p(90)": 181.64988000000002 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 50, + "min": 50, + "max": 50 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 127.48294999999999, + "max": 1487.4188, + "p(90)": 175.54750000000016, + "p(95)": 209.204255, + "avg": 139.3668943421095, + "min": 101.9511 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-summary.json new file mode 100644 index 0000000..f77a478 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "passes": 9046, + "fails": 0, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa" + }, + { + "passes": 9046, + "fails": 0, + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979" + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 9046, + "fails": 0 + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 9046, + "fails": 0 + }, + { + "fails": 0, + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 3015 + }, + { + "passes": 3015, + "fails": 0, + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516" + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 3015, + "fails": 0 + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 3015, + "fails": 0 + }, + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 9047, + "fails": 0 + }, + { + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 9047, + "fails": 0, + "name": "cost_summary: response is JSON" + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 9047, + "fails": 0 + }, + { + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 9047, + "fails": 0, + "name": "cost_summary: no auth or server error" + }, + { + "passes": 7537, + "fails": 1, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b" + }, + { + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 7537, + "fails": 1, + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON" + }, + { + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 7537, + "fails": 1, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid" + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 7537, + "fails": 1 + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 1508, + "fails": 0 + }, + { + "passes": 1508, + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe" + }, + { + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 1508, + "fails": 0 + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 1508, + "fails": 0 + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 181080.376 + }, + "metrics": { + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "min": 107.9767, + "med": 130.4875, + "max": 1792.5767, + "p(90)": 180.04752000000005, + "p(95)": 218.51911999999996, + "avg": 148.4589563820268 + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 153.54279737330873, + "min": 115.1806, + "med": 130.7686, + "max": 1831.0342, + "p(90)": 185.28279000000003, + "p(95)": 226.4119699999999 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0 + } + }, + "http_req_connecting": { + "values": { + "avg": 0.2033668380036479, + "min": 0, + "med": 0, + "max": 128.3665, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend", + "contains": "time" + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 279126831, + "rate": 1541452.6806593332 + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 30154, + "rate": 166.5227379470429 + } + }, + "http_req_receiving": { + "contains": "time", + "values": { + "avg": 49.614278690101166, + "min": 0, + "med": 0.4015, + "max": 6062.7696, + "p(90)": 118.3341, + "p(95)": 228.39542999999998 + }, + "type": "trend" + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 128.3665, + "p(90)": 0, + "p(95)": 0, + "avg": 0.20368251699552323, + "min": 0 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 3.3161996352180403E-05, + "passes": 1, + "fails": 30154 + } + }, + "endpoint_cost_summary_duration": { + "values": { + "avg": 144.33511214767327, + "min": 114.3393, + "med": 129.8515, + "max": 1211.1026, + "p(90)": 176.2036000000001, + "p(95)": 209.34790999999993 + }, + "type": "trend", + "contains": "time" + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 0.7361, + "p(90)": 0, + "p(95)": 0, + "avg": 7.107279058199305E-05 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 107.9767, + "med": 125.5411, + "max": 2255.2562, + "p(90)": 239.3538, + "p(95)": 552.9870799999978, + "avg": 179.86870663349927 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 30154, + "rate": 166.5227379470429 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 198.0757285368436, + "min": 107.9767, + "med": 133.94335, + "max": 6205.0197, + "p(90)": 275.75085, + "p(95)": 397.13922499999967 + } + }, + "checks": { + "values": { + "rate": 0.9999668369038933, + "passes": 120612, + "fails": 4 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate", + "contains": "default" + }, + "measured_failures": { + "thresholds": { + "rate<0.01": { + "ok": true + } + }, + "type": "rate", + "contains": "default", + "values": { + "fails": 30153, + "rate": 3.316309610665252E-05, + "passes": 1 + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 107.9767, + "med": 133.9433, + "max": 6205.0197, + "p(90)": 275.7474000000001, + "p(95)": 397.12874999999985, + "avg": 198.07330614491732 + } + }, + "iteration_duration": { + "contains": "time", + "values": { + "p(90)": 376.19429, + "p(95)": 500.211154999998, + "avg": 298.6949059660431, + "min": 209.9327, + "med": 234.3594, + "max": 6305.9135 + }, + "type": "trend" + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 10, + "min": 10, + "max": 50 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 152.94021067639255, + "min": 109.4582, + "med": 123.84905, + "max": 2020.6022, + "p(90)": 178.74674999999993, + "p(95)": 237.7946249999996 + } + }, + "http_reqs": { + "values": { + "count": 30155, + "rate": 166.5282603566054 + }, + "type": "counter", + "contains": "default" + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 464.02755, + "p(95)": 782.7374, + "avg": 302.4648887685178, + "min": 114.8763, + "med": 246.51715, + "max": 6205.0197 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 19342569, + "rate": 106817.58800854268 + } + }, + "vus_max": { + "contains": "default", + "values": { + "value": 50, + "min": 50, + "max": 50 + }, + "type": "gauge" + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 198.05799357299142, + "min": 107.9767, + "med": 133.9426, + "max": 6205.0197, + "p(90)": 275.72208, + "p(95)": 397.04061499999995 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-51vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-51vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..42e1643 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-51vus-post-rollup-cloudwatch.json @@ -0,0 +1,227 @@ +{ + "startTime": "2026-09-10T20:17:40Z", + "endTime": "2026-09-10T20:20:42Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.2880415260002257, + "p90": 0.07948940468971274, + "p50": 0.018442464722367446, + "p95": 0.10820152849797136 + } + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1505583141337567, + "p90": 0.0730106115252081, + "p50": 0.018088536052210898, + "p95": 0.10037498725255879 + } + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.14906399169684623, + "p90": 0.06219236034512958, + "p50": 0.016825326654889554, + "p95": 0.08929295067891424 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 22.529106200595077, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 79.64221700032552, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 82.50581868489583, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 0.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 10.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 3.7731762981225745, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 55.76862150039186, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 73.39999999999999, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 104902656.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 78884864.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 78188544.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 0.005927272727272727, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 0.0013580246913580246, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-51vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-51vus-post-rollup-summary.json new file mode 100644 index 0000000..f902dbb --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-51vus-post-rollup-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 11233, + "fails": 0, + "name": "cost_summary: status is 200" + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 11233, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 11233 + }, + { + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 11233, + "fails": 0, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error" + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 1873, + "fails": 0 + }, + { + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 1873 + }, + { + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 1873, + "fails": 0, + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid" + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 1873, + "fails": 0 + }, + { + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 9360, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200" + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 9360, + "fails": 0 + }, + { + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 9360, + "fails": 0 + }, + { + "passes": 9360, + "fails": 0, + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2" + }, + { + "passes": 3743, + "fails": 0, + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c" + }, + { + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 3743, + "fails": 0, + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON" + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 3743, + "fails": 0 + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 3743, + "fails": 0 + }, + { + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 11233, + "fails": 0, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200" + }, + { + "passes": 11233, + "fails": 0, + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979" + }, + { + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 11233, + "fails": 0, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid" + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 11233, + "fails": 0 + } + ] + }, + "options": { + "summaryTimeUnit": "", + "noColor": false, + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ] + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 181052.8133 + }, + "metrics": { + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0.16349902518494777, + "min": 0, + "med": 0, + "max": 130.52 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 113.2606, + "med": 131.298, + "max": 548.9019, + "p(90)": 183.78562000000002, + "p(95)": 211.28477999999998, + "avg": 143.49691190243084 + } + }, + "endpoint_cost_by_service_duration": { + "contains": "time", + "values": { + "avg": 145.81175856837612, + "min": 112.9281, + "med": 133.13580000000002, + "max": 584.9632, + "p(90)": 187.61186000000004, + "p(95)": 215.67958499999992 + }, + "type": "trend" + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 37442 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "iterations": { + "values": { + "count": 37442, + "rate": 206.80153662102748 + }, + "type": "counter", + "contains": "default" + }, + "http_req_failed": { + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 37443 + }, + "type": "rate" + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "rate": 206.80705987129778, + "count": 37443 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 245.3787644196366, + "min": 208.3688, + "med": 233.0397, + "max": 1224.5342, + "p(90)": 286.15191, + "p(95)": 313.79029499999996 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 144.80129183537312, + "min": 108.2092, + "med": 132.58065, + "max": 1124.1081, + "p(90)": 185.71535, + "p(95)": 213.46366499999993 + } + }, + "data_sent": { + "contains": "data", + "values": { + "count": 24017470, + "rate": 132654.49766971392 + }, + "type": "counter" + }, + "http_req_blocked": { + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0.1638940175733782, + "min": 0, + "med": 0, + "max": 131.0549 + }, + "type": "trend" + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 149.79524967506393, + "min": 118.8314, + "med": 135.6302, + "max": 1124.1081, + "p(90)": 193.12204000000003, + "p(95)": 221.32451999999998 + } + }, + "http_req_duration": { + "contains": "time", + "values": { + "avg": 144.81406223593302, + "min": 108.2092, + "med": 132.5813, + "max": 1124.1081, + "p(90)": 185.72392000000002, + "p(95)": 213.5090700000001 + }, + "type": "trend" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 12, + "min": 12, + "max": 51 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "max": 1124.1081, + "p(90)": 185.72392000000002, + "p(95)": 213.5090700000001, + "avg": 144.81406223593302, + "min": 108.2092, + "med": 132.5813 + } + }, + "vus_max": { + "contains": "default", + "values": { + "value": 51, + "min": 51, + "max": 51 + }, + "type": "gauge" + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "avg": 5.44667895200705E-05, + "min": 0, + "med": 0, + "max": 0.5154, + "p(90)": 0, + "p(95)": 0 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 346605630, + "rate": 1914389.6395892126 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 149768, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "http_req_receiving": { + "contains": "time", + "values": { + "p(95)": 2.5885500000000006, + "avg": 0.8456026974334306, + "min": 0, + "med": 0, + "max": 974.5422, + "p(90)": 1.5779800000000004 + }, + "type": "trend" + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 37442, + "rate": 206.80153662102748 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "max": 644.1557, + "p(90)": 184.51766, + "p(95)": 212.19169000000008, + "avg": 143.9684050717098, + "min": 108.2092, + "med": 132.1555 + } + }, + "endpoint_aws_account_list_duration": { + "values": { + "max": 818.7096, + "p(90)": 168.78276000000014, + "p(95)": 195.50673999999995, + "avg": 135.9329380443495, + "min": 108.2092, + "med": 126.1689 + }, + "type": "trend", + "contains": "time" + }, + "endpoint_sync_history_duration": { + "values": { + "max": 747.2143, + "p(90)": 171.70332, + "p(95)": 193.89965999999998, + "avg": 135.3465406300053, + "min": 108.6792, + "med": 125.0237 + }, + "type": "trend", + "contains": "time" + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-52vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-52vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..1fbe2d6 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-52vus-post-rollup-cloudwatch.json @@ -0,0 +1,227 @@ +{ + "startTime": "2026-09-10T20:06:18Z", + "endTime": "2026-09-10T20:09:20Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.22347255991764695, + "p90": 0.07572715771518361, + "p50": 0.015252685325852097, + "p95": 0.11130110310856753 + } + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.15085225788166384, + "p90": 0.06767277685608598, + "p50": 0.016104680093595163, + "p95": 0.09582022982669502 + } + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.14168660851949375, + "p90": 0.06248335774006523, + "p50": 0.01614740385775519, + "p95": 0.09204288027000343 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 48.21265488455538, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 80.6585210164388, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 83.10693359375, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 7.877604166666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 10.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 12.99209973665789, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 71.10221777555444, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 74.48651530042811, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 90566656.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 86020096.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 85389312.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 0.0012, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 0.0002666666666666667, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 0.00125, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 0.00035087719298245617, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 0.00834983498349835, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-52vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-52vus-post-rollup-summary.json new file mode 100644 index 0000000..263f42c --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-52vus-post-rollup-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 11189, + "fails": 0, + "name": "cost_summary: status is 200" + }, + { + "passes": 11189, + "fails": 0, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c" + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 11189, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 11189 + }, + { + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 9324 + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 9324, + "fails": 0 + }, + { + "passes": 9324, + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937" + }, + { + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 9324, + "fails": 0, + "name": "cost_by_service: no auth or server error" + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 3730, + "fails": 0 + }, + { + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 3730, + "fails": 0, + "name": "aws_account_list: response is JSON" + }, + { + "passes": 3730, + "fails": 0, + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2" + }, + { + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 3730, + "fails": 0, + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error" + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 1865, + "fails": 0 + }, + { + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 1865, + "fails": 0 + }, + { + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 1865, + "fails": 0 + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 1865, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 11190 + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 11190, + "fails": 0 + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 11190, + "fails": 0 + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 11190, + "fails": 0 + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdErrTTY": true, + "testRunDurationMs": 180969.1261, + "isStdOutTTY": true + }, + "metrics": { + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 37298, + "rate": 206.10145389877084 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 140.69879482573762, + "min": 113.8867, + "med": 125.94675000000001, + "max": 2107.8066, + "p(90)": 172.31829, + "p(95)": 202.19666999999987 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 132.07105, + "max": 3974.3104, + "p(90)": 200.10179000000002, + "p(95)": 241.32360000000003, + "avg": 150.50175418521098, + "min": 111.8465 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 142.67178198230442, + "min": 116.4939, + "med": 131.0257, + "max": 644.4354, + "p(90)": 181.6774600000001, + "p(95)": 211.14314 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 52, + "min": 52, + "max": 52 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 149192, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 345284996, + "rate": 1907977.3629961733 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 1203.7148, + "p(90)": 166.0654000000001, + "p(95)": 196.26023999999998, + "avg": 137.15631957104554, + "min": 111.8465, + "med": 124.3088 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 3698.6537, + "p(90)": 1.2074600000000013, + "p(95)": 2.739839999999999, + "avg": 6.71136261025764 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 130.3903, + "p(90)": 0, + "p(95)": 0, + "avg": 0.17347992707579293 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 37298 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "rate": 206.10145389877084, + "count": 37298 + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 145.47898524238545, + "min": 118.9518, + "med": 131.71305, + "max": 3026.4441, + "p(90)": 186.33369000000002, + "p(95)": 217.41836500000002 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 241.3436699999999, + "avg": 150.5142719643958, + "min": 111.8465, + "med": 132.0713, + "max": 3974.3104, + "p(90)": 200.1154600000001 + } + }, + "http_req_waiting": { + "values": { + "med": 131.5655, + "max": 3025.9005, + "p(90)": 183.85038000000006, + "p(95)": 213.9809, + "avg": 143.80277442022552, + "min": 111.8465 + }, + "type": "trend", + "contains": "time" + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "max": 130.3903, + "p(90)": 0, + "p(95)": 0, + "avg": 0.17381027373388025, + "min": 0, + "med": 0 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 52, + "min": 52, + "max": 52 + } + }, + "iteration_duration": { + "contains": "time", + "values": { + "avg": 251.08215306182612, + "min": 211.8823, + "med": 232.5004, + "max": 4074.3826, + "p(90)": 300.60742000000005, + "p(95)": 341.854545 + }, + "type": "trend" + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 37299, + "rate": 206.10697970320805 + } + }, + "http_req_duration": { + "contains": "time", + "values": { + "max": 3974.3104, + "p(90)": 200.1154600000001, + "p(95)": 241.3436699999999, + "avg": 150.5142719643958, + "min": 111.8465, + "med": 132.0713 + }, + "type": "trend" + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 37299 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0.00013493391243733075, + "min": 0, + "med": 0, + "max": 0.6267, + "p(90)": 0 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 168.0081108042897, + "min": 119.5791, + "med": 135.8696, + "max": 3974.3104, + "p(90)": 244.30289, + "p(95)": 260.6085999999999 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 23925076, + "rate": 132205.2911212019 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-55vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-55vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..f396a77 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-55vus-post-rollup-cloudwatch.json @@ -0,0 +1,239 @@ +{ + "startTime": "2026-09-10T18:55:06Z", + "endTime": "2026-09-10T18:58:08Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Sum": 1.0, + "Unit": "Count" + } + ] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.2361971697087722, + "p90": 0.09394422412950405, + "p50": 0.018468971656289565, + "p95": 0.12256630720052263 + } + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1764701783888151, + "p90": 0.09835390058525331, + "p50": 0.018012820219698068, + "p95": 0.12446378768216686 + } + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.16258226163573955, + "p90": 0.09320919325511794, + "p50": 0.01827358178096868, + "p95": 0.11612541236155473 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 65.79698944091797, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 87.73880004882812, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 89.22252655029297, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 6.819661458333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 10.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 27.479211451615594, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 76.36630476476377, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 77.6624631428143, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 134.7748501, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 80838656.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 79757312.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 78917632.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 0.0007086614173228346, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 0.0004761904761904762, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 0.012427843803056027, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 0.00024390243902439024, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-55vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-55vus-post-rollup-summary.json new file mode 100644 index 0000000..e06c26c --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-55vus-post-rollup-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 3933, + "fails": 0, + "name": "aws_account_list: status is 200" + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 3933, + "fails": 0 + }, + { + "fails": 0, + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 3933 + }, + { + "passes": 3933, + "fails": 0, + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8" + }, + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 11801, + "fails": 0 + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 11801, + "fails": 0 + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 11801, + "fails": 0 + }, + { + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 11801, + "fails": 0, + "name": "cost_summary: no auth or server error" + }, + { + "passes": 9833, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b" + }, + { + "fails": 0, + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 9833 + }, + { + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 9833, + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid" + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 9833, + "fails": 0 + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 1967, + "fails": 0 + }, + { + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 1967 + }, + { + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 1967, + "fails": 0 + }, + { + "fails": 0, + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 1967 + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 11799, + "fails": 1 + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 11799, + "fails": 1 + }, + { + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 11799, + "fails": 1, + "name": "cost_timeseries: response shape is valid" + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 11799, + "fails": 1 + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "testRunDurationMs": 181170.258, + "isStdOutTTY": true, + "isStdErrTTY": true + }, + "metrics": { + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 1211.7814, + "p(90)": 213.3031, + "p(95)": 239.30273999999997, + "avg": 151.35395052497645, + "min": 109.6584, + "med": 133.701 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 225.61909999999997, + "avg": 143.05961601423465, + "min": 109.9668, + "med": 127.3207, + "max": 784.503, + "p(90)": 201.78064 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 235.709, + "avg": 148.7976235149566, + "min": 113.9269, + "med": 132.4747, + "max": 497.5059, + "p(90)": 210.2992 + } + }, + "http_req_duration{expected_response:true}": { + "contains": "time", + "values": { + "avg": 151.35486167946175, + "min": 109.6584, + "med": 133.70105, + "max": 1211.7814, + "p(90)": 213.30355000000003, + "p(95)": 239.30302999999998 + }, + "type": "trend" + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 26, + "min": 26, + "max": 55 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 1.5909945570102841, + "min": 0, + "med": 0, + "max": 1057.4258, + "p(90)": 1.623119999999999, + "p(95)": 2.646619999999998 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 1211.7814, + "p(90)": 223.59256000000005, + "p(95)": 249.16418999999996, + "avg": 158.53912683050916, + "min": 115.5146, + "med": 137.18075 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 39335, + "rate": 217.11621120504228 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "avg": 149.7628461853296, + "min": 109.6584, + "med": 133.2451, + "max": 719.0125, + "p(90)": 211.53426, + "p(95)": 237.0313 + } + }, + "measured_failures": { + "values": { + "rate": 2.542329791020491E-05, + "passes": 1, + "fails": 39333 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + }, + "type": "rate", + "contains": "default" + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.16881004703190544, + "min": 0, + "med": 0, + "max": 129.1976, + "p(90)": 0, + "p(95)": 0 + } + }, + "data_received": { + "values": { + "count": 364087935, + "rate": 2009645.175865456 + }, + "type": "counter", + "contains": "data" + }, + "measured_duration": { + "values": { + "avg": 151.33989913560657, + "min": 109.6584, + "med": 133.7006, + "max": 1211.7814, + "p(90)": 213.29191000000012, + "p(95)": 239.28955999999997 + }, + "type": "trend", + "contains": "time" + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 251.92874859663345, + "min": 210.1985, + "med": 234.12035, + "max": 1312.2911, + "p(90)": 313.79856000000007, + "p(95)": 339.663715 + } + }, + "checks": { + "contains": "default", + "values": { + "rate": 0.9999745767020898, + "passes": 157332, + "fails": 4 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0 + } + }, + "iterations": { + "contains": "default", + "values": { + "count": 39334, + "rate": 217.1106915352519 + }, + "type": "counter" + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 39334, + "rate": 217.1106915352519 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 196.85150000000004, + "p(95)": 220.77591999999999, + "avg": 142.8809517925248, + "min": 109.6584, + "med": 127.9144, + "max": 925.1758 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 124.2498, + "p(90)": 0, + "p(95)": 0, + "avg": 0.16835549510613954 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 55, + "min": 55, + "max": 55 + } + }, + "http_req_failed": { + "values": { + "rate": 2.542265158256006E-05, + "passes": 1, + "fails": 39334 + }, + "type": "rate", + "contains": "default" + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 112.8023, + "med": 133.5092, + "max": 738.4481, + "p(90)": 214.0598400000001, + "p(95)": 238.10482, + "avg": 150.7914260958001 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.00010978263632896913, + "min": 0, + "med": 0, + "max": 0.5845, + "p(90)": 0, + "p(95)": 0 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 25231080, + "rate": 139267.23005494644 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-cloudwatch.json new file mode 100644 index 0000000..791be12 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-cloudwatch.json @@ -0,0 +1,233 @@ +{ + "startTime": "2026-09-10T13:49:14Z", + "endTime": "2026-09-10T13:52:16Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.38503836010724485, + "p90": 0.24790600143524694, + "p50": 0.13583234992459062, + "p95": 0.29491442946251023 + } + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.4718735570654435, + "p90": 0.2517015409611456, + "p50": 0.1351305709310395, + "p95": 0.2891127463841507 + } + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.4130869283655758, + "p90": 0.24940696483845395, + "p50": 0.13200396833054012, + "p95": 0.280391269297301 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 8.759881378461918, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 9.105795860290527, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 9.782214800516764, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 5.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 5.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 5.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 17.9, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 83.13481817121723, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 83.66523905544354, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 120.18349128333334, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 86781952.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 96403456.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 92573696.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 0.0009090909090909091, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 0.00025, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 0.0009523809523809524, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 0.00018518518518518518, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 0.00012987012987012987, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..ae32a06 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-post-rollup-cloudwatch.json @@ -0,0 +1,227 @@ +{ + "startTime": "2026-09-10T17:31:43Z", + "endTime": "2026-09-10T17:34:45Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1952222781114497, + "p90": 0.014861946841870696, + "p50": 0.011576437432551786, + "p95": 0.016001061270904927 + } + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.04436199798816148, + "p90": 0.015123646812044922, + "p50": 0.011758133175178308, + "p95": 0.016941720825547756 + } + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.04278065045980197, + "p90": 0.014847077393205982, + "p50": 0.01128631323548445, + "p95": 0.01627129332777227 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 4.182211247816062, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 16.581934928894043, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 11.510798454284668, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 6.070963541666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 5.598958333333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 5.989583333333333, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 0.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 2.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 3.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 3.991666666666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 9.153908944010393, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 11.447199479748544, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 82169856.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 83734528.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 89088000.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 0.0004237288135593221, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 0.00013043478260869564, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 0.001858974358974359, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 0.014932432432432433, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-post-rollup-summary.json new file mode 100644 index 0000000..3d1aad5 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-post-rollup-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 1137, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 1137 + }, + { + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 1137, + "fails": 0, + "name": "cost_summary: response shape is valid" + }, + { + "fails": 0, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 1137 + }, + { + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 947, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200" + }, + { + "passes": 947, + "fails": 0, + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d" + }, + { + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 947, + "fails": 0, + "name": "cost_by_service: response shape is valid" + }, + { + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 947, + "fails": 0, + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error" + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 1137, + "fails": 0 + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 1137, + "fails": 0 + }, + { + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 1137, + "fails": 0, + "name": "cost_timeseries: response shape is valid" + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 1137, + "fails": 0 + }, + { + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 379, + "fails": 0, + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200" + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 379, + "fails": 0 + }, + { + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 379, + "fails": 0, + "name": "aws_account_list: response shape is valid" + }, + { + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 379, + "fails": 0, + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error" + }, + { + "passes": 190, + "fails": 0, + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb" + }, + { + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 190 + }, + { + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 190, + "fails": 0, + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid" + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 190, + "fails": 0 + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 180967.6034 + }, + "metrics": { + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 132.9634, + "p(95)": 136.13060000000002, + "avg": 127.81529649169079, + "min": 109.3059, + "med": 125.8793, + "max": 943.6198 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 127.70312102022855, + "min": 115.3949, + "med": 125.0232, + "max": 480.4279, + "p(90)": 130.47238, + "p(95)": 135.9221 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 5, + "min": 5, + "max": 5 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 121.65926473684209, + "min": 111.465, + "med": 120.08824999999999, + "max": 187.5883, + "p(90)": 125.19625, + "p(95)": 127.319445 + } + }, + "http_req_sending": { + "values": { + "max": 0.5284, + "p(90)": 0, + "p(95)": 0, + "avg": 0.0002722236876813506, + "min": 0, + "med": 0 + }, + "type": "trend", + "contains": "time" + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 3790, + "rate": 20.942975034171226 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "passes": 0, + "fails": 3791, + "rate": 0 + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.18622978106040625, + "min": 0, + "med": 0, + "max": 125.4608, + "p(90)": 0, + "p(95)": 0 + } + }, + "measured_failures": { + "values": { + "rate": 0, + "passes": 0, + "fails": 3790 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + }, + "type": "rate", + "contains": "default" + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 120.1431, + "p(90)": 0, + "p(95)": 0, + "avg": 0.18337327881825374, + "min": 0 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 137.08394701846956, + "min": 109.3059, + "med": 126.63165000000001, + "max": 1203.5025, + "p(90)": 135.13379, + "p(95)": 180.6777899999998 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 2431365, + "rate": 13435.360552495442 + } + }, + "http_req_tls_handshaking": { + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + }, + "type": "trend" + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "med": 0.3258, + "max": 1058.9299, + "p(90)": 4.2874, + "p(95)": 8.092400000000001, + "avg": 9.400878765497245, + "min": 0 + } + }, + "http_req_duration": { + "values": { + "p(90)": 135.1499, + "p(95)": 181.70295, + "avg": 137.21644748087562, + "min": 109.3059, + "med": 126.6335, + "max": 1203.5025 + }, + "type": "trend", + "contains": "time" + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 35086969, + "rate": 193885.36036721367 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 3790, + "rate": 20.942975034171226 + } + }, + "iteration_duration": { + "contains": "time", + "values": { + "avg": 237.67025042216397, + "min": 211.675, + "med": 227.04895, + "max": 1303.9652, + "p(90)": 235.61726000000002, + "p(95)": 281.07308499999965 + }, + "type": "trend" + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 124.33560976253294, + "min": 109.3059, + "med": 120.0058, + "max": 660.801, + "p(90)": 125.74108, + "p(95)": 129.26656999999994 + } + }, + "vus": { + "values": { + "value": 5, + "min": 5, + "max": 5 + }, + "type": "gauge", + "contains": "default" + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 136.37514, + "avg": 129.45352111932436, + "min": 116.6179, + "med": 126.515, + "max": 658.8627, + "p(90)": 132.72353999999999 + } + }, + "checks": { + "contains": "default", + "values": { + "rate": 1, + "passes": 15160, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate" + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 3791, + "rate": 20.948500885103726 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "med": 126.6335, + "max": 1203.5025, + "p(90)": 135.1499, + "p(95)": 181.70295, + "avg": 137.21644748087562, + "min": 109.3059 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 243.0198, + "p(95)": 252.71936, + "avg": 159.6471149516271, + "min": 114.1595, + "med": 129.5399, + "max": 1203.5025 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-summary.json new file mode 100644 index 0000000..0e7b20e --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-summary.json @@ -0,0 +1,492 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 751, + "fails": 0, + "name": "cost_summary: status is 200" + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 751, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 751 + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 751, + "fails": 0 + }, + { + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 625, + "fails": 0 + }, + { + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 625, + "fails": 0, + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON" + }, + { + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 625, + "fails": 0, + "name": "cost_by_service: response shape is valid" + }, + { + "fails": 0, + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 625 + }, + { + "passes": 751, + "fails": 0, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa" + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 751, + "fails": 0 + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 751, + "fails": 0 + }, + { + "passes": 751, + "fails": 0, + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7" + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 250, + "fails": 0 + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 250, + "fails": 0 + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 250, + "fails": 0 + }, + { + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 250, + "fails": 0, + "name": "aws_account_list: no auth or server error" + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 125, + "fails": 0 + }, + { + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 125, + "fails": 0 + }, + { + "fails": 0, + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 125 + }, + { + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 125, + "fails": 0, + "name": "sync_history: no auth or server error" + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 181246.516 + }, + "metrics": { + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 2503 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0.2873751498202157, + "min": 0, + "med": 0, + "max": 122.9711 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 126.96905000000001, + "max": 210.701, + "p(90)": 141.20443999999998, + "p(95)": 147.67658, + "avg": 129.9706844, + "min": 115.5086 + }, + "thresholds": { + "p(99)<500": { + "ok": true + }, + "p(95)<200": { + "ok": true + } + } + }, + "http_req_blocked": { + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0.2913048341989613, + "min": 0, + "med": 0, + "max": 123.8153 + }, + "type": "trend" + }, + "vus": { + "values": { + "value": 4, + "min": 4, + "max": 5 + }, + "type": "gauge", + "contains": "default" + }, + "iteration_duration": { + "contains": "time", + "values": { + "avg": 360.29667230215784, + "min": 215.999, + "med": 352.35685, + "max": 907.3306, + "p(90)": 468.78731, + "p(95)": 500.6701699999999 + }, + "type": "trend" + }, + "data_sent": { + "values": { + "count": 1605198, + "rate": 8856.435066591845 + }, + "type": "counter", + "contains": "data" + }, + "endpoint_cost_by_service_duration": { + "values": { + "min": 207.9014, + "med": 309.3012, + "max": 735.927, + "p(90)": 389.20082, + "p(95)": 443.75378, + "avg": 318.1573131199999 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 276.96930332889474, + "min": 180.1102, + "med": 253.077, + "max": 806.0354, + "p(90)": 377.8986, + "p(95)": 420.2709 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "max": 806.0354, + "p(90)": 368.35678, + "p(95)": 399.7149600000001, + "avg": 259.8228172992397, + "min": 115.5086, + "med": 252.2266 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 5, + "min": 5, + "max": 5 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 248.8785, + "max": 649.5329, + "p(90)": 325.7928, + "p(95)": 352.39430000000004, + "avg": 257.7986651131828, + "min": 173.3294 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_req_duration": { + "values": { + "avg": 259.8228172992397, + "min": 115.5086, + "med": 252.2266, + "max": 806.0354, + "p(90)": 368.35678, + "p(95)": 399.7149600000001 + }, + "type": "trend", + "contains": "time" + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 2502, + "rate": 13.804403280226362 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 2502, + "rate": 13.804403280226362 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 131.2268, + "max": 188.4988, + "p(90)": 144.42888000000002, + "p(95)": 150.74277999999998, + "avg": 133.3218672, + "min": 119.8368 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 2503, + "rate": 13.809920627660505 + } + }, + "data_received": { + "contains": "data", + "values": { + "rate": 127823.13012847098, + "count": 23167497 + }, + "type": "counter" + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 368.27666, + "p(95)": 398.8659849999998, + "avg": 259.6390765387679, + "min": 115.5086, + "med": 252.17865, + "max": 806.0354 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + } + }, + "http_req_receiving": { + "contains": "time", + "values": { + "med": 0, + "max": 121.1964, + "p(90)": 2.976320000000002, + "p(95)": 5.138820000000006, + "avg": 1.1028236116660006, + "min": 0 + }, + "type": "trend" + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 2502 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 10008, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "http_req_waiting": { + "contains": "time", + "values": { + "avg": 258.7199936875754, + "min": 113.0042, + "med": 251.0832, + "max": 805.5317, + "p(90)": 366.9243, + "p(95)": 397.2793000000001 + }, + "type": "trend" + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-7vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-7vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..8a7f0d7 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-7vus-post-rollup-cloudwatch.json @@ -0,0 +1,227 @@ +{ + "startTime": "2026-09-10T18:06:08Z", + "endTime": "2026-09-10T18:09:10Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.048875775470276064, + "p90": 0.01470869187793284, + "p50": 0.011105136446586394, + "p95": 0.016513708598530772 + } + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.036291967035372186, + "p90": 0.014591710538548358, + "p50": 0.011155320525666462, + "p95": 0.015682558011621873 + } + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.03823544103971422, + "p90": 0.014827862505333743, + "p50": 0.011094414541070859, + "p95": 0.01750815776761174 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 12.90796184539795, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 17.11275291442871, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 16.957215627034504, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 7.275390625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 6.34765625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 6.331380208333333, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 4.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 2.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 3.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 9.954977488744372, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 15.041666666666666, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 14.291666666666666, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 93843456.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 92295168.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 80420864.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 0.0003275109170305677, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 0.00037037037037037035, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 0.0008823529411764705, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 0.005150501672240803, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 0.00023529411764705883, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 0.0003846153846153846, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-7vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-7vus-post-rollup-summary.json new file mode 100644 index 0000000..74bb09c --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-7vus-post-rollup-summary.json @@ -0,0 +1,444 @@ +{ + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 181023.486 + }, + "metrics": { + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "rate": 19472.953912731522, + "count": 3525062 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 234.01264, + "p(95)": 236.85278000000005, + "avg": 229.42284242038224, + "min": 212.2446, + "med": 227.2499, + "max": 1432.478 + } + }, + "http_req_waiting": { + "contains": "time", + "values": { + "avg": 127.79697032387158, + "min": 110.5408, + "med": 125.9812, + "max": 987.1489, + "p(90)": 132.99095, + "p(95)": 135.45057500000001 + }, + "type": "trend" + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 1332.3934, + "p(90)": 135.10252, + "p(95)": 139.9969, + "avg": 133.0907663432384, + "min": 118.2188, + "med": 129.1766 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 0.5045, + "p(90)": 0, + "p(95)": 0, + "avg": 9.179403202328966E-05, + "min": 0 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 1, + "min": 1, + "max": 7 + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 5495, + "rate": 30.355177228218885 + } + }, + "data_received": { + "contains": "data", + "values": { + "count": 50878079, + "rate": 281057.88991379825 + }, + "type": "counter" + }, + "endpoint_aws_account_list_duration": { + "values": { + "min": 111.7288, + "med": 120.0673, + "max": 253.2063, + "p(90)": 125.3419, + "p(95)": 126.09172, + "avg": 121.70351748633873 + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 127.39056747572792, + "min": 117.2137, + "med": 125.1144, + "max": 312.1902, + "p(90)": 130.81339, + "p(95)": 134.53433 + } + }, + "http_req_duration{expected_response:true}": { + "contains": "time", + "values": { + "med": 126.86619999999999, + "max": 1332.3934, + "p(90)": 133.58935, + "p(95)": 136.363925, + "avg": 128.97286906841325, + "min": 111.7288 + }, + "type": "trend" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 5496 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 111.7288, + "med": 126.866, + "max": 1332.3934, + "p(90)": 133.58687999999998, + "p(95)": 136.32460000000003, + "avg": 128.87314520473143 + } + }, + "checks": { + "values": { + "rate": 1, + "passes": 21980, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate", + "contains": "default" + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "max": 125.6214, + "p(90)": 0, + "p(95)": 0, + "avg": 0.17479599708879184, + "min": 0, + "med": 0 + } + }, + "measured_failures": { + "contains": "default", + "values": { + "fails": 5495, + "rate": 0, + "passes": 0 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + }, + "type": "rate" + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 133.58935, + "p(95)": 136.363925, + "avg": 128.97286906841325, + "min": 111.7288, + "med": 126.86619999999999, + "max": 1332.3934 + } + }, + "iterations": { + "values": { + "count": 5495, + "rate": 30.355177228218885 + }, + "type": "counter", + "contains": "default" + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "max": 345.2445, + "p(90)": 3.10895, + "p(95)": 4.7539, + "avg": 1.1758069505094617, + "min": 0, + "med": 0.32935000000000003 + } + }, + "endpoint_cost_by_service_duration": { + "values": { + "p(95)": 135.993755, + "avg": 129.82906957787498, + "min": 116.9609, + "med": 126.53725, + "max": 962.727, + "p(90)": 132.00417000000002 + }, + "type": "trend", + "contains": "time" + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 7, + "min": 7, + "max": 7 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 204.954, + "p(90)": 125.98084, + "p(95)": 130.56608, + "avg": 122.00447745454548, + "min": 112.7169, + "med": 120.1279 + } + }, + "http_req_blocked": { + "contains": "time", + "values": { + "max": 125.6214, + "p(90)": 0, + "p(95)": 0, + "avg": 0.17668147743813684, + "min": 0, + "med": 0 + }, + "type": "trend" + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "rate": 30.360701373301364, + "count": 5496 + } + } + }, + "root_group": { + "checks": [ + { + "fails": 0, + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 549 + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 549, + "fails": 0 + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 549, + "fails": 0 + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 549, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 1648 + }, + { + "passes": 1648, + "fails": 0, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c" + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 1648, + "fails": 0 + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 1648, + "fails": 0 + }, + { + "passes": 1374, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b" + }, + { + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 1374, + "fails": 0, + "name": "cost_by_service: response is JSON" + }, + { + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 1374, + "fails": 0 + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 1374, + "fails": 0 + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 1649, + "fails": 0 + }, + { + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 1649, + "fails": 0, + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON" + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 1649, + "fails": 0 + }, + { + "passes": 1649, + "fails": 0, + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7" + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 275, + "fails": 0 + }, + { + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 275, + "fails": 0, + "name": "sync_history: response is JSON" + }, + { + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 275, + "fails": 0, + "name": "sync_history: response shape is valid" + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 275, + "fails": 0 + } + ], + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [] + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/cloudwatch-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/cloudwatch-summary.json new file mode 100644 index 0000000..0dd2eb8 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/cloudwatch-summary.json @@ -0,0 +1,998 @@ +{ + "startTime": "2026-09-10T12:13:57Z", + "endTime": "2026-09-10T12:31:58Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.5005213825997035, + "p90": 0.49081191581736405, + "p50": 0.0016069181341690968, + "p95": 0.4961825739755273 + } + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.5117978895329298, + "p90": 1.1432096936062883, + "p50": 0.5024037003108845, + "p95": 1.2832487783847044 + } + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.8888792236095295, + "p90": 1.5602898584288212, + "p50": 1.1604053865967086, + "p95": 1.6704390030265635 + } + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.8139916951359347, + "p90": 1.530789537650868, + "p50": 1.1505018056846472, + "p95": 1.6206587563758006 + } + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.8450457291684828, + "p90": 1.5547777747106735, + "p50": 1.1348863117813814, + "p95": 1.672208938285089 + } + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.8853114255683752, + "p90": 1.5299423163676742, + "p50": 1.1440282056693523, + "p95": 1.653717772894364 + } + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.771241587019642, + "p90": 1.4986302469710606, + "p50": 1.1469635044910165, + "p95": 1.5969743068030005 + } + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.9176353638449173, + "p90": 1.5645161460898938, + "p50": 1.1572194284423918, + "p95": 1.6994982694971987 + } + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.9035583575561894, + "p90": 1.562309040847082, + "p50": 1.161570698549713, + "p95": 1.6793597591796334 + } + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.8626015031124603, + "p90": 1.5232402809184402, + "p50": 1.1245384911787704, + "p95": 1.6335054497057249 + } + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.7890379680264643, + "p90": 1.5100691302246265, + "p50": 1.119703632920338, + "p95": 1.6178605165757192 + } + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 4.242044367626646, + "p90": 3.7482320375318006, + "p50": 1.7648199460882255, + "p95": 3.976938307208984 + } + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 5.683100647998581, + "p90": 5.263568926527421, + "p50": 4.42094810691806, + "p95": 5.4352360097699215 + } + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 6.145181394704302, + "p90": 5.81238796610425, + "p50": 5.244265437348053, + "p95": 5.980980693111135 + } + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 6.0988860981981965, + "p90": 5.772456731265545, + "p50": 5.242209055535328, + "p95": 5.951587426084212 + } + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 6.269539225425209, + "p90": 5.927756138694554, + "p50": 5.267723250809718, + "p95": 6.0471514482254705 + } + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.17863320565535443, + "p90": 0.04408555720637314, + "p50": 0.002900799462199063, + "p95": 0.07173479423210939 + } + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.24798102219877785, + "p90": 0.05944444542334447, + "p50": 0.003232158232002402, + "p95": 0.08606966966592673 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 0.032596416771411896, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 13.574704806009928, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 12.062845865885416, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 14.789092063903809, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 12.07803757985433, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 11.818715413411459, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 12.702068964640299, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 11.432299931844076, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 12.210658073425293, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 11.095791816711426, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 11.91440455118815, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 11.007564862569174, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 12.858075141906738, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 13.510810534159342, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 13.549819310506185, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 13.187480926513672, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 68.82544199625652, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 77.06313578287761, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 5.46875, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 4.98046875, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 5.826822916666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 6.005859375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 6.087239583333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 6.15234375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 6.168619791666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 6.184895833333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 6.233723958333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 6.25, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 6.25, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 6.25, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 6.34765625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 6.380208333333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 6.4453125, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 6.4453125, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 6.54296875, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 0.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 0.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 0.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 3.7148711456129537, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 35.14198529289156, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 99.52334127764537, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 99.80999683328055, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 99.80833333333334, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 99.6018591014343, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 99.76488636174152, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 99.79177078127601, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 99.69984992496248, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 99.59166666666665, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 99.50833333333334, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 99.66683325004163, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 99.78322494580623, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 99.74846329396479, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 99.77832594419814, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 99.58668733230004, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 61.16870562352078, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 4.627313656828414, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 136.89453651666668, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 127.89600651666667, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 118.8959455, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 111.75111935, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 86867968.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 84504576.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 92930048.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 85712896.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 79523840.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 82714624.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 79966208.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 90292224.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 81620992.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 82665472.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 84733952.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 81457152.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 88825856.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 82919424.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 79323136.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 79519744.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 128004096.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 113164288.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 0.00012195121951219512, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 0.0003111111111111111, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 0.0021, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 0.0014893617021276594, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 9.900990099009902e-05, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 0.00039999999999999996, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 0.00015267175572519084, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 0.00029761904761904765, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 0.0005952380952380953, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 0.00013157894736842105, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 0.00019047619047619048, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 0.0002631578947368421, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 0.008726968174204356, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 0.00022222222222222223, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 0.0003225806451612903, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 0.003851351351351351, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 0.00125, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 0.0004878048780487805, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 0.004918625678119349, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 0.0004166666666666667, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 0.00020408163265306123, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 0.004720720720720721, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/dataset.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/dataset.json new file mode 100644 index 0000000..a4758ac --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/dataset.json @@ -0,0 +1,15 @@ +{ + "dataset": "underflow-api-benchmark-v1", + "dateRange": { + "from": "2025-01-01", + "to": "2025-12-31" + }, + "servicesPerAccountPerDay": 50, + "users": 10, + "workspaces": 10, + "workspaceMembers": 10, + "awsAccounts": 200, + "costSyncRuns": 200, + "costSnapshots": 3650000, + "costRollups": 182500 +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/environment.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/environment.json new file mode 100644 index 0000000..99d5b6e --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/environment.json @@ -0,0 +1,108 @@ +{ + "gitSha": "2624a6ccafc427521745b1180764fcde963f90bc", + "repositoryEquivalentSha": "f6c356d5e38409b5dfcd44908af68cc968335631", + "imageDigest": "sha256:b1eec7b5c7652dfab0fe8259ab66f9b0682e228930da769c8d1688961680e644", + "testTimestamp": "2026-09-10T21:07:35Z", + "awsRegion": "us-east-1", + "availabilityZones": [ + "us-east-1a", + "us-east-1b" + ], + "infrastructure": { + "ecs": { + "cpu": 512, + "memoryMiB": 1024, + "desiredCount": 1 + }, + "rds": { + "class": "db.t4g.micro", + "engine": "postgres", + "engineVersion": "16.13", + "storageGiB": 20, + "multiAz": false, + "publiclyAccessible": false + } + }, + "nodeVersion": "v25.9.0", + "k6Version": "k6 v2.2.0 (commit/00a9a1b7f5, go1.26.5, linux/amd64)", + "dataset": "underflow-api-benchmark-v1", + "profiles": { + "smoke": { + "vus": 2, + "duration": "30s" + }, + "supportedCapacity": { + "vus": 45, + "duration": "15m", + "serverSideTargetsMs": { + "p50": 30, + "p95": 100, + "p99": 250 + } + }, + "normalLoad": { + "stages": [ + [ + "1m", + 25 + ], + [ + "3m", + 25 + ], + [ + "1m", + 50 + ], + [ + "5m", + 50 + ], + [ + "2m", + 100 + ], + [ + "5m", + 100 + ], + [ + "1m", + 0 + ] + ] + }, + "stress": { + "stages": [ + [ + "2m", + 100 + ], + [ + "3m", + 100 + ], + [ + "1m", + 150 + ], + [ + "3m", + 150 + ], + [ + "1m", + 200 + ], + [ + "3m", + 200 + ], + [ + "1m", + 0 + ] + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-covering-index.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-covering-index.json new file mode 100644 index 0000000..9d14bea --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-covering-index.json @@ -0,0 +1,606 @@ +{ + "benchmarkDiagnostic": true, + "stats": { + "total_size": "1423 MB", + "table_size": "439 MB", + "indexes_size": "984 MB", + "n_live_tup": "3650010", + "seq_scan": "96", + "idx_scan": "7" + }, + "plans": { + "summary": [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Finalize", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 21816.24, + "Total Cost": 21816.25, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 1662.094, + "Actual Total Time": 1680.351, + "Actual Rows": 1, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 3417, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 1535.725, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 21816.01, + "Total Cost": 21816.22, + "Plan Rows": 2, + "Plan Width": 64, + "Actual Startup Time": 1651.804, + "Actual Total Time": 1680.322, + "Actual Rows": 3, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Single Copy": false, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 3417, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 1535.725, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 20816.01, + "Total Cost": 20816.02, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 1644.889, + "Actual Total Time": 1644.89, + "Actual Rows": 1, + "Actual Loops": 3, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 3417, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 1535.725, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [], + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_cost_snapshots_workspace_date_reporting", + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 0.56, + "Total Cost": 20047.62, + "Plan Rows": 153677, + "Plan Width": 10, + "Actual Startup Time": 2.398, + "Actual Total Time": 1618.116, + "Actual Rows": 121667, + "Actual Loops": 3, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 3417, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 1535.725, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 158, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 5.082, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 16.606, + "Triggers": [], + "Execution Time": 1681.568 + } + ], + "timeseries": [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Sorted", + "Partial Mode": "Finalize", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1000.58, + "Total Cost": 22300.91, + "Plan Rows": 365, + "Plan Width": 68, + "Actual Startup Time": 56.454, + "Actual Total Time": 111, + "Actual Rows": 365, + "Actual Loops": 1, + "Group Key": [ + "usage_date" + ], + "Shared Hit Blocks": 3421, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather Merge", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1000.58, + "Total Cost": 22289.05, + "Plan Rows": 730, + "Plan Width": 68, + "Actual Startup Time": 56.395, + "Actual Total Time": 110.506, + "Actual Rows": 732, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Shared Hit Blocks": 3421, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Sorted", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.56, + "Total Cost": 21204.76, + "Plan Rows": 365, + "Plan Width": 68, + "Actual Startup Time": 0.346, + "Actual Total Time": 46.68, + "Actual Rows": 244, + "Actual Loops": 3, + "Group Key": [ + "usage_date" + ], + "Shared Hit Blocks": 3421, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [], + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_cost_snapshots_workspace_date_reporting", + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 0.56, + "Total Cost": 20047.62, + "Plan Rows": 153677, + "Plan Width": 14, + "Actual Startup Time": 0.034, + "Actual Total Time": 23.068, + "Actual Rows": 121667, + "Actual Loops": 3, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 3421, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 9, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 7.226, + "Triggers": [], + "Execution Time": 112.415 + } + ], + "byService": [ + { + "Plan": { + "Node Type": "Sort", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 22216.96, + "Total Cost": 22217.09, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 105.622, + "Actual Total Time": 108.875, + "Actual Rows": 50, + "Actual Loops": 1, + "Sort Key": [ + "(sum(amount)) DESC" + ], + "Sort Method": "quicksort", + "Sort Space Used": 27, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 3440, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Sorted", + "Partial Mode": "Finalize", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 22202.26, + "Total Cost": 22215.55, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 104.774, + "Actual Total Time": 108.19, + "Actual Rows": 50, + "Actual Loops": 1, + "Group Key": [ + "service_name" + ], + "Shared Hit Blocks": 3437, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather Merge", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 22202.26, + "Total Cost": 22213.93, + "Plan Rows": 100, + "Plan Width": 85, + "Actual Startup Time": 104.756, + "Actual Total Time": 108.068, + "Actual Rows": 150, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Shared Hit Blocks": 3437, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 21202.24, + "Total Cost": 21202.36, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 96.739, + "Actual Total Time": 96.745, + "Actual Rows": 50, + "Actual Loops": 3, + "Sort Key": [ + "service_name" + ], + "Sort Method": "quicksort", + "Sort Space Used": 30, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 3437, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [ + { + "Worker Number": 0, + "Sort Method": "quicksort", + "Sort Space Used": 30, + "Sort Space Type": "Memory" + }, + { + "Worker Number": 1, + "Sort Method": "quicksort", + "Sort Space Used": 30, + "Sort Space Type": "Memory" + } + ], + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Hashed", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 21200.2, + "Total Cost": 21200.82, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 92.664, + "Actual Total Time": 92.682, + "Actual Rows": 50, + "Actual Loops": 3, + "Group Key": [ + "service_name" + ], + "Planned Partitions": 0, + "HashAgg Batches": 1, + "Peak Memory Usage": 48, + "Disk Usage": 0, + "Shared Hit Blocks": 3421, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [ + { + "Worker Number": 0, + "HashAgg Batches": 1, + "Peak Memory Usage": 48, + "Disk Usage": 0 + }, + { + "Worker Number": 1, + "HashAgg Batches": 1, + "Peak Memory Usage": 48, + "Disk Usage": 0 + } + ], + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_cost_snapshots_workspace_date_reporting", + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 0.56, + "Total Cost": 20047.62, + "Plan Rows": 153677, + "Plan Width": 31, + "Actual Startup Time": 0.043, + "Actual Total Time": 47.768, + "Actual Rows": 121667, + "Actual Loops": 3, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 3421, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 26, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 0.9, + "Triggers": [], + "Execution Time": 108.981 + } + ] + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-index-cold.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-index-cold.json new file mode 100644 index 0000000..2c4415e --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-index-cold.json @@ -0,0 +1,171 @@ +{ + "benchmarkDiagnostic": true, + "stats": { + "total_size": "1328 MB", + "table_size": "439 MB", + "indexes_size": "889 MB", + "n_live_tup": "3650010", + "seq_scan": "94", + "idx_scan": "2" + }, + "plan": [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Finalize", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 16626.75, + "Total Cost": 16626.76, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 515.76, + "Actual Total Time": 565.052, + "Actual Rows": 1, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 2203, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 435.312, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 16626.53, + "Total Cost": 16626.74, + "Plan Rows": 2, + "Plan Width": 64, + "Actual Startup Time": 514.577, + "Actual Total Time": 565.03, + "Actual Rows": 3, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Single Copy": false, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 2203, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 435.312, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 15626.53, + "Total Cost": 15626.54, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 501.883, + "Actual Total Time": 501.884, + "Actual Rows": 1, + "Actual Loops": 3, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 2203, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 435.312, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [], + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_cost_snapshots_workspace_date", + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 0.56, + "Total Cost": 14870.31, + "Plan Rows": 151244, + "Plan Width": 10, + "Actual Startup Time": 1.895, + "Actual Total Time": 479.762, + "Actual Rows": 121667, + "Actual Loops": 3, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 2203, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 435.312, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 154, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 2.536, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 10.875, + "Triggers": [], + "Execution Time": 565.241 + } + ] +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-index-warm.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-index-warm.json new file mode 100644 index 0000000..aa57883 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-index-warm.json @@ -0,0 +1,171 @@ +{ + "benchmarkDiagnostic": true, + "stats": { + "total_size": "1328 MB", + "table_size": "439 MB", + "indexes_size": "889 MB", + "n_live_tup": "3650010", + "seq_scan": "94", + "idx_scan": "5" + }, + "plan": [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Finalize", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 16626.75, + "Total Cost": 16626.76, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 300.848, + "Actual Total Time": 304.182, + "Actual Rows": 1, + "Actual Loops": 1, + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 16626.53, + "Total Cost": 16626.74, + "Plan Rows": 2, + "Plan Width": 64, + "Actual Startup Time": 300.834, + "Actual Total Time": 304.17, + "Actual Rows": 3, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Single Copy": false, + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 15626.53, + "Total Cost": 15626.54, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 289.377, + "Actual Total Time": 289.378, + "Actual Rows": 1, + "Actual Loops": 3, + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [], + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_cost_snapshots_workspace_date", + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 0.56, + "Total Cost": 14870.31, + "Plan Rows": 151244, + "Plan Width": 10, + "Actual Startup Time": 1.269, + "Actual Total Time": 267.45, + "Actual Rows": 121667, + "Actual Loops": 3, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 157, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 5.763, + "Triggers": [], + "Execution Time": 304.281 + } + ] +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-rollup.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-rollup.json new file mode 100644 index 0000000..236e045 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-rollup.json @@ -0,0 +1,386 @@ +{ + "benchmarkDiagnostic": true, + "stats": [ + { + "relname": "cost_snapshots", + "total_size": "1423 MB", + "table_size": "439 MB", + "indexes_size": "984 MB", + "n_live_tup": "3650010", + "seq_scan": "96", + "idx_scan": "49394" + }, + { + "relname": "workspace_cost_daily_rollups", + "total_size": "52 MB", + "table_size": "16 MB", + "indexes_size": "36 MB", + "n_live_tup": "182500", + "seq_scan": "3", + "idx_scan": "0" + } + ], + "plans": { + "summary": [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Simple", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1709.58, + "Total Cost": 1709.59, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 61.707, + "Actual Total Time": 61.708, + "Actual Rows": 1, + "Actual Loops": 1, + "Shared Hit Blocks": 306, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 2.775, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_workspace_cost_daily_rollups_workspace_service_date", + "Relation Name": "workspace_cost_daily_rollups", + "Alias": "workspace_cost_daily_rollups", + "Startup Cost": 0.42, + "Total Cost": 1618.47, + "Plan Rows": 18222, + "Plan Width": 12, + "Actual Startup Time": 0.103, + "Actual Total Time": 57.74, + "Actual Rows": 18250, + "Actual Loops": 1, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 306, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 2.775, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 149, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 6, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 13.424, + "Triggers": [], + "Execution Time": 63.087 + } + ], + "timeseries": [ + { + "Plan": { + "Node Type": "Sort", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1775.23, + "Total Cost": 1776.14, + "Plan Rows": 365, + "Plan Width": 68, + "Actual Startup Time": 10.673, + "Actual Total Time": 10.702, + "Actual Rows": 365, + "Actual Loops": 1, + "Sort Key": [ + "usage_date" + ], + "Sort Method": "quicksort", + "Sort Space Used": 39, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 312, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Hashed", + "Partial Mode": "Simple", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1755.13, + "Total Cost": 1759.69, + "Plan Rows": 365, + "Plan Width": 68, + "Actual Startup Time": 9.726, + "Actual Total Time": 9.835, + "Actual Rows": 365, + "Actual Loops": 1, + "Group Key": [ + "usage_date" + ], + "Planned Partitions": 0, + "HashAgg Batches": 1, + "Peak Memory Usage": 285, + "Disk Usage": 0, + "Shared Hit Blocks": 309, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_workspace_cost_daily_rollups_workspace_service_date", + "Relation Name": "workspace_cost_daily_rollups", + "Alias": "workspace_cost_daily_rollups", + "Startup Cost": 0.42, + "Total Cost": 1618.47, + "Plan Rows": 18222, + "Plan Width": 16, + "Actual Startup Time": 0.021, + "Actual Total Time": 4.254, + "Actual Rows": 18250, + "Actual Loops": 1, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 309, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 20, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 1.307, + "Triggers": [], + "Execution Time": 11.025 + } + ], + "byService": [ + { + "Plan": { + "Node Type": "Sort", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1757.17, + "Total Cost": 1757.29, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 8.085, + "Actual Total Time": 8.089, + "Actual Rows": 50, + "Actual Loops": 1, + "Sort Key": [ + "(sum(total_amount)) DESC" + ], + "Sort Method": "quicksort", + "Sort Space Used": 27, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 312, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Sorted", + "Partial Mode": "Simple", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 1755.76, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 0.192, + "Actual Total Time": 8.046, + "Actual Rows": 50, + "Actual Loops": 1, + "Group Key": [ + "service_name" + ], + "Shared Hit Blocks": 309, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_workspace_cost_daily_rollups_workspace_service_date", + "Relation Name": "workspace_cost_daily_rollups", + "Alias": "workspace_cost_daily_rollups", + "Startup Cost": 0.42, + "Total Cost": 1618.47, + "Plan Rows": 18222, + "Plan Width": 33, + "Actual Startup Time": 0.018, + "Actual Total Time": 3.916, + "Actual Rows": 18250, + "Actual Loops": 1, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 309, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 20, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 0.18, + "Triggers": [], + "Execution Time": 8.155 + } + ] + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-all-cost-queries-after-index.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-all-cost-queries-after-index.json new file mode 100644 index 0000000..03b00f0 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-all-cost-queries-after-index.json @@ -0,0 +1,638 @@ +{ + "benchmarkDiagnostic": true, + "stats": { + "total_size": "1328 MB", + "table_size": "439 MB", + "indexes_size": "889 MB", + "n_live_tup": "3650010", + "seq_scan": "94", + "idx_scan": "199" + }, + "plans": { + "summary": [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Finalize", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 16626.75, + "Total Cost": 16626.76, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 269.255, + "Actual Total Time": 332.514, + "Actual Rows": 1, + "Actual Loops": 1, + "Shared Hit Blocks": 2206, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 2.112, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 16626.53, + "Total Cost": 16626.74, + "Plan Rows": 2, + "Plan Width": 64, + "Actual Startup Time": 269.239, + "Actual Total Time": 332.5, + "Actual Rows": 3, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Single Copy": false, + "Shared Hit Blocks": 2206, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 2.112, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 15626.53, + "Total Cost": 15626.54, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 255.513, + "Actual Total Time": 255.514, + "Actual Rows": 1, + "Actual Loops": 3, + "Shared Hit Blocks": 2206, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 2.112, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [], + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_cost_snapshots_workspace_date", + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 0.56, + "Total Cost": 14870.31, + "Plan Rows": 151244, + "Plan Width": 10, + "Actual Startup Time": 0.872, + "Actual Total Time": 232.371, + "Actual Rows": 121667, + "Actual Loops": 3, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 2206, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 2.112, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 135, + "Shared Read Blocks": 22, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 9.65, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 18.291, + "Triggers": [], + "Execution Time": 333.558 + } + ], + "timeseries": [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Sorted", + "Partial Mode": "Finalize", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1000.58, + "Total Cost": 17105.34, + "Plan Rows": 365, + "Plan Width": 68, + "Actual Startup Time": 81.49, + "Actual Total Time": 173.273, + "Actual Rows": 365, + "Actual Loops": 1, + "Group Key": [ + "usage_date" + ], + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather Merge", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1000.58, + "Total Cost": 17093.48, + "Plan Rows": 730, + "Plan Width": 68, + "Actual Startup Time": 81.396, + "Actual Total Time": 172.774, + "Actual Rows": 731, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Sorted", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.56, + "Total Cost": 16009.2, + "Plan Rows": 365, + "Plan Width": 68, + "Actual Startup Time": 0.283, + "Actual Total Time": 44.615, + "Actual Rows": 244, + "Actual Loops": 3, + "Group Key": [ + "usage_date" + ], + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [], + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_cost_snapshots_workspace_date", + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 0.56, + "Total Cost": 14870.31, + "Plan Rows": 151244, + "Plan Width": 14, + "Actual Startup Time": 0.043, + "Actual Total Time": 20.963, + "Actual Rows": 121667, + "Actual Loops": 3, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 9, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 10.911, + "Triggers": [], + "Execution Time": 177.469 + } + ], + "byService": [ + { + "Plan": { + "Node Type": "Sort", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 74400.47, + "Total Cost": 74400.6, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 4752.793, + "Actual Total Time": 4870.886, + "Actual Rows": 50, + "Actual Loops": 1, + "Sort Key": [ + "(sum(amount)) DESC" + ], + "Sort Method": "quicksort", + "Sort Space Used": 27, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 7860, + "Shared Read Blocks": 16821, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 13188.305, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Sorted", + "Partial Mode": "Finalize", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 74385.77, + "Total Cost": 74399.06, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 4750.929, + "Actual Total Time": 4869.189, + "Actual Rows": 50, + "Actual Loops": 1, + "Group Key": [ + "service_name" + ], + "Shared Hit Blocks": 7857, + "Shared Read Blocks": 16821, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 13188.305, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather Merge", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 74385.77, + "Total Cost": 74397.44, + "Plan Rows": 100, + "Plan Width": 85, + "Actual Startup Time": 4749.694, + "Actual Total Time": 4867.855, + "Actual Rows": 150, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Shared Hit Blocks": 7857, + "Shared Read Blocks": 16821, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 13188.305, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 73385.75, + "Total Cost": 73385.87, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 4680.725, + "Actual Total Time": 4681.286, + "Actual Rows": 50, + "Actual Loops": 3, + "Sort Key": [ + "service_name" + ], + "Sort Method": "quicksort", + "Sort Space Used": 30, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 7857, + "Shared Read Blocks": 16821, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 13188.305, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [ + { + "Worker Number": 0, + "Sort Method": "quicksort", + "Sort Space Used": 30, + "Sort Space Type": "Memory" + }, + { + "Worker Number": 1, + "Sort Method": "quicksort", + "Sort Space Used": 30, + "Sort Space Type": "Memory" + } + ], + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Hashed", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 73383.71, + "Total Cost": 73384.33, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 4655.295, + "Actual Total Time": 4655.324, + "Actual Rows": 50, + "Actual Loops": 3, + "Group Key": [ + "service_name" + ], + "Planned Partitions": 0, + "HashAgg Batches": 1, + "Peak Memory Usage": 48, + "Disk Usage": 0, + "Shared Hit Blocks": 7841, + "Shared Read Blocks": 16821, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 13188.305, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [ + { + "Worker Number": 0, + "HashAgg Batches": 1, + "Peak Memory Usage": 48, + "Disk Usage": 0 + }, + { + "Worker Number": 1, + "HashAgg Batches": 1, + "Peak Memory Usage": 48, + "Disk Usage": 0 + } + ], + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 13448.61, + "Total Cost": 72249.38, + "Plan Rows": 151244, + "Plan Width": 31, + "Actual Startup Time": 14.672, + "Actual Total Time": 4583.582, + "Actual Rows": 121667, + "Actual Loops": 3, + "Recheck Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Exact Heap Blocks": 8323, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 7841, + "Shared Read Blocks": 16821, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 13188.305, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [], + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "idx_cost_snapshots_workspace_date", + "Startup Cost": 0, + "Total Cost": 13357.87, + "Plan Rows": 362985, + "Plan Width": 0, + "Actual Startup Time": 30.291, + "Actual Total Time": 30.291, + "Actual Rows": 365000, + "Actual Loops": 1, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Shared Hit Blocks": 2202, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 26, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 8.288, + "Triggers": [], + "Execution Time": 4881.756 + } + ] + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/load-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/load-summary.json new file mode 100644 index 0000000..e101bdd --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/load-summary.json @@ -0,0 +1,492 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "passes": 4877, + "fails": 19448, + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b" + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 24177, + "fails": 148 + }, + { + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 4877, + "fails": 19448, + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid" + }, + { + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 5025, + "fails": 19300, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error" + }, + { + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 4896, + "fails": 19429, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200" + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 24196, + "fails": 129 + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 4896, + "fails": 19429 + }, + { + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 5025, + "fails": 19300, + "name": "cost_timeseries: no auth or server error" + }, + { + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 4088, + "fails": 16183 + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 20172, + "fails": 99 + }, + { + "passes": 4088, + "fails": 16183, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937" + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 4187, + "fails": 16084 + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 1633, + "fails": 6475 + }, + { + "passes": 8067, + "fails": 41, + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516" + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 1633, + "fails": 6475 + }, + { + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 1674, + "fails": 6434, + "name": "aws_account_list: no auth or server error" + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 817, + "fails": 3238 + }, + { + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 4034, + "fails": 21 + }, + { + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 817, + "fails": 3238, + "name": "sync_history: response shape is valid" + }, + { + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 838, + "fails": 3217, + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error" + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 1080807.8306 + }, + "metrics": { + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "rate": 173805.80958200176, + "count": 187850680 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 81085, + "rate": 75.02258746125706 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0.4008713186325292, + "passes": 130017, + "fails": 194319 + }, + "thresholds": { + "rate>0.99": { + "ok": false + } + } + }, + "http_req_sending": { + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0.00014903619658383175, + "min": 0, + "med": 0, + "max": 1.4342 + }, + "type": "trend", + "contains": "time" + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0.7988283899611519, + "passes": 64773, + "fails": 16312 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 51730585, + "rate": 47862.888790583864 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 21200.8377, + "p(90)": 1574.81915, + "p(95)": 4862.471445, + "avg": 794.498336821066, + "min": 197.2772, + "med": 220.36725 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 100, + "min": 100, + "max": 100 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 6241.0414, + "p(90)": 1311.5007200000005, + "p(95)": 4267.424819999999, + "avg": 566.8355523247702, + "min": 0, + "med": 119.8733 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 136.7955, + "p(90)": 0, + "p(95)": 0, + "avg": 0.1457691730899674 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 790.09768, + "p(95)": 3839.2237799999994, + "avg": 478.35285334155384, + "min": 0, + "med": 120.1497, + "max": 5705.7234 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 81084, + "rate": 75.02166222739801 + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 6559.6784, + "p(90)": 1635.6005, + "p(95)": 4475.634749999999, + "avg": 615.5000295989349, + "min": 0, + "med": 119.9605 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 580.3618943072064, + "min": 0, + "med": 119.9342, + "max": 6650.735, + "p(90)": 1435.03636, + "p(95)": 4360.035120000001 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "med": 119.7364, + "max": 6559.6784, + "p(90)": 1422.2633, + "p(95)": 4321.152080000003, + "avg": 577.394677929325, + "min": 0 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 1770.1792, + "p(90)": 1.0123, + "p(95)": 2.6079600000000034, + "avg": 2.967067341678489 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 2371.683743225847, + "min": 113.3143, + "med": 1430.217, + "max": 6650.735, + "p(90)": 5432.37344, + "p(95)": 5670.554255 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 4562.528579999998, + "avg": 613.7160610071961, + "min": 0, + "med": 119.9344, + "max": 6650.735, + "p(90)": 1528.07138 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 81084, + "rate": 75.02166222739801 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "fails": 16311, + "rate": 0.7988382418232943, + "passes": 64773 + }, + "thresholds": { + "rate<0.01": { + "ok": false + } + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "max": 136.7955, + "p(90)": 0, + "p(95)": 0, + "avg": 0.14598503422334597, + "min": 0, + "med": 0 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 0, + "max": 100 + } + }, + "measured_duration": { + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + }, + "type": "trend", + "contains": "time", + "values": { + "avg": 580.3614398907285, + "min": 0, + "med": 119.9342, + "max": 6650.735, + "p(90)": 1435.03682, + "p(95)": 4360.062264999995 + } + }, + "endpoint_aws_account_list_duration": { + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + }, + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 119.94300000000001, + "max": 5714.5888, + "p(90)": 827.86022, + "p(95)": 3877.8526899999965, + "avg": 484.0388600764678 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/rollup.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/rollup.json new file mode 100644 index 0000000..0b161d8 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/rollup.json @@ -0,0 +1,4 @@ +{ + "costRollupBackfill": true, + "rows": 182500 +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-after-covering-index.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-after-covering-index.json new file mode 100644 index 0000000..e9b5a7e --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-after-covering-index.json @@ -0,0 +1,492 @@ +{ + "metrics": { + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 200, + "rate": 6.444449128842911 + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 2.0129830845771144, + "min": 0, + "med": 0, + "max": 172.3786, + "p(90)": 0 + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 234.504014, + "min": 204.2331, + "med": 213.54180000000002, + "max": 608.7454, + "p(90)": 261.0743, + "p(95)": 304.49527999999987 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 385.54878999999994, + "avg": 301.729654, + "min": 220.0796, + "med": 293.27065, + "max": 709.2555, + "p(90)": 351.67377 + } + }, + "endpoint_cost_summary_duration": { + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time", + "values": { + "max": 485.3711, + "p(90)": 231.30100000000002, + "p(95)": 243.76201999999995, + "avg": 198.0279983333333, + "min": 175.4515, + "med": 184.13275 + } + }, + "endpoint_cost_timeseries_duration": { + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time", + "values": { + "avg": 211.13826333333336, + "min": 185.184, + "med": 193.88445000000002, + "max": 412.1376, + "p(90)": 268.74838, + "p(95)": 290.67519 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "avg": 1.7500567164179104, + "min": 0, + "med": 0, + "max": 120.5372, + "p(90)": 0, + "p(95)": 0 + } + }, + "endpoint_sync_history_duration": { + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time", + "values": { + "avg": 123.16491999999998, + "min": 119.9965, + "med": 124.04105, + "max": 127.3809, + "p(90)": 125.15826, + "p(95)": 126.26957999999999 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "passes": 800, + "fails": 0, + "rate": 1 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 1853054, + "rate": 59709.56117999436 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 193.07495, + "max": 608.7454, + "p(90)": 251.50535, + "p(95)": 281.23817499999996, + "avg": 200.1448755, + "min": 119.7259 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + } + }, + "http_req_duration{expected_response:true}": { + "contains": "time", + "values": { + "p(90)": 253.2707, + "p(95)": 284.7907, + "avg": 202.45341592039804, + "min": 119.7259, + "med": 193.1059, + "max": 664.1615 + }, + "type": "trend" + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 2.1177621890547256, + "min": 0, + "med": 0.3143, + "max": 123.5184, + "p(90)": 3.3307, + "p(95)": 6.2356 + } + }, + "vus": { + "values": { + "value": 1, + "min": 1, + "max": 2 + }, + "type": "gauge", + "contains": "default" + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "measured_failures": { + "thresholds": { + "rate<0.01": { + "ok": true + } + }, + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 200 + } + }, + "data_sent": { + "contains": "data", + "values": { + "count": 128572, + "rate": 4142.878566967954 + }, + "type": "counter" + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "rate": 6.444449128842911, + "count": 200 + } + }, + "http_reqs": { + "values": { + "count": 201, + "rate": 6.476671374487125 + }, + "type": "counter", + "contains": "default" + }, + "http_req_waiting": { + "values": { + "avg": 200.33565373134343, + "min": 116.6615, + "med": 191.3569, + "max": 664.1615, + "p(90)": 250.8553, + "p(95)": 265.0447 + }, + "type": "trend", + "contains": "time" + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 201 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 124.6481, + "max": 156.9006, + "p(90)": 128.21906, + "p(95)": 130.64032, + "avg": 126.10747500000002, + "min": 119.7259 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 253.2707, + "p(95)": 284.7907, + "avg": 202.45341592039804, + "min": 119.7259, + "med": 193.1059, + "max": 664.1615 + } + } + }, + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 60, + "fails": 0, + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200" + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 60, + "fails": 0 + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 60, + "fails": 0 + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 60, + "fails": 0 + }, + { + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 60, + "fails": 0, + "name": "cost_timeseries: status is 200" + }, + { + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 60, + "fails": 0, + "name": "cost_timeseries: response is JSON" + }, + { + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 60, + "fails": 0, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid" + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 60, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 50 + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 50, + "fails": 0 + }, + { + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 50, + "fails": 0 + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 50, + "fails": 0 + }, + { + "fails": 0, + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 20 + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 20, + "fails": 0 + }, + { + "passes": 20, + "fails": 0, + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2" + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 20, + "fails": 0 + }, + { + "passes": 10, + "fails": 0, + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb" + }, + { + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 10, + "fails": 0 + }, + { + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 10, + "fails": 0, + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid" + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 10, + "fails": 0 + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 31034.4602 + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-after-index.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-after-index.json new file mode 100644 index 0000000..90d6216 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-after-index.json @@ -0,0 +1,436 @@ +{ + "root_group": { + "checks": [ + { + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 30, + "fails": 0, + "name": "cost_summary: status is 200" + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 30, + "fails": 0 + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 30, + "fails": 0 + }, + { + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 30, + "fails": 0, + "name": "cost_summary: no auth or server error" + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 30, + "fails": 0 + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 30, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 30 + }, + { + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 30, + "fails": 0, + "name": "cost_timeseries: no auth or server error" + }, + { + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 6, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200" + }, + { + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 6, + "fails": 0, + "name": "cost_by_service: response is JSON" + }, + { + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 6 + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 6, + "fails": 0 + } + ], + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "testRunDurationMs": 32600.2426, + "isStdOutTTY": true, + "isStdErrTTY": true + }, + "metrics": { + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 4.079570149253732, + "min": 0, + "med": 0, + "max": 119.0283, + "p(90)": 2.504759999999999, + "p(95)": 3.777379999999998 + } + }, + "http_req_waiting": { + "values": { + "avg": 851.3495925373134, + "min": 170.7098, + "med": 184.2804, + "max": 7773.1857, + "p(90)": 431.4635799999993, + "p(95)": 7498.600769999999 + }, + "type": "trend", + "contains": "time" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 42894, + "rate": 1315.757079672775 + } + }, + "endpoint_cost_by_service_duration": { + "values": { + "p(90)": 7774.282999999999, + "p(95)": 7774.34245, + "avg": 7530.107083333333, + "min": 7155.4914, + "med": 7569.6728, + "max": 7774.4019 + }, + "thresholds": { + "p(99)<500": { + "ok": false + }, + "p(95)<200": { + "ok": false + } + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 186.0958, + "max": 322.6081, + "p(90)": 209.28229000000002, + "p(95)": 265.0318499999998, + "avg": 196.08340333333334, + "min": 180.212 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 172.9595, + "med": 184.4398, + "max": 7774.4019, + "p(90)": 311.96915, + "p(95)": 7510.936875, + "avg": 857.6684045454547 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 723048, + "rate": 22179.221451560607 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 66 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_req_duration{expected_response:true}": { + "values": { + "p(95)": 7499.357049999999, + "avg": 855.4291626865672, + "min": 172.9595, + "med": 184.5992, + "max": 7774.4019, + "p(90)": 476.62053999999944 + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_summary_duration": { + "contains": "time", + "values": { + "med": 179.01010000000002, + "max": 247.3465, + "p(90)": 201.02833, + "p(95)": 214.19035499999998, + "avg": 184.76567000000003, + "min": 172.9595 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend" + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 855.4291626865672, + "min": 172.9595, + "med": 184.5992, + "max": 7774.4019, + "p(90)": 476.62053999999944, + "p(95)": 7499.357049999999 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 66, + "rate": 2.0245248113583054 + } + }, + "http_req_connecting": { + "values": { + "avg": 5.227479104477612, + "min": 0, + "med": 0, + "max": 119.6825, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend", + "contains": "time" + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 156.021, + "p(90)": 0, + "p(95)": 0, + "avg": 5.8316462686567165 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 449.3528, + "p(95)": 7611.205099999999, + "avg": 961.6354909090909, + "min": 273.3362, + "med": 285.00735, + "max": 7874.4922 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "rate": 2.055199429712219, + "count": 67 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 66, + "rate": 2.0245248113583054 + } + }, + "checks": { + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 264, + "fails": 0 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 67 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-failed-20260909T201023Z.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-failed-20260909T201023Z.json new file mode 100644 index 0000000..184cf27 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-failed-20260909T201023Z.json @@ -0,0 +1,380 @@ +{ + "root_group": { + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 28, + "fails": 0, + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200" + }, + { + "passes": 28, + "fails": 0, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c" + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 28, + "fails": 0 + }, + { + "passes": 28, + "fails": 0, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f" + } + ], + "name": "", + "path": "" + }, + "options": { + "noColor": false, + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "" + }, + "state": { + "isStdErrTTY": true, + "testRunDurationMs": 32254.7316, + "isStdOutTTY": true + }, + "metrics": { + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 28, + "rate": 0.8680896913741487 + } + }, + "http_req_duration{expected_response:true}": { + "contains": "time", + "values": { + "avg": 2082.0090206896552, + "min": 720.7265, + "med": 2012.1423, + "max": 3256.4249, + "p(90)": 2636.60236, + "p(95)": 3007.8793399999986 + }, + "type": "trend" + }, + "iterations": { + "values": { + "count": 28, + "rate": 0.8680896913741487 + }, + "type": "counter", + "contains": "default" + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 28 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "avg": 12.376496551724138, + "min": 0, + "med": 0, + "max": 120.3326, + "p(90)": 23.858579999999915, + "p(95)": 119.2929 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 119.2929, + "avg": 14.423544827586209, + "min": 0, + "med": 0, + "max": 179.697, + "p(90)": 23.858579999999915 + } + }, + "endpoint_cost_timeseries_duration": { + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend" + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 2636.60409, + "p(95)": 3038.817934999999, + "avg": 2130.626253571428, + "min": 1794.0838, + "med": 2038.5707, + "max": 3256.4249 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 2130.626253571428, + "min": 1794.0838, + "med": 2038.5707, + "max": 3256.4249, + "p(90)": 2636.60409, + "p(95)": 3038.817934999999 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 29 + } + }, + "data_received": { + "contains": "data", + "values": { + "rate": 501.50781598815104, + "count": 16176 + }, + "type": "counter" + }, + "endpoint_sync_history_duration": { + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 112, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "http_req_receiving": { + "contains": "time", + "values": { + "avg": 0.15027931034482755, + "min": 0, + "med": 0, + "max": 0.7449, + "p(90)": 0.49267999999999995, + "p(95)": 0.55106 + }, + "type": "trend" + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 1894.2972, + "med": 2138.90485, + "max": 3476.0488, + "p(90)": 2736.9842700000004, + "p(95)": 3216.715584999999, + "avg": 2239.4733535714286 + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 2636.60236, + "p(95)": 3007.8793399999986, + "avg": 2082.0090206896552, + "min": 720.7265, + "med": 2012.1423, + "max": 3256.4249 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 18314, + "rate": 567.79266456522 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 29, + "rate": 0.8990928946375111 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "min": 720.7265, + "med": 2011.7252, + "max": 3255.8919, + "p(90)": 2636.60236, + "p(95)": 3007.6159999999986, + "avg": 2081.85874137931 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary.json new file mode 100644 index 0000000..90d6216 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary.json @@ -0,0 +1,436 @@ +{ + "root_group": { + "checks": [ + { + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 30, + "fails": 0, + "name": "cost_summary: status is 200" + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 30, + "fails": 0 + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 30, + "fails": 0 + }, + { + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 30, + "fails": 0, + "name": "cost_summary: no auth or server error" + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 30, + "fails": 0 + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 30, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 30 + }, + { + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 30, + "fails": 0, + "name": "cost_timeseries: no auth or server error" + }, + { + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 6, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200" + }, + { + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 6, + "fails": 0, + "name": "cost_by_service: response is JSON" + }, + { + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 6 + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 6, + "fails": 0 + } + ], + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "testRunDurationMs": 32600.2426, + "isStdOutTTY": true, + "isStdErrTTY": true + }, + "metrics": { + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 4.079570149253732, + "min": 0, + "med": 0, + "max": 119.0283, + "p(90)": 2.504759999999999, + "p(95)": 3.777379999999998 + } + }, + "http_req_waiting": { + "values": { + "avg": 851.3495925373134, + "min": 170.7098, + "med": 184.2804, + "max": 7773.1857, + "p(90)": 431.4635799999993, + "p(95)": 7498.600769999999 + }, + "type": "trend", + "contains": "time" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 42894, + "rate": 1315.757079672775 + } + }, + "endpoint_cost_by_service_duration": { + "values": { + "p(90)": 7774.282999999999, + "p(95)": 7774.34245, + "avg": 7530.107083333333, + "min": 7155.4914, + "med": 7569.6728, + "max": 7774.4019 + }, + "thresholds": { + "p(99)<500": { + "ok": false + }, + "p(95)<200": { + "ok": false + } + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 186.0958, + "max": 322.6081, + "p(90)": 209.28229000000002, + "p(95)": 265.0318499999998, + "avg": 196.08340333333334, + "min": 180.212 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 172.9595, + "med": 184.4398, + "max": 7774.4019, + "p(90)": 311.96915, + "p(95)": 7510.936875, + "avg": 857.6684045454547 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 723048, + "rate": 22179.221451560607 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 66 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_req_duration{expected_response:true}": { + "values": { + "p(95)": 7499.357049999999, + "avg": 855.4291626865672, + "min": 172.9595, + "med": 184.5992, + "max": 7774.4019, + "p(90)": 476.62053999999944 + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_summary_duration": { + "contains": "time", + "values": { + "med": 179.01010000000002, + "max": 247.3465, + "p(90)": 201.02833, + "p(95)": 214.19035499999998, + "avg": 184.76567000000003, + "min": 172.9595 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend" + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 855.4291626865672, + "min": 172.9595, + "med": 184.5992, + "max": 7774.4019, + "p(90)": 476.62053999999944, + "p(95)": 7499.357049999999 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 66, + "rate": 2.0245248113583054 + } + }, + "http_req_connecting": { + "values": { + "avg": 5.227479104477612, + "min": 0, + "med": 0, + "max": 119.6825, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend", + "contains": "time" + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 156.021, + "p(90)": 0, + "p(95)": 0, + "avg": 5.8316462686567165 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 449.3528, + "p(95)": 7611.205099999999, + "avg": 961.6354909090909, + "min": 273.3362, + "med": 285.00735, + "max": 7874.4922 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "rate": 2.055199429712219, + "count": 67 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 66, + "rate": 2.0245248113583054 + } + }, + "checks": { + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 264, + "fails": 0 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 67 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/teardown-verification.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/teardown-verification.json new file mode 100644 index 0000000..36f0e81 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/teardown-verification.json @@ -0,0 +1,19 @@ +{ + "verifiedAt": "2026-09-11T07:18:38Z", + "terraformApplyDestroyed": 31, + "terraformStateResources": 0, + "prefixScopedInventory": { + "ecsClusters": 0, + "activeTaskDefinitions": 0, + "rdsInstances": 0, + "ecrRepositories": 0, + "loadBalancers": 0, + "targetGroups": 0, + "logGroups": 0, + "vpcs": 0, + "securityGroups": 0, + "iamRoles": 0, + "secretsIncludingPendingDeletion": 0 + }, + "localSensitiveArtifactsRemoved": true +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/terraform-plan.txt b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/terraform-plan.txt new file mode 100644 index 0000000..e15c86e --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/terraform-plan.txt @@ -0,0 +1,995 @@ + +Terraform used the selected providers to generate the following execution +plan. Resource actions are indicated with the following symbols: + + create + <= read (data resources) + +Terraform will perform the following actions: + + # data.aws_iam_policy_document.task_execution_secrets will be read during apply + # (config refers to values not yet known) + <= data "aws_iam_policy_document" "task_execution_secrets" { + + id = (known after apply) + + json = (known after apply) + + minified_json = (known after apply) + + + statement { + + actions = [ + + "secretsmanager:GetSecretValue", + ] + + resources = [ + + (known after apply), + ] + + sid = "ReadBenchmarkRuntimeSecret" + } + } + + # aws_cloudwatch_log_group.api will be created + + resource "aws_cloudwatch_log_group" "api" { + + arn = (known after apply) + + deletion_protection_enabled = (known after apply) + + id = (known after apply) + + log_group_class = (known after apply) + + name = "/ecs/underflow-api-bench-c0cf1f68/api" + + name_prefix = (known after apply) + + region = "us-east-1" + + retention_in_days = 3 + + skip_destroy = false + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-api-logs" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-api-logs" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + } + + # aws_db_instance.postgres will be created + + resource "aws_db_instance" "postgres" { + + address = (known after apply) + + allocated_storage = 20 + + apply_immediately = true + + arn = (known after apply) + + auto_minor_version_upgrade = true + + availability_zone = (known after apply) + + backup_retention_period = 0 + + backup_target = (known after apply) + + backup_window = (known after apply) + + ca_cert_identifier = (known after apply) + + character_set_name = (known after apply) + + copy_tags_to_snapshot = false + + database_insights_mode = (known after apply) + + db_name = "underflow_benchmark" + + db_subnet_group_name = "underflow-api-bench-c0cf1f68-db-subnets" + + dedicated_log_volume = false + + delete_automated_backups = true + + deletion_protection = false + + domain_fqdn = (known after apply) + + endpoint = (known after apply) + + engine = "postgres" + + engine_lifecycle_support = (known after apply) + + engine_version = "16.13" + + engine_version_actual = (known after apply) + + hosted_zone_id = (known after apply) + + id = (known after apply) + + identifier = "underflow-api-bench-c0cf1f68-postgres" + + identifier_prefix = (known after apply) + + instance_class = "db.t4g.micro" + + iops = (known after apply) + + kms_key_id = (known after apply) + + latest_restorable_time = (known after apply) + + license_model = (known after apply) + + listener_endpoint = (known after apply) + + maintenance_window = (known after apply) + + master_user_secret = (known after apply) + + master_user_secret_kms_key_id = (known after apply) + + monitoring_interval = 0 + + monitoring_role_arn = (known after apply) + + multi_az = false + + nchar_character_set_name = (known after apply) + + network_type = (known after apply) + + option_group_name = (known after apply) + + parameter_group_name = (known after apply) + + password = (sensitive value) + + password_wo = (write-only attribute) + + performance_insights_enabled = false + + performance_insights_kms_key_id = (known after apply) + + performance_insights_retention_period = (known after apply) + + port = 5432 + + publicly_accessible = false + + region = "us-east-1" + + replica_mode = (known after apply) + + replicas = (known after apply) + + resource_id = (known after apply) + + skip_final_snapshot = true + + snapshot_identifier = (known after apply) + + status = (known after apply) + + storage_encrypted = true + + storage_throughput = (known after apply) + + storage_type = "gp3" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-postgres" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-postgres" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + timezone = (known after apply) + + upgrade_rollout_order = (known after apply) + + username = "underflow_benchmark" + + vpc_security_group_ids = (known after apply) + } + + # aws_db_subnet_group.benchmark will be created + + resource "aws_db_subnet_group" "benchmark" { + + arn = (known after apply) + + description = "Managed by Terraform" + + id = (known after apply) + + name = "underflow-api-bench-c0cf1f68-db-subnets" + + name_prefix = (known after apply) + + region = "us-east-1" + + subnet_ids = (known after apply) + + supported_network_types = (known after apply) + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-db-subnets" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-db-subnets" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_ecr_repository.api will be created + + resource "aws_ecr_repository" "api" { + + arn = (known after apply) + + force_delete = true + + id = (known after apply) + + image_tag_mutability = "IMMUTABLE" + + name = "underflow-api-bench-c0cf1f68-api" + + region = "us-east-1" + + registry_id = (known after apply) + + repository_url = (known after apply) + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-api" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-api" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + + image_scanning_configuration { + + scan_on_push = true + } + } + + # aws_ecs_cluster.benchmark will be created + + resource "aws_ecs_cluster" "benchmark" { + + arn = (known after apply) + + id = (known after apply) + + name = "underflow-api-bench-c0cf1f68-cluster" + + region = "us-east-1" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-cluster" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-cluster" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + + setting { + + name = "containerInsights" + + value = "enabled" + } + } + + # aws_ecs_service.api will be created + + resource "aws_ecs_service" "api" { + + arn = (known after apply) + + availability_zone_rebalancing = (known after apply) + + cluster = (known after apply) + + deployment_maximum_percent = 200 + + deployment_minimum_healthy_percent = 100 + + desired_count = 1 + + enable_ecs_managed_tags = false + + enable_execute_command = false + + health_check_grace_period_seconds = 90 + + iam_role = (known after apply) + + id = (known after apply) + + launch_type = "FARGATE" + + name = "underflow-api-bench-c0cf1f68-api" + + platform_version = (known after apply) + + region = "us-east-1" + + scheduling_strategy = "REPLICA" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-api" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-api" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + task_definition = (known after apply) + + triggers = (known after apply) + + wait_for_steady_state = true + + + deployment_circuit_breaker { + + enable = true + + rollback = true + } + + + deployment_configuration (known after apply) + + + load_balancer { + + container_name = "api" + + container_port = 3080 + + target_group_arn = (known after apply) + # (1 unchanged attribute hidden) + } + + + network_configuration { + + assign_public_ip = true + + security_groups = (known after apply) + + subnets = (known after apply) + } + } + + # aws_ecs_task_definition.api will be created + + resource "aws_ecs_task_definition" "api" { + + arn = (known after apply) + + arn_without_revision = (known after apply) + + container_definitions = (known after apply) + + cpu = "512" + + enable_fault_injection = (known after apply) + + execution_role_arn = (known after apply) + + family = "underflow-api-bench-c0cf1f68-api" + + id = (known after apply) + + memory = "1024" + + network_mode = "awsvpc" + + region = "us-east-1" + + requires_compatibilities = [ + + "FARGATE", + ] + + revision = (known after apply) + + skip_destroy = false + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-api" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-api" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + task_role_arn = (known after apply) + + track_latest = false + + + runtime_platform { + + cpu_architecture = "X86_64" + + operating_system_family = "LINUX" + } + } + + # aws_iam_role.task_execution will be created + + resource "aws_iam_role" "task_execution" { + + arn = (known after apply) + + assume_role_policy = jsonencode( + { + + Statement = [ + + { + + Action = "sts:AssumeRole" + + Effect = "Allow" + + Principal = { + + Service = "ecs-tasks.amazonaws.com" + } + }, + ] + + Version = "2012-10-17" + } + ) + + create_date = (known after apply) + + force_detach_policies = false + + id = (known after apply) + + managed_policy_arns = (known after apply) + + max_session_duration = 3600 + + name = "underflow-api-bench-c0cf1f68-task-execution" + + name_prefix = (known after apply) + + path = "/" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-task-execution" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-task-execution" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + unique_id = (known after apply) + + + inline_policy (known after apply) + } + + # aws_iam_role.task_runtime will be created + + resource "aws_iam_role" "task_runtime" { + + arn = (known after apply) + + assume_role_policy = jsonencode( + { + + Statement = [ + + { + + Action = "sts:AssumeRole" + + Effect = "Allow" + + Principal = { + + Service = "ecs-tasks.amazonaws.com" + } + }, + ] + + Version = "2012-10-17" + } + ) + + create_date = (known after apply) + + force_detach_policies = false + + id = (known after apply) + + managed_policy_arns = (known after apply) + + max_session_duration = 3600 + + name = "underflow-api-bench-c0cf1f68-task-runtime" + + name_prefix = (known after apply) + + path = "/" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-task-runtime" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-task-runtime" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + unique_id = (known after apply) + + + inline_policy (known after apply) + } + + # aws_iam_role_policy.task_execution_secrets will be created + + resource "aws_iam_role_policy" "task_execution_secrets" { + + id = (known after apply) + + name = "underflow-api-bench-c0cf1f68-read-runtime-secret" + + name_prefix = (known after apply) + + policy = (known after apply) + + role = (known after apply) + } + + # aws_iam_role_policy_attachment.task_execution will be created + + resource "aws_iam_role_policy_attachment" "task_execution" { + + id = (known after apply) + + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + + role = "underflow-api-bench-c0cf1f68-task-execution" + } + + # aws_internet_gateway.benchmark will be created + + resource "aws_internet_gateway" "benchmark" { + + arn = (known after apply) + + id = (known after apply) + + owner_id = (known after apply) + + region = "us-east-1" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-igw" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-igw" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_lb.api will be created + + resource "aws_lb" "api" { + + arn = (known after apply) + + arn_suffix = (known after apply) + + client_keep_alive = 3600 + + desync_mitigation_mode = "defensive" + + dns_name = (known after apply) + + drop_invalid_header_fields = false + + enable_deletion_protection = false + + enable_http2 = true + + enable_prefix_for_ipv6_source_nat = (known after apply) + + enable_tls_version_and_cipher_suite_headers = false + + enable_waf_fail_open = false + + enable_xff_client_port = false + + enable_zonal_shift = false + + enforce_security_group_inbound_rules_on_private_link_traffic = (known after apply) + + id = (known after apply) + + idle_timeout = 60 + + internal = false + + ip_address_type = (known after apply) + + load_balancer_type = "application" + + name = "underflow-api-bench-c0cf1f68-alb" + + name_prefix = (known after apply) + + preserve_host_header = false + + region = "us-east-1" + + secondary_ips_auto_assigned_per_subnet = (known after apply) + + security_groups = (known after apply) + + subnets = (known after apply) + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-alb" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-alb" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + + xff_header_processing_mode = "append" + + zone_id = (known after apply) + + + subnet_mapping (known after apply) + } + + # aws_lb_listener.http will be created + + resource "aws_lb_listener" "http" { + + arn = (known after apply) + + id = (known after apply) + + load_balancer_arn = (known after apply) + + port = 80 + + protocol = "HTTP" + + region = "us-east-1" + + routing_http_request_x_amzn_mtls_clientcert_header_name = (known after apply) + + routing_http_request_x_amzn_mtls_clientcert_issuer_header_name = (known after apply) + + routing_http_request_x_amzn_mtls_clientcert_leaf_header_name = (known after apply) + + routing_http_request_x_amzn_mtls_clientcert_serial_number_header_name = (known after apply) + + routing_http_request_x_amzn_mtls_clientcert_subject_header_name = (known after apply) + + routing_http_request_x_amzn_mtls_clientcert_validity_header_name = (known after apply) + + routing_http_request_x_amzn_tls_cipher_suite_header_name = (known after apply) + + routing_http_request_x_amzn_tls_version_header_name = (known after apply) + + routing_http_response_access_control_allow_credentials_header_value = (known after apply) + + routing_http_response_access_control_allow_headers_header_value = (known after apply) + + routing_http_response_access_control_allow_methods_header_value = (known after apply) + + routing_http_response_access_control_allow_origin_header_value = (known after apply) + + routing_http_response_access_control_expose_headers_header_value = (known after apply) + + routing_http_response_access_control_max_age_header_value = (known after apply) + + routing_http_response_content_security_policy_header_value = (known after apply) + + routing_http_response_server_enabled = (known after apply) + + routing_http_response_strict_transport_security_header_value = (known after apply) + + routing_http_response_x_content_type_options_header_value = (known after apply) + + routing_http_response_x_frame_options_header_value = (known after apply) + + ssl_policy = (known after apply) + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-http" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-http" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + tcp_idle_timeout_seconds = (known after apply) + + + default_action { + + order = (known after apply) + + target_group_arn = (known after apply) + + type = "forward" + } + + + mutual_authentication (known after apply) + } + + # aws_lb_target_group.api will be created + + resource "aws_lb_target_group" "api" { + + arn = (known after apply) + + arn_suffix = (known after apply) + + connection_termination = (known after apply) + + deregistration_delay = "300" + + id = (known after apply) + + ip_address_type = (known after apply) + + lambda_multi_value_headers_enabled = false + + load_balancer_arns = (known after apply) + + load_balancing_algorithm_type = (known after apply) + + load_balancing_anomaly_mitigation = (known after apply) + + load_balancing_cross_zone_enabled = (known after apply) + + name = "underflow-api-bench-c0cf1f68-tg" + + name_prefix = (known after apply) + + port = 3080 + + preserve_client_ip = (known after apply) + + protocol = "HTTP" + + protocol_version = (known after apply) + + proxy_protocol_v2 = false + + region = "us-east-1" + + slow_start = 0 + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-tg" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-tg" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + target_type = "ip" + + vpc_id = (known after apply) + + + health_check { + + enabled = true + + healthy_threshold = 2 + + interval = 30 + + matcher = "200" + + path = "/api/v1/health" + + port = "traffic-port" + + protocol = "HTTP" + + timeout = 5 + + unhealthy_threshold = 3 + } + + + stickiness (known after apply) + + + target_failover (known after apply) + + + target_group_health (known after apply) + + + target_health_state (known after apply) + } + + # aws_route_table.public will be created + + resource "aws_route_table" "public" { + + arn = (known after apply) + + id = (known after apply) + + owner_id = (known after apply) + + propagating_vgws = (known after apply) + + region = "us-east-1" + + route = [ + + { + + cidr_block = "[REDACTED_IPV4]/0" + + gateway_id = (known after apply) + # (12 unchanged attributes hidden) + }, + ] + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-public" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-public" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_route_table_association.public[0] will be created + + resource "aws_route_table_association" "public" { + + id = (known after apply) + + region = "us-east-1" + + route_table_id = (known after apply) + + subnet_id = (known after apply) + } + + # aws_route_table_association.public[1] will be created + + resource "aws_route_table_association" "public" { + + id = (known after apply) + + region = "us-east-1" + + route_table_id = (known after apply) + + subnet_id = (known after apply) + } + + # aws_secretsmanager_secret.runtime will be created + + resource "aws_secretsmanager_secret" "runtime" { + + arn = (known after apply) + + description = "Disposable API benchmark runtime secrets." + + force_overwrite_replica_secret = false + + id = (known after apply) + + name = "underflow-api-bench-c0cf1f68-runtime" + + name_prefix = (known after apply) + + policy = (known after apply) + + recovery_window_in_days = 0 + + region = "us-east-1" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-runtime" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-runtime" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + + replica (known after apply) + } + + # aws_secretsmanager_secret_version.runtime will be created + + resource "aws_secretsmanager_secret_version" "runtime" { + + arn = (known after apply) + + has_secret_string_wo = (known after apply) + + id = (known after apply) + + region = "us-east-1" + + secret_arn = (known after apply) + + secret_id = (known after apply) + + secret_string = (sensitive value) + + secret_string_wo = (write-only attribute) + + version_id = (known after apply) + + version_stages = (known after apply) + } + + # aws_security_group.alb will be created + + resource "aws_security_group" "alb" { + + arn = (known after apply) + + description = "Benchmark load generator ingress only." + + egress = [ + + { + + cidr_blocks = [ + + "[REDACTED_IPV4]/0", + ] + + from_port = 0 + + ipv6_cidr_blocks = [] + + prefix_list_ids = [] + + protocol = "-1" + + security_groups = [] + + self = false + + to_port = 0 + # (1 unchanged attribute hidden) + }, + ] + + id = (known after apply) + + ingress = [ + + { + + cidr_blocks = [ + + "[REDACTED_IPV4]/32", + ] + + description = "HTTP from the explicitly configured load generator" + + from_port = 80 + + ipv6_cidr_blocks = [] + + prefix_list_ids = [] + + protocol = "tcp" + + security_groups = [] + + self = false + + to_port = 80 + }, + ] + + name = "underflow-api-bench-c0cf1f68-alb" + + name_prefix = (known after apply) + + owner_id = (known after apply) + + region = "us-east-1" + + revoke_rules_on_delete = false + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-alb" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-alb" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_security_group.ecs will be created + + resource "aws_security_group" "ecs" { + + arn = (known after apply) + + description = "Benchmark API tasks; no direct public ingress." + + egress = [ + + { + + cidr_blocks = [ + + "[REDACTED_IPV4]/0", + ] + + from_port = 0 + + ipv6_cidr_blocks = [] + + prefix_list_ids = [] + + protocol = "-1" + + security_groups = [] + + self = false + + to_port = 0 + # (1 unchanged attribute hidden) + }, + ] + + id = (known after apply) + + ingress = [ + + { + + cidr_blocks = [] + + description = "API traffic from the benchmark ALB only" + + from_port = 3080 + + ipv6_cidr_blocks = [] + + prefix_list_ids = [] + + protocol = "tcp" + + security_groups = (known after apply) + + self = false + + to_port = 3080 + }, + ] + + name = "underflow-api-bench-c0cf1f68-ecs" + + name_prefix = (known after apply) + + owner_id = (known after apply) + + region = "us-east-1" + + revoke_rules_on_delete = false + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-ecs" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-ecs" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_security_group.rds will be created + + resource "aws_security_group" "rds" { + + arn = (known after apply) + + description = "Benchmark PostgreSQL ingress from API tasks only." + + egress = [ + + { + + cidr_blocks = [ + + "[REDACTED_IPV4]/0", + ] + + from_port = 0 + + ipv6_cidr_blocks = [] + + prefix_list_ids = [] + + protocol = "-1" + + security_groups = [] + + self = false + + to_port = 0 + # (1 unchanged attribute hidden) + }, + ] + + id = (known after apply) + + ingress = [ + + { + + cidr_blocks = [] + + description = "PostgreSQL from benchmark API tasks only" + + from_port = 5432 + + ipv6_cidr_blocks = [] + + prefix_list_ids = [] + + protocol = "tcp" + + security_groups = (known after apply) + + self = false + + to_port = 5432 + }, + ] + + name = "underflow-api-bench-c0cf1f68-rds" + + name_prefix = (known after apply) + + owner_id = (known after apply) + + region = "us-east-1" + + revoke_rules_on_delete = false + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-rds" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-rds" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_subnet.public[0] will be created + + resource "aws_subnet" "public" { + + arn = (known after apply) + + assign_ipv6_address_on_creation = false + + availability_zone = "us-east-1a" + + availability_zone_id = (known after apply) + + cidr_block = "[REDACTED_IPV4]/24" + + enable_dns64 = false + + enable_resource_name_dns_a_record_on_launch = false + + enable_resource_name_dns_aaaa_record_on_launch = false + + id = (known after apply) + + ipv6_cidr_block = (known after apply) + + ipv6_cidr_block_association_id = (known after apply) + + ipv6_native = false + + map_public_ip_on_launch = true + + owner_id = (known after apply) + + private_dns_hostname_type_on_launch = (known after apply) + + region = "us-east-1" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-public-1" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-public-1" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_subnet.public[1] will be created + + resource "aws_subnet" "public" { + + arn = (known after apply) + + assign_ipv6_address_on_creation = false + + availability_zone = "us-east-1b" + + availability_zone_id = (known after apply) + + cidr_block = "[REDACTED_IPV4]/24" + + enable_dns64 = false + + enable_resource_name_dns_a_record_on_launch = false + + enable_resource_name_dns_aaaa_record_on_launch = false + + id = (known after apply) + + ipv6_cidr_block = (known after apply) + + ipv6_cidr_block_association_id = (known after apply) + + ipv6_native = false + + map_public_ip_on_launch = true + + owner_id = (known after apply) + + private_dns_hostname_type_on_launch = (known after apply) + + region = "us-east-1" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-public-2" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-public-2" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_vpc.benchmark will be created + + resource "aws_vpc" "benchmark" { + + arn = (known after apply) + + cidr_block = "[REDACTED_IPV4]/16" + + default_network_acl_id = (known after apply) + + default_route_table_id = (known after apply) + + default_security_group_id = (known after apply) + + dhcp_options_id = (known after apply) + + enable_dns_hostnames = true + + enable_dns_support = true + + enable_network_address_usage_metrics = (known after apply) + + id = (known after apply) + + instance_tenancy = "default" + + ipv6_association_id = (known after apply) + + ipv6_cidr_block = (known after apply) + + ipv6_cidr_block_network_border_group = (known after apply) + + main_route_table_id = (known after apply) + + owner_id = (known after apply) + + region = "us-east-1" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-vpc" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-vpc" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + } + + # random_password.benchmark will be created + + resource "random_password" "benchmark" { + + bcrypt_hash = (sensitive value) + + id = (known after apply) + + length = 24 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + numeric = true + + result = (sensitive value) + + special = false + + upper = true + } + + # random_password.csrf will be created + + resource "random_password" "csrf" { + + bcrypt_hash = (sensitive value) + + id = (known after apply) + + length = 48 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + numeric = true + + result = (sensitive value) + + special = false + + upper = true + } + + # random_password.database will be created + + resource "random_password" "database" { + + bcrypt_hash = (sensitive value) + + id = (known after apply) + + length = 32 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + numeric = true + + override_special = "!#$%&*+-.:=?@_" + + result = (sensitive value) + + special = true + + upper = true + } + + # random_password.jwt_access will be created + + resource "random_password" "jwt_access" { + + bcrypt_hash = (sensitive value) + + id = (known after apply) + + length = 64 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + numeric = true + + result = (sensitive value) + + special = false + + upper = true + } + + # random_password.jwt_refresh will be created + + resource "random_password" "jwt_refresh" { + + bcrypt_hash = (sensitive value) + + id = (known after apply) + + length = 64 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + numeric = true + + result = (sensitive value) + + special = false + + upper = true + } + +Plan: 31 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + alb_dns_name = (known after apply) + + api_base_url = (known after apply) + + availability_zones = [ + + "us-east-1a", + + "us-east-1b", + ] + + aws_region = "us-east-1" + + ecr_repository_url = (known after apply) + + ecs_cluster_name = "underflow-api-bench-c0cf1f68-cluster" + + ecs_security_group_id = (known after apply) + + ecs_service_name = "underflow-api-bench-c0cf1f68-api" + + load_balancer_arn_suffix = (known after apply) + + rds_identifier = "underflow-api-bench-c0cf1f68-postgres" + + resource_prefix = "underflow-api-bench-c0cf1f68" + + runtime_secret_arn = (sensitive value) + + subnet_ids = [ + + (known after apply), + + (known after apply), + ] + + target_group_arn_suffix = (known after apply) + + task_definition_arn = (known after apply) diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/terraform-state-list.txt b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/terraform-state-list.txt new file mode 100644 index 0000000..71a0fc2 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/terraform-state-list.txt @@ -0,0 +1,34 @@ +data.aws_availability_zones.available +data.aws_iam_policy_document.ecs_tasks_assume_role +data.aws_iam_policy_document.task_execution_secrets +aws_cloudwatch_log_group.api +aws_db_instance.postgres +aws_db_subnet_group.benchmark +aws_ecr_repository.api +aws_ecs_cluster.benchmark +aws_ecs_service.api +aws_ecs_task_definition.api +aws_iam_role.task_execution +aws_iam_role.task_runtime +aws_iam_role_policy.task_execution_secrets +aws_iam_role_policy_attachment.task_execution +aws_internet_gateway.benchmark +aws_lb.api +aws_lb_listener.http +aws_lb_target_group.api +aws_route_table.public +aws_route_table_association.public[0] +aws_route_table_association.public[1] +aws_secretsmanager_secret.runtime +aws_secretsmanager_secret_version.runtime +aws_security_group.alb +aws_security_group.ecs +aws_security_group.rds +aws_subnet.public[0] +aws_subnet.public[1] +aws_vpc.benchmark +random_password.benchmark +random_password.csrf +random_password.database +random_password.jwt_access +random_password.jwt_refresh diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/warmup-after-covering-index.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/warmup-after-covering-index.json new file mode 100644 index 0000000..4b93a52 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/warmup-after-covering-index.json @@ -0,0 +1,492 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 120, + "fails": 0 + }, + { + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 120, + "fails": 0, + "name": "cost_summary: response is JSON" + }, + { + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 120, + "fails": 0, + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid" + }, + { + "fails": 0, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 120 + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 120, + "fails": 0 + }, + { + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 120, + "fails": 0, + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON" + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 120, + "fails": 0 + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 120, + "fails": 0 + }, + { + "passes": 100, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b" + }, + { + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 100, + "fails": 0, + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON" + }, + { + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 100, + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid" + }, + { + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 100, + "fails": 0, + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error" + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 40, + "fails": 0 + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 40, + "fails": 0 + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 40, + "fails": 0 + }, + { + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 40, + "fails": 0, + "name": "aws_account_list: no auth or server error" + }, + { + "fails": 0, + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 20 + }, + { + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 20, + "fails": 0 + }, + { + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 20, + "fails": 0, + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid" + }, + { + "fails": 0, + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 20 + } + ] + }, + "options": { + "noColor": false, + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "" + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 61152.0884 + }, + "metrics": { + "vus_max": { + "values": { + "value": 2, + "min": 2, + "max": 2 + }, + "type": "gauge", + "contains": "default" + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "rate": 6.557421185308203, + "count": 401 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 256862, + "rate": 4200.379851622532 + } + }, + "vus": { + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + }, + "type": "gauge" + }, + "endpoint_sync_history_duration": { + "values": { + "max": 130.7253, + "p(90)": 129.31742, + "p(95)": 130.30882, + "avg": 124.50813999999998, + "min": 119.9934, + "med": 125.0402 + }, + "thresholds": { + "p(99)<500": { + "ok": true + }, + "p(95)<200": { + "ok": true + } + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_timeseries_duration": { + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time", + "values": { + "max": 434.0696, + "p(90)": 231.75937000000005, + "p(95)": 306.80621499999995, + "avg": 208.06491833333328, + "min": 179.4055, + "med": 194.57015 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 203.2400591666666, + "min": 176.923, + "med": 187.61725, + "max": 379.5257, + "p(90)": 241.04909000000004, + "p(95)": 270.70661 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 203.0791, + "med": 217.70815, + "max": 358.0709, + "p(90)": 265.58596000000006, + "p(95)": 302.73309499999993, + "avg": 231.28093599999988 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 118.434, + "p(90)": 0, + "p(95)": 0, + "avg": 0.8739768079800498 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 401 + } + }, + "data_received": { + "contains": "data", + "values": { + "count": 3704464, + "rate": 60577.88207933059 + }, + "type": "counter" + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 400 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "min": 114.2297, + "med": 195.4294, + "max": 724.7384, + "p(90)": 246.434, + "p(95)": 274.0703, + "avg": 201.48684488778053 + } + }, + "http_req_duration": { + "values": { + "p(95)": 274.0703, + "avg": 201.48684488778053, + "min": 114.2297, + "med": 195.4294, + "max": 724.7384, + "p(90)": 246.434 + }, + "type": "trend", + "contains": "time" + }, + "http_req_blocked": { + "contains": "time", + "values": { + "avg": 0.896937157107232, + "min": 0, + "med": 0, + "max": 127.4808, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend" + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 400, + "rate": 6.541068514023145 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 400, + "rate": 6.541068514023145 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 246.21125, + "p(95)": 271.78782999999993, + "avg": 200.17871599999998, + "min": 114.2297, + "med": 195.31595, + "max": 434.0696 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 125.58075, + "max": 209.3966, + "p(90)": 134.1675, + "p(95)": 135.64968000000002, + "avg": 127.41581749999997, + "min": 114.2297 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "checks": { + "contains": "default", + "values": { + "rate": 1, + "passes": 1600, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 1.702756608478804, + "min": 0, + "med": 0.3967, + "max": 119.1295, + "p(90)": 3.8914, + "p(95)": 5.5716 + } + }, + "http_req_waiting": { + "contains": "time", + "values": { + "avg": 199.7840882793018, + "min": 114.2297, + "med": 194.3901, + "max": 723.9436, + "p(90)": 243.9218, + "p(95)": 271.6677 + }, + "type": "trend" + }, + "iteration_duration": { + "contains": "time", + "values": { + "avg": 301.2783107499998, + "min": 214.3706, + "med": 296.28955, + "max": 644.4486, + "p(90)": 346.88457, + "p(95)": 372.1879999999999 + }, + "type": "trend" + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/warmup-after-index-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/warmup-after-index-summary.json new file mode 100644 index 0000000..d2248b9 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/warmup-after-index-summary.json @@ -0,0 +1,311 @@ +{ + "metrics": { + "data_sent": { + "values": { + "count": 0, + "rate": 0 + }, + "type": "counter", + "contains": "data" + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 0, + "rate": 0 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_req_connecting": { + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend", + "contains": "time" + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 1, + "rate": 0.047426040346869126 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 0, + "min": 0, + "max": 0 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + } + }, + "endpoint_aws_account_list_duration": { + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time" + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 0 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0 + } + }, + "http_req_failed": { + "contains": "default", + "values": { + "rate": 1, + "passes": 1, + "fails": 0 + }, + "type": "rate" + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "endpoint_cost_summary_duration": { + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend" + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + } + } + }, + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 21085.4626 + } +} \ No newline at end of file diff --git a/benchmarks/underflow-api/scripts/check-capacity-result.sh b/benchmarks/underflow-api/scripts/check-capacity-result.sh new file mode 100644 index 0000000..0457e43 --- /dev/null +++ b/benchmarks/underflow-api/scripts/check-capacity-result.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 K6_SUMMARY CLOUDWATCH_SUMMARY" >&2 + exit 2 +fi + +K6_SUMMARY="$1" +CLOUDWATCH_SUMMARY="$2" + +for command in jq; do + command -v "$command" >/dev/null 2>&1 || { + echo "$command is required" >&2 + exit 2 + } +done + +[[ -f "$K6_SUMMARY" ]] || { echo "missing k6 summary: $K6_SUMMARY" >&2; exit 2; } +[[ -f "$CLOUDWATCH_SUMMARY" ]] || { echo "missing CloudWatch summary: $CLOUDWATCH_SUMMARY" >&2; exit 2; } + +jq -e ' + .metrics.measured_failures.values.rate < 0.01 and + .metrics.checks.values.rate > 0.99 +' "$K6_SUMMARY" >/dev/null || { + echo "FAIL: functional thresholds were not met" >&2 + exit 1 +} + +jq -e ' + .metrics.alb_target_response.datapoints as $points | + ($points | length) > 0 and + all($points[]; + .ExtendedStatistics.p50 < 0.030 and + .ExtendedStatistics.p95 < 0.100 and + .ExtendedStatistics.p99 < 0.250 + ) +' "$CLOUDWATCH_SUMMARY" >/dev/null || { + echo "FAIL: ALB server-side targets were not met in every one-minute datapoint" >&2 + exit 1 +} + +jq -r ' + .metrics.alb_target_response.datapoints | + "PASS: \(length) datapoints; worst p50=\(map(.ExtendedStatistics.p50) | max * 1000) ms, p95=\(map(.ExtendedStatistics.p95) | max * 1000) ms, p99=\(map(.ExtendedStatistics.p99) | max * 1000) ms" +' "$CLOUDWATCH_SUMMARY" diff --git a/benchmarks/underflow-api/scripts/collect-aws-metadata.sh b/benchmarks/underflow-api/scripts/collect-aws-metadata.sh new file mode 100644 index 0000000..75fce7e --- /dev/null +++ b/benchmarks/underflow-api/scripts/collect-aws-metadata.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +set -euo pipefail + +required=(AWS_REGION ECS_CLUSTER ECS_SERVICE RDS_IDENTIFIER ALB_ARN_SUFFIX TARGET_GROUP_ARN_SUFFIX START_TIME END_TIME OUTPUT_FILE) +for name in "${required[@]}"; do + [[ -n "${!name:-}" ]] || { echo "$name is required" >&2; exit 2; } +done + +command -v aws >/dev/null || { echo "aws is required" >&2; exit 2; } +if command -v py >/dev/null; then + PYTHON_COMMAND=(py -3) +elif command -v python3 >/dev/null; then + PYTHON_COMMAND=(python3) +elif command -v python >/dev/null; then + PYTHON_COMMAND=(python) +else + echo "python3, py, or python is required" >&2 + exit 2 +fi + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +collect_metric() { + local key="$1" namespace="$2" metric="$3" statistic="$4" + shift 4 + aws cloudwatch get-metric-statistics \ + --region "$AWS_REGION" \ + --namespace "$namespace" \ + --metric-name "$metric" \ + --dimensions "$@" \ + --start-time "$START_TIME" \ + --end-time "$END_TIME" \ + --period 60 \ + --statistics "$statistic" \ + --output json > "$TMP_DIR/$key.json" +} + +collect_percentile_metric() { + local key="$1" namespace="$2" metric="$3" + shift 3 + aws cloudwatch get-metric-statistics \ + --region "$AWS_REGION" \ + --namespace "$namespace" \ + --metric-name "$metric" \ + --dimensions "$@" \ + --start-time "$START_TIME" \ + --end-time "$END_TIME" \ + --period 60 \ + --extended-statistics p50 p90 p95 p99 \ + --output json > "$TMP_DIR/$key.json" +} + +collect_metric ecs_cpu AWS/ECS CPUUtilization Average \ + Name=ClusterName,Value="$ECS_CLUSTER" Name=ServiceName,Value="$ECS_SERVICE" +collect_metric ecs_memory AWS/ECS MemoryUtilization Average \ + Name=ClusterName,Value="$ECS_CLUSTER" Name=ServiceName,Value="$ECS_SERVICE" +collect_metric ecs_running_tasks ECS/ContainerInsights RunningTaskCount Average \ + Name=ClusterName,Value="$ECS_CLUSTER" Name=ServiceName,Value="$ECS_SERVICE" +collect_percentile_metric alb_target_response AWS/ApplicationELB TargetResponseTime \ + Name=LoadBalancer,Value="$ALB_ARN_SUFFIX" Name=TargetGroup,Value="$TARGET_GROUP_ARN_SUFFIX" +collect_metric alb_4xx AWS/ApplicationELB HTTPCode_ELB_4XX_Count Sum \ + Name=LoadBalancer,Value="$ALB_ARN_SUFFIX" +collect_metric alb_5xx AWS/ApplicationELB HTTPCode_ELB_5XX_Count Sum \ + Name=LoadBalancer,Value="$ALB_ARN_SUFFIX" +collect_metric alb_target_5xx AWS/ApplicationELB HTTPCode_Target_5XX_Count Sum \ + Name=LoadBalancer,Value="$ALB_ARN_SUFFIX" Name=TargetGroup,Value="$TARGET_GROUP_ARN_SUFFIX" +collect_metric rds_cpu AWS/RDS CPUUtilization Average Name=DBInstanceIdentifier,Value="$RDS_IDENTIFIER" +collect_metric rds_connections AWS/RDS DatabaseConnections Average Name=DBInstanceIdentifier,Value="$RDS_IDENTIFIER" +collect_metric rds_free_memory AWS/RDS FreeableMemory Average Name=DBInstanceIdentifier,Value="$RDS_IDENTIFIER" +collect_metric rds_read_latency AWS/RDS ReadLatency Average Name=DBInstanceIdentifier,Value="$RDS_IDENTIFIER" +collect_metric rds_write_latency AWS/RDS WriteLatency Average Name=DBInstanceIdentifier,Value="$RDS_IDENTIFIER" +collect_metric rds_burst_balance AWS/RDS BurstBalance Average Name=DBInstanceIdentifier,Value="$RDS_IDENTIFIER" +collect_metric rds_cpu_credit AWS/RDS CPUCreditBalance Average Name=DBInstanceIdentifier,Value="$RDS_IDENTIFIER" + +aws ecs describe-services \ + --region "$AWS_REGION" \ + --cluster "$ECS_CLUSTER" \ + --services "$ECS_SERVICE" \ + --query 'services[0].{desiredCount:desiredCount,runningCount:runningCount,pendingCount:pendingCount}' \ + --output json > "$TMP_DIR/ecs_tasks.json" + +"${PYTHON_COMMAND[@]}" - "$TMP_DIR" "$OUTPUT_FILE" "$START_TIME" "$END_TIME" <<'PY' +import json +import pathlib +import sys + +source = pathlib.Path(sys.argv[1]) +output = pathlib.Path(sys.argv[2]) +payload = { + "startTime": sys.argv[3], + "endTime": sys.argv[4], + "periodSeconds": 60, + "ecsTaskCountsAtCollection": json.loads((source / "ecs_tasks.json").read_text()), + "metrics": {}, +} +for path in sorted(source.glob("*.json")): + if path.name == "ecs_tasks.json": + continue + metric = json.loads(path.read_text()) + payload["metrics"][path.stem] = { + "label": metric.get("Label"), + "datapoints": sorted(metric.get("Datapoints", []), key=lambda item: item["Timestamp"]), + } +output.parent.mkdir(parents=True, exist_ok=True) +output.write_text(json.dumps(payload, indent=2) + "\n") +PY + +echo "Saved CloudWatch summary to $OUTPUT_FILE" diff --git a/benchmarks/underflow-api/scripts/collect-environment.sh b/benchmarks/underflow-api/scripts/collect-environment.sh new file mode 100644 index 0000000..7c89eca --- /dev/null +++ b/benchmarks/underflow-api/scripts/collect-environment.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +required=(GIT_SHA IMAGE_DIGEST AWS_REGION RESULTS_DIR TF_ROOT) +for name in "${required[@]}"; do + [[ -n "${!name:-}" ]] || { echo "$name is required" >&2; exit 2; } +done + +for command in jq terraform node; do + command -v "$command" >/dev/null || { echo "$command is required" >&2; exit 2; } +done + +mkdir -p "$RESULTS_DIR" +if [[ -z "${K6_VERSION:-}" ]]; then + if command -v k6 >/dev/null; then + K6_VERSION="$(k6 version | head -n1)" + elif command -v docker >/dev/null; then + K6_VERSION="$(docker run --rm grafana/k6:latest version | head -n1)" + else + echo "k6 or Docker is required" >&2 + exit 2 + fi +fi + +jq -n \ + --arg gitSha "$GIT_SHA" \ + --arg imageDigest "$IMAGE_DIGEST" \ + --arg timestamp "$(date -u +%FT%TZ)" \ + --arg region "$AWS_REGION" \ + --argjson availabilityZones "$(terraform -chdir="$TF_ROOT" output -json availability_zones)" \ + --arg nodeVersion "$(node --version)" \ + --arg k6Version "$K6_VERSION" \ + '{ + gitSha: $gitSha, + imageDigest: $imageDigest, + testTimestamp: $timestamp, + awsRegion: $region, + availabilityZones: $availabilityZones, + infrastructure: { + ecs: {cpu: 512, memoryMiB: 1024, desiredCount: 1}, + rds: { + class: "db.t4g.micro", + engine: "postgres", + engineVersion: "16.13", + storageGiB: 20, + multiAz: false, + publiclyAccessible: false + } + }, + nodeVersion: $nodeVersion, + k6Version: $k6Version, + dataset: "underflow-api-benchmark-v1", + profiles: { + smoke: {vus: 2, duration: "30s"}, + capacityDiscovery: {levels: [2,3,4,5], durationPerLevel: "3m"}, + normalLoad: {stages: [["1m",25],["3m",25],["1m",50],["5m",50],["2m",100],["5m",100],["1m",0]]}, + stress: {stages: [["2m",100],["3m",100],["1m",150],["3m",150],["1m",200],["3m",200],["1m",0]]} + } + }' > "$RESULTS_DIR/environment.json" + +jq -e . "$RESULTS_DIR/environment.json" >/dev/null +echo "Saved environment metadata to $RESULTS_DIR/environment.json" diff --git a/benchmarks/underflow-api/scripts/generate-results.sh b/benchmarks/underflow-api/scripts/generate-results.sh new file mode 100644 index 0000000..406b15f --- /dev/null +++ b/benchmarks/underflow-api/scripts/generate-results.sh @@ -0,0 +1,232 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 RESULTS_DIRECTORY OUTPUT_MARKDOWN" >&2 + exit 2 +fi + +if command -v py >/dev/null; then + PYTHON_COMMAND=(py -3) +elif command -v python3 >/dev/null; then + PYTHON_COMMAND=(python3) +elif command -v python >/dev/null; then + PYTHON_COMMAND=(python) +else + echo "python3, py, or python is required" >&2 + exit 2 +fi + +"${PYTHON_COMMAND[@]}" - "$1" "$2" <<'PY' +import json +import pathlib +import sys + +directory = pathlib.Path(sys.argv[1]) +output = pathlib.Path(sys.argv[2]) + +def read(name): + path = directory / name + if not path.exists(): + raise SystemExit(f"missing required evidence: {path}") + return json.loads(path.read_text(encoding="utf-8")) + +def values(summary, metric): + try: + return summary["metrics"][metric]["values"] + except KeyError as error: + raise SystemExit(f"missing metric {metric}: {error}") from error + +def points(cloudwatch, metric): + result = cloudwatch["metrics"][metric]["datapoints"] + if not result: + raise SystemExit(f"missing CloudWatch datapoints for {metric}") + return result + +def worst_percentile(cloudwatch, percentile): + return max( + point["ExtendedStatistics"][percentile] * 1000 + for point in points(cloudwatch, "alb_target_response") + ) + +def maximum_average(cloudwatch, metric): + return max(point["Average"] for point in points(cloudwatch, metric)) + +def minimum_average(cloudwatch, metric): + return min(point["Average"] for point in points(cloudwatch, metric)) + +def count_breaches(cloudwatch, percentile, target_ms): + return sum( + point["ExtendedStatistics"][percentile] * 1000 >= target_ms + for point in points(cloudwatch, "alb_target_response") + ) + +def sum_metric(cloudwatch, metric): + return sum(point.get("Sum", 0) for point in cloudwatch["metrics"][metric]["datapoints"]) + +environment = read("environment.json") +dataset = read("dataset.json") +rollup = read("rollup.json") +smoke = read("smoke-summary.json") +invalid_load = read("load-summary.json") +supported = read("capacity-45vus-post-rollup-soak-summary.json") +supported_cw = read("capacity-45vus-post-rollup-soak-cloudwatch.json") +failed = read("capacity-50vus-post-rollup-soak-summary.json") +failed_cw = read("capacity-50vus-post-rollup-soak-cloudwatch.json") +explain = read("explain-after-rollup.json") + +supported_requests = values(supported, "measured_requests") +supported_failures = values(supported, "measured_failures") +supported_checks = values(supported, "checks") +failed_requests = values(failed, "measured_requests") +failed_failures = values(failed, "measured_failures") +failed_checks = values(failed, "checks") + +supported_p50 = worst_percentile(supported_cw, "p50") +supported_p95 = worst_percentile(supported_cw, "p95") +supported_p99 = worst_percentile(supported_cw, "p99") +failed_p50 = worst_percentile(failed_cw, "p50") +failed_p95 = worst_percentile(failed_cw, "p95") +failed_p99 = worst_percentile(failed_cw, "p99") + +supported_passes = ( + supported_failures["rate"] < 0.01 + and supported_checks["rate"] > 0.99 + and supported_p50 < 30 + and supported_p95 < 100 + and supported_p99 < 250 +) +failed_passes = ( + failed_failures["rate"] < 0.01 + and failed_checks["rate"] > 0.99 + and failed_p50 < 30 + and failed_p95 < 100 + and failed_p99 < 250 +) +if not supported_passes: + raise SystemExit("45-VU soak evidence does not meet the documented targets") +if failed_passes: + raise SystemExit("50-VU soak evidence unexpectedly meets every documented target") +if int(dataset["costRollups"]) != int(rollup["rows"]): + raise SystemExit("dataset and rollup evidence disagree") + +raw_rows = int(dataset["costSnapshots"]) +rollup_rows = int(dataset["costRollups"]) +records_label = f"{raw_rows / 1_000_000:.2f}M" +resume = ( + "Built and benchmarked a multi-tenant AWS cost-monitoring API against " + f"{records_label} cost records, sustaining {supported_requests['rate']:.2f} requests/second " + f"with {supported_p95:.2f} ms worst-minute p95 server-side latency under 45 concurrent virtual users." +) + +plans = explain["plans"] +plan_times = { + name: plans[name][0]["Execution Time"] + for name in ("summary", "timeseries", "byService") +} +stats = {row["relname"]: row for row in explain["stats"]} + +endpoint_metrics = [ + ("Cost summary", "endpoint_cost_summary_duration"), + ("Cost timeseries", "endpoint_cost_timeseries_duration"), + ("Cost by service", "endpoint_cost_by_service_duration"), + ("AWS account list", "endpoint_aws_account_list_duration"), + ("Sync history", "endpoint_sync_history_duration"), +] + +lines = [ + "# Underflow API benchmark results", + "", + "## Objective", + "", + "Measure authenticated, PostgreSQL-backed cost-reporting reads on an isolated, disposable AWS configuration and establish a repeatable server-side capacity boundary.", + "", + "## Tested commit", + "", + f"The deployed image was built from local commit `{environment['gitSha']}` using image digest `{environment['imageDigest']}`. After an evidence-only history rewrite removed a bearer token from an earlier commit, the code-equivalent repository commit is `{environment['repositoryEquivalentSha']}`.", + "", + "## Infrastructure", + "", + "One ECS Fargate API task with 0.5 vCPU and 1,024 MiB memory, backed by a private, single-AZ `db.t4g.micro` PostgreSQL 16.13 instance with 20 GiB encrypted storage in `us-east-1`.", + "", + "## Dataset", + "", + f"{raw_rows:,} deterministic synthetic cost snapshots across {dataset['workspaces']} workspaces and {dataset['awsAccounts']} synthetic AWS accounts. Workspace/date/service reads use {rollup_rows:,} daily rollup rows, a {raw_rows / rollup_rows:.0f}x row-count reduction.", + "", + "## Workload", + "", + "Bearer-authenticated read traffic distributed as 30% cost summary, 30% timeseries, 25% by service, 10% AWS account list, and 5% sync history. Login occurs once during setup and is excluded from measured requests. The supported and failing boundary runs each lasted 15 minutes.", + "", + "## Acceptance targets", + "", + "Measured failure rate `<1%`, checks pass rate `>99%`, and every one-minute ALB `TargetResponseTime` datapoint p50 `<30 ms`, p95 `<100 ms`, and p99 `<250 ms`. ALB target response time excludes load-generator-to-ALB Internet transit.", + "", + "## Capacity results", + "", + "| Run | Duration | Requests | Requests/s | Failure rate | Checks | Worst p50 | Worst p95 | Worst p99 | Result |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |", + f"| 45 VUs | 15m | {supported_requests['count']:,} | {supported_requests['rate']:.2f} | {supported_failures['rate'] * 100:.4f}% | {supported_checks['rate'] * 100:.4f}% | {supported_p50:.2f} ms | {supported_p95:.2f} ms | {supported_p99:.2f} ms | Pass |", + f"| 50 VUs | 15m | {failed_requests['count']:,} | {failed_requests['rate']:.2f} | {failed_failures['rate'] * 100:.4f}% | {failed_checks['rate'] * 100:.4f}% | {failed_p50:.2f} ms | {failed_p95:.2f} ms | {failed_p99:.2f} ms | Fail |", + "", + f"The documented configuration therefore sustained at least **{supported_requests['rate']:.2f} requests/second at 45 continuously active VUs**. The first sustained failing level tested was 50 VUs; this establishes a tested boundary, not a claim that 45 VUs equals 45 registered users or that 45 is the absolute maximum.", + "", + "## Initial smoke and normal-load outcomes", + "", + f"The initial 2-VU smoke run completed {values(smoke, 'measured_requests')['count']:,} measured requests with {values(smoke, 'measured_failures')['rate'] * 100:.3f}% functional failures and {values(smoke, 'checks')['rate'] * 100:.3f}% checks passed. It exposed the pre-rollup cost-query latency bottleneck.", + "", + f"The original ramped normal-load attempt recorded {values(invalid_load, 'measured_failures')['rate'] * 100:.2f}% failures after the load generator's public IP changed and was therefore invalid for capacity claims. It is retained as failure evidence but excluded from the supported result. A separate stress profile was not run after the fixed-load capacity boundary was established.", + "", + "## Database optimization evidence", + "", + f"The raw `cost_snapshots` relation occupied {stats['cost_snapshots']['total_size']}; the rollup relation occupied {stats['workspace_cost_daily_rollups']['total_size']}. Post-rollup full-year `EXPLAIN (ANALYZE, BUFFERS)` execution times were {plan_times['summary']:.3f} ms for summary, {plan_times['timeseries']:.3f} ms for timeseries, and {plan_times['byService']:.3f} ms for by-service. All used index-only scans.", + "", + "## Client-observed endpoint diagnostics", + "", + "These values include Internet transit from the load-generator location and are retained for diagnosis, not backend acceptance.", + "", + "| Endpoint | Average | Median | p90 | p95 | Maximum |", + "| --- | ---: | ---: | ---: | ---: | ---: |", +] +for label, metric in endpoint_metrics: + metric_values = values(supported, metric) + lines.append( + f"| {label} | {metric_values['avg']:.2f} ms | {metric_values['med']:.2f} ms | " + f"{metric_values['p(90)']:.2f} ms | {metric_values['p(95)']:.2f} ms | {metric_values['max']:.2f} ms |" + ) + +lines.extend([ + "", + "## AWS resource metrics at the supported level", + "", + f"Peak one-minute ECS CPU was {maximum_average(supported_cw, 'ecs_cpu'):.2f}% and ECS memory was {maximum_average(supported_cw, 'ecs_memory'):.2f}%. Peak RDS CPU was {maximum_average(supported_cw, 'rds_cpu'):.2f}%, the minimum average freeable memory was {minimum_average(supported_cw, 'rds_free_memory') / 1024 / 1024:.2f} MiB, and database connections peaked at {maximum_average(supported_cw, 'rds_connections'):.0f}. The ALB recorded {sum_metric(supported_cw, 'alb_5xx'):.0f} load-balancer-generated 5xx responses and {sum_metric(supported_cw, 'alb_target_5xx'):.0f} target-generated 5xx responses during {supported_requests['count']:,} measured requests.", + "", + "## Observed bottleneck", + "", + f"At 50 VUs, p95 exceeded 100 ms in {count_breaches(failed_cw, 'p95', 100)} of 15 one-minute periods and p99 exceeded 250 ms in {count_breaches(failed_cw, 'p99', 250)} period. Peak one-minute ECS CPU reached {maximum_average(failed_cw, 'ecs_cpu'):.2f}% and RDS CPU reached {maximum_average(failed_cw, 'rds_cpu'):.2f}%. Functional reliability still met its target, so server-side tail latency—not widespread request failure—defined the tested capacity boundary.", + "", + "## Limitations", + "", + "Synthetic data; one AWS region; one Fargate task; HTTP restricted to one changing public CIDR; one load-generator location; a single-AZ burstable database; 15-minute confirmation windows; no production AWS integrations; and no post-boundary stress run. Results apply only to the documented configuration and workload.", + "", + "## Exact reproduction", + "", + "See `../../README.md`. The final confirmation command was:", + "", + "```bash", + "CAPACITY_LABEL=post-rollup-soak \\", + " bash benchmarks/underflow-api/scripts/run-capacity-test.sh 45 15m", + "```", + "", + "## Evidence-backed résumé bullet", + "", + resume, + "", + "```latex", + "\\item Built and benchmarked a multi-tenant AWS cost-monitoring API against \\textbf{" + records_label + " cost records}, sustaining \\textbf{" + f"{supported_requests['rate']:.2f} requests/second" + "} with \\textbf{" + f"{supported_p95:.2f} ms worst-minute p95 server-side latency" + "} under \\textbf{45 concurrent virtual users}.", + "```", + "", +]) + +output.write_text("\n".join(lines), encoding="utf-8") +print(f"Generated {output} from the preserved 45-VU pass and 50-VU failure evidence") +PY diff --git a/benchmarks/underflow-api/scripts/run-capacity-test.sh b/benchmarks/underflow-api/scripts/run-capacity-test.sh new file mode 100644 index 0000000..50b5a69 --- /dev/null +++ b/benchmarks/underflow-api/scripts/run-capacity-test.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +set -euo pipefail + +CAPACITY_VUS="${1:-7}" +CAPACITY_DURATION="${2:-3m}" +CAPACITY_LABEL="${CAPACITY_LABEL:-post-rollup}" +CLOUDWATCH_WAIT_SECONDS="${CLOUDWATCH_WAIT_SECONDS:-120}" + +if [[ ! "$CAPACITY_VUS" =~ ^[1-9][0-9]*$ ]] || (( CAPACITY_VUS > 100 )); then + echo "VUs must be an integer from 1 through 100" >&2 + exit 2 +fi + +if [[ ! "$CAPACITY_LABEL" =~ ^[A-Za-z0-9._-]+$ ]]; then + echo "CAPACITY_LABEL may contain only letters, numbers, dots, underscores, and hyphens" >&2 + exit 2 +fi + +for command in aws curl jq k6 terraform tr; do + command -v "$command" >/dev/null 2>&1 || { + echo "$command is required" >&2 + exit 2 + } +done + +REPO_ROOT="${REPO_ROOT:-$(git rev-parse --show-toplevel)}" +TF_ROOT="${TF_ROOT:-$REPO_ROOT/infra/terraform/envs/api-benchmark}" + +tf_output() { + terraform -chdir="$TF_ROOT" output -raw "$1" | tr -d '\r' +} + +AWS_REGION="$(tf_output aws_region)" +PREFIX="$(tf_output resource_prefix)" +BENCHMARK_ID="${PREFIX#underflow-api-bench-}" +BASE_URL="$(tf_output api_base_url)" + +if [[ -z "${BENCHMARK_RESULTS_DIR:-${RESULTS_DIR:-}}" ]]; then + RESULTS_BASE="$REPO_ROOT/benchmarks/underflow-api/results" + shopt -s nullglob + existing_results=("$RESULTS_BASE"/*-"$BENCHMARK_ID") + shopt -u nullglob + if (( ${#existing_results[@]} > 0 )); then + RESULTS_DIR="${existing_results[${#existing_results[@]} - 1]}" + else + RESULTS_DIR="$RESULTS_BASE/$(date -u +%F)-$BENCHMARK_ID" + fi +else + RESULTS_DIR="${BENCHMARK_RESULTS_DIR:-${RESULTS_DIR:-}}" +fi +mkdir -p "$RESULTS_DIR" + +RUN_NAME="capacity-${CAPACITY_VUS}vus-${CAPACITY_LABEL}" +K6_SUMMARY="$RESULTS_DIR/${RUN_NAME}-summary.json" +CLOUDWATCH_SUMMARY="$RESULTS_DIR/${RUN_NAME}-cloudwatch.json" + +if [[ -e "$K6_SUMMARY" || -e "$CLOUDWATCH_SUMMARY" ]]; then + echo "Refusing to overwrite existing evidence for $RUN_NAME" >&2 + echo "Set CAPACITY_LABEL to a new label before rerunning." >&2 + exit 1 +fi + +bash "$REPO_ROOT/benchmarks/underflow-api/scripts/update-load-test-ip.sh" +curl --fail --silent --show-error "$BASE_URL/api/v1/health" >/dev/null + +TEST_EMAIL="benchmark+01@example.invalid" +WORKSPACE_ID="20000000-0000-4000-8000-000000000001" +TEST_PASSWORD="$(aws secretsmanager get-secret-value \ + --region "$AWS_REGION" \ + --secret-id "$PREFIX-runtime" \ + --query SecretString \ + --output text | jq -r '.BENCHMARK_PASSWORD')" +[[ -n "$TEST_PASSWORD" && "$TEST_PASSWORD" != "null" ]] || { + echo "Benchmark password was missing from Secrets Manager" >&2 + exit 1 +} + +echo "Running $CAPACITY_VUS VUs for $CAPACITY_DURATION" +CAPACITY_START="$(date -u +%FT%TZ)" +set +e +RESULTS_DIR="$RESULTS_DIR" \ +SUMMARY_NAME="${RUN_NAME}-summary.json" \ +BASE_URL="$BASE_URL" \ +TEST_EMAIL="$TEST_EMAIL" \ +TEST_PASSWORD="$TEST_PASSWORD" \ +WORKSPACE_ID="$WORKSPACE_ID" \ +CAPACITY_VUS="$CAPACITY_VUS" \ +CAPACITY_DURATION="$CAPACITY_DURATION" \ + k6 run "$REPO_ROOT/benchmarks/underflow-api/k6/capacity.js" +K6_EXIT=$? +set -e +CAPACITY_END="$(date -u +%FT%TZ)" +unset TEST_PASSWORD + +echo "Waiting ${CLOUDWATCH_WAIT_SECONDS}s for CloudWatch datapoints" +sleep "$CLOUDWATCH_WAIT_SECONDS" + +AWS_REGION="$AWS_REGION" \ +ECS_CLUSTER="$(tf_output ecs_cluster_name)" \ +ECS_SERVICE="$(tf_output ecs_service_name)" \ +RDS_IDENTIFIER="$(tf_output rds_identifier)" \ +ALB_ARN_SUFFIX="$(tf_output load_balancer_arn_suffix)" \ +TARGET_GROUP_ARN_SUFFIX="$(tf_output target_group_arn_suffix)" \ +START_TIME="$CAPACITY_START" \ +END_TIME="$CAPACITY_END" \ +OUTPUT_FILE="$CLOUDWATCH_SUMMARY" \ + bash "$REPO_ROOT/benchmarks/underflow-api/scripts/collect-aws-metadata.sh" + +echo "Functional summary" +jq '{ + iterations: .metrics.iterations.values, + requests: .metrics.measured_requests.values, + failures: .metrics.measured_failures.values, + checks: .metrics.checks.values +}' "$K6_SUMMARY" + +echo "Server and resource summary" +jq '{ + alb_latency: .metrics.alb_target_response.datapoints, + rds_cpu: .metrics.rds_cpu.datapoints, + rds_memory: .metrics.rds_free_memory.datapoints, + ecs_cpu: .metrics.ecs_cpu.datapoints +}' "$CLOUDWATCH_SUMMARY" + +if (( K6_EXIT != 0 )); then + echo "FAIL: k6 exited with status $K6_EXIT" >&2 + exit 1 +fi + +bash "$REPO_ROOT/benchmarks/underflow-api/scripts/check-capacity-result.sh" \ + "$K6_SUMMARY" "$CLOUDWATCH_SUMMARY" + +echo "Evidence saved to:" +echo " $K6_SUMMARY" +echo " $CLOUDWATCH_SUMMARY" diff --git a/benchmarks/underflow-api/scripts/run-one-off-task.sh b/benchmarks/underflow-api/scripts/run-one-off-task.sh new file mode 100644 index 0000000..86d0c3e --- /dev/null +++ b/benchmarks/underflow-api/scripts/run-one-off-task.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +set -euo pipefail + +MODE="${1:-}" +if [[ "$MODE" != "migrate" && "$MODE" != "seed" && "$MODE" != "rollup" && "$MODE" != "explain" ]]; then + echo "usage: $0 migrate|seed|rollup|explain" >&2 + exit 2 +fi + +for command in aws terraform jq; do + command -v "$command" >/dev/null || { echo "$command is required" >&2; exit 2; } +done + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +TF_ROOT="$ROOT_DIR/infra/terraform/envs/api-benchmark" +RESULTS_DIR="${BENCHMARK_RESULTS_DIR:-$ROOT_DIR/benchmarks/underflow-api/results/local-evidence}" +mkdir -p "$RESULTS_DIR" + +tf_output() { + terraform -chdir="$TF_ROOT" output -raw "$1" | tr -d '\r' +} + +REGION="$(tf_output aws_region)" +CLUSTER="$(tf_output ecs_cluster_name)" +TASK_DEFINITION="$(tf_output task_definition_arn)" +SECURITY_GROUP="$(tf_output ecs_security_group_id)" +mapfile -t SUBNETS < <(terraform -chdir="$TF_ROOT" output -json subnet_ids | jq -r '.[]') +SUBNET_LIST="$(IFS=,; echo "${SUBNETS[*]}")" + +if [[ "$MODE" == "migrate" ]]; then + COMMAND_JSON='["node","dist/db/migrate.js"]' + ENVIRONMENT_JSON='[]' +elif [[ "$MODE" == "seed" ]]; then + COMMAND_JSON='["node","dist/scripts/seed-benchmark.js"]' + ENVIRONMENT_JSON='[{"name":"ALLOW_BENCHMARK_SEED","value":"true"}]' +elif [[ "$MODE" == "rollup" ]]; then + COMMAND_JSON='["node","dist/scripts/backfill-cost-rollups.js"]' + ENVIRONMENT_JSON='[{"name":"ALLOW_COST_ROLLUP_BACKFILL","value":"true"}]' +else + EXPLAIN_SCRIPT='import { pool } from "./dist/config/db.js"; +const workspaceId = "20000000-0000-4000-8000-000000000001"; +const parameters = [workspaceId, "2025-01-01", "2025-12-31"]; +const queries = { + summary: `SELECT COALESCE(SUM(total_amount), 0) AS total_amount, + COALESCE(MAX(currency), '\''USD'\'') AS currency + FROM workspace_cost_daily_rollups + WHERE workspace_id = $1 AND usage_date BETWEEN $2 AND $3`, + timeseries: `SELECT usage_date, SUM(total_amount) AS total_amount, + COALESCE(MAX(currency), '\''USD'\'') AS currency + FROM workspace_cost_daily_rollups + WHERE workspace_id = $1 AND usage_date BETWEEN $2 AND $3 + GROUP BY usage_date ORDER BY usage_date ASC`, + byService: `SELECT service_name, SUM(total_amount) AS total_amount, + COALESCE(MAX(currency), '\''USD'\'') AS currency + FROM workspace_cost_daily_rollups + WHERE workspace_id = $1 AND usage_date BETWEEN $2 AND $3 + GROUP BY service_name ORDER BY total_amount DESC`, +}; +const plans = {}; +for (const [name, sql] of Object.entries(queries)) { + const result = await pool.query(`EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT JSON) ${sql}`, parameters); + plans[name] = result.rows[0]["QUERY PLAN"]; +} +const stats = await pool.query(`SELECT relname, + pg_size_pretty(pg_total_relation_size(relid)) AS total_size, + pg_size_pretty(pg_relation_size(relid)) AS table_size, + pg_size_pretty(pg_indexes_size(relid)) AS indexes_size, + n_live_tup, seq_scan, idx_scan + FROM pg_stat_user_tables + WHERE relname IN ('\''cost_snapshots'\'', '\''workspace_cost_daily_rollups'\'') + ORDER BY relname`); +console.log(JSON.stringify({ benchmarkDiagnostic: true, stats: stats.rows, plans }, null, 2)); +await pool.end();' + COMMAND_JSON="$(jq -cn --arg script "$EXPLAIN_SCRIPT" '["node","--input-type=module","--eval",$script]')" + ENVIRONMENT_JSON='[]' +fi + +OVERRIDES="$(jq -cn \ + --argjson command "$COMMAND_JSON" \ + --argjson environment "$ENVIRONMENT_JSON" \ + '{containerOverrides:[{name:"api",command:$command,environment:$environment}]}')" + +TASK_ARN="$(aws ecs run-task \ + --region "$REGION" \ + --cluster "$CLUSTER" \ + --task-definition "$TASK_DEFINITION" \ + --launch-type FARGATE \ + --network-configuration "awsvpcConfiguration={subnets=[$SUBNET_LIST],securityGroups=[$SECURITY_GROUP],assignPublicIp=ENABLED}" \ + --overrides "$OVERRIDES" \ + --query 'tasks[0].taskArn' \ + --output text)" + +if [[ -z "$TASK_ARN" || "$TASK_ARN" == "None" ]]; then + echo "ECS did not return a task ARN" >&2 + exit 1 +fi + +TASK_ID="${TASK_ARN##*/}" +echo "Waiting for benchmark $MODE task $TASK_ID" +aws ecs wait tasks-stopped --region "$REGION" --cluster "$CLUSTER" --tasks "$TASK_ARN" + +EXIT_CODE="$(aws ecs describe-tasks \ + --region "$REGION" \ + --cluster "$CLUSTER" \ + --tasks "$TASK_ARN" \ + --query 'tasks[0].containers[?name==`api`].exitCode | [0]' \ + --output text)" + +LOG_GROUP="/ecs/$(tf_output resource_prefix)/api" +LOG_STREAM="ecs/api/$TASK_ID" +RAW_LOG="$(mktemp)" +trap 'rm -f "$RAW_LOG"' EXIT + +# Git Bash otherwise rewrites /ecs/... into a Windows filesystem path before +# invoking the native AWS CLI. Scope the override to this command so Terraform +# still receives converted -chdir paths. +MSYS_NO_PATHCONV=1 aws logs get-log-events \ + --region "$REGION" \ + --log-group-name "$LOG_GROUP" \ + --log-stream-name "$LOG_STREAM" \ + --output json > "$RAW_LOG" + +"$ROOT_DIR/benchmarks/underflow-api/scripts/sanitize-results.sh" \ + "$RAW_LOG" "$RESULTS_DIR/${MODE}-task.log" + +if [[ "$EXIT_CODE" != "0" ]]; then + echo "Benchmark $MODE task failed with exit code $EXIT_CODE; see sanitized log" >&2 + exit 1 +fi + +if [[ "$MODE" == "seed" ]]; then + jq '[.events[].message | fromjson? | select(.dataset == "underflow-api-benchmark-v1")] | last' \ + "$RESULTS_DIR/seed-task.log" > "$RESULTS_DIR/dataset.json" + jq -e '.costSnapshots == 3650000 and .costRollups == 182500 and .users == 10 and .workspaces == 10 and .awsAccounts == 200' \ + "$RESULTS_DIR/dataset.json" >/dev/null +elif [[ "$MODE" == "rollup" ]]; then + jq '[.events[].message | fromjson? | select(.costRollupBackfill == true)] | last' \ + "$RESULTS_DIR/rollup-task.log" > "$RESULTS_DIR/rollup.json" + jq -e '.rows == 182500' "$RESULTS_DIR/rollup.json" >/dev/null + + if [[ -f "$RESULTS_DIR/dataset.json" ]]; then + DATASET_TMP="$(mktemp)" + jq --slurpfile rollup "$RESULTS_DIR/rollup.json" \ + '.costRollups = $rollup[0].rows' "$RESULTS_DIR/dataset.json" > "$DATASET_TMP" + mv "$DATASET_TMP" "$RESULTS_DIR/dataset.json" + fi +fi + +echo "Benchmark $MODE task completed with exit code 0" diff --git a/benchmarks/underflow-api/scripts/sanitize-results.sh b/benchmarks/underflow-api/scripts/sanitize-results.sh new file mode 100644 index 0000000..4a8aa33 --- /dev/null +++ b/benchmarks/underflow-api/scripts/sanitize-results.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 INPUT_FILE OUTPUT_FILE" >&2 + exit 2 +fi + +if command -v py >/dev/null; then + PYTHON_COMMAND=(py -3) +elif command -v python3 >/dev/null; then + PYTHON_COMMAND=(python3) +elif command -v python >/dev/null; then + PYTHON_COMMAND=(python) +else + echo "python3, py, or python is required" >&2 + exit 2 +fi + +"${PYTHON_COMMAND[@]}" - "$1" "$2" <<'PY' +import pathlib +import re +import sys + +source = pathlib.Path(sys.argv[1]) +target = pathlib.Path(sys.argv[2]) +text = source.read_text(encoding="utf-8", errors="replace") + +patterns = [ + (r"(?i)arn:aws:secretsmanager:[^\s\"']+", "[REDACTED_SECRET_ARN]"), + (r"(?i)\b[A-Z0-9]{20}\b", "[REDACTED_AWS_ACCESS_KEY]"), + (r"(?i)\beyJ[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\b", "[REDACTED_TOKEN]"), + (r"\b\d{12}\.dkr\.ecr\.[a-z0-9-]+\.amazonaws\.com\b", "[REDACTED_ECR_HOST]"), + (r"(?i)\b[a-z0-9-]+\.[a-z0-9-]+\.rds\.amazonaws\.com\b", "[REDACTED_RDS_HOST]"), + (r"\b(?:\d{1,3}\.){3}\d{1,3}\b", "[REDACTED_IPV4]"), + (r"(?i)\b(?![a-z0-9.+_-]+@example\.invalid\b)[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}\b", "[REDACTED_EMAIL]"), + (r"(?i)(terraform\.tfstate(?:\.[^\s\"']+)?)", "[REDACTED_TERRAFORM_STATE]"), + (r"(? $2" diff --git a/benchmarks/underflow-api/scripts/update-load-test-ip.sh b/benchmarks/underflow-api/scripts/update-load-test-ip.sh new file mode 100644 index 0000000..255939d --- /dev/null +++ b/benchmarks/underflow-api/scripts/update-load-test-ip.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +set -euo pipefail + +for command in curl jq sed terraform; do + command -v "$command" >/dev/null 2>&1 || { + echo "$command is required" >&2 + exit 2 + } +done + +REPO_ROOT="${REPO_ROOT:-$(git rev-parse --show-toplevel)}" +TF_ROOT="${TF_ROOT:-$REPO_ROOT/infra/terraform/envs/api-benchmark}" +TFVARS="$TF_ROOT/terraform.tfvars" +PLAN_NAME="ip-update.tfplan" + +[[ -f "$TFVARS" ]] || { + echo "Missing $TFVARS" >&2 + exit 2 +} + +PUBLIC_IP="$(curl --noproxy '*' --fail --silent --show-error \ + https://checkip.amazonaws.com | tr -d '\r\n')" + +if [[ ! "$PUBLIC_IP" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then + echo "Public IP lookup returned an invalid IPv4 address: $PUBLIC_IP" >&2 + exit 1 +fi + +IFS=. read -r -a octets <<< "$PUBLIC_IP" +for octet in "${octets[@]}"; do + if ((10#$octet > 255)); then + echo "Public IP lookup returned an invalid IPv4 address: $PUBLIC_IP" >&2 + exit 1 + fi +done + +CURRENT_CIDR="$(sed -nE 's/^[[:space:]]*load_test_cidr[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/p' "$TFVARS")" +NEW_CIDR="$PUBLIC_IP/32" + +if [[ "$CURRENT_CIDR" == "$NEW_CIDR" ]]; then + echo "ALB already allows the current public IP: $NEW_CIDR" +else + cp "$TFVARS" "$TFVARS.bak" + sed -i \ + "s|^[[:space:]]*load_test_cidr[[:space:]]*=.*$|load_test_cidr = \"$NEW_CIDR\"|" \ + "$TFVARS" + echo "Updated load_test_cidr: ${CURRENT_CIDR:-} -> $NEW_CIDR" +fi + +terraform -chdir="$TF_ROOT" plan -out="$PLAN_NAME" + +PLAN_JSON="$(terraform -chdir="$TF_ROOT" show -json "$PLAN_NAME")" +if ! jq -e ' + [.resource_changes[]? + | select(.mode == "managed" and .change.actions != ["no-op"])] as $changes + | ($changes | length) <= 1 + and all($changes[]; + .address == "aws_security_group.alb" + and .change.actions == ["update"]) +' >/dev/null <<< "$PLAN_JSON"; then + echo "Refusing to apply: the plan changes resources other than aws_security_group.alb." >&2 + terraform -chdir="$TF_ROOT" show -no-color "$PLAN_NAME" >&2 + if [[ -f "$TFVARS.bak" ]]; then + mv "$TFVARS.bak" "$TFVARS" + fi + rm -f "$TF_ROOT/$PLAN_NAME" + exit 1 +fi + +CONFIRM_IP="$(curl --noproxy '*' --fail --silent --show-error \ + https://checkip.amazonaws.com | tr -d '\r\n')" +if [[ "$CONFIRM_IP" != "$PUBLIC_IP" ]]; then + echo "Public IP changed during planning ($PUBLIC_IP -> $CONFIRM_IP); refusing to apply." >&2 + if [[ -f "$TFVARS.bak" ]]; then + mv "$TFVARS.bak" "$TFVARS" + fi + rm -f "$TF_ROOT/$PLAN_NAME" + exit 1 +fi + +terraform -chdir="$TF_ROOT" apply "$PLAN_NAME" +rm -f "$TF_ROOT/$PLAN_NAME" "$TFVARS.bak" + +BASE_URL="$(terraform -chdir="$TF_ROOT" output -raw api_base_url | tr -d '\r')" +echo "Applied ALB ingress CIDR $NEW_CIDR" +curl --fail --silent --show-error "$BASE_URL/api/v1/health" +echo +echo "Health check passed: $BASE_URL/api/v1/health" diff --git a/infra/terraform/envs/api-benchmark/README.md b/infra/terraform/envs/api-benchmark/README.md new file mode 100644 index 0000000..c11830b --- /dev/null +++ b/infra/terraform/envs/api-benchmark/README.md @@ -0,0 +1,9 @@ +# Isolated API benchmark infrastructure + +This Terraform root owns only the disposable `underflow-api-bench-` environment. It uses local state in this directory and has no backend, module, data source, or state reference to the production stack. Never run these commands from `envs/production`. + +Create an ignored `terraform.tfvars` from the example. `load_test_cidr` is required and rejects `0.0.0.0/0`; use the load generator's public `/32`. `image_tag` must be the Git SHA used to build the API image. + +Before any apply, run the validation and full-plan checks documented in [`benchmarks/underflow-api/README.md`](../../../../benchmarks/underflow-api/README.md). The first apply targets only this root's ECR repository so an immutable commit-tagged image can be pushed. After the push, run and review a fresh full plan before the full apply. + +All secret values are generated, stored in local Terraform state, and injected from one disposable Secrets Manager secret. State and `.tfvars` are ignored and must never be committed. The secret has zero-day recovery, ECR uses `force_delete`, and RDS has no backups, final snapshot, public access, Multi-AZ, or deletion protection. diff --git a/infra/terraform/envs/api-benchmark/compute.tf b/infra/terraform/envs/api-benchmark/compute.tf new file mode 100644 index 0000000..eeabc81 --- /dev/null +++ b/infra/terraform/envs/api-benchmark/compute.tf @@ -0,0 +1,146 @@ +resource "aws_ecr_repository" "api" { + name = "${local.name_prefix}-api" + image_tag_mutability = "IMMUTABLE" + force_delete = true + + image_scanning_configuration { + scan_on_push = true + } + + tags = { Name = "${local.name_prefix}-api" } +} + +resource "aws_cloudwatch_log_group" "api" { + name = "/ecs/${local.name_prefix}/api" + retention_in_days = 3 + tags = { Name = "${local.name_prefix}-api-logs" } +} + +resource "aws_ecs_cluster" "benchmark" { + name = "${local.name_prefix}-cluster" + + setting { + name = "containerInsights" + value = "enabled" + } + + tags = { Name = "${local.name_prefix}-cluster" } +} + +resource "aws_lb" "api" { + name = substr("${local.name_prefix}-alb", 0, 32) + internal = false + load_balancer_type = "application" + security_groups = [aws_security_group.alb.id] + subnets = aws_subnet.public[*].id + + tags = { Name = "${local.name_prefix}-alb" } +} + +resource "aws_lb_target_group" "api" { + name = substr("${local.name_prefix}-tg", 0, 32) + port = 3080 + protocol = "HTTP" + target_type = "ip" + vpc_id = aws_vpc.benchmark.id + + health_check { + enabled = true + path = "/api/v1/health" + matcher = "200" + interval = 30 + timeout = 5 + healthy_threshold = 2 + unhealthy_threshold = 3 + } + + tags = { Name = "${local.name_prefix}-tg" } +} + +resource "aws_lb_listener" "http" { + load_balancer_arn = aws_lb.api.arn + port = 80 + protocol = "HTTP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.api.arn + } + + tags = { Name = "${local.name_prefix}-http" } +} + +resource "aws_ecs_task_definition" "api" { + family = "${local.name_prefix}-api" + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + cpu = "512" + memory = "1024" + execution_role_arn = aws_iam_role.task_execution.arn + task_role_arn = aws_iam_role.task_runtime.arn + + container_definitions = jsonencode([ + { + name = "api" + image = "${aws_ecr_repository.api.repository_url}:${var.image_tag}" + essential = true + command = ["node", "dist/server.js"] + portMappings = [{ + containerPort = 3080 + hostPort = 3080 + protocol = "tcp" + }] + environment = local.container_environment + secrets = local.container_secrets + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.api.name + awslogs-region = var.aws_region + awslogs-stream-prefix = "ecs" + } + } + } + ]) + + runtime_platform { + cpu_architecture = "X86_64" + operating_system_family = "LINUX" + } + + tags = { Name = "${local.name_prefix}-api" } +} + +resource "aws_ecs_service" "api" { + name = "${local.name_prefix}-api" + cluster = aws_ecs_cluster.benchmark.id + task_definition = aws_ecs_task_definition.api.arn + desired_count = 1 + launch_type = "FARGATE" + health_check_grace_period_seconds = 90 + wait_for_steady_state = true + + network_configuration { + subnets = aws_subnet.public[*].id + security_groups = [aws_security_group.ecs.id] + assign_public_ip = true + } + + load_balancer { + target_group_arn = aws_lb_target_group.api.arn + container_name = "api" + container_port = 3080 + } + + deployment_circuit_breaker { + enable = true + rollback = true + } + + depends_on = [ + aws_lb_listener.http, + aws_secretsmanager_secret_version.runtime, + ] + + tags = { Name = "${local.name_prefix}-api" } +} diff --git a/infra/terraform/envs/api-benchmark/database.tf b/infra/terraform/envs/api-benchmark/database.tf new file mode 100644 index 0000000..a567b88 --- /dev/null +++ b/infra/terraform/envs/api-benchmark/database.tf @@ -0,0 +1,75 @@ +resource "random_password" "database" { + length = 32 + special = true + override_special = "!#$%&*+-.:=?_" +} + +resource "random_password" "csrf" { + length = 48 + special = false +} + +resource "random_password" "jwt_access" { + length = 64 + special = false +} + +resource "random_password" "jwt_refresh" { + length = 64 + special = false +} + +resource "random_password" "benchmark" { + length = 24 + special = false +} + +resource "aws_db_subnet_group" "benchmark" { + name = "${local.name_prefix}-db-subnets" + subnet_ids = aws_subnet.public[*].id + tags = { Name = "${local.name_prefix}-db-subnets" } +} + +resource "aws_db_instance" "postgres" { + identifier = "${local.name_prefix}-postgres" + engine = "postgres" + engine_version = var.db_engine_version + instance_class = var.db_instance_class + allocated_storage = 20 + storage_type = "gp3" + storage_encrypted = true + db_name = "underflow_benchmark" + username = "underflow_benchmark" + password = random_password.database.result + port = 5432 + db_subnet_group_name = aws_db_subnet_group.benchmark.name + vpc_security_group_ids = [aws_security_group.rds.id] + publicly_accessible = false + multi_az = false + backup_retention_period = 0 + deletion_protection = false + skip_final_snapshot = true + auto_minor_version_upgrade = true + apply_immediately = true + performance_insights_enabled = var.performance_insights_enabled + + tags = { Name = "${local.name_prefix}-postgres" } +} + +resource "aws_secretsmanager_secret" "runtime" { + name = "${local.name_prefix}-runtime" + description = "Disposable API benchmark runtime secrets." + recovery_window_in_days = 0 + tags = { Name = "${local.name_prefix}-runtime" } +} + +resource "aws_secretsmanager_secret_version" "runtime" { + secret_id = aws_secretsmanager_secret.runtime.id + secret_string = jsonencode({ + DATABASE_URL = "postgresql://underflow_benchmark:${urlencode(random_password.database.result)}@${aws_db_instance.postgres.address}:5432/underflow_benchmark" + CSRF_SECRET = random_password.csrf.result + JWT_ACCESS_SECRET = random_password.jwt_access.result + JWT_REFRESH_SECRET = random_password.jwt_refresh.result + BENCHMARK_PASSWORD = random_password.benchmark.result + }) +} diff --git a/infra/terraform/envs/api-benchmark/iam.tf b/infra/terraform/envs/api-benchmark/iam.tf new file mode 100644 index 0000000..c452c2f --- /dev/null +++ b/infra/terraform/envs/api-benchmark/iam.tf @@ -0,0 +1,41 @@ +data "aws_iam_policy_document" "ecs_tasks_assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ecs-tasks.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "task_execution" { + name = "${local.name_prefix}-task-execution" + assume_role_policy = data.aws_iam_policy_document.ecs_tasks_assume_role.json + tags = { Name = "${local.name_prefix}-task-execution" } +} + +resource "aws_iam_role_policy_attachment" "task_execution" { + role = aws_iam_role.task_execution.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" +} + +data "aws_iam_policy_document" "task_execution_secrets" { + statement { + sid = "ReadBenchmarkRuntimeSecret" + actions = ["secretsmanager:GetSecretValue"] + resources = [aws_secretsmanager_secret.runtime.arn] + } +} + +resource "aws_iam_role_policy" "task_execution_secrets" { + name = "${local.name_prefix}-read-runtime-secret" + role = aws_iam_role.task_execution.id + policy = data.aws_iam_policy_document.task_execution_secrets.json +} + +resource "aws_iam_role" "task_runtime" { + name = "${local.name_prefix}-task-runtime" + assume_role_policy = data.aws_iam_policy_document.ecs_tasks_assume_role.json + tags = { Name = "${local.name_prefix}-task-runtime" } +} diff --git a/infra/terraform/envs/api-benchmark/locals.tf b/infra/terraform/envs/api-benchmark/locals.tf new file mode 100644 index 0000000..40072fa --- /dev/null +++ b/infra/terraform/envs/api-benchmark/locals.tf @@ -0,0 +1,45 @@ +locals { + name_prefix = "underflow-api-bench-${var.benchmark_id}" + + required_tags = { + project = "underflow" + environment = "api-benchmark" + purpose = "disposable-load-test" + managed = "terraform" + } + + container_environment = [ + { name = "PORT", value = "3080" }, + { name = "NODE_ENV", value = "production" }, + { name = "DATABASE_SSL_ENABLED", value = "true" }, + { name = "DATABASE_SSL_REJECT_UNAUTHORIZED", value = "false" }, + { name = "AWS_REGION", value = var.aws_region }, + { name = "AWS_SES_REGION", value = var.aws_region }, + { name = "AWS_SES_ACCESS_KEY_ID", value = "" }, + { name = "AWS_SES_SECRET_ACCESS_KEY", value = "" }, + { name = "EMAIL_PROVIDER", value = "console" }, + { name = "BILLING_ENABLED", value = "false" }, + { name = "STRIPE_SECRET_KEY", value = "" }, + { name = "STRIPE_WEBHOOK_SECRET", value = "" }, + { name = "STRIPE_SUCCESS_URL", value = "" }, + { name = "STRIPE_CANCEL_URL", value = "" }, + { name = "CLIENT_URL", value = "http://${aws_lb.api.dns_name}" }, + { name = "AUTH_COOKIE_DOMAIN", value = "" }, + { name = "AUTH_COOKIE_SAME_SITE", value = "lax" }, + { name = "LOG_LEVEL", value = "info" }, + { name = "COST_SYNC_LOOKBACK_DAYS", value = "30" }, + ] + + container_secrets = [ + for key in [ + "DATABASE_URL", + "CSRF_SECRET", + "JWT_ACCESS_SECRET", + "JWT_REFRESH_SECRET", + "BENCHMARK_PASSWORD", + ] : { + name = key + valueFrom = "${aws_secretsmanager_secret.runtime.arn}:${key}::" + } + ] +} diff --git a/infra/terraform/envs/api-benchmark/networking.tf b/infra/terraform/envs/api-benchmark/networking.tf new file mode 100644 index 0000000..6a41a6a --- /dev/null +++ b/infra/terraform/envs/api-benchmark/networking.tf @@ -0,0 +1,114 @@ +data "aws_availability_zones" "available" { + state = "available" +} + +resource "aws_vpc" "benchmark" { + cidr_block = "10.90.0.0/16" + enable_dns_support = true + enable_dns_hostnames = true + + tags = { Name = "${local.name_prefix}-vpc" } +} + +resource "aws_internet_gateway" "benchmark" { + vpc_id = aws_vpc.benchmark.id + tags = { Name = "${local.name_prefix}-igw" } +} + +resource "aws_subnet" "public" { + count = 2 + + vpc_id = aws_vpc.benchmark.id + availability_zone = data.aws_availability_zones.available.names[count.index] + cidr_block = cidrsubnet(aws_vpc.benchmark.cidr_block, 8, count.index) + map_public_ip_on_launch = true + + tags = { Name = "${local.name_prefix}-public-${count.index + 1}" } +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.benchmark.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.benchmark.id + } + + tags = { Name = "${local.name_prefix}-public" } +} + +resource "aws_route_table_association" "public" { + count = 2 + + subnet_id = aws_subnet.public[count.index].id + route_table_id = aws_route_table.public.id +} + +resource "aws_security_group" "alb" { + name = "${local.name_prefix}-alb" + description = "Benchmark load generator ingress only." + vpc_id = aws_vpc.benchmark.id + + ingress { + description = "HTTP from the explicitly configured load generator" + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = [var.load_test_cidr] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { Name = "${local.name_prefix}-alb" } +} + +resource "aws_security_group" "ecs" { + name = "${local.name_prefix}-ecs" + description = "Benchmark API tasks; no direct public ingress." + vpc_id = aws_vpc.benchmark.id + + ingress { + description = "API traffic from the benchmark ALB only" + from_port = 3080 + to_port = 3080 + protocol = "tcp" + security_groups = [aws_security_group.alb.id] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { Name = "${local.name_prefix}-ecs" } +} + +resource "aws_security_group" "rds" { + name = "${local.name_prefix}-rds" + description = "Benchmark PostgreSQL ingress from API tasks only." + vpc_id = aws_vpc.benchmark.id + + ingress { + description = "PostgreSQL from benchmark API tasks only" + from_port = 5432 + to_port = 5432 + protocol = "tcp" + security_groups = [aws_security_group.ecs.id] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { Name = "${local.name_prefix}-rds" } +} diff --git a/infra/terraform/envs/api-benchmark/outputs.tf b/infra/terraform/envs/api-benchmark/outputs.tf new file mode 100644 index 0000000..b2e30e4 --- /dev/null +++ b/infra/terraform/envs/api-benchmark/outputs.tf @@ -0,0 +1,60 @@ +output "aws_region" { + value = var.aws_region +} + +output "resource_prefix" { + value = local.name_prefix +} + +output "ecr_repository_url" { + value = aws_ecr_repository.api.repository_url +} + +output "ecs_cluster_name" { + value = aws_ecs_cluster.benchmark.name +} + +output "ecs_service_name" { + value = aws_ecs_service.api.name +} + +output "task_definition_arn" { + value = aws_ecs_task_definition.api.arn +} + +output "alb_dns_name" { + value = aws_lb.api.dns_name +} + +output "api_base_url" { + value = "http://${aws_lb.api.dns_name}" +} + +output "rds_identifier" { + value = aws_db_instance.postgres.identifier +} + +output "ecs_security_group_id" { + value = aws_security_group.ecs.id +} + +output "subnet_ids" { + value = aws_subnet.public[*].id +} + +output "load_balancer_arn_suffix" { + value = aws_lb.api.arn_suffix +} + +output "target_group_arn_suffix" { + value = aws_lb_target_group.api.arn_suffix +} + +output "availability_zones" { + value = aws_subnet.public[*].availability_zone +} + +output "runtime_secret_arn" { + value = aws_secretsmanager_secret.runtime.arn + sensitive = true +} diff --git a/infra/terraform/envs/api-benchmark/terraform.tfvars.example b/infra/terraform/envs/api-benchmark/terraform.tfvars.example new file mode 100644 index 0000000..c89d18f --- /dev/null +++ b/infra/terraform/envs/api-benchmark/terraform.tfvars.example @@ -0,0 +1,7 @@ +aws_region = "us-east-1" +benchmark_id = "a1b2c3d4" +load_test_cidr = "203.0.113.10/32" +image_tag = "c0cf1f6" + +# Keep disabled unless support and any cost have been checked for the selected class. +performance_insights_enabled = false diff --git a/infra/terraform/envs/api-benchmark/variables.tf b/infra/terraform/envs/api-benchmark/variables.tf new file mode 100644 index 0000000..6eaac7a --- /dev/null +++ b/infra/terraform/envs/api-benchmark/variables.tf @@ -0,0 +1,57 @@ +variable "aws_region" { + description = "AWS region in which to create the disposable benchmark environment." + type = string + default = "us-east-1" +} + +variable "benchmark_id" { + description = "Unique lowercase identifier appended to every named benchmark resource." + type = string + + validation { + condition = can(regex("^[a-z0-9]{4,8}$", var.benchmark_id)) + error_message = "benchmark_id must contain 4-8 lowercase letters or digits." + } +} + +variable "load_test_cidr" { + description = "Single explicit IPv4 CIDR allowed to reach the benchmark ALB on port 80." + type = string + + validation { + condition = ( + can(cidrnetmask(var.load_test_cidr)) && + strcontains(var.load_test_cidr, "/") && + var.load_test_cidr != "0.0.0.0/0" + ) + error_message = "load_test_cidr must be an explicit IPv4 CIDR and cannot be 0.0.0.0/0." + } +} + +variable "image_tag" { + description = "Immutable Git SHA tag pushed to the benchmark ECR repository." + type = string + + validation { + condition = can(regex("^[0-9a-f]{7,40}$", var.image_tag)) + error_message = "image_tag must be a 7-40 character lowercase hexadecimal Git SHA." + } +} + +variable "db_instance_class" { + description = "Single-AZ disposable PostgreSQL instance class." + type = string + default = "db.t4g.micro" +} + +variable "db_engine_version" { + description = "PostgreSQL version compatible with the application." + type = string + default = "16.13" +} + +variable "performance_insights_enabled" { + description = "Enable RDS Performance Insights only after confirming class support and cost." + type = bool + default = false +} diff --git a/infra/terraform/envs/api-benchmark/versions.tf b/infra/terraform/envs/api-benchmark/versions.tf new file mode 100644 index 0000000..5b9e7e7 --- /dev/null +++ b/infra/terraform/envs/api-benchmark/versions.tf @@ -0,0 +1,22 @@ +terraform { + required_version = ">= 1.6.0, < 2.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + } +} + +provider "aws" { + region = var.aws_region + + default_tags { + tags = local.required_tags + } +}