From ae0099f491a0582c8b53a29e32a8b9c096f80c60 Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Wed, 9 Sep 2026 21:44:34 +0100 Subject: [PATCH 01/19] chore(git): ignore local benchmark commit helper --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3a8821c..230baae 100644 --- a/.gitignore +++ b/.gitignore @@ -50,4 +50,7 @@ temp/ UI.html -response.json \ No newline at end of file +response.json + +# Local commit orchestration helpers +/commit-benchmark-changes.sh From 2ec4a8c809a06028ec5f5a918aeafa809e4cba5e Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Wed, 9 Sep 2026 21:44:34 +0100 Subject: [PATCH 02/19] feat(api): add deterministic benchmark data seeder --- apps/api/package.json | 1 + apps/api/src/scripts/seed-benchmark.ts | 227 +++++++++++++++++++++++++ 2 files changed, 228 insertions(+) create mode 100644 apps/api/src/scripts/seed-benchmark.ts diff --git a/apps/api/package.json b/apps/api/package.json index f14fe22..dbf9be4 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -11,6 +11,7 @@ "jobs": "tsx src/jobs/worker.ts", "migrate": "tsx src/db/migrate.ts", "seed:demo": "node src/scripts/seed-demo.js", + "seed:benchmark": "tsx src/scripts/seed-benchmark.ts", "build": "tsc -p tsconfig.json && node scripts/copy-build-assets.mjs", "build:lambda": "npm run build && node scripts/build-lambda-package.mjs", "start": "node dist/server.js", diff --git a/apps/api/src/scripts/seed-benchmark.ts b/apps/api/src/scripts/seed-benchmark.ts new file mode 100644 index 0000000..b48e8b1 --- /dev/null +++ b/apps/api/src/scripts/seed-benchmark.ts @@ -0,0 +1,227 @@ +const BENCHMARK_GUARD = "ALLOW_BENCHMARK_SEED"; +const EXPECTED_COST_SNAPSHOTS = 3_650_000; + +const assertBenchmarkGuard = (): void => { + if (process.env[BENCHMARK_GUARD] !== "true") { + const environment = process.env.NODE_ENV ?? "development"; + throw new Error( + `Refusing to seed benchmark data in ${environment}: set ${BENCHMARK_GUARD}=true explicitly`, + ); + } +}; + +assertBenchmarkGuard(); + +const benchmarkPassword = process.env.BENCHMARK_PASSWORD; + +if (!benchmarkPassword || benchmarkPassword.length < 12) { + throw new Error("BENCHMARK_PASSWORD must contain at least 12 characters"); +} + +const { pool } = await import("../config/db.js"); +const { hashPassword } = await import("../utils/password.js"); + +const benchmarkUuid = (kind: number, sequence: number): string => + `${kind}0000000-0000-4000-8000-${String(sequence).padStart(12, "0")}`; + +const userIds = Array.from({ length: 10 }, (_, index) => benchmarkUuid(1, index + 1)); +const workspaceIds = Array.from( + { length: 10 }, + (_, index) => benchmarkUuid(2, index + 1), +); + +const run = async (): Promise => { + const passwordHash = await hashPassword(benchmarkPassword); + const client = await pool.connect(); + + try { + await client.query("BEGIN"); + + // The fixed UUID ranges and reserved example.invalid identities are the complete + // ownership boundary for this disposable dataset. + await client.query("DELETE FROM workspaces WHERE id = ANY($1::uuid[])", [workspaceIds]); + await client.query( + `DELETE FROM users + WHERE id = ANY($1::uuid[]) + OR email = ANY( + SELECT format('benchmark+%s@example.invalid', lpad(i::text, 2, '0')) + FROM generate_series(1, 10) AS i + )`, + [userIds], + ); + + await client.query( + `INSERT INTO users ( + id, email, password_hash, first_name, last_name, role, + is_active, is_email_verified, password_changed_at, session_version + ) + SELECT + format('10000000-0000-4000-8000-%s', lpad(i::text, 12, '0'))::uuid, + format('benchmark+%s@example.invalid', lpad(i::text, 2, '0')), + $1, + 'Benchmark', + format('User %s', lpad(i::text, 2, '0')), + 'customer', TRUE, TRUE, TIMESTAMP '2025-01-01 00:00:00', 1 + FROM generate_series(1, 10) AS i`, + [passwordHash], + ); + + await client.query( + `INSERT INTO workspaces (id, name, slug, owner_user_id, created_at, updated_at) + SELECT + format('20000000-0000-4000-8000-%s', lpad(i::text, 12, '0'))::uuid, + format('Benchmark Workspace %s', lpad(i::text, 2, '0')), + format('benchmark-workspace-%s', lpad(i::text, 2, '0')), + format('10000000-0000-4000-8000-%s', lpad(i::text, 12, '0'))::uuid, + TIMESTAMP '2025-01-01 00:00:00', TIMESTAMP '2025-01-01 00:00:00' + FROM generate_series(1, 10) AS i`, + ); + + await client.query( + `INSERT INTO workspace_members (id, workspace_id, user_id, role, created_at) + SELECT + format('21000000-0000-4000-8000-%s', lpad(i::text, 12, '0'))::uuid, + format('20000000-0000-4000-8000-%s', lpad(i::text, 12, '0'))::uuid, + format('10000000-0000-4000-8000-%s', lpad(i::text, 12, '0'))::uuid, + 'owner', TIMESTAMP '2025-01-01 00:00:00' + FROM generate_series(1, 10) AS i`, + ); + + await client.query( + `INSERT INTO aws_accounts ( + id, workspace_id, name, aws_account_id, role_arn, external_id, + status, last_verified_at, last_sync_at, created_at, updated_at + ) + SELECT + format( + '30000000-0000-4000-8000-%s', + lpad((((workspace_number - 1) * 20) + account_number)::text, 12, '0') + )::uuid, + format( + '20000000-0000-4000-8000-%s', + lpad(workspace_number::text, 12, '0') + )::uuid, + format('Synthetic Account %s-%s', workspace_number, lpad(account_number::text, 2, '0')), + lpad((workspace_number * 1000 + account_number)::text, 12, '0'), + format( + 'arn:aws:iam::%s:role/UnderflowSyntheticBenchmark', + lpad((workspace_number * 1000 + account_number)::text, 12, '0') + ), + format('benchmark-external-%s-%s', workspace_number, account_number), + 'verified', TIMESTAMP '2025-01-01 00:00:00', + TIMESTAMP '2025-12-31 23:59:00', TIMESTAMP '2025-01-01 00:00:00', + TIMESTAMP '2025-12-31 23:59:00' + FROM generate_series(1, 10) AS workspace_number + CROSS JOIN generate_series(1, 20) AS account_number`, + ); + + await client.query( + `INSERT INTO cost_sync_runs ( + id, aws_account_id, status, started_at, finished_at, error_message + ) + SELECT + format('40000000-0000-4000-8000-%s', lpad(sequence_number::text, 12, '0'))::uuid, + format('30000000-0000-4000-8000-%s', lpad(sequence_number::text, 12, '0'))::uuid, + 'completed', TIMESTAMP '2025-12-31 23:55:00', + TIMESTAMP '2025-12-31 23:59:00', NULL + FROM generate_series(1, 200) AS sequence_number`, + ); + + await client.query( + `INSERT INTO cost_snapshots ( + workspace_id, aws_account_id, usage_date, service_name, amount, currency, created_at + ) + SELECT + format( + '20000000-0000-4000-8000-%s', + lpad(workspace_number::text, 12, '0') + )::uuid, + format( + '30000000-0000-4000-8000-%s', + lpad((((workspace_number - 1) * 20) + account_number)::text, 12, '0') + )::uuid, + DATE '2025-01-01' + day_offset, + format('Synthetic Service %s', lpad(service_number::text, 2, '0')), + ( + ((workspace_number * 100000 + account_number * 1000 + day_offset * 10 + service_number) + % 500000) + 1 + )::numeric / 100, + 'USD', TIMESTAMP '2025-01-01 00:00:00' + FROM generate_series(1, 10) AS workspace_number + CROSS JOIN generate_series(1, 20) AS account_number + CROSS JOIN generate_series(0, 364) AS day_offset + CROSS JOIN generate_series(1, 50) AS service_number`, + ); + + const countsResult = await client.query<{ + users: string; + workspaces: string; + workspace_members: string; + aws_accounts: string; + cost_sync_runs: string; + cost_snapshots: string; + }>( + `SELECT + (SELECT COUNT(*) FROM users WHERE id = ANY($1::uuid[])) AS users, + (SELECT COUNT(*) FROM workspaces WHERE id = ANY($2::uuid[])) AS workspaces, + (SELECT COUNT(*) FROM workspace_members WHERE workspace_id = ANY($2::uuid[])) AS workspace_members, + (SELECT COUNT(*) FROM aws_accounts WHERE workspace_id = ANY($2::uuid[])) AS aws_accounts, + (SELECT COUNT(*) FROM cost_sync_runs csr + JOIN aws_accounts aa ON aa.id = csr.aws_account_id + WHERE aa.workspace_id = ANY($2::uuid[])) AS cost_sync_runs, + (SELECT COUNT(*) FROM cost_snapshots WHERE workspace_id = ANY($2::uuid[])) AS cost_snapshots`, + [userIds, workspaceIds], + ); + + const row = countsResult.rows[0]; + const counts = { + users: Number(row?.users ?? 0), + workspaces: Number(row?.workspaces ?? 0), + workspaceMembers: Number(row?.workspace_members ?? 0), + awsAccounts: Number(row?.aws_accounts ?? 0), + costSyncRuns: Number(row?.cost_sync_runs ?? 0), + costSnapshots: Number(row?.cost_snapshots ?? 0), + }; + + if ( + counts.users !== 10 || + counts.workspaces !== 10 || + counts.workspaceMembers !== 10 || + counts.awsAccounts !== 200 || + counts.costSyncRuns !== 200 || + counts.costSnapshots !== EXPECTED_COST_SNAPSHOTS + ) { + throw new Error(`Unexpected benchmark row counts: ${JSON.stringify(counts)}`); + } + + await client.query( + "ANALYZE users, workspaces, workspace_members, aws_accounts, cost_sync_runs, cost_snapshots", + ); + await client.query("COMMIT"); + + console.log( + JSON.stringify({ + dataset: "underflow-api-benchmark-v1", + dateRange: { from: "2025-01-01", to: "2025-12-31" }, + servicesPerAccountPerDay: 50, + ...counts, + }), + ); + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } finally { + client.release(); + await pool.end(); + } +}; + +run().catch((error: unknown) => { + console.error( + JSON.stringify({ + message: "Benchmark seed failed", + error: error instanceof Error ? error.message : "Unknown error", + }), + ); + process.exitCode = 1; +}); From 4ee9b72eea7d6638b5763c3e13f753341eefd30b Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Wed, 9 Sep 2026 21:44:34 +0100 Subject: [PATCH 03/19] feat(terraform): establish isolated benchmark configuration --- infra/terraform/envs/api-benchmark/locals.tf | 45 +++++++++++++++ .../api-benchmark/terraform.tfvars.example | 7 +++ .../terraform/envs/api-benchmark/variables.tf | 57 +++++++++++++++++++ .../terraform/envs/api-benchmark/versions.tf | 22 +++++++ 4 files changed, 131 insertions(+) create mode 100644 infra/terraform/envs/api-benchmark/locals.tf create mode 100644 infra/terraform/envs/api-benchmark/terraform.tfvars.example create mode 100644 infra/terraform/envs/api-benchmark/variables.tf create mode 100644 infra/terraform/envs/api-benchmark/versions.tf diff --git a/infra/terraform/envs/api-benchmark/locals.tf b/infra/terraform/envs/api-benchmark/locals.tf new file mode 100644 index 0000000..40072fa --- /dev/null +++ b/infra/terraform/envs/api-benchmark/locals.tf @@ -0,0 +1,45 @@ +locals { + name_prefix = "underflow-api-bench-${var.benchmark_id}" + + required_tags = { + project = "underflow" + environment = "api-benchmark" + purpose = "disposable-load-test" + managed = "terraform" + } + + container_environment = [ + { name = "PORT", value = "3080" }, + { name = "NODE_ENV", value = "production" }, + { name = "DATABASE_SSL_ENABLED", value = "true" }, + { name = "DATABASE_SSL_REJECT_UNAUTHORIZED", value = "false" }, + { name = "AWS_REGION", value = var.aws_region }, + { name = "AWS_SES_REGION", value = var.aws_region }, + { name = "AWS_SES_ACCESS_KEY_ID", value = "" }, + { name = "AWS_SES_SECRET_ACCESS_KEY", value = "" }, + { name = "EMAIL_PROVIDER", value = "console" }, + { name = "BILLING_ENABLED", value = "false" }, + { name = "STRIPE_SECRET_KEY", value = "" }, + { name = "STRIPE_WEBHOOK_SECRET", value = "" }, + { name = "STRIPE_SUCCESS_URL", value = "" }, + { name = "STRIPE_CANCEL_URL", value = "" }, + { name = "CLIENT_URL", value = "http://${aws_lb.api.dns_name}" }, + { name = "AUTH_COOKIE_DOMAIN", value = "" }, + { name = "AUTH_COOKIE_SAME_SITE", value = "lax" }, + { name = "LOG_LEVEL", value = "info" }, + { name = "COST_SYNC_LOOKBACK_DAYS", value = "30" }, + ] + + container_secrets = [ + for key in [ + "DATABASE_URL", + "CSRF_SECRET", + "JWT_ACCESS_SECRET", + "JWT_REFRESH_SECRET", + "BENCHMARK_PASSWORD", + ] : { + name = key + valueFrom = "${aws_secretsmanager_secret.runtime.arn}:${key}::" + } + ] +} diff --git a/infra/terraform/envs/api-benchmark/terraform.tfvars.example b/infra/terraform/envs/api-benchmark/terraform.tfvars.example new file mode 100644 index 0000000..c89d18f --- /dev/null +++ b/infra/terraform/envs/api-benchmark/terraform.tfvars.example @@ -0,0 +1,7 @@ +aws_region = "us-east-1" +benchmark_id = "a1b2c3d4" +load_test_cidr = "203.0.113.10/32" +image_tag = "c0cf1f6" + +# Keep disabled unless support and any cost have been checked for the selected class. +performance_insights_enabled = false diff --git a/infra/terraform/envs/api-benchmark/variables.tf b/infra/terraform/envs/api-benchmark/variables.tf new file mode 100644 index 0000000..6eaac7a --- /dev/null +++ b/infra/terraform/envs/api-benchmark/variables.tf @@ -0,0 +1,57 @@ +variable "aws_region" { + description = "AWS region in which to create the disposable benchmark environment." + type = string + default = "us-east-1" +} + +variable "benchmark_id" { + description = "Unique lowercase identifier appended to every named benchmark resource." + type = string + + validation { + condition = can(regex("^[a-z0-9]{4,8}$", var.benchmark_id)) + error_message = "benchmark_id must contain 4-8 lowercase letters or digits." + } +} + +variable "load_test_cidr" { + description = "Single explicit IPv4 CIDR allowed to reach the benchmark ALB on port 80." + type = string + + validation { + condition = ( + can(cidrnetmask(var.load_test_cidr)) && + strcontains(var.load_test_cidr, "/") && + var.load_test_cidr != "0.0.0.0/0" + ) + error_message = "load_test_cidr must be an explicit IPv4 CIDR and cannot be 0.0.0.0/0." + } +} + +variable "image_tag" { + description = "Immutable Git SHA tag pushed to the benchmark ECR repository." + type = string + + validation { + condition = can(regex("^[0-9a-f]{7,40}$", var.image_tag)) + error_message = "image_tag must be a 7-40 character lowercase hexadecimal Git SHA." + } +} + +variable "db_instance_class" { + description = "Single-AZ disposable PostgreSQL instance class." + type = string + default = "db.t4g.micro" +} + +variable "db_engine_version" { + description = "PostgreSQL version compatible with the application." + type = string + default = "16.13" +} + +variable "performance_insights_enabled" { + description = "Enable RDS Performance Insights only after confirming class support and cost." + type = bool + default = false +} diff --git a/infra/terraform/envs/api-benchmark/versions.tf b/infra/terraform/envs/api-benchmark/versions.tf new file mode 100644 index 0000000..5b9e7e7 --- /dev/null +++ b/infra/terraform/envs/api-benchmark/versions.tf @@ -0,0 +1,22 @@ +terraform { + required_version = ">= 1.6.0, < 2.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + } +} + +provider "aws" { + region = var.aws_region + + default_tags { + tags = local.required_tags + } +} From 937f594791bb4fc8e20a8ec149104f50f9a83e47 Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Wed, 9 Sep 2026 21:44:34 +0100 Subject: [PATCH 04/19] feat(terraform): add disposable benchmark networking --- .../envs/api-benchmark/networking.tf | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 infra/terraform/envs/api-benchmark/networking.tf diff --git a/infra/terraform/envs/api-benchmark/networking.tf b/infra/terraform/envs/api-benchmark/networking.tf new file mode 100644 index 0000000..6a41a6a --- /dev/null +++ b/infra/terraform/envs/api-benchmark/networking.tf @@ -0,0 +1,114 @@ +data "aws_availability_zones" "available" { + state = "available" +} + +resource "aws_vpc" "benchmark" { + cidr_block = "10.90.0.0/16" + enable_dns_support = true + enable_dns_hostnames = true + + tags = { Name = "${local.name_prefix}-vpc" } +} + +resource "aws_internet_gateway" "benchmark" { + vpc_id = aws_vpc.benchmark.id + tags = { Name = "${local.name_prefix}-igw" } +} + +resource "aws_subnet" "public" { + count = 2 + + vpc_id = aws_vpc.benchmark.id + availability_zone = data.aws_availability_zones.available.names[count.index] + cidr_block = cidrsubnet(aws_vpc.benchmark.cidr_block, 8, count.index) + map_public_ip_on_launch = true + + tags = { Name = "${local.name_prefix}-public-${count.index + 1}" } +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.benchmark.id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.benchmark.id + } + + tags = { Name = "${local.name_prefix}-public" } +} + +resource "aws_route_table_association" "public" { + count = 2 + + subnet_id = aws_subnet.public[count.index].id + route_table_id = aws_route_table.public.id +} + +resource "aws_security_group" "alb" { + name = "${local.name_prefix}-alb" + description = "Benchmark load generator ingress only." + vpc_id = aws_vpc.benchmark.id + + ingress { + description = "HTTP from the explicitly configured load generator" + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = [var.load_test_cidr] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { Name = "${local.name_prefix}-alb" } +} + +resource "aws_security_group" "ecs" { + name = "${local.name_prefix}-ecs" + description = "Benchmark API tasks; no direct public ingress." + vpc_id = aws_vpc.benchmark.id + + ingress { + description = "API traffic from the benchmark ALB only" + from_port = 3080 + to_port = 3080 + protocol = "tcp" + security_groups = [aws_security_group.alb.id] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { Name = "${local.name_prefix}-ecs" } +} + +resource "aws_security_group" "rds" { + name = "${local.name_prefix}-rds" + description = "Benchmark PostgreSQL ingress from API tasks only." + vpc_id = aws_vpc.benchmark.id + + ingress { + description = "PostgreSQL from benchmark API tasks only" + from_port = 5432 + to_port = 5432 + protocol = "tcp" + security_groups = [aws_security_group.ecs.id] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { Name = "${local.name_prefix}-rds" } +} From 606983550a5fcfce5169808dcca80cc1a7070a2e Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Wed, 9 Sep 2026 21:44:35 +0100 Subject: [PATCH 05/19] feat(terraform): add benchmark postgres and secrets --- .../terraform/envs/api-benchmark/database.tf | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 infra/terraform/envs/api-benchmark/database.tf diff --git a/infra/terraform/envs/api-benchmark/database.tf b/infra/terraform/envs/api-benchmark/database.tf new file mode 100644 index 0000000..a567b88 --- /dev/null +++ b/infra/terraform/envs/api-benchmark/database.tf @@ -0,0 +1,75 @@ +resource "random_password" "database" { + length = 32 + special = true + override_special = "!#$%&*+-.:=?_" +} + +resource "random_password" "csrf" { + length = 48 + special = false +} + +resource "random_password" "jwt_access" { + length = 64 + special = false +} + +resource "random_password" "jwt_refresh" { + length = 64 + special = false +} + +resource "random_password" "benchmark" { + length = 24 + special = false +} + +resource "aws_db_subnet_group" "benchmark" { + name = "${local.name_prefix}-db-subnets" + subnet_ids = aws_subnet.public[*].id + tags = { Name = "${local.name_prefix}-db-subnets" } +} + +resource "aws_db_instance" "postgres" { + identifier = "${local.name_prefix}-postgres" + engine = "postgres" + engine_version = var.db_engine_version + instance_class = var.db_instance_class + allocated_storage = 20 + storage_type = "gp3" + storage_encrypted = true + db_name = "underflow_benchmark" + username = "underflow_benchmark" + password = random_password.database.result + port = 5432 + db_subnet_group_name = aws_db_subnet_group.benchmark.name + vpc_security_group_ids = [aws_security_group.rds.id] + publicly_accessible = false + multi_az = false + backup_retention_period = 0 + deletion_protection = false + skip_final_snapshot = true + auto_minor_version_upgrade = true + apply_immediately = true + performance_insights_enabled = var.performance_insights_enabled + + tags = { Name = "${local.name_prefix}-postgres" } +} + +resource "aws_secretsmanager_secret" "runtime" { + name = "${local.name_prefix}-runtime" + description = "Disposable API benchmark runtime secrets." + recovery_window_in_days = 0 + tags = { Name = "${local.name_prefix}-runtime" } +} + +resource "aws_secretsmanager_secret_version" "runtime" { + secret_id = aws_secretsmanager_secret.runtime.id + secret_string = jsonencode({ + DATABASE_URL = "postgresql://underflow_benchmark:${urlencode(random_password.database.result)}@${aws_db_instance.postgres.address}:5432/underflow_benchmark" + CSRF_SECRET = random_password.csrf.result + JWT_ACCESS_SECRET = random_password.jwt_access.result + JWT_REFRESH_SECRET = random_password.jwt_refresh.result + BENCHMARK_PASSWORD = random_password.benchmark.result + }) +} From f33bf0e1fe367c4d91b2edf71a8a7c4adddcab86 Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Wed, 9 Sep 2026 21:44:35 +0100 Subject: [PATCH 06/19] feat(terraform): deploy benchmark API on ECS --- infra/terraform/envs/api-benchmark/compute.tf | 146 ++++++++++++++++++ infra/terraform/envs/api-benchmark/iam.tf | 41 +++++ infra/terraform/envs/api-benchmark/outputs.tf | 60 +++++++ 3 files changed, 247 insertions(+) create mode 100644 infra/terraform/envs/api-benchmark/compute.tf create mode 100644 infra/terraform/envs/api-benchmark/iam.tf create mode 100644 infra/terraform/envs/api-benchmark/outputs.tf diff --git a/infra/terraform/envs/api-benchmark/compute.tf b/infra/terraform/envs/api-benchmark/compute.tf new file mode 100644 index 0000000..eeabc81 --- /dev/null +++ b/infra/terraform/envs/api-benchmark/compute.tf @@ -0,0 +1,146 @@ +resource "aws_ecr_repository" "api" { + name = "${local.name_prefix}-api" + image_tag_mutability = "IMMUTABLE" + force_delete = true + + image_scanning_configuration { + scan_on_push = true + } + + tags = { Name = "${local.name_prefix}-api" } +} + +resource "aws_cloudwatch_log_group" "api" { + name = "/ecs/${local.name_prefix}/api" + retention_in_days = 3 + tags = { Name = "${local.name_prefix}-api-logs" } +} + +resource "aws_ecs_cluster" "benchmark" { + name = "${local.name_prefix}-cluster" + + setting { + name = "containerInsights" + value = "enabled" + } + + tags = { Name = "${local.name_prefix}-cluster" } +} + +resource "aws_lb" "api" { + name = substr("${local.name_prefix}-alb", 0, 32) + internal = false + load_balancer_type = "application" + security_groups = [aws_security_group.alb.id] + subnets = aws_subnet.public[*].id + + tags = { Name = "${local.name_prefix}-alb" } +} + +resource "aws_lb_target_group" "api" { + name = substr("${local.name_prefix}-tg", 0, 32) + port = 3080 + protocol = "HTTP" + target_type = "ip" + vpc_id = aws_vpc.benchmark.id + + health_check { + enabled = true + path = "/api/v1/health" + matcher = "200" + interval = 30 + timeout = 5 + healthy_threshold = 2 + unhealthy_threshold = 3 + } + + tags = { Name = "${local.name_prefix}-tg" } +} + +resource "aws_lb_listener" "http" { + load_balancer_arn = aws_lb.api.arn + port = 80 + protocol = "HTTP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.api.arn + } + + tags = { Name = "${local.name_prefix}-http" } +} + +resource "aws_ecs_task_definition" "api" { + family = "${local.name_prefix}-api" + network_mode = "awsvpc" + requires_compatibilities = ["FARGATE"] + cpu = "512" + memory = "1024" + execution_role_arn = aws_iam_role.task_execution.arn + task_role_arn = aws_iam_role.task_runtime.arn + + container_definitions = jsonencode([ + { + name = "api" + image = "${aws_ecr_repository.api.repository_url}:${var.image_tag}" + essential = true + command = ["node", "dist/server.js"] + portMappings = [{ + containerPort = 3080 + hostPort = 3080 + protocol = "tcp" + }] + environment = local.container_environment + secrets = local.container_secrets + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.api.name + awslogs-region = var.aws_region + awslogs-stream-prefix = "ecs" + } + } + } + ]) + + runtime_platform { + cpu_architecture = "X86_64" + operating_system_family = "LINUX" + } + + tags = { Name = "${local.name_prefix}-api" } +} + +resource "aws_ecs_service" "api" { + name = "${local.name_prefix}-api" + cluster = aws_ecs_cluster.benchmark.id + task_definition = aws_ecs_task_definition.api.arn + desired_count = 1 + launch_type = "FARGATE" + health_check_grace_period_seconds = 90 + wait_for_steady_state = true + + network_configuration { + subnets = aws_subnet.public[*].id + security_groups = [aws_security_group.ecs.id] + assign_public_ip = true + } + + load_balancer { + target_group_arn = aws_lb_target_group.api.arn + container_name = "api" + container_port = 3080 + } + + deployment_circuit_breaker { + enable = true + rollback = true + } + + depends_on = [ + aws_lb_listener.http, + aws_secretsmanager_secret_version.runtime, + ] + + tags = { Name = "${local.name_prefix}-api" } +} diff --git a/infra/terraform/envs/api-benchmark/iam.tf b/infra/terraform/envs/api-benchmark/iam.tf new file mode 100644 index 0000000..c452c2f --- /dev/null +++ b/infra/terraform/envs/api-benchmark/iam.tf @@ -0,0 +1,41 @@ +data "aws_iam_policy_document" "ecs_tasks_assume_role" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ecs-tasks.amazonaws.com"] + } + } +} + +resource "aws_iam_role" "task_execution" { + name = "${local.name_prefix}-task-execution" + assume_role_policy = data.aws_iam_policy_document.ecs_tasks_assume_role.json + tags = { Name = "${local.name_prefix}-task-execution" } +} + +resource "aws_iam_role_policy_attachment" "task_execution" { + role = aws_iam_role.task_execution.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" +} + +data "aws_iam_policy_document" "task_execution_secrets" { + statement { + sid = "ReadBenchmarkRuntimeSecret" + actions = ["secretsmanager:GetSecretValue"] + resources = [aws_secretsmanager_secret.runtime.arn] + } +} + +resource "aws_iam_role_policy" "task_execution_secrets" { + name = "${local.name_prefix}-read-runtime-secret" + role = aws_iam_role.task_execution.id + policy = data.aws_iam_policy_document.task_execution_secrets.json +} + +resource "aws_iam_role" "task_runtime" { + name = "${local.name_prefix}-task-runtime" + assume_role_policy = data.aws_iam_policy_document.ecs_tasks_assume_role.json + tags = { Name = "${local.name_prefix}-task-runtime" } +} diff --git a/infra/terraform/envs/api-benchmark/outputs.tf b/infra/terraform/envs/api-benchmark/outputs.tf new file mode 100644 index 0000000..b2e30e4 --- /dev/null +++ b/infra/terraform/envs/api-benchmark/outputs.tf @@ -0,0 +1,60 @@ +output "aws_region" { + value = var.aws_region +} + +output "resource_prefix" { + value = local.name_prefix +} + +output "ecr_repository_url" { + value = aws_ecr_repository.api.repository_url +} + +output "ecs_cluster_name" { + value = aws_ecs_cluster.benchmark.name +} + +output "ecs_service_name" { + value = aws_ecs_service.api.name +} + +output "task_definition_arn" { + value = aws_ecs_task_definition.api.arn +} + +output "alb_dns_name" { + value = aws_lb.api.dns_name +} + +output "api_base_url" { + value = "http://${aws_lb.api.dns_name}" +} + +output "rds_identifier" { + value = aws_db_instance.postgres.identifier +} + +output "ecs_security_group_id" { + value = aws_security_group.ecs.id +} + +output "subnet_ids" { + value = aws_subnet.public[*].id +} + +output "load_balancer_arn_suffix" { + value = aws_lb.api.arn_suffix +} + +output "target_group_arn_suffix" { + value = aws_lb_target_group.api.arn_suffix +} + +output "availability_zones" { + value = aws_subnet.public[*].availability_zone +} + +output "runtime_secret_arn" { + value = aws_secretsmanager_secret.runtime.arn + sensitive = true +} From 423947b370d8cf04d5f8d6a959dee462e53ad917 Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Wed, 9 Sep 2026 21:44:35 +0100 Subject: [PATCH 07/19] feat(benchmark): add authenticated k6 workloads --- benchmarks/underflow-api/k6/common.js | 158 ++++++++++++++++++++++++++ benchmarks/underflow-api/k6/load.js | 27 +++++ benchmarks/underflow-api/k6/smoke.js | 18 +++ benchmarks/underflow-api/k6/stress.js | 27 +++++ 4 files changed, 230 insertions(+) create mode 100644 benchmarks/underflow-api/k6/common.js create mode 100644 benchmarks/underflow-api/k6/load.js create mode 100644 benchmarks/underflow-api/k6/smoke.js create mode 100644 benchmarks/underflow-api/k6/stress.js diff --git a/benchmarks/underflow-api/k6/common.js b/benchmarks/underflow-api/k6/common.js new file mode 100644 index 0000000..ffa9383 --- /dev/null +++ b/benchmarks/underflow-api/k6/common.js @@ -0,0 +1,158 @@ +import http from "k6/http"; +import { check, sleep } from "k6"; +import exec from "k6/execution"; +import { Counter, Rate, Trend } from "k6/metrics"; + +const measuredRequests = new Counter("measured_requests"); +const measuredFailures = new Rate("measured_failures"); +const measuredDuration = new Trend("measured_duration", true); + +const endpointTrends = { + cost_summary: new Trend("endpoint_cost_summary_duration", true), + cost_timeseries: new Trend("endpoint_cost_timeseries_duration", true), + cost_by_service: new Trend("endpoint_cost_by_service_duration", true), + aws_account_list: new Trend("endpoint_aws_account_list_duration", true), + sync_history: new Trend("endpoint_sync_history_duration", true), +}; + +const requiredEnvironment = ["BASE_URL", "TEST_EMAIL", "TEST_PASSWORD", "WORKSPACE_ID"]; + +export const assertEnvironment = () => { + for (const name of requiredEnvironment) { + if (!__ENV[name]) { + throw new Error(`${name} is required`); + } + } + if (!/^https?:\/\//.test(__ENV.BASE_URL)) { + throw new Error("BASE_URL must start with http:// or https://"); + } +}; + +export const standardThresholds = (failureThreshold = "rate<0.01", abortOnFailure = false) => ({ + measured_failures: abortOnFailure + ? [{ threshold: failureThreshold, abortOnFail: true, delayAbortEval: "30s" }] + : [failureThreshold], + measured_duration: ["p(95)<200", "p(99)<500"], + checks: ["rate>0.99"], + endpoint_cost_summary_duration: ["p(95)<200", "p(99)<500"], + endpoint_cost_timeseries_duration: ["p(95)<200", "p(99)<500"], + endpoint_cost_by_service_duration: ["p(95)<200", "p(99)<500"], + endpoint_aws_account_list_duration: ["p(95)<200", "p(99)<500"], + endpoint_sync_history_duration: ["p(95)<200", "p(99)<500"], +}); + +export const authenticate = () => { + assertEnvironment(); + const response = http.post( + `${__ENV.BASE_URL}/api/v1/auth/mobile/login`, + JSON.stringify({ email: __ENV.TEST_EMAIL, password: __ENV.TEST_PASSWORD }), + { + headers: { "Content-Type": "application/json" }, + tags: { name: "auth_setup", scope: "setup" }, + }, + ); + + if (response.status !== 200) { + throw new Error(`Authentication failed with HTTP ${response.status}`); + } + + let payload; + try { + payload = response.json(); + } catch (_error) { + throw new Error("Authentication returned invalid JSON"); + } + + const accessToken = payload?.tokens?.accessToken; + if (typeof accessToken !== "string" || accessToken.length === 0) { + throw new Error("Authentication response did not contain an access token"); + } + + return { accessToken }; +}; + +const requestDefinitions = [ + { + upperBound: 30, + name: "cost_summary", + path: `/api/v1/workspaces/${__ENV.WORKSPACE_ID}/costs/summary?from=2025-01-01&to=2025-12-31`, + shape: (body) => typeof body?.summary?.totalAmount === "number" && body.summary.currency === "USD", + }, + { + upperBound: 60, + name: "cost_timeseries", + path: `/api/v1/workspaces/${__ENV.WORKSPACE_ID}/costs/timeseries?from=2025-01-01&to=2025-12-31`, + shape: (body) => Array.isArray(body?.points) && body.points.length === 365, + }, + { + upperBound: 85, + name: "cost_by_service", + path: `/api/v1/workspaces/${__ENV.WORKSPACE_ID}/costs/by-service?from=2025-01-01&to=2025-12-31`, + shape: (body) => Array.isArray(body?.services) && body.services.length === 50, + }, + { + upperBound: 95, + name: "aws_account_list", + path: `/api/v1/workspaces/${__ENV.WORKSPACE_ID}/aws-accounts`, + shape: (body) => Array.isArray(body?.awsAccounts) && body.awsAccounts.length === 20, + }, + { + upperBound: 100, + name: "sync_history", + path: `/api/v1/workspaces/${__ENV.WORKSPACE_ID}/sync-history?limit=25`, + shape: (body) => Array.isArray(body?.syncRuns) && body.syncRuns.length > 0, + }, +]; + +const chooseRequest = () => { + const bucket = exec.scenario.iterationInTest % 100; + return requestDefinitions.find((definition) => bucket < definition.upperBound); +}; + +export const runMeasuredRequest = (data) => { + const definition = chooseRequest(); + if (!definition) { + throw new Error("No request definition selected"); + } + + const response = http.get(`${__ENV.BASE_URL}${definition.path}`, { + headers: { + Authorization: `Bearer ${data.accessToken}`, + Accept: "application/json", + }, + tags: { name: definition.name, endpoint: definition.name, scope: "measured" }, + }); + + let body = null; + let validJson = true; + try { + body = response.json(); + } catch (_error) { + validJson = false; + } + + const validStatus = response.status === 200; + const validShape = validJson && definition.shape(body); + const passed = check(response, { + [`${definition.name}: status is 200`]: () => validStatus, + [`${definition.name}: response is JSON`]: () => validJson, + [`${definition.name}: response shape is valid`]: () => validShape, + [`${definition.name}: no auth or server error`]: () => + response.status !== 401 && response.status !== 403 && response.status < 500, + }); + + measuredRequests.add(1); + measuredFailures.add(!passed || !validStatus || !validJson || !validShape); + measuredDuration.add(response.timings.duration); + endpointTrends[definition.name].add(response.timings.duration); + sleep(Number(__ENV.THINK_TIME_SECONDS ?? "0.1")); +}; + +export const writeSummary = (data, defaultName) => { + const resultsDirectory = (__ENV.RESULTS_DIR ?? "results").replace(/[\\/]$/, ""); + const summaryName = __ENV.SUMMARY_NAME ?? defaultName; + return { + [`${resultsDirectory}/${summaryName}`]: JSON.stringify(data, null, 2), + stdout: `Saved compact k6 summary to ${resultsDirectory}/${summaryName}\n`, + }; +}; diff --git a/benchmarks/underflow-api/k6/load.js b/benchmarks/underflow-api/k6/load.js new file mode 100644 index 0000000..9466287 --- /dev/null +++ b/benchmarks/underflow-api/k6/load.js @@ -0,0 +1,27 @@ +import { authenticate, runMeasuredRequest, standardThresholds, writeSummary } from "./common.js"; + +export const options = { + scenarios: { + normal_load: { + executor: "ramping-vus", + startVUs: 0, + gracefulRampDown: "30s", + stages: [ + { duration: "1m", target: 25 }, + { duration: "3m", target: 25 }, + { duration: "1m", target: 50 }, + { duration: "5m", target: 50 }, + { duration: "2m", target: 100 }, + { duration: "5m", target: 100 }, + { duration: "1m", target: 0 }, + ], + }, + }, + thresholds: standardThresholds(), +}; + +export const setup = authenticate; + +export default runMeasuredRequest; + +export const handleSummary = (data) => writeSummary(data, "load-summary.json"); diff --git a/benchmarks/underflow-api/k6/smoke.js b/benchmarks/underflow-api/k6/smoke.js new file mode 100644 index 0000000..7c36b00 --- /dev/null +++ b/benchmarks/underflow-api/k6/smoke.js @@ -0,0 +1,18 @@ +import { authenticate, runMeasuredRequest, standardThresholds, writeSummary } from "./common.js"; + +export const options = { + scenarios: { + smoke: { + executor: "constant-vus", + vus: Number(__ENV.SMOKE_VUS ?? "2"), + duration: __ENV.SMOKE_DURATION ?? "30s", + }, + }, + thresholds: standardThresholds(), +}; + +export const setup = authenticate; + +export default runMeasuredRequest; + +export const handleSummary = (data) => writeSummary(data, "smoke-summary.json"); diff --git a/benchmarks/underflow-api/k6/stress.js b/benchmarks/underflow-api/k6/stress.js new file mode 100644 index 0000000..3ec4645 --- /dev/null +++ b/benchmarks/underflow-api/k6/stress.js @@ -0,0 +1,27 @@ +import { authenticate, runMeasuredRequest, standardThresholds, writeSummary } from "./common.js"; + +export const options = { + scenarios: { + stress: { + executor: "ramping-vus", + startVUs: 0, + gracefulRampDown: "15s", + stages: [ + { duration: "2m", target: 100 }, + { duration: "3m", target: 100 }, + { duration: "1m", target: 150 }, + { duration: "3m", target: 150 }, + { duration: "1m", target: 200 }, + { duration: "3m", target: 200 }, + { duration: "1m", target: 0 }, + ], + }, + }, + thresholds: standardThresholds("rate<0.05", true), +}; + +export const setup = authenticate; + +export default runMeasuredRequest; + +export const handleSummary = (data) => writeSummary(data, "stress-summary.json"); From a57beb433898e9b25e86f4e3e31b53c1a1e00de1 Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Wed, 9 Sep 2026 21:44:35 +0100 Subject: [PATCH 08/19] feat(benchmark): add execution and evidence tooling --- benchmarks/underflow-api/results/.gitkeep | 1 + .../scripts/collect-aws-metadata.sh | 94 ++++++++++ .../scripts/collect-environment.sh | 61 +++++++ .../underflow-api/scripts/generate-results.sh | 160 ++++++++++++++++++ .../underflow-api/scripts/run-one-off-task.sh | 98 +++++++++++ .../underflow-api/scripts/sanitize-results.sh | 52 ++++++ 6 files changed, 466 insertions(+) create mode 100644 benchmarks/underflow-api/results/.gitkeep create mode 100644 benchmarks/underflow-api/scripts/collect-aws-metadata.sh create mode 100644 benchmarks/underflow-api/scripts/collect-environment.sh create mode 100644 benchmarks/underflow-api/scripts/generate-results.sh create mode 100644 benchmarks/underflow-api/scripts/run-one-off-task.sh create mode 100644 benchmarks/underflow-api/scripts/sanitize-results.sh diff --git a/benchmarks/underflow-api/results/.gitkeep b/benchmarks/underflow-api/results/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/benchmarks/underflow-api/results/.gitkeep @@ -0,0 +1 @@ + diff --git a/benchmarks/underflow-api/scripts/collect-aws-metadata.sh b/benchmarks/underflow-api/scripts/collect-aws-metadata.sh new file mode 100644 index 0000000..e763eff --- /dev/null +++ b/benchmarks/underflow-api/scripts/collect-aws-metadata.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +set -euo pipefail + +required=(AWS_REGION ECS_CLUSTER ECS_SERVICE RDS_IDENTIFIER ALB_ARN_SUFFIX TARGET_GROUP_ARN_SUFFIX START_TIME END_TIME OUTPUT_FILE) +for name in "${required[@]}"; do + [[ -n "${!name:-}" ]] || { echo "$name is required" >&2; exit 2; } +done + +command -v aws >/dev/null || { echo "aws is required" >&2; exit 2; } +if command -v py >/dev/null; then + PYTHON_COMMAND=(py -3) +elif command -v python3 >/dev/null; then + PYTHON_COMMAND=(python3) +elif command -v python >/dev/null; then + PYTHON_COMMAND=(python) +else + echo "python3, py, or python is required" >&2 + exit 2 +fi + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +collect_metric() { + local key="$1" namespace="$2" metric="$3" statistic="$4" + shift 4 + aws cloudwatch get-metric-statistics \ + --region "$AWS_REGION" \ + --namespace "$namespace" \ + --metric-name "$metric" \ + --dimensions "$@" \ + --start-time "$START_TIME" \ + --end-time "$END_TIME" \ + --period 60 \ + --statistics "$statistic" \ + --output json > "$TMP_DIR/$key.json" +} + +collect_metric ecs_cpu AWS/ECS CPUUtilization Average \ + Name=ClusterName,Value="$ECS_CLUSTER" Name=ServiceName,Value="$ECS_SERVICE" +collect_metric ecs_memory AWS/ECS MemoryUtilization Average \ + Name=ClusterName,Value="$ECS_CLUSTER" Name=ServiceName,Value="$ECS_SERVICE" +collect_metric ecs_running_tasks ECS/ContainerInsights RunningTaskCount Average \ + Name=ClusterName,Value="$ECS_CLUSTER" Name=ServiceName,Value="$ECS_SERVICE" +collect_metric alb_target_response AWS/ApplicationELB TargetResponseTime Average \ + Name=LoadBalancer,Value="$ALB_ARN_SUFFIX" Name=TargetGroup,Value="$TARGET_GROUP_ARN_SUFFIX" +collect_metric alb_4xx AWS/ApplicationELB HTTPCode_ELB_4XX_Count Sum \ + Name=LoadBalancer,Value="$ALB_ARN_SUFFIX" +collect_metric alb_5xx AWS/ApplicationELB HTTPCode_ELB_5XX_Count Sum \ + Name=LoadBalancer,Value="$ALB_ARN_SUFFIX" +collect_metric alb_target_5xx AWS/ApplicationELB HTTPCode_Target_5XX_Count Sum \ + Name=LoadBalancer,Value="$ALB_ARN_SUFFIX" Name=TargetGroup,Value="$TARGET_GROUP_ARN_SUFFIX" +collect_metric rds_cpu AWS/RDS CPUUtilization Average Name=DBInstanceIdentifier,Value="$RDS_IDENTIFIER" +collect_metric rds_connections AWS/RDS DatabaseConnections Average Name=DBInstanceIdentifier,Value="$RDS_IDENTIFIER" +collect_metric rds_free_memory AWS/RDS FreeableMemory Average Name=DBInstanceIdentifier,Value="$RDS_IDENTIFIER" +collect_metric rds_read_latency AWS/RDS ReadLatency Average Name=DBInstanceIdentifier,Value="$RDS_IDENTIFIER" +collect_metric rds_write_latency AWS/RDS WriteLatency Average Name=DBInstanceIdentifier,Value="$RDS_IDENTIFIER" +collect_metric rds_burst_balance AWS/RDS BurstBalance Average Name=DBInstanceIdentifier,Value="$RDS_IDENTIFIER" +collect_metric rds_cpu_credit AWS/RDS CPUCreditBalance Average Name=DBInstanceIdentifier,Value="$RDS_IDENTIFIER" + +aws ecs describe-services \ + --region "$AWS_REGION" \ + --cluster "$ECS_CLUSTER" \ + --services "$ECS_SERVICE" \ + --query 'services[0].{desiredCount:desiredCount,runningCount:runningCount,pendingCount:pendingCount}' \ + --output json > "$TMP_DIR/ecs_tasks.json" + +"${PYTHON_COMMAND[@]}" - "$TMP_DIR" "$OUTPUT_FILE" "$START_TIME" "$END_TIME" <<'PY' +import json +import pathlib +import sys + +source = pathlib.Path(sys.argv[1]) +output = pathlib.Path(sys.argv[2]) +payload = { + "startTime": sys.argv[3], + "endTime": sys.argv[4], + "periodSeconds": 60, + "ecsTaskCountsAtCollection": json.loads((source / "ecs_tasks.json").read_text()), + "metrics": {}, +} +for path in sorted(source.glob("*.json")): + if path.name == "ecs_tasks.json": + continue + metric = json.loads(path.read_text()) + payload["metrics"][path.stem] = { + "label": metric.get("Label"), + "datapoints": sorted(metric.get("Datapoints", []), key=lambda item: item["Timestamp"]), + } +output.parent.mkdir(parents=True, exist_ok=True) +output.write_text(json.dumps(payload, indent=2) + "\n") +PY + +echo "Saved CloudWatch summary to $OUTPUT_FILE" diff --git a/benchmarks/underflow-api/scripts/collect-environment.sh b/benchmarks/underflow-api/scripts/collect-environment.sh new file mode 100644 index 0000000..e7e5e70 --- /dev/null +++ b/benchmarks/underflow-api/scripts/collect-environment.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +required=(GIT_SHA IMAGE_DIGEST AWS_REGION RESULTS_DIR TF_ROOT) +for name in "${required[@]}"; do + [[ -n "${!name:-}" ]] || { echo "$name is required" >&2; exit 2; } +done + +for command in jq terraform node; do + command -v "$command" >/dev/null || { echo "$command is required" >&2; exit 2; } +done + +mkdir -p "$RESULTS_DIR" +if [[ -z "${K6_VERSION:-}" ]]; then + if command -v k6 >/dev/null; then + K6_VERSION="$(k6 version | head -n1)" + elif command -v docker >/dev/null; then + K6_VERSION="$(docker run --rm grafana/k6:latest version | head -n1)" + else + echo "k6 or Docker is required" >&2 + exit 2 + fi +fi + +jq -n \ + --arg gitSha "$GIT_SHA" \ + --arg imageDigest "$IMAGE_DIGEST" \ + --arg timestamp "$(date -u +%FT%TZ)" \ + --arg region "$AWS_REGION" \ + --argjson availabilityZones "$(terraform -chdir="$TF_ROOT" output -json availability_zones)" \ + --arg nodeVersion "$(node --version)" \ + --arg k6Version "$K6_VERSION" \ + '{ + gitSha: $gitSha, + imageDigest: $imageDigest, + testTimestamp: $timestamp, + awsRegion: $region, + availabilityZones: $availabilityZones, + infrastructure: { + ecs: {cpu: 512, memoryMiB: 1024, desiredCount: 1}, + rds: { + class: "db.t4g.micro", + engine: "postgres", + engineVersion: "16.13", + storageGiB: 20, + multiAz: false, + publiclyAccessible: false + } + }, + nodeVersion: $nodeVersion, + k6Version: $k6Version, + dataset: "underflow-api-benchmark-v1", + profiles: { + smoke: {vus: 2, duration: "30s"}, + normalLoad: {stages: [["1m",25],["3m",25],["1m",50],["5m",50],["2m",100],["5m",100],["1m",0]]}, + stress: {stages: [["2m",100],["3m",100],["1m",150],["3m",150],["1m",200],["3m",200],["1m",0]]} + } + }' > "$RESULTS_DIR/environment.json" + +jq -e . "$RESULTS_DIR/environment.json" >/dev/null +echo "Saved environment metadata to $RESULTS_DIR/environment.json" diff --git a/benchmarks/underflow-api/scripts/generate-results.sh b/benchmarks/underflow-api/scripts/generate-results.sh new file mode 100644 index 0000000..473d137 --- /dev/null +++ b/benchmarks/underflow-api/scripts/generate-results.sh @@ -0,0 +1,160 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 RESULTS_DIRECTORY OUTPUT_MARKDOWN" >&2 + exit 2 +fi + +if command -v py >/dev/null; then + PYTHON_COMMAND=(py -3) +elif command -v python3 >/dev/null; then + PYTHON_COMMAND=(python3) +elif command -v python >/dev/null; then + PYTHON_COMMAND=(python) +else + echo "python3, py, or python is required" >&2 + exit 2 +fi + +"${PYTHON_COMMAND[@]}" - "$1" "$2" <<'PY' +import json +import pathlib +import sys + +directory = pathlib.Path(sys.argv[1]) +output = pathlib.Path(sys.argv[2]) + +def read(name, required=True): + path = directory / name + if not path.exists(): + if required: + raise SystemExit(f"missing required evidence: {path}") + return None + return json.loads(path.read_text()) + +environment = read("environment.json") +dataset = read("dataset.json") +smoke = read("smoke-summary.json") +load = read("load-summary.json") +stress = read("stress-summary.json", required=False) +cloudwatch = read("cloudwatch-summary.json") + +def values(summary, metric): + try: + return summary["metrics"][metric]["values"] + except KeyError as error: + raise SystemExit(f"missing metric {metric}: {error}") from error + +def metric_row(label, summary): + requests = values(summary, "measured_requests") + duration = values(summary, "measured_duration") + failures = values(summary, "measured_failures") + checks = values(summary, "checks") + return ( + f"| {label} | {requests['count']:.0f} | {requests['rate']:.2f} | " + f"{duration['med']:.2f} | {duration['p(90)']:.2f} | {duration['p(95)']:.2f} | " + f"{duration['p(99)']:.2f} | {duration['max']:.2f} | " + f"{failures['rate'] * 100:.3f}% | {checks['rate'] * 100:.3f}% |" + ) + +endpoint_metrics = [ + ("Cost summary", "endpoint_cost_summary_duration"), + ("Cost timeseries", "endpoint_cost_timeseries_duration"), + ("Cost by service", "endpoint_cost_by_service_duration"), + ("AWS account list", "endpoint_aws_account_list_duration"), + ("Sync history", "endpoint_sync_history_duration"), +] + +load_duration = values(load, "measured_duration") +load_requests = values(load, "measured_requests") +vus = int(values(load, "vus_max")["max"]) +cost_rows = int(dataset["costSnapshots"]) +records_label = f"{cost_rows / 1_000_000:.2f}M" if cost_rows >= 1_000_000 else f"{cost_rows:,}" +resume = ( + "Built and benchmarked a multi-tenant AWS cost-monitoring API against " + f"{records_label} cost records, sustaining {load_requests['rate']:.2f} requests/second " + f"with {load_duration['p(95)']:.2f} ms p95 latency under {vus} concurrent users." +) + +lines = [ + "# Underflow API benchmark results", + "", + "## Objective", + "", + "Measure authenticated, PostgreSQL-backed cost-reporting reads on the documented disposable AWS configuration.", + "", + "## Tested commit", + "", + f"`{environment['gitSha']}` using image digest `{environment['imageDigest']}`.", + "", + "## Infrastructure", + "", + "On the documented one-task ECS and db.t4g.micro configuration described in `environment.json`.", + "", + "## Dataset", + "", + f"{cost_rows:,} deterministic synthetic cost snapshots across {dataset['workspaces']} workspaces and {dataset['awsAccounts']} synthetic AWS accounts.", + "", + "## Workload", + "", + "30% summary, 30% timeseries, 25% by service, 10% AWS account list, and 5% sync history, with bearer-token authentication on every measured request.", + "", + "## Thresholds", + "", + "Failure rate < 1%, p95 < 200 ms, p99 < 500 ms, and checks pass rate > 99%.", + "", + "## Results", + "", + "| Run | Requests | Requests/s | p50 ms | p90 ms | p95 ms | p99 ms | Max ms | Failures | Checks passed |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + metric_row("Smoke", smoke), + metric_row("Normal load", load), +] +if stress is not None: + lines.append(metric_row("Stress", stress)) + +lines.extend([ + "", + "## Per-endpoint results (normal load)", + "", + "| Endpoint | p95 ms | p99 ms | Max ms |", + "| --- | ---: | ---: | ---: |", +]) +for label, metric in endpoint_metrics: + metric_values = values(load, metric) + lines.append( + f"| {label} | {metric_values['p(95)']:.2f} | {metric_values['p(99)']:.2f} | {metric_values['max']:.2f} |" + ) + +lines.extend([ + "", + "## AWS resource metrics", + "", + "Captured for the measured interval in `cloudwatch-summary.json`.", + "", + "## Observed bottlenecks", + "", + "Interpret from the preserved k6 and CloudWatch evidence; do not generalize beyond this configuration.", + "", + "## Limitations", + "", + "Synthetic data, one AWS region, one Fargate task, one load-generator location, HTTP restricted by CIDR, and a single-AZ burstable RDS instance.", + "", + "## Exact reproduction commands", + "", + "See `../README.md` and the immutable environment metadata in this directory.", + "", + "## Evidence-backed résumé bullet", + "", + resume, + "", + "```latex", + "\\item " + resume.replace("%", "\\%"), + "```", + "", +]) + +output.write_text("\n".join(lines), encoding="utf-8") +print(f"Generated {output} from preserved result values") +PY diff --git a/benchmarks/underflow-api/scripts/run-one-off-task.sh b/benchmarks/underflow-api/scripts/run-one-off-task.sh new file mode 100644 index 0000000..0ed79cb --- /dev/null +++ b/benchmarks/underflow-api/scripts/run-one-off-task.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +set -euo pipefail + +MODE="${1:-}" +if [[ "$MODE" != "migrate" && "$MODE" != "seed" ]]; then + echo "usage: $0 migrate|seed" >&2 + exit 2 +fi + +for command in aws terraform jq; do + command -v "$command" >/dev/null || { echo "$command is required" >&2; exit 2; } +done + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +TF_ROOT="$ROOT_DIR/infra/terraform/envs/api-benchmark" +RESULTS_DIR="${BENCHMARK_RESULTS_DIR:-$ROOT_DIR/benchmarks/underflow-api/results/local-evidence}" +mkdir -p "$RESULTS_DIR" + +tf_output() { + terraform -chdir="$TF_ROOT" output -raw "$1" | tr -d '\r' +} + +REGION="$(tf_output aws_region)" +CLUSTER="$(tf_output ecs_cluster_name)" +TASK_DEFINITION="$(tf_output task_definition_arn)" +SECURITY_GROUP="$(tf_output ecs_security_group_id)" +mapfile -t SUBNETS < <(terraform -chdir="$TF_ROOT" output -json subnet_ids | jq -r '.[]') +SUBNET_LIST="$(IFS=,; echo "${SUBNETS[*]}")" + +if [[ "$MODE" == "migrate" ]]; then + COMMAND_JSON='["node","dist/db/migrate.js"]' + ENVIRONMENT_JSON='[]' +else + COMMAND_JSON='["node","dist/scripts/seed-benchmark.js"]' + ENVIRONMENT_JSON='[{"name":"ALLOW_BENCHMARK_SEED","value":"true"}]' +fi + +OVERRIDES="$(jq -cn \ + --argjson command "$COMMAND_JSON" \ + --argjson environment "$ENVIRONMENT_JSON" \ + '{containerOverrides:[{name:"api",command:$command,environment:$environment}]}')" + +TASK_ARN="$(aws ecs run-task \ + --region "$REGION" \ + --cluster "$CLUSTER" \ + --task-definition "$TASK_DEFINITION" \ + --launch-type FARGATE \ + --network-configuration "awsvpcConfiguration={subnets=[$SUBNET_LIST],securityGroups=[$SECURITY_GROUP],assignPublicIp=ENABLED}" \ + --overrides "$OVERRIDES" \ + --query 'tasks[0].taskArn' \ + --output text)" + +if [[ -z "$TASK_ARN" || "$TASK_ARN" == "None" ]]; then + echo "ECS did not return a task ARN" >&2 + exit 1 +fi + +TASK_ID="${TASK_ARN##*/}" +echo "Waiting for benchmark $MODE task $TASK_ID" +aws ecs wait tasks-stopped --region "$REGION" --cluster "$CLUSTER" --tasks "$TASK_ARN" + +EXIT_CODE="$(aws ecs describe-tasks \ + --region "$REGION" \ + --cluster "$CLUSTER" \ + --tasks "$TASK_ARN" \ + --query 'tasks[0].containers[?name==`api`].exitCode | [0]' \ + --output text)" + +LOG_GROUP="/ecs/$(tf_output resource_prefix)/api" +LOG_STREAM="ecs/api/$TASK_ID" +RAW_LOG="$(mktemp)" +trap 'rm -f "$RAW_LOG"' EXIT + +# Git Bash otherwise rewrites /ecs/... into a Windows filesystem path before +# invoking the native AWS CLI. Scope the override to this command so Terraform +# still receives converted -chdir paths. +MSYS_NO_PATHCONV=1 aws logs get-log-events \ + --region "$REGION" \ + --log-group-name "$LOG_GROUP" \ + --log-stream-name "$LOG_STREAM" \ + --output json > "$RAW_LOG" + +"$ROOT_DIR/benchmarks/underflow-api/scripts/sanitize-results.sh" \ + "$RAW_LOG" "$RESULTS_DIR/${MODE}-task.log" + +if [[ "$EXIT_CODE" != "0" ]]; then + echo "Benchmark $MODE task failed with exit code $EXIT_CODE; see sanitized log" >&2 + exit 1 +fi + +if [[ "$MODE" == "seed" ]]; then + jq '[.events[].message | fromjson? | select(.dataset == "underflow-api-benchmark-v1")] | last' \ + "$RESULTS_DIR/seed-task.log" > "$RESULTS_DIR/dataset.json" + jq -e '.costSnapshots == 3650000 and .users == 10 and .workspaces == 10 and .awsAccounts == 200' \ + "$RESULTS_DIR/dataset.json" >/dev/null +fi + +echo "Benchmark $MODE task completed with exit code 0" diff --git a/benchmarks/underflow-api/scripts/sanitize-results.sh b/benchmarks/underflow-api/scripts/sanitize-results.sh new file mode 100644 index 0000000..4a8aa33 --- /dev/null +++ b/benchmarks/underflow-api/scripts/sanitize-results.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 INPUT_FILE OUTPUT_FILE" >&2 + exit 2 +fi + +if command -v py >/dev/null; then + PYTHON_COMMAND=(py -3) +elif command -v python3 >/dev/null; then + PYTHON_COMMAND=(python3) +elif command -v python >/dev/null; then + PYTHON_COMMAND=(python) +else + echo "python3, py, or python is required" >&2 + exit 2 +fi + +"${PYTHON_COMMAND[@]}" - "$1" "$2" <<'PY' +import pathlib +import re +import sys + +source = pathlib.Path(sys.argv[1]) +target = pathlib.Path(sys.argv[2]) +text = source.read_text(encoding="utf-8", errors="replace") + +patterns = [ + (r"(?i)arn:aws:secretsmanager:[^\s\"']+", "[REDACTED_SECRET_ARN]"), + (r"(?i)\b[A-Z0-9]{20}\b", "[REDACTED_AWS_ACCESS_KEY]"), + (r"(?i)\beyJ[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\.[a-zA-Z0-9_-]+\b", "[REDACTED_TOKEN]"), + (r"\b\d{12}\.dkr\.ecr\.[a-z0-9-]+\.amazonaws\.com\b", "[REDACTED_ECR_HOST]"), + (r"(?i)\b[a-z0-9-]+\.[a-z0-9-]+\.rds\.amazonaws\.com\b", "[REDACTED_RDS_HOST]"), + (r"\b(?:\d{1,3}\.){3}\d{1,3}\b", "[REDACTED_IPV4]"), + (r"(?i)\b(?![a-z0-9.+_-]+@example\.invalid\b)[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}\b", "[REDACTED_EMAIL]"), + (r"(?i)(terraform\.tfstate(?:\.[^\s\"']+)?)", "[REDACTED_TERRAFORM_STATE]"), + (r"(? $2" From c9624a3c67d0430d581e6fc250b8b5ec8303e712 Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Wed, 9 Sep 2026 21:44:35 +0100 Subject: [PATCH 09/19] docs(benchmark): document isolated AWS benchmark workflow --- benchmarks/underflow-api/README.md | 296 +++++++++++++++++++ benchmarks/underflow-api/RESULTS_TEMPLATE.md | 55 ++++ infra/terraform/envs/api-benchmark/README.md | 9 + 3 files changed, 360 insertions(+) create mode 100644 benchmarks/underflow-api/README.md create mode 100644 benchmarks/underflow-api/RESULTS_TEMPLATE.md create mode 100644 infra/terraform/envs/api-benchmark/README.md diff --git a/benchmarks/underflow-api/README.md b/benchmarks/underflow-api/README.md new file mode 100644 index 0000000..e0c3db1 --- /dev/null +++ b/benchmarks/underflow-api/README.md @@ -0,0 +1,296 @@ +# Underflow authenticated API benchmark + +This benchmark measures the real bearer-authenticated, PostgreSQL-backed cost-reporting paths against a deterministic synthetic dataset. It deliberately excludes login latency from the measured metric while retaining bearer authentication—and therefore the database-backed user lookup—on every measured request. + +No result in this directory should be treated as measured evidence until all commands complete and a timestamped result directory contains the required JSON files. A failed k6 threshold is evidence and must be preserved as-is. + +## Safety boundary + +The only authorized Terraform root is `infra/terraform/envs/api-benchmark`. It has local, isolated state and does not reference production state or modules. Every named resource begins with `underflow-api-bench-` and AWS provider default tags apply: + +- `project = underflow` +- `environment = api-benchmark` +- `purpose = disposable-load-test` +- `managed = terraform` + +Never run these commands from `infra/terraform/envs/production`. Never use production data or real AWS account integrations. The benchmark creates one VPC, two public subnets, an internet gateway and route table, three narrowly scoped security groups, ECR, one ECS cluster/service/task definition, one ALB/listener/target group, one log group, one non-public single-AZ RDS instance, IAM execution/runtime roles, and one zero-recovery Secrets Manager secret. It creates no NAT gateway, private subnet, DNS, certificate, HTTPS listener, SES, Lambda, EventBridge, worker, autoscaling, Stripe, S3, or CloudFront resource. + +## Prerequisites and run identity + +Commands below assume Bash plus AWS CLI v2, Docker with BuildKit, Terraform, jq, Python 3, and a native k6 installation available in the same shell that runs the benchmark. Docker is used to build the API image, but not to run k6. Keep shell history disabled while handling the generated benchmark password. + +```bash +export REPO_ROOT="$(git rev-parse --show-toplevel)" +export TF_ROOT="$REPO_ROOT/infra/terraform/envs/api-benchmark" +export GIT_SHA="$(git -C "$REPO_ROOT" rev-parse HEAD)" +export SHORT_SHA="$(printf '%s' "$GIT_SHA" | cut -c1-8)" +export BENCHMARK_ID="$SHORT_SHA" +export AWS_REGION="us-east-1" +export LOAD_TEST_IP="$(curl --fail --silent https://checkip.amazonaws.com | tr -d '\r\n')" +export LOAD_TEST_CIDR="$LOAD_TEST_IP/32" # public /32 of this host, where native k6 runs +export RUN_DATE="$(date -u +%F)" +export RESULTS_DIR="$REPO_ROOT/benchmarks/underflow-api/results/${RUN_DATE}-${SHORT_SHA}" +mkdir -p "$RESULTS_DIR" +command -v k6 >/dev/null +k6 version +``` + +Record the clean baseline before editing or deployment: + +```bash +git -C "$REPO_ROOT" status --short +git -C "$REPO_ROOT" rev-parse HEAD +cd "$REPO_ROOT/apps/api" +npm ci +npm run build +npm test +# Only when a disposable PostgreSQL DATABASE_URL is configured: +npm run test:db +``` + +## Configure and validate isolated Terraform + +Create the ignored `terraform.tfvars`; do not commit it: + +```bash +cat > "$TF_ROOT/terraform.tfvars" </dev/null && tfsec "$TF_ROOT" + +terraform -chdir="$TF_ROOT" plan -out=tfplan +terraform -chdir="$TF_ROOT" show -no-color tfplan > "$RESULTS_DIR/terraform-plan.txt.raw" +"$REPO_ROOT/benchmarks/underflow-api/scripts/sanitize-results.sh" \ + "$RESULTS_DIR/terraform-plan.txt.raw" "$RESULTS_DIR/terraform-plan.txt" +rm "$RESULTS_DIR/terraform-plan.txt.raw" +``` + +Before applying, read the plan and confirm: + +```bash +export PREFIX="underflow-api-bench-$BENCHMARK_ID" +terraform -chdir="$TF_ROOT" show -no-color tfplan | less +terraform -chdir="$TF_ROOT" show -json tfplan | jq -r \ + '.resource_changes[] | select(.mode == "managed") | [.type, .name, (.change.actions | join(","))] | @tsv' +``` + +The review must show only this root's benchmark resources, the benchmark prefix on every name-capable AWS resource, the four tags on every tag-capable resource, non-public RDS, ECS port 3080 ingress only from the ALB security group, ALB port 80 ingress only from `load_test_cidr`, and none of the excluded service types. Terraform state must be local to this root. + +## Bootstrap ECR, build, and deploy + +The reviewed configuration is applied first to only its disposable ECR resource so the immutable image exists before ECS starts. This target is in the isolated benchmark root; it does not address any existing Underflow resource. + +```bash +terraform -chdir="$TF_ROOT" apply -target=aws_ecr_repository.api +export ECR_REPOSITORY="$(terraform -chdir="$TF_ROOT" output -raw ecr_repository_url)" +aws ecr get-login-password --region "$AWS_REGION" | \ + docker login --username AWS --password-stdin "${ECR_REPOSITORY%%/*}" +docker buildx build --platform linux/amd64 \ + --tag "$ECR_REPOSITORY:$GIT_SHA" --push "$REPO_ROOT/apps/api" +export IMAGE_DIGEST="$(aws ecr describe-images --region "$AWS_REGION" \ + --repository-name "${ECR_REPOSITORY##*/}" --image-ids imageTag="$GIT_SHA" \ + --query 'imageDetails[0].imageDigest' --output text)" +test -n "$IMAGE_DIGEST" && test "$IMAGE_DIGEST" != "None" + +# Refresh and review the complete plan after the immutable image exists. +terraform -chdir="$TF_ROOT" plan -out=tfplan +terraform -chdir="$TF_ROOT" show -no-color tfplan | less +terraform -chdir="$TF_ROOT" apply tfplan +aws ecs wait services-stable --region "$AWS_REGION" \ + --cluster "$(terraform -chdir="$TF_ROOT" output -raw ecs_cluster_name)" \ + --services "$(terraform -chdir="$TF_ROOT" output -raw ecs_service_name)" +export BASE_URL="$(terraform -chdir="$TF_ROOT" output -raw api_base_url)" +test "$(curl --silent --output /dev/null --write-out '%{http_code}' "$BASE_URL/api/v1/health")" = "200" +``` + +Record only sanitized, non-secret environment metadata. Do not save the full `terraform output -json`, because it includes a sensitive secret ARN. The helper script creates and validates the metadata file: + +```bash +"$REPO_ROOT/benchmarks/underflow-api/scripts/collect-environment.sh" +jq . "$RESULTS_DIR/environment.json" +``` + +## Migrate and seed + +Both commands launch one-off Fargate tasks from the deployed API task definition with identical subnets, security group, image, environment, and Secrets Manager injection. The runner waits for task completion and requires container exit code zero. + +```bash +export BENCHMARK_RESULTS_DIR="$RESULTS_DIR" +"$REPO_ROOT/benchmarks/underflow-api/scripts/run-one-off-task.sh" migrate +"$REPO_ROOT/benchmarks/underflow-api/scripts/run-one-off-task.sh" seed +jq -e '.users == 10 and .workspaces == 10 and .awsAccounts == 200 and .costSnapshots == 3650000' \ + "$RESULTS_DIR/dataset.json" +``` + +The seeder requires `ALLOW_BENCHMARK_SEED=true`, uses only reserved UUIDs and `example.invalid` identities, bulk-loads with PostgreSQL set operations, replaces only its fixed dataset, verifies exact counts, and runs `ANALYZE`. The generated password is never logged. The first identity is `benchmark+01@example.invalid`, and its workspace is `20000000-0000-4000-8000-000000000001`. + +## Preflight and k6 execution + +Run native k6 from this host, outside ECS and outside Docker. The ALB permits only `load_test_cidr`, so it must be this host's current public `/32`; if the address changes, update `terraform.tfvars`, review a new plan, and apply it before continuing. Fetch the generated synthetic password into an environment variable without echoing it, then disable shell history and verify one healthy task, the seed counts, and health before testing. + +```bash +set +o history +export TEST_EMAIL="benchmark+01@example.invalid" +export WORKSPACE_ID="20000000-0000-4000-8000-000000000001" +export TEST_PASSWORD="$(aws secretsmanager get-secret-value --region "$AWS_REGION" \ + --secret-id "$PREFIX-runtime" --query SecretString --output text | jq -r '.BENCHMARK_PASSWORD')" +test -n "$TEST_PASSWORD" + +aws ecs describe-services --region "$AWS_REGION" \ + --cluster "$(terraform -chdir="$TF_ROOT" output -raw ecs_cluster_name)" \ + --services "$(terraform -chdir="$TF_ROOT" output -raw ecs_service_name)" \ + --query 'services[0].{desired:desiredCount,running:runningCount,pending:pendingCount}' +curl --fail --silent "$BASE_URL/api/v1/health" + +export RESULTS_DIR +k6 run "$REPO_ROOT/benchmarks/underflow-api/k6/smoke.js" + +# Unmeasured 60-second warm-up; preserve separately from smoke and normal load. +SMOKE_DURATION=60s SUMMARY_NAME=warmup-summary.json \ + k6 run "$REPO_ROOT/benchmarks/underflow-api/k6/smoke.js" +sleep 30 + +export LOAD_START="$(date -u +%FT%TZ)" +k6 run "$REPO_ROOT/benchmarks/underflow-api/k6/load.js" +export LOAD_END="$(date -u +%FT%TZ)" +``` + +The scripts use deterministic iteration buckets for the 30/30/25/10/5 distribution, validate status, JSON, response shape, and authorization/server errors, and save compact summaries through `handleSummary`. Overall measured thresholds are failure rate `<1%`, p95 `<200 ms`, p99 `<500 ms`, and checks `>99%`; per-endpoint p95/p99 values are retained. Login is tagged as setup and excluded from `measured_*` metrics. + +Run stress only when the normal profile completes, thresholds are reviewed, the service remains healthy, and the load generator is not saturated: + +```bash +export STRESS_START="$(date -u +%FT%TZ)" +k6 run "$REPO_ROOT/benchmarks/underflow-api/k6/stress.js" +export STRESS_END="$(date -u +%FT%TZ)" +``` + +Stress aborts after a sustained measured failure rate above 5%. Treat its breaking point as configuration-specific, never as a fabricated supported-user claim. Record load-generator CPU/network separately if it approaches saturation. + +## Collect CloudWatch evidence and generate results + +Collect metrics immediately after each measured interval (use distinct output names if collecting smoke and stress separately). The required final `cloudwatch-summary.json` should cover normal load. + +```bash +AWS_REGION="$AWS_REGION" \ +ECS_CLUSTER="$(terraform -chdir="$TF_ROOT" output -raw ecs_cluster_name)" \ +ECS_SERVICE="$(terraform -chdir="$TF_ROOT" output -raw ecs_service_name)" \ +RDS_IDENTIFIER="$(terraform -chdir="$TF_ROOT" output -raw rds_identifier)" \ +ALB_ARN_SUFFIX="$(terraform -chdir="$TF_ROOT" output -raw load_balancer_arn_suffix)" \ +TARGET_GROUP_ARN_SUFFIX="$(terraform -chdir="$TF_ROOT" output -raw target_group_arn_suffix)" \ +START_TIME="$LOAD_START" END_TIME="$LOAD_END" \ +OUTPUT_FILE="$RESULTS_DIR/cloudwatch-summary.json" \ + "$REPO_ROOT/benchmarks/underflow-api/scripts/collect-aws-metadata.sh" + +"$REPO_ROOT/benchmarks/underflow-api/scripts/generate-results.sh" \ + "$RESULTS_DIR" "$RESULTS_DIR/RESULTS.md" +``` + +The generator refuses missing evidence and mechanically derives requests/second, p95, concurrency, dataset size, and the résumé bullet from preserved JSON. Review `RESULTS.md` for supported bottleneck observations; do not replace failed thresholds or missing stress evidence with estimates. + +Required timestamped evidence is: + +```text +environment.json +dataset.json +smoke-summary.json +load-summary.json +stress-summary.json # only if safely performed +cloudwatch-summary.json +RESULTS.md +``` + +## Repository validation and secret scan + +Before teardown, verify the repository and evidence while the environment is still available for diagnosis: + +```bash +cd "$REPO_ROOT/apps/api" +npm run build +npm test +terraform -chdir="$TF_ROOT" fmt -check -recursive +terraform -chdir="$TF_ROOT" validate + +node --check "$REPO_ROOT/benchmarks/underflow-api/k6/common.js" +node --check "$REPO_ROOT/benchmarks/underflow-api/k6/smoke.js" +node --check "$REPO_ROOT/benchmarks/underflow-api/k6/load.js" +node --check "$REPO_ROOT/benchmarks/underflow-api/k6/stress.js" +bash -n "$REPO_ROOT/benchmarks/underflow-api/scripts/"*.sh + +git -C "$REPO_ROOT" diff --check +git -C "$REPO_ROOT" status --short +git -C "$REPO_ROOT" diff --stat +git -C "$REPO_ROOT" diff -- . ':!infra/terraform/envs/production' + +# These scans must print nothing. Review any match before proceeding. +git -C "$REPO_ROOT" ls-files --others --exclude-standard --cached | \ + grep -E '(\.tfstate|\.tfvars$|\.tfplan$)' || true +rg -n --hidden --glob '!**/.terraform/**' --glob '!**/node_modules/**' \ + '(AKIA[0-9A-Z]{16}|ASIA[0-9A-Z]{16}|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.|[0-9]{12}\.dkr\.ecr\.|\.rds\.amazonaws\.com)' \ + "$REPO_ROOT/infra/terraform/envs/api-benchmark" "$REPO_ROOT/benchmarks/underflow-api" || true +``` + +The synthetic 12-digit account-number generator in the seeder is expected source code; no real account number may appear in code or evidence. + +## Mandatory Terraform-only destruction + +Do not begin until the timestamped evidence exists outside `.terraform`, is sanitized, and `RESULTS.md` has been generated. Destroy only from the exact isolated root. + +```bash +test "$(cd "$TF_ROOT" && pwd)" = "$TF_ROOT" +for file in environment.json dataset.json smoke-summary.json load-summary.json cloudwatch-summary.json RESULTS.md; do + test -s "$RESULTS_DIR/$file" +done + +terraform -chdir="$TF_ROOT" output -json | \ + jq '{aws_region,resource_prefix,ecs_cluster_name,ecs_service_name,rds_identifier,availability_zones}' \ + > "$RESULTS_DIR/final-terraform-outputs.json" +terraform -chdir="$TF_ROOT" state list | tee "$RESULTS_DIR/terraform-state-list.txt" +terraform -chdir="$TF_ROOT" state list | grep -E 'envs.production|platform_stack' && exit 1 || true + +# Stop only active one-off tasks in this isolated benchmark cluster. Service +# tasks have a `service:` group and are left for Terraform to remove. +export CLUSTER="$(terraform -chdir="$TF_ROOT" output -raw ecs_cluster_name)" +export TASK_ARNS="$(aws ecs list-tasks --region "$AWS_REGION" --cluster "$CLUSTER" \ + --desired-status RUNNING --family "$PREFIX-api" --query 'taskArns[]' --output text)" +if [[ -n "$TASK_ARNS" && "$TASK_ARNS" != "None" ]]; then + for task in $(aws ecs describe-tasks --region "$AWS_REGION" --cluster "$CLUSTER" \ + --tasks $TASK_ARNS --output json | \ + jq -r '.tasks[] | select(.group | startswith("family:")) | .taskArn'); do + aws ecs stop-task --region "$AWS_REGION" --cluster "$CLUSTER" --task "$task" \ + --reason 'benchmark teardown' >/dev/null + done +fi + +terraform -chdir="$TF_ROOT" plan -destroy -out=destroy.tfplan +terraform -chdir="$TF_ROOT" show -no-color destroy.tfplan | less +terraform -chdir="$TF_ROOT" apply destroy.tfplan +test -z "$(terraform -chdir="$TF_ROOT" state list)" +``` + +Finally verify by benchmark prefix that ECS, ELBv2, RDS, ECR, Secrets Manager, CloudWatch Logs, and EC2/VPC resources are absent. Queries must use `$PREFIX`; never delete unrelated matches manually. + +```bash +aws ecs list-clusters --region "$AWS_REGION" --query "clusterArns[?contains(@, '$PREFIX')]" +aws elbv2 describe-load-balancers --region "$AWS_REGION" --query "LoadBalancers[?contains(LoadBalancerName, '$PREFIX')]" +aws rds describe-db-instances --region "$AWS_REGION" --query "DBInstances[?contains(DBInstanceIdentifier, '$PREFIX')]" +aws ecr describe-repositories --region "$AWS_REGION" --query "repositories[?contains(repositoryName, '$PREFIX')]" +aws secretsmanager list-secrets --region "$AWS_REGION" --include-planned-deletion \ + --query "SecretList[?contains(Name, '$PREFIX')]" +aws logs describe-log-groups --region "$AWS_REGION" --log-group-name-prefix "/ecs/$PREFIX" +aws ec2 describe-vpcs --region "$AWS_REGION" \ + --filters "Name=tag:Name,Values=$PREFIX*" --query 'Vpcs[].VpcId' +aws ecs list-tasks --region "$AWS_REGION" --cluster "$PREFIX-cluster" --desired-status RUNNING 2>/dev/null || true +``` + +All result arrays must be empty. Remove ignored `terraform.tfvars`, plan files, local state, and any temporary raw logs only after successful destruction; retain sanitized timestamped results and repository changes. Do not commit or push without explicit authorization. diff --git a/benchmarks/underflow-api/RESULTS_TEMPLATE.md b/benchmarks/underflow-api/RESULTS_TEMPLATE.md new file mode 100644 index 0000000..3ab1c75 --- /dev/null +++ b/benchmarks/underflow-api/RESULTS_TEMPLATE.md @@ -0,0 +1,55 @@ +# Underflow API benchmark results + +> This template intentionally contains no performance claims. Generate the completed file from captured summaries with `scripts/generate-results.sh`. + +## Objective + +Measure authenticated, PostgreSQL-backed cost-reporting reads on the documented disposable AWS configuration. + +## Tested commit + +See `environment.json`. + +## Infrastructure + +See `environment.json`. Results apply only to the documented one-task ECS and single-AZ RDS configuration. + +## Dataset + +See `dataset.json` for exact verified counts. + +## Workload + +30% summary, 30% timeseries, 25% by service, 10% AWS account list, and 5% sync history. Authentication occurs before measurement; every measured request still uses bearer authentication and performs the application's PostgreSQL user lookup. + +## Thresholds + +Measured failure rate `< 1%`, measured request latency p95 `< 200 ms`, p99 `< 500 ms`, and checks pass rate `> 99%`. + +## Results + +Generated from `smoke-summary.json`, `load-summary.json`, and, when safely run, `stress-summary.json`. + +## Per-endpoint results + +Generated from named endpoint trend metrics. + +## AWS resource metrics + +See `cloudwatch-summary.json`. + +## Observed bottlenecks + +Record only observations supported by k6 and CloudWatch evidence. + +## Limitations + +Synthetic data, one region, one Fargate task, one load-generator location, HTTP restricted by CIDR, and a single-AZ burstable RDS instance. + +## Reproduction commands + +See [`README.md`](README.md). + +## Evidence-backed résumé bullet + +Generated mechanically only after a completed normal-load run. diff --git a/infra/terraform/envs/api-benchmark/README.md b/infra/terraform/envs/api-benchmark/README.md new file mode 100644 index 0000000..c11830b --- /dev/null +++ b/infra/terraform/envs/api-benchmark/README.md @@ -0,0 +1,9 @@ +# Isolated API benchmark infrastructure + +This Terraform root owns only the disposable `underflow-api-bench-` environment. It uses local state in this directory and has no backend, module, data source, or state reference to the production stack. Never run these commands from `envs/production`. + +Create an ignored `terraform.tfvars` from the example. `load_test_cidr` is required and rejects `0.0.0.0/0`; use the load generator's public `/32`. `image_tag` must be the Git SHA used to build the API image. + +Before any apply, run the validation and full-plan checks documented in [`benchmarks/underflow-api/README.md`](../../../../benchmarks/underflow-api/README.md). The first apply targets only this root's ECR repository so an immutable commit-tagged image can be pushed. After the push, run and review a fresh full plan before the full apply. + +All secret values are generated, stored in local Terraform state, and injected from one disposable Secrets Manager secret. State and `.tfvars` are ignored and must never be committed. The secret has zero-day recovery, ECR uses `force_delete`, and RDS has no backups, final snapshot, public access, Multi-AZ, or deletion protection. From b23a4a5f290342b2d47225fa5bf5295ed33fa2b3 Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Wed, 9 Sep 2026 22:15:10 +0100 Subject: [PATCH 10/19] feat(benchmark): add postgres query-plan diagnostics --- benchmarks/underflow-api/README.md | 9 ++++++ .../underflow-api/scripts/run-one-off-task.sh | 28 +++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/benchmarks/underflow-api/README.md b/benchmarks/underflow-api/README.md index e0c3db1..7a217e2 100644 --- a/benchmarks/underflow-api/README.md +++ b/benchmarks/underflow-api/README.md @@ -134,6 +134,15 @@ jq -e '.users == 10 and .workspaces == 10 and .awsAccounts == 200 and .costSnaps The seeder requires `ALLOW_BENCHMARK_SEED=true`, uses only reserved UUIDs and `example.invalid` identities, bulk-loads with PostgreSQL set operations, replaces only its fixed dataset, verifies exact counts, and runs `ANALYZE`. The generated password is never logged. The first identity is `benchmark+01@example.invalid`, and its workspace is `20000000-0000-4000-8000-000000000001`. +To capture a baseline PostgreSQL execution plan for the full-year cost-summary query without exposing RDS publicly, run the diagnostic one-off task and preserve its sanitized output: + +```bash +export BENCHMARK_RESULTS_DIR="$RESULTS_DIR" +"$REPO_ROOT/benchmarks/underflow-api/scripts/run-one-off-task.sh" explain +``` + +The resulting `explain-task.log` contains `EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT JSON)` plus sanitized `cost_snapshots` table/index size and scan counters. Capture this baseline before adding an index or changing the query. + ## Preflight and k6 execution Run native k6 from this host, outside ECS and outside Docker. The ALB permits only `load_test_cidr`, so it must be this host's current public `/32`; if the address changes, update `terraform.tfvars`, review a new plan, and apply it before continuing. Fetch the generated synthetic password into an environment variable without echoing it, then disable shell history and verify one healthy task, the seed counts, and health before testing. diff --git a/benchmarks/underflow-api/scripts/run-one-off-task.sh b/benchmarks/underflow-api/scripts/run-one-off-task.sh index 0ed79cb..8de52d6 100644 --- a/benchmarks/underflow-api/scripts/run-one-off-task.sh +++ b/benchmarks/underflow-api/scripts/run-one-off-task.sh @@ -2,8 +2,8 @@ set -euo pipefail MODE="${1:-}" -if [[ "$MODE" != "migrate" && "$MODE" != "seed" ]]; then - echo "usage: $0 migrate|seed" >&2 +if [[ "$MODE" != "migrate" && "$MODE" != "seed" && "$MODE" != "explain" ]]; then + echo "usage: $0 migrate|seed|explain" >&2 exit 2 fi @@ -30,9 +30,31 @@ SUBNET_LIST="$(IFS=,; echo "${SUBNETS[*]}")" if [[ "$MODE" == "migrate" ]]; then COMMAND_JSON='["node","dist/db/migrate.js"]' ENVIRONMENT_JSON='[]' -else +elif [[ "$MODE" == "seed" ]]; then COMMAND_JSON='["node","dist/scripts/seed-benchmark.js"]' ENVIRONMENT_JSON='[{"name":"ALLOW_BENCHMARK_SEED","value":"true"}]' +else + EXPLAIN_SCRIPT='import { pool } from "./dist/config/db.js"; +const workspaceId = "20000000-0000-4000-8000-000000000001"; +const explain = await pool.query(`EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT JSON) + SELECT COALESCE(SUM(amount), 0) AS total_amount, + COALESCE(MAX(currency), '\''USD'\'') AS currency + FROM cost_snapshots + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3 + AND ($4::uuid IS NULL OR aws_account_id = $4::uuid)`, + [workspaceId, "2025-01-01", "2025-12-31", null]); +const stats = await pool.query(`SELECT + pg_size_pretty(pg_total_relation_size('\''cost_snapshots'\'')) AS total_size, + pg_size_pretty(pg_relation_size('\''cost_snapshots'\'')) AS table_size, + pg_size_pretty(pg_indexes_size('\''cost_snapshots'\'')) AS indexes_size, + n_live_tup, seq_scan, idx_scan + FROM pg_stat_user_tables + WHERE relname = '\''cost_snapshots'\''`); +console.log(JSON.stringify({ benchmarkDiagnostic: true, stats: stats.rows[0], plan: explain.rows[0]["QUERY PLAN"] }, null, 2)); +await pool.end();' + COMMAND_JSON="$(jq -cn --arg script "$EXPLAIN_SCRIPT" '["node","--input-type=module","--eval",$script]')" + ENVIRONMENT_JSON='[]' fi OVERRIDES="$(jq -cn \ From 30d5c182fe243c4407b4b253fdc7401269e43f62 Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Wed, 9 Sep 2026 22:15:19 +0100 Subject: [PATCH 11/19] perf(api): index workspace cost queries by date --- .../003_add_cost_snapshots_workspace_date_index.sql | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 apps/api/src/db/migrations/003_add_cost_snapshots_workspace_date_index.sql diff --git a/apps/api/src/db/migrations/003_add_cost_snapshots_workspace_date_index.sql b/apps/api/src/db/migrations/003_add_cost_snapshots_workspace_date_index.sql new file mode 100644 index 0000000..9e09324 --- /dev/null +++ b/apps/api/src/db/migrations/003_add_cost_snapshots_workspace_date_index.sql @@ -0,0 +1,8 @@ +-- The benchmark summary and reporting queries filter by workspace and date +-- without always filtering by AWS account. The original unique index orders +-- aws_account_id before usage_date, so it cannot efficiently serve that path. +CREATE INDEX IF NOT EXISTS idx_cost_snapshots_workspace_date + ON cost_snapshots (workspace_id, usage_date) + INCLUDE (amount, currency); + +ANALYZE cost_snapshots; From 4dae7ea1e9a4be5cd5a1d45f33e42836422d8bb2 Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Thu, 10 Sep 2026 11:30:11 +0100 Subject: [PATCH 12/19] fix(benchmark): distribute smoke traffic and expand diagnostics --- benchmarks/underflow-api/README.md | 2 +- benchmarks/underflow-api/k6/common.js | 5 ++- .../underflow-api/scripts/run-one-off-task.sh | 35 ++++++++++++++----- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/benchmarks/underflow-api/README.md b/benchmarks/underflow-api/README.md index 7a217e2..2817cb7 100644 --- a/benchmarks/underflow-api/README.md +++ b/benchmarks/underflow-api/README.md @@ -141,7 +141,7 @@ export BENCHMARK_RESULTS_DIR="$RESULTS_DIR" "$REPO_ROOT/benchmarks/underflow-api/scripts/run-one-off-task.sh" explain ``` -The resulting `explain-task.log` contains `EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT JSON)` plus sanitized `cost_snapshots` table/index size and scan counters. Capture this baseline before adding an index or changing the query. +The resulting `explain-task.log` contains `EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT JSON)` for summary, timeseries, and by-service queries, plus sanitized `cost_snapshots` table/index size and scan counters. Capture this baseline before adding an index or changing a query. ## Preflight and k6 execution diff --git a/benchmarks/underflow-api/k6/common.js b/benchmarks/underflow-api/k6/common.js index ffa9383..4404e78 100644 --- a/benchmarks/underflow-api/k6/common.js +++ b/benchmarks/underflow-api/k6/common.js @@ -105,7 +105,10 @@ const requestDefinitions = [ ]; const chooseRequest = () => { - const bucket = exec.scenario.iterationInTest % 100; + // Multiplication by a value coprime to 100 deterministically spreads short + // runs across the complete weighted distribution instead of exhausting each + // endpoint's contiguous bucket in order. + const bucket = (exec.scenario.iterationInTest * 37) % 100; return requestDefinitions.find((definition) => bucket < definition.upperBound); }; diff --git a/benchmarks/underflow-api/scripts/run-one-off-task.sh b/benchmarks/underflow-api/scripts/run-one-off-task.sh index 8de52d6..8ff978c 100644 --- a/benchmarks/underflow-api/scripts/run-one-off-task.sh +++ b/benchmarks/underflow-api/scripts/run-one-off-task.sh @@ -36,14 +36,31 @@ elif [[ "$MODE" == "seed" ]]; then else EXPLAIN_SCRIPT='import { pool } from "./dist/config/db.js"; const workspaceId = "20000000-0000-4000-8000-000000000001"; -const explain = await pool.query(`EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT JSON) - SELECT COALESCE(SUM(amount), 0) AS total_amount, - COALESCE(MAX(currency), '\''USD'\'') AS currency - FROM cost_snapshots - WHERE workspace_id = $1 - AND usage_date BETWEEN $2 AND $3 - AND ($4::uuid IS NULL OR aws_account_id = $4::uuid)`, - [workspaceId, "2025-01-01", "2025-12-31", null]); +const parameters = [workspaceId, "2025-01-01", "2025-12-31", null]; +const queries = { + summary: `SELECT COALESCE(SUM(amount), 0) AS total_amount, + COALESCE(MAX(currency), '\''USD'\'') AS currency + FROM cost_snapshots + WHERE workspace_id = $1 AND usage_date BETWEEN $2 AND $3 + AND ($4::uuid IS NULL OR aws_account_id = $4::uuid)`, + timeseries: `SELECT usage_date, SUM(amount) AS total_amount, + COALESCE(MAX(currency), '\''USD'\'') AS currency + FROM cost_snapshots + WHERE workspace_id = $1 AND usage_date BETWEEN $2 AND $3 + AND ($4::uuid IS NULL OR aws_account_id = $4::uuid) + GROUP BY usage_date ORDER BY usage_date ASC`, + byService: `SELECT service_name, SUM(amount) AS total_amount, + COALESCE(MAX(currency), '\''USD'\'') AS currency + FROM cost_snapshots + WHERE workspace_id = $1 AND usage_date BETWEEN $2 AND $3 + AND ($4::uuid IS NULL OR aws_account_id = $4::uuid) + GROUP BY service_name ORDER BY total_amount DESC`, +}; +const plans = {}; +for (const [name, sql] of Object.entries(queries)) { + const result = await pool.query(`EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT JSON) ${sql}`, parameters); + plans[name] = result.rows[0]["QUERY PLAN"]; +} const stats = await pool.query(`SELECT pg_size_pretty(pg_total_relation_size('\''cost_snapshots'\'')) AS total_size, pg_size_pretty(pg_relation_size('\''cost_snapshots'\'')) AS table_size, @@ -51,7 +68,7 @@ const stats = await pool.query(`SELECT n_live_tup, seq_scan, idx_scan FROM pg_stat_user_tables WHERE relname = '\''cost_snapshots'\''`); -console.log(JSON.stringify({ benchmarkDiagnostic: true, stats: stats.rows[0], plan: explain.rows[0]["QUERY PLAN"] }, null, 2)); +console.log(JSON.stringify({ benchmarkDiagnostic: true, stats: stats.rows[0], plans }, null, 2)); await pool.end();' COMMAND_JSON="$(jq -cn --arg script "$EXPLAIN_SCRIPT" '["node","--input-type=module","--eval",$script]')" ENVIRONMENT_JSON='[]' From 57fde35be6bc7a6c697ab264a31b933fb313e816 Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Thu, 10 Sep 2026 11:30:29 +0100 Subject: [PATCH 13/19] perf(api): cover cost reporting queries --- .../migrations/004_cover_cost_reporting_queries.sql | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 apps/api/src/db/migrations/004_cover_cost_reporting_queries.sql diff --git a/apps/api/src/db/migrations/004_cover_cost_reporting_queries.sql b/apps/api/src/db/migrations/004_cover_cost_reporting_queries.sql new file mode 100644 index 0000000..aec7289 --- /dev/null +++ b/apps/api/src/db/migrations/004_cover_cost_reporting_queries.sql @@ -0,0 +1,12 @@ +-- The by-service report needs service_name in addition to the values covered +-- by the workspace/date index. Including it allows all three cost-reporting +-- queries to avoid fetching the matching rows from the heap. +CREATE INDEX IF NOT EXISTS idx_cost_snapshots_workspace_date_reporting + ON cost_snapshots (workspace_id, usage_date) + INCLUDE (service_name, amount, currency); + +-- The reporting index is a strict covering replacement for the narrower index +-- introduced by migration 003. +DROP INDEX IF EXISTS idx_cost_snapshots_workspace_date; + +ANALYZE cost_snapshots; From 0047e782b94198b102b1af979041e77c7f161d08 Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Thu, 10 Sep 2026 16:23:15 +0100 Subject: [PATCH 14/19] feat(benchmark): add server-side capacity discovery --- benchmarks/underflow-api/README.md | 40 ++++++++- benchmarks/underflow-api/k6/capacity.js | 29 ++++++ .../scripts/collect-aws-metadata.sh | 17 +++- .../scripts/collect-environment.sh | 1 + .../scripts/update-load-test-ip.sh | 88 +++++++++++++++++++ 5 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 benchmarks/underflow-api/k6/capacity.js create mode 100644 benchmarks/underflow-api/scripts/update-load-test-ip.sh diff --git a/benchmarks/underflow-api/README.md b/benchmarks/underflow-api/README.md index 2817cb7..7d827d4 100644 --- a/benchmarks/underflow-api/README.md +++ b/benchmarks/underflow-api/README.md @@ -86,6 +86,12 @@ terraform -chdir="$TF_ROOT" show -json tfplan | jq -r \ The review must show only this root's benchmark resources, the benchmark prefix on every name-capable AWS resource, the four tags on every tag-capable resource, non-public RDS, ECS port 3080 ingress only from the ALB security group, ALB port 80 ingress only from `load_test_cidr`, and none of the excluded service types. Terraform state must be local to this root. +If the native load generator's public IP changes after deployment, update only the ALB ingress rule and verify health with the guarded helper. It refuses to apply a plan that changes any managed resource other than `aws_security_group.alb`: + +```bash +bash "$REPO_ROOT/benchmarks/underflow-api/scripts/update-load-test-ip.sh" +``` + ## Bootstrap ECR, build, and deploy The reviewed configuration is applied first to only its disposable ECR resource so the immutable image exists before ECS starts. This target is in the isolated benchmark root; it does not address any existing Underflow resource. @@ -174,7 +180,37 @@ k6 run "$REPO_ROOT/benchmarks/underflow-api/k6/load.js" export LOAD_END="$(date -u +%FT%TZ)" ``` -The scripts use deterministic iteration buckets for the 30/30/25/10/5 distribution, validate status, JSON, response shape, and authorization/server errors, and save compact summaries through `handleSummary`. Overall measured thresholds are failure rate `<1%`, p95 `<200 ms`, p99 `<500 ms`, and checks `>99%`; per-endpoint p95/p99 values are retained. Login is tagged as setup and excluded from `measured_*` metrics. +The scripts use deterministic iteration buckets for the 30/30/25/10/5 distribution, validate status, JSON, response shape, and authorization/server errors, and save compact summaries through `handleSummary`. k6 retains the client-observed failure rate `<1%`, p95 `<200 ms`, p99 `<500 ms`, and checks `>99%` thresholds; per-endpoint values are retained. Because a developer-machine run includes location-dependent Internet transit, the primary backend latency reported by this benchmark is the ALB `TargetResponseTime` p50/p90/p95/p99 collected from CloudWatch. Client-observed k6 latency remains evidence and must be labeled with the load-generator location rather than relabeled or discarded. Login is tagged as setup and excluded from `measured_*` metrics. + +### Capacity discovery on the baseline database class + +When the normal profile saturates the documented `db.t4g.micro`, preserve that failed run and determine the configuration's actual capacity with independent constant-load levels. Test 2, 3, 4, and 5 VUs, stopping once a level fails or once the first failing level above the highest passing level is established. A passing level has no functional failures and every one-minute ALB `TargetResponseTime` p95 below 200 ms and p99 below 500 ms. Refresh the ALB ingress CIDR before each level; never update it during a measured interval. + +For each level, run the following block after setting `CAPACITY_VUS` to 2, 3, 4, or 5: + +```bash +bash "$REPO_ROOT/benchmarks/underflow-api/scripts/update-load-test-ip.sh" +export CAPACITY_VUS=5 +export CAPACITY_START="$(date -u +%FT%TZ)" +CAPACITY_VUS="$CAPACITY_VUS" CAPACITY_DURATION=3m \ + k6 run "$REPO_ROOT/benchmarks/underflow-api/k6/capacity.js" +export CAPACITY_END="$(date -u +%FT%TZ)" + +# Allow one-minute CloudWatch statistics to become available. +sleep 120 + +AWS_REGION="$AWS_REGION" \ +ECS_CLUSTER="$(terraform -chdir="$TF_ROOT" output -raw ecs_cluster_name | tr -d '\r')" \ +ECS_SERVICE="$(terraform -chdir="$TF_ROOT" output -raw ecs_service_name | tr -d '\r')" \ +RDS_IDENTIFIER="$(terraform -chdir="$TF_ROOT" output -raw rds_identifier | tr -d '\r')" \ +ALB_ARN_SUFFIX="$(terraform -chdir="$TF_ROOT" output -raw load_balancer_arn_suffix | tr -d '\r')" \ +TARGET_GROUP_ARN_SUFFIX="$(terraform -chdir="$TF_ROOT" output -raw target_group_arn_suffix | tr -d '\r')" \ +START_TIME="$CAPACITY_START" END_TIME="$CAPACITY_END" \ +OUTPUT_FILE="$RESULTS_DIR/capacity-${CAPACITY_VUS}vus-cloudwatch.json" \ + "$REPO_ROOT/benchmarks/underflow-api/scripts/collect-aws-metadata.sh" +``` + +The k6 summary is saved as `capacity-vus-summary.json`. A run aborted for network failures is invalid, must be preserved as such, and cannot establish capacity. The supported level is the highest fully valid level whose server-side threshold passes; do not interpolate or claim the next failing level. Run stress only when the normal profile completes, thresholds are reviewed, the service remains healthy, and the load generator is not saturated: @@ -188,7 +224,7 @@ Stress aborts after a sustained measured failure rate above 5%. Treat its breaki ## Collect CloudWatch evidence and generate results -Collect metrics immediately after each measured interval (use distinct output names if collecting smoke and stress separately). The required final `cloudwatch-summary.json` should cover normal load. +Collect metrics immediately after each measured interval (use distinct output names if collecting smoke and stress separately). The required final `cloudwatch-summary.json` should cover normal load. ALB target-response datapoints contain server-side p50, p90, p95, and p99 extended statistics for each 60-second period; do not describe these as end-to-end client latency. ```bash AWS_REGION="$AWS_REGION" \ diff --git a/benchmarks/underflow-api/k6/capacity.js b/benchmarks/underflow-api/k6/capacity.js new file mode 100644 index 0000000..4a8695a --- /dev/null +++ b/benchmarks/underflow-api/k6/capacity.js @@ -0,0 +1,29 @@ +import { authenticate, runMeasuredRequest, standardThresholds, writeSummary } from "./common.js"; + +const capacityVus = Number(__ENV.CAPACITY_VUS ?? "5"); +if (!Number.isInteger(capacityVus) || capacityVus < 1 || capacityVus > 100) { + throw new Error("CAPACITY_VUS must be an integer from 1 through 100"); +} + +const capacityDuration = __ENV.CAPACITY_DURATION ?? "3m"; + +export const options = { + scenarios: { + capacity: { + executor: "constant-vus", + vus: capacityVus, + duration: capacityDuration, + gracefulStop: "30s", + }, + }, + // Abort sustained functional failures so a changed public IP does not turn + // the remainder of a capacity run into misleading fast failures. + thresholds: standardThresholds("rate<0.01", true), +}; + +export const setup = authenticate; + +export default runMeasuredRequest; + +export const handleSummary = (data) => + writeSummary(data, `capacity-${capacityVus}vus-summary.json`); diff --git a/benchmarks/underflow-api/scripts/collect-aws-metadata.sh b/benchmarks/underflow-api/scripts/collect-aws-metadata.sh index e763eff..75fce7e 100644 --- a/benchmarks/underflow-api/scripts/collect-aws-metadata.sh +++ b/benchmarks/underflow-api/scripts/collect-aws-metadata.sh @@ -36,13 +36,28 @@ collect_metric() { --output json > "$TMP_DIR/$key.json" } +collect_percentile_metric() { + local key="$1" namespace="$2" metric="$3" + shift 3 + aws cloudwatch get-metric-statistics \ + --region "$AWS_REGION" \ + --namespace "$namespace" \ + --metric-name "$metric" \ + --dimensions "$@" \ + --start-time "$START_TIME" \ + --end-time "$END_TIME" \ + --period 60 \ + --extended-statistics p50 p90 p95 p99 \ + --output json > "$TMP_DIR/$key.json" +} + collect_metric ecs_cpu AWS/ECS CPUUtilization Average \ Name=ClusterName,Value="$ECS_CLUSTER" Name=ServiceName,Value="$ECS_SERVICE" collect_metric ecs_memory AWS/ECS MemoryUtilization Average \ Name=ClusterName,Value="$ECS_CLUSTER" Name=ServiceName,Value="$ECS_SERVICE" collect_metric ecs_running_tasks ECS/ContainerInsights RunningTaskCount Average \ Name=ClusterName,Value="$ECS_CLUSTER" Name=ServiceName,Value="$ECS_SERVICE" -collect_metric alb_target_response AWS/ApplicationELB TargetResponseTime Average \ +collect_percentile_metric alb_target_response AWS/ApplicationELB TargetResponseTime \ Name=LoadBalancer,Value="$ALB_ARN_SUFFIX" Name=TargetGroup,Value="$TARGET_GROUP_ARN_SUFFIX" collect_metric alb_4xx AWS/ApplicationELB HTTPCode_ELB_4XX_Count Sum \ Name=LoadBalancer,Value="$ALB_ARN_SUFFIX" diff --git a/benchmarks/underflow-api/scripts/collect-environment.sh b/benchmarks/underflow-api/scripts/collect-environment.sh index e7e5e70..7c89eca 100644 --- a/benchmarks/underflow-api/scripts/collect-environment.sh +++ b/benchmarks/underflow-api/scripts/collect-environment.sh @@ -52,6 +52,7 @@ jq -n \ dataset: "underflow-api-benchmark-v1", profiles: { smoke: {vus: 2, duration: "30s"}, + capacityDiscovery: {levels: [2,3,4,5], durationPerLevel: "3m"}, normalLoad: {stages: [["1m",25],["3m",25],["1m",50],["5m",50],["2m",100],["5m",100],["1m",0]]}, stress: {stages: [["2m",100],["3m",100],["1m",150],["3m",150],["1m",200],["3m",200],["1m",0]]} } diff --git a/benchmarks/underflow-api/scripts/update-load-test-ip.sh b/benchmarks/underflow-api/scripts/update-load-test-ip.sh new file mode 100644 index 0000000..255939d --- /dev/null +++ b/benchmarks/underflow-api/scripts/update-load-test-ip.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +set -euo pipefail + +for command in curl jq sed terraform; do + command -v "$command" >/dev/null 2>&1 || { + echo "$command is required" >&2 + exit 2 + } +done + +REPO_ROOT="${REPO_ROOT:-$(git rev-parse --show-toplevel)}" +TF_ROOT="${TF_ROOT:-$REPO_ROOT/infra/terraform/envs/api-benchmark}" +TFVARS="$TF_ROOT/terraform.tfvars" +PLAN_NAME="ip-update.tfplan" + +[[ -f "$TFVARS" ]] || { + echo "Missing $TFVARS" >&2 + exit 2 +} + +PUBLIC_IP="$(curl --noproxy '*' --fail --silent --show-error \ + https://checkip.amazonaws.com | tr -d '\r\n')" + +if [[ ! "$PUBLIC_IP" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then + echo "Public IP lookup returned an invalid IPv4 address: $PUBLIC_IP" >&2 + exit 1 +fi + +IFS=. read -r -a octets <<< "$PUBLIC_IP" +for octet in "${octets[@]}"; do + if ((10#$octet > 255)); then + echo "Public IP lookup returned an invalid IPv4 address: $PUBLIC_IP" >&2 + exit 1 + fi +done + +CURRENT_CIDR="$(sed -nE 's/^[[:space:]]*load_test_cidr[[:space:]]*=[[:space:]]*"([^"]+)".*/\1/p' "$TFVARS")" +NEW_CIDR="$PUBLIC_IP/32" + +if [[ "$CURRENT_CIDR" == "$NEW_CIDR" ]]; then + echo "ALB already allows the current public IP: $NEW_CIDR" +else + cp "$TFVARS" "$TFVARS.bak" + sed -i \ + "s|^[[:space:]]*load_test_cidr[[:space:]]*=.*$|load_test_cidr = \"$NEW_CIDR\"|" \ + "$TFVARS" + echo "Updated load_test_cidr: ${CURRENT_CIDR:-} -> $NEW_CIDR" +fi + +terraform -chdir="$TF_ROOT" plan -out="$PLAN_NAME" + +PLAN_JSON="$(terraform -chdir="$TF_ROOT" show -json "$PLAN_NAME")" +if ! jq -e ' + [.resource_changes[]? + | select(.mode == "managed" and .change.actions != ["no-op"])] as $changes + | ($changes | length) <= 1 + and all($changes[]; + .address == "aws_security_group.alb" + and .change.actions == ["update"]) +' >/dev/null <<< "$PLAN_JSON"; then + echo "Refusing to apply: the plan changes resources other than aws_security_group.alb." >&2 + terraform -chdir="$TF_ROOT" show -no-color "$PLAN_NAME" >&2 + if [[ -f "$TFVARS.bak" ]]; then + mv "$TFVARS.bak" "$TFVARS" + fi + rm -f "$TF_ROOT/$PLAN_NAME" + exit 1 +fi + +CONFIRM_IP="$(curl --noproxy '*' --fail --silent --show-error \ + https://checkip.amazonaws.com | tr -d '\r\n')" +if [[ "$CONFIRM_IP" != "$PUBLIC_IP" ]]; then + echo "Public IP changed during planning ($PUBLIC_IP -> $CONFIRM_IP); refusing to apply." >&2 + if [[ -f "$TFVARS.bak" ]]; then + mv "$TFVARS.bak" "$TFVARS" + fi + rm -f "$TF_ROOT/$PLAN_NAME" + exit 1 +fi + +terraform -chdir="$TF_ROOT" apply "$PLAN_NAME" +rm -f "$TF_ROOT/$PLAN_NAME" "$TFVARS.bak" + +BASE_URL="$(terraform -chdir="$TF_ROOT" output -raw api_base_url | tr -d '\r')" +echo "Applied ALB ingress CIDR $NEW_CIDR" +curl --fail --silent --show-error "$BASE_URL/api/v1/health" +echo +echo "Health check passed: $BASE_URL/api/v1/health" From f6c356d5e38409b5dfcd44908af68cc968335631 Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Thu, 10 Sep 2026 16:53:39 +0100 Subject: [PATCH 15/19] perf(api): add daily cost rollups --- apps/api/package.json | 1 + .../005_add_workspace_cost_daily_rollups.sql | 13 ++ apps/api/src/repositories/cost.repository.ts | 171 +++++++++++++----- apps/api/src/scripts/backfill-cost-rollups.ts | 56 ++++++ apps/api/src/scripts/seed-benchmark.ts | 22 ++- apps/api/src/services/cost.service.ts | 4 + .../src/test/cost-monitor.integration.spec.ts | 37 ++++ benchmarks/underflow-api/README.md | 13 +- .../underflow-api/scripts/run-one-off-task.sh | 52 ++++-- 9 files changed, 300 insertions(+), 69 deletions(-) create mode 100644 apps/api/src/db/migrations/005_add_workspace_cost_daily_rollups.sql create mode 100644 apps/api/src/scripts/backfill-cost-rollups.ts diff --git a/apps/api/package.json b/apps/api/package.json index dbf9be4..0dce9cc 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -12,6 +12,7 @@ "migrate": "tsx src/db/migrate.ts", "seed:demo": "node src/scripts/seed-demo.js", "seed:benchmark": "tsx src/scripts/seed-benchmark.ts", + "backfill:cost-rollups": "tsx src/scripts/backfill-cost-rollups.ts", "build": "tsc -p tsconfig.json && node scripts/copy-build-assets.mjs", "build:lambda": "npm run build && node scripts/build-lambda-package.mjs", "start": "node dist/server.js", diff --git a/apps/api/src/db/migrations/005_add_workspace_cost_daily_rollups.sql b/apps/api/src/db/migrations/005_add_workspace_cost_daily_rollups.sql new file mode 100644 index 0000000..e331778 --- /dev/null +++ b/apps/api/src/db/migrations/005_add_workspace_cost_daily_rollups.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS workspace_cost_daily_rollups ( + workspace_id UUID NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE, + usage_date DATE NOT NULL, + service_name VARCHAR(255) NOT NULL, + total_amount DECIMAL(18, 6) NOT NULL, + currency VARCHAR(10) NOT NULL DEFAULT 'USD', + updated_at TIMESTAMP NOT NULL DEFAULT NOW(), + PRIMARY KEY (workspace_id, usage_date, service_name) +); + +CREATE INDEX IF NOT EXISTS idx_workspace_cost_daily_rollups_workspace_service_date + ON workspace_cost_daily_rollups (workspace_id, service_name, usage_date) + INCLUDE (total_amount, currency); diff --git a/apps/api/src/repositories/cost.repository.ts b/apps/api/src/repositories/cost.repository.ts index 26e9ae4..00d40bf 100644 --- a/apps/api/src/repositories/cost.repository.ts +++ b/apps/api/src/repositories/cost.repository.ts @@ -1,3 +1,5 @@ +import type { PoolClient } from "pg"; + import { pool } from "../config/db.js"; import type { CostQueryInput, @@ -8,6 +10,31 @@ import type { TimeseriesCostPoint, } from "../types/aws-account.types.js"; +const rebuildWorkspaceRollups = async ( + client: PoolClient, + workspaceId: string, + from: string, + to: string, +): Promise => { + await client.query( + `DELETE FROM workspace_cost_daily_rollups + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3`, + [workspaceId, from, to], + ); + await client.query( + `INSERT INTO workspace_cost_daily_rollups ( + workspace_id, usage_date, service_name, total_amount, currency, updated_at + ) + SELECT workspace_id, usage_date, service_name, SUM(amount), MAX(currency), NOW() + FROM cost_snapshots + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3 + GROUP BY workspace_id, usage_date, service_name`, + [workspaceId, from, to], + ); +}; + export const costRepository = { async createSyncRun(awsAccountId: string): Promise { const result = await pool.query( @@ -45,6 +72,8 @@ export const costRepository = { async replaceSnapshots(input: { workspaceId: string; awsAccountId: string; + from: string; + to: string; entries: Array<{ usageDate: string; serviceName: string; @@ -52,35 +81,69 @@ export const costRepository = { currency: string; }>; }): Promise { - for (const entry of input.entries) { - await pool.query( - `INSERT INTO cost_snapshots (workspace_id, aws_account_id, usage_date, service_name, amount, currency) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT (workspace_id, aws_account_id, usage_date, service_name) - DO UPDATE SET amount = EXCLUDED.amount, currency = EXCLUDED.currency`, - [ - input.workspaceId, - input.awsAccountId, - entry.usageDate, - entry.serviceName, - entry.amount, - entry.currency, - ], + const client = await pool.connect(); + + try { + await client.query("BEGIN"); + // Account syncs are independently locked, so serialize mutations and + // rollup refreshes for the same workspace. + await client.query( + "SELECT pg_advisory_xact_lock(hashtext('cost-rollup:' || $1::text))", + [input.workspaceId], ); - } + await client.query( + `DELETE FROM cost_snapshots + WHERE workspace_id = $1 + AND aws_account_id = $2 + AND usage_date BETWEEN $3 AND $4`, + [input.workspaceId, input.awsAccountId, input.from, input.to], + ); + + for (const entry of input.entries) { + await client.query( + `INSERT INTO cost_snapshots (workspace_id, aws_account_id, usage_date, service_name, amount, currency) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (workspace_id, aws_account_id, usage_date, service_name) + DO UPDATE SET amount = EXCLUDED.amount, currency = EXCLUDED.currency`, + [ + input.workspaceId, + input.awsAccountId, + entry.usageDate, + entry.serviceName, + entry.amount, + entry.currency, + ], + ); + } - return input.entries.length; + await rebuildWorkspaceRollups(client, input.workspaceId, input.from, input.to); + await client.query("COMMIT"); + return input.entries.length; + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } finally { + client.release(); + } }, async getSummary(workspaceId: string, input: CostQueryInput): Promise { - const result = await pool.query( - `SELECT COALESCE(SUM(amount), 0) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency - FROM cost_snapshots - WHERE workspace_id = $1 - AND usage_date BETWEEN $2 AND $3 - AND ($4::uuid IS NULL OR aws_account_id = $4::uuid)`, - [workspaceId, input.from, input.to, input.awsAccountId ?? null], - ); + const result = input.awsAccountId + ? await pool.query( + `SELECT COALESCE(SUM(amount), 0) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency + FROM cost_snapshots + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3 + AND aws_account_id = $4::uuid`, + [workspaceId, input.from, input.to, input.awsAccountId], + ) + : await pool.query( + `SELECT COALESCE(SUM(total_amount), 0) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency + FROM workspace_cost_daily_rollups + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3`, + [workspaceId, input.from, input.to], + ); return { totalAmount: Number(result.rows[0]?.total_amount ?? 0), @@ -94,16 +157,26 @@ export const costRepository = { workspaceId: string, input: CostQueryInput, ): Promise { - const result = await pool.query( - `SELECT service_name, SUM(amount) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency - FROM cost_snapshots - WHERE workspace_id = $1 - AND usage_date BETWEEN $2 AND $3 - AND ($4::uuid IS NULL OR aws_account_id = $4::uuid) - GROUP BY service_name - ORDER BY total_amount DESC`, - [workspaceId, input.from, input.to, input.awsAccountId ?? null], - ); + const result = input.awsAccountId + ? await pool.query( + `SELECT service_name, SUM(amount) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency + FROM cost_snapshots + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3 + AND aws_account_id = $4::uuid + GROUP BY service_name + ORDER BY total_amount DESC`, + [workspaceId, input.from, input.to, input.awsAccountId], + ) + : await pool.query( + `SELECT service_name, SUM(total_amount) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency + FROM workspace_cost_daily_rollups + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3 + GROUP BY service_name + ORDER BY total_amount DESC`, + [workspaceId, input.from, input.to], + ); return result.rows.map((row) => ({ serviceName: String(row.service_name), @@ -116,16 +189,26 @@ export const costRepository = { workspaceId: string, input: CostQueryInput, ): Promise { - const result = await pool.query( - `SELECT usage_date, SUM(amount) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency - FROM cost_snapshots - WHERE workspace_id = $1 - AND usage_date BETWEEN $2 AND $3 - AND ($4::uuid IS NULL OR aws_account_id = $4::uuid) - GROUP BY usage_date - ORDER BY usage_date ASC`, - [workspaceId, input.from, input.to, input.awsAccountId ?? null], - ); + const result = input.awsAccountId + ? await pool.query( + `SELECT usage_date, SUM(amount) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency + FROM cost_snapshots + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3 + AND aws_account_id = $4::uuid + GROUP BY usage_date + ORDER BY usage_date ASC`, + [workspaceId, input.from, input.to, input.awsAccountId], + ) + : await pool.query( + `SELECT usage_date, SUM(total_amount) AS total_amount, COALESCE(MAX(currency), 'USD') AS currency + FROM workspace_cost_daily_rollups + WHERE workspace_id = $1 + AND usage_date BETWEEN $2 AND $3 + GROUP BY usage_date + ORDER BY usage_date ASC`, + [workspaceId, input.from, input.to], + ); return result.rows.map((row) => ({ usageDate: String(row.usage_date).slice(0, 10), diff --git a/apps/api/src/scripts/backfill-cost-rollups.ts b/apps/api/src/scripts/backfill-cost-rollups.ts new file mode 100644 index 0000000..6d9f53c --- /dev/null +++ b/apps/api/src/scripts/backfill-cost-rollups.ts @@ -0,0 +1,56 @@ +if (process.env.ALLOW_COST_ROLLUP_BACKFILL !== "true") { + throw new Error("Refusing to backfill cost rollups without ALLOW_COST_ROLLUP_BACKFILL=true"); +} + +const { pool } = await import("../config/db.js"); + +const run = async (): Promise => { + const client = await pool.connect(); + + try { + await client.query("BEGIN"); + await client.query( + "SELECT pg_advisory_xact_lock(hashtext('cost-rollup-backfill'))", + ); + await client.query("LOCK TABLE cost_snapshots IN SHARE MODE"); + await client.query("LOCK TABLE workspace_cost_daily_rollups IN EXCLUSIVE MODE"); + await client.query("DELETE FROM workspace_cost_daily_rollups"); + const result = await client.query<{ count: string }>( + `WITH inserted AS ( + INSERT INTO workspace_cost_daily_rollups ( + workspace_id, usage_date, service_name, total_amount, currency, updated_at + ) + SELECT workspace_id, usage_date, service_name, SUM(amount), MAX(currency), NOW() + FROM cost_snapshots + GROUP BY workspace_id, usage_date, service_name + RETURNING 1 + ) + SELECT COUNT(*)::text AS count FROM inserted`, + ); + await client.query("ANALYZE workspace_cost_daily_rollups"); + await client.query("COMMIT"); + + console.log( + JSON.stringify({ + costRollupBackfill: true, + rows: Number(result.rows[0]?.count ?? 0), + }), + ); + } catch (error) { + await client.query("ROLLBACK"); + throw error; + } finally { + client.release(); + await pool.end(); + } +}; + +run().catch((error: unknown) => { + console.error( + JSON.stringify({ + message: "Cost rollup backfill failed", + error: error instanceof Error ? error.message : "Unknown error", + }), + ); + process.exitCode = 1; +}); diff --git a/apps/api/src/scripts/seed-benchmark.ts b/apps/api/src/scripts/seed-benchmark.ts index b48e8b1..fe26e29 100644 --- a/apps/api/src/scripts/seed-benchmark.ts +++ b/apps/api/src/scripts/seed-benchmark.ts @@ -1,5 +1,6 @@ const BENCHMARK_GUARD = "ALLOW_BENCHMARK_SEED"; const EXPECTED_COST_SNAPSHOTS = 3_650_000; +const EXPECTED_COST_ROLLUPS = 182_500; const assertBenchmarkGuard = (): void => { if (process.env[BENCHMARK_GUARD] !== "true") { @@ -153,6 +154,17 @@ const run = async (): Promise => { CROSS JOIN generate_series(1, 50) AS service_number`, ); + await client.query( + `INSERT INTO workspace_cost_daily_rollups ( + workspace_id, usage_date, service_name, total_amount, currency, updated_at + ) + SELECT workspace_id, usage_date, service_name, SUM(amount), MAX(currency), NOW() + FROM cost_snapshots + WHERE workspace_id = ANY($1::uuid[]) + GROUP BY workspace_id, usage_date, service_name`, + [workspaceIds], + ); + const countsResult = await client.query<{ users: string; workspaces: string; @@ -160,6 +172,7 @@ const run = async (): Promise => { aws_accounts: string; cost_sync_runs: string; cost_snapshots: string; + cost_rollups: string; }>( `SELECT (SELECT COUNT(*) FROM users WHERE id = ANY($1::uuid[])) AS users, @@ -169,7 +182,8 @@ const run = async (): Promise => { (SELECT COUNT(*) FROM cost_sync_runs csr JOIN aws_accounts aa ON aa.id = csr.aws_account_id WHERE aa.workspace_id = ANY($2::uuid[])) AS cost_sync_runs, - (SELECT COUNT(*) FROM cost_snapshots WHERE workspace_id = ANY($2::uuid[])) AS cost_snapshots`, + (SELECT COUNT(*) FROM cost_snapshots WHERE workspace_id = ANY($2::uuid[])) AS cost_snapshots, + (SELECT COUNT(*) FROM workspace_cost_daily_rollups WHERE workspace_id = ANY($2::uuid[])) AS cost_rollups`, [userIds, workspaceIds], ); @@ -181,6 +195,7 @@ const run = async (): Promise => { awsAccounts: Number(row?.aws_accounts ?? 0), costSyncRuns: Number(row?.cost_sync_runs ?? 0), costSnapshots: Number(row?.cost_snapshots ?? 0), + costRollups: Number(row?.cost_rollups ?? 0), }; if ( @@ -189,13 +204,14 @@ const run = async (): Promise => { counts.workspaceMembers !== 10 || counts.awsAccounts !== 200 || counts.costSyncRuns !== 200 || - counts.costSnapshots !== EXPECTED_COST_SNAPSHOTS + counts.costSnapshots !== EXPECTED_COST_SNAPSHOTS || + counts.costRollups !== EXPECTED_COST_ROLLUPS ) { throw new Error(`Unexpected benchmark row counts: ${JSON.stringify(counts)}`); } await client.query( - "ANALYZE users, workspaces, workspace_members, aws_accounts, cost_sync_runs, cost_snapshots", + "ANALYZE users, workspaces, workspace_members, aws_accounts, cost_sync_runs, cost_snapshots, workspace_cost_daily_rollups", ); await client.query("COMMIT"); diff --git a/apps/api/src/services/cost.service.ts b/apps/api/src/services/cost.service.ts index 13cf67b..3d33e28 100644 --- a/apps/api/src/services/cost.service.ts +++ b/apps/api/src/services/cost.service.ts @@ -66,6 +66,8 @@ export const costService = { const recordsSynced = await costRepository.replaceSnapshots({ workspaceId: awsAccount.workspaceId, awsAccountId: awsAccount.id, + from: syncRange.from, + to: syncRange.to, entries, }); @@ -164,6 +166,8 @@ export const costService = { await costRepository.replaceSnapshots({ workspaceId: account.workspaceId, awsAccountId: account.id, + from: range.from, + to: range.to, entries, }); diff --git a/apps/api/src/test/cost-monitor.integration.spec.ts b/apps/api/src/test/cost-monitor.integration.spec.ts index 8d07a2c..87b0ba8 100644 --- a/apps/api/src/test/cost-monitor.integration.spec.ts +++ b/apps/api/src/test/cost-monitor.integration.spec.ts @@ -77,6 +77,7 @@ const truncateTables = async (): Promise => { notification_deliveries, alert_events, budget_alerts, + workspace_cost_daily_rollups, cost_snapshots, cost_sync_runs, aws_accounts, @@ -171,6 +172,24 @@ test("workspace, aws account, sync, reporting, and alerts work together against assert.equal(summaryResponse.status, 200); assert.equal(summaryResponse.body.summary.totalAmount, 120); + const rollupResult = await pool.query( + `SELECT COUNT(*) AS count, SUM(total_amount) AS total_amount + FROM workspace_cost_daily_rollups + WHERE workspace_id = $1`, + [workspaceId], + ); + assert.equal(Number(rollupResult.rows[0]?.count ?? 0), 3); + assert.equal(Number(rollupResult.rows[0]?.total_amount ?? 0), 120); + + const accountSummaryResponse = await request(app) + .get( + `/api/v1/workspaces/${workspaceId}/costs/summary?from=${dayOne}&to=${dayThree}&awsAccountId=${awsAccountId}`, + ) + .set("Authorization", `Bearer ${accessToken}`); + + assert.equal(accountSummaryResponse.status, 200); + assert.equal(accountSummaryResponse.body.summary.totalAmount, 120); + const byServiceResponse = await request(app) .get(`/api/v1/workspaces/${workspaceId}/costs/by-service?from=${dayOne}&to=${dayThree}`) .set("Authorization", `Bearer ${accessToken}`); @@ -199,6 +218,24 @@ test("workspace, aws account, sync, reporting, and alerts work together against assert.equal(Number(alertEventsResult.rows[0]?.count ?? 0), 1); assert.equal(Number(deliveriesResult.rows[0]?.count ?? 0), 1); + + costExplorerService.fetchCostData = async () => []; + const emptyReplacementResponse = await request(app) + .post(`/api/v1/aws-accounts/${awsAccountId}/sync`) + .set("Authorization", `Bearer ${accessToken}`) + .send({ from: dayOne, to: dayThree }); + + assert.equal(emptyReplacementResponse.status, 200); + assert.equal(emptyReplacementResponse.body.recordsSynced, 0); + + const replacedCounts = await pool.query( + `SELECT + (SELECT COUNT(*) FROM cost_snapshots WHERE workspace_id = $1) AS snapshots, + (SELECT COUNT(*) FROM workspace_cost_daily_rollups WHERE workspace_id = $1) AS rollups`, + [workspaceId], + ); + assert.equal(Number(replacedCounts.rows[0]?.snapshots ?? 0), 0); + assert.equal(Number(replacedCounts.rows[0]?.rollups ?? 0), 0); } finally { costExplorerService.fetchCostData = originalFetch; } diff --git a/benchmarks/underflow-api/README.md b/benchmarks/underflow-api/README.md index 7d827d4..e26f67d 100644 --- a/benchmarks/underflow-api/README.md +++ b/benchmarks/underflow-api/README.md @@ -134,11 +134,20 @@ Both commands launch one-off Fargate tasks from the deployed API task definition export BENCHMARK_RESULTS_DIR="$RESULTS_DIR" "$REPO_ROOT/benchmarks/underflow-api/scripts/run-one-off-task.sh" migrate "$REPO_ROOT/benchmarks/underflow-api/scripts/run-one-off-task.sh" seed -jq -e '.users == 10 and .workspaces == 10 and .awsAccounts == 200 and .costSnapshots == 3650000' \ +jq -e '.users == 10 and .workspaces == 10 and .awsAccounts == 200 and .costSnapshots == 3650000 and .costRollups == 182500' \ "$RESULTS_DIR/dataset.json" ``` -The seeder requires `ALLOW_BENCHMARK_SEED=true`, uses only reserved UUIDs and `example.invalid` identities, bulk-loads with PostgreSQL set operations, replaces only its fixed dataset, verifies exact counts, and runs `ANALYZE`. The generated password is never logged. The first identity is `benchmark+01@example.invalid`, and its workspace is `20000000-0000-4000-8000-000000000001`. +The seeder requires `ALLOW_BENCHMARK_SEED=true`, uses only reserved UUIDs and `example.invalid` identities, bulk-loads with PostgreSQL set operations, replaces only its fixed dataset, verifies exact counts, builds 182,500 workspace/date/service rollups from the 3,650,000 raw snapshots, and runs `ANALYZE`. The generated password is never logged. The first identity is `benchmark+01@example.invalid`, and its workspace is `20000000-0000-4000-8000-000000000001`. + +After deploying the rollup migration over an already-seeded benchmark database, backfill without reseeding the raw dataset: + +```bash +export BENCHMARK_RESULTS_DIR="$RESULTS_DIR" +"$REPO_ROOT/benchmarks/underflow-api/scripts/run-one-off-task.sh" rollup +``` + +The guarded backfill rebuilds all rollups transactionally and records its sanitized output in `rollup-task.log`. Normal synchronization refreshes only the affected workspace/date range under a workspace-scoped transaction lock. Workspace-wide summary, timeseries, and by-service reads use rollups; account-filtered reads continue to use raw snapshots. To capture a baseline PostgreSQL execution plan for the full-year cost-summary query without exposing RDS publicly, run the diagnostic one-off task and preserve its sanitized output: diff --git a/benchmarks/underflow-api/scripts/run-one-off-task.sh b/benchmarks/underflow-api/scripts/run-one-off-task.sh index 8ff978c..86d0c3e 100644 --- a/benchmarks/underflow-api/scripts/run-one-off-task.sh +++ b/benchmarks/underflow-api/scripts/run-one-off-task.sh @@ -2,8 +2,8 @@ set -euo pipefail MODE="${1:-}" -if [[ "$MODE" != "migrate" && "$MODE" != "seed" && "$MODE" != "explain" ]]; then - echo "usage: $0 migrate|seed|explain" >&2 +if [[ "$MODE" != "migrate" && "$MODE" != "seed" && "$MODE" != "rollup" && "$MODE" != "explain" ]]; then + echo "usage: $0 migrate|seed|rollup|explain" >&2 exit 2 fi @@ -33,27 +33,27 @@ if [[ "$MODE" == "migrate" ]]; then elif [[ "$MODE" == "seed" ]]; then COMMAND_JSON='["node","dist/scripts/seed-benchmark.js"]' ENVIRONMENT_JSON='[{"name":"ALLOW_BENCHMARK_SEED","value":"true"}]' +elif [[ "$MODE" == "rollup" ]]; then + COMMAND_JSON='["node","dist/scripts/backfill-cost-rollups.js"]' + ENVIRONMENT_JSON='[{"name":"ALLOW_COST_ROLLUP_BACKFILL","value":"true"}]' else EXPLAIN_SCRIPT='import { pool } from "./dist/config/db.js"; const workspaceId = "20000000-0000-4000-8000-000000000001"; -const parameters = [workspaceId, "2025-01-01", "2025-12-31", null]; +const parameters = [workspaceId, "2025-01-01", "2025-12-31"]; const queries = { - summary: `SELECT COALESCE(SUM(amount), 0) AS total_amount, + summary: `SELECT COALESCE(SUM(total_amount), 0) AS total_amount, COALESCE(MAX(currency), '\''USD'\'') AS currency - FROM cost_snapshots - WHERE workspace_id = $1 AND usage_date BETWEEN $2 AND $3 - AND ($4::uuid IS NULL OR aws_account_id = $4::uuid)`, - timeseries: `SELECT usage_date, SUM(amount) AS total_amount, + FROM workspace_cost_daily_rollups + WHERE workspace_id = $1 AND usage_date BETWEEN $2 AND $3`, + timeseries: `SELECT usage_date, SUM(total_amount) AS total_amount, COALESCE(MAX(currency), '\''USD'\'') AS currency - FROM cost_snapshots + FROM workspace_cost_daily_rollups WHERE workspace_id = $1 AND usage_date BETWEEN $2 AND $3 - AND ($4::uuid IS NULL OR aws_account_id = $4::uuid) GROUP BY usage_date ORDER BY usage_date ASC`, - byService: `SELECT service_name, SUM(amount) AS total_amount, + byService: `SELECT service_name, SUM(total_amount) AS total_amount, COALESCE(MAX(currency), '\''USD'\'') AS currency - FROM cost_snapshots + FROM workspace_cost_daily_rollups WHERE workspace_id = $1 AND usage_date BETWEEN $2 AND $3 - AND ($4::uuid IS NULL OR aws_account_id = $4::uuid) GROUP BY service_name ORDER BY total_amount DESC`, }; const plans = {}; @@ -61,14 +61,15 @@ for (const [name, sql] of Object.entries(queries)) { const result = await pool.query(`EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT JSON) ${sql}`, parameters); plans[name] = result.rows[0]["QUERY PLAN"]; } -const stats = await pool.query(`SELECT - pg_size_pretty(pg_total_relation_size('\''cost_snapshots'\'')) AS total_size, - pg_size_pretty(pg_relation_size('\''cost_snapshots'\'')) AS table_size, - pg_size_pretty(pg_indexes_size('\''cost_snapshots'\'')) AS indexes_size, +const stats = await pool.query(`SELECT relname, + pg_size_pretty(pg_total_relation_size(relid)) AS total_size, + pg_size_pretty(pg_relation_size(relid)) AS table_size, + pg_size_pretty(pg_indexes_size(relid)) AS indexes_size, n_live_tup, seq_scan, idx_scan FROM pg_stat_user_tables - WHERE relname = '\''cost_snapshots'\''`); -console.log(JSON.stringify({ benchmarkDiagnostic: true, stats: stats.rows[0], plans }, null, 2)); + WHERE relname IN ('\''cost_snapshots'\'', '\''workspace_cost_daily_rollups'\'') + ORDER BY relname`); +console.log(JSON.stringify({ benchmarkDiagnostic: true, stats: stats.rows, plans }, null, 2)); await pool.end();' COMMAND_JSON="$(jq -cn --arg script "$EXPLAIN_SCRIPT" '["node","--input-type=module","--eval",$script]')" ENVIRONMENT_JSON='[]' @@ -130,8 +131,19 @@ fi if [[ "$MODE" == "seed" ]]; then jq '[.events[].message | fromjson? | select(.dataset == "underflow-api-benchmark-v1")] | last' \ "$RESULTS_DIR/seed-task.log" > "$RESULTS_DIR/dataset.json" - jq -e '.costSnapshots == 3650000 and .users == 10 and .workspaces == 10 and .awsAccounts == 200' \ + jq -e '.costSnapshots == 3650000 and .costRollups == 182500 and .users == 10 and .workspaces == 10 and .awsAccounts == 200' \ "$RESULTS_DIR/dataset.json" >/dev/null +elif [[ "$MODE" == "rollup" ]]; then + jq '[.events[].message | fromjson? | select(.costRollupBackfill == true)] | last' \ + "$RESULTS_DIR/rollup-task.log" > "$RESULTS_DIR/rollup.json" + jq -e '.rows == 182500' "$RESULTS_DIR/rollup.json" >/dev/null + + if [[ -f "$RESULTS_DIR/dataset.json" ]]; then + DATASET_TMP="$(mktemp)" + jq --slurpfile rollup "$RESULTS_DIR/rollup.json" \ + '.costRollups = $rollup[0].rows' "$RESULTS_DIR/dataset.json" > "$DATASET_TMP" + mv "$DATASET_TMP" "$RESULTS_DIR/dataset.json" + fi fi echo "Benchmark $MODE task completed with exit code 0" From f37ea205b6d4c946230b3e762f41cf81d0024994 Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Fri, 11 Sep 2026 04:57:56 +0100 Subject: [PATCH 16/19] feat(benchmark): automate server-side capacity validation --- benchmarks/underflow-api/README.md | 41 +++- benchmarks/underflow-api/RESULTS_TEMPLATE.md | 2 +- benchmarks/underflow-api/k6/common.js | 10 +- .../scripts/check-capacity-result.sh | 46 ++++ .../underflow-api/scripts/generate-results.sh | 202 ++++++++++++------ .../scripts/run-capacity-test.sh | 135 ++++++++++++ 6 files changed, 352 insertions(+), 84 deletions(-) create mode 100644 benchmarks/underflow-api/scripts/check-capacity-result.sh create mode 100644 benchmarks/underflow-api/scripts/run-capacity-test.sh diff --git a/benchmarks/underflow-api/README.md b/benchmarks/underflow-api/README.md index e26f67d..ab8c91e 100644 --- a/benchmarks/underflow-api/README.md +++ b/benchmarks/underflow-api/README.md @@ -4,6 +4,10 @@ This benchmark measures the real bearer-authenticated, PostgreSQL-backed cost-re No result in this directory should be treated as measured evidence until all commands complete and a timestamped result directory contains the required JSON files. A failed k6 threshold is evidence and must be preserved as-is. +## Final measured result + +The completed benchmark sustained **170.90 requests/second for 15 minutes at 45 continuously active VUs** with a 0.0325% functional failure rate. Across every one-minute ALB server-side datapoint, the worst p50 was 15.22 ms, p95 was 89.89 ms, and p99 was 226.96 ms. The 50-VU soak was the first sustained failing level because its p95 and p99 targets were breached. See [`results/2026-09-10-c0cf1f68/RESULTS.md`](results/2026-09-10-c0cf1f68/RESULTS.md) for the generated report and limitations; VUs are continuously active workers, not registered-user counts. + ## Safety boundary The only authorized Terraform root is `infra/terraform/envs/api-benchmark`. It has local, isolated state and does not reference production state or modules. Every named resource begins with `underflow-api-bench-` and AWS provider default tags apply: @@ -189,13 +193,21 @@ k6 run "$REPO_ROOT/benchmarks/underflow-api/k6/load.js" export LOAD_END="$(date -u +%FT%TZ)" ``` -The scripts use deterministic iteration buckets for the 30/30/25/10/5 distribution, validate status, JSON, response shape, and authorization/server errors, and save compact summaries through `handleSummary`. k6 retains the client-observed failure rate `<1%`, p95 `<200 ms`, p99 `<500 ms`, and checks `>99%` thresholds; per-endpoint values are retained. Because a developer-machine run includes location-dependent Internet transit, the primary backend latency reported by this benchmark is the ALB `TargetResponseTime` p50/p90/p95/p99 collected from CloudWatch. Client-observed k6 latency remains evidence and must be labeled with the load-generator location rather than relabeled or discarded. Login is tagged as setup and excluded from `measured_*` metrics. +The scripts use deterministic iteration buckets for the 30/30/25/10/5 distribution, validate status, JSON, response shape, and authorization/server errors, and save compact summaries through `handleSummary`. k6 enforces the client-observed failure rate `<1%` and checks `>99%`; its end-to-end latency and per-endpoint values are retained as diagnostic evidence but are not used for backend acceptance. Because a developer-machine run includes location-dependent Internet transit, backend latency targets are evaluated from the ALB `TargetResponseTime` p50/p90/p95/p99 collected from CloudWatch. The required server-side targets are p50 `<30 ms`, p95 `<100 ms`, and p99 `<250 ms` in every one-minute datapoint. Client-observed k6 latency must be labeled with the load-generator location rather than relabeled or discarded. Login is tagged as setup and excluded from `measured_*` metrics. ### Capacity discovery on the baseline database class -When the normal profile saturates the documented `db.t4g.micro`, preserve that failed run and determine the configuration's actual capacity with independent constant-load levels. Test 2, 3, 4, and 5 VUs, stopping once a level fails or once the first failing level above the highest passing level is established. A passing level has no functional failures and every one-minute ALB `TargetResponseTime` p95 below 200 ms and p99 below 500 ms. Refresh the ALB ingress CIDR before each level; never update it during a measured interval. +When the normal profile saturates the documented `db.t4g.micro`, preserve that failed run and determine the configuration's actual capacity with independent constant-load levels. Test increasing VU levels, stopping once the first failing level above the highest passing level is established. A passing level has a measured failure rate below 1%, a checks pass rate above 99%, and every one-minute ALB `TargetResponseTime` p50 below 30 ms, p95 below 100 ms, and p99 below 250 ms. Refresh the ALB ingress CIDR before each level; never update it during a measured interval. + +The all-in-one runner performs the IP refresh, health and credential preflight, k6 run, CloudWatch wait and collection, and final capacity check. It defaults to 7 VUs for 3 minutes; the optional arguments are VUs and duration: + +```bash +bash "$REPO_ROOT/benchmarks/underflow-api/scripts/run-capacity-test.sh" 7 3m +``` -For each level, run the following block after setting `CAPACITY_VUS` to 2, 3, 4, or 5: +It refuses to overwrite existing evidence. Set a distinct label when intentionally repeating a level, for example `CAPACITY_LABEL=post-rollup-repeat`. + +The equivalent manual sequence follows for troubleshooting or inspecting each step: ```bash bash "$REPO_ROOT/benchmarks/underflow-api/scripts/update-load-test-ip.sh" @@ -217,9 +229,13 @@ TARGET_GROUP_ARN_SUFFIX="$(terraform -chdir="$TF_ROOT" output -raw target_group_ START_TIME="$CAPACITY_START" END_TIME="$CAPACITY_END" \ OUTPUT_FILE="$RESULTS_DIR/capacity-${CAPACITY_VUS}vus-cloudwatch.json" \ "$REPO_ROOT/benchmarks/underflow-api/scripts/collect-aws-metadata.sh" + +"$REPO_ROOT/benchmarks/underflow-api/scripts/check-capacity-result.sh" \ + "$RESULTS_DIR/capacity-${CAPACITY_VUS}vus-summary.json" \ + "$RESULTS_DIR/capacity-${CAPACITY_VUS}vus-cloudwatch.json" ``` -The k6 summary is saved as `capacity-vus-summary.json`. A run aborted for network failures is invalid, must be preserved as such, and cannot establish capacity. The supported level is the highest fully valid level whose server-side threshold passes; do not interpolate or claim the next failing level. +The k6 summary is saved as `capacity-vus-summary.json`. The checker requires functional success plus p50 `<30 ms`, p95 `<100 ms`, and p99 `<250 ms` in every collected ALB minute. A run aborted for network failures is invalid, must be preserved as such, and cannot establish capacity. The supported level is the highest fully valid level whose server-side threshold passes; do not interpolate or claim the next failing level. Run stress only when the normal profile completes, thresholds are reviewed, the service remains healthy, and the load generator is not saturated: @@ -252,15 +268,19 @@ OUTPUT_FILE="$RESULTS_DIR/cloudwatch-summary.json" \ The generator refuses missing evidence and mechanically derives requests/second, p95, concurrency, dataset size, and the résumé bullet from preserved JSON. Review `RESULTS.md` for supported bottleneck observations; do not replace failed thresholds or missing stress evidence with estimates. -Required timestamped evidence is: +Required final evidence for this capacity-discovery run is: ```text environment.json dataset.json smoke-summary.json -load-summary.json -stress-summary.json # only if safely performed -cloudwatch-summary.json +load-summary.json # retained invalid run, excluded from capacity claims +capacity-45vus-post-rollup-soak-summary.json +capacity-45vus-post-rollup-soak-cloudwatch.json +capacity-50vus-post-rollup-soak-summary.json +capacity-50vus-post-rollup-soak-cloudwatch.json +explain-after-rollup.json +rollup.json RESULTS.md ``` @@ -306,9 +326,8 @@ for file in environment.json dataset.json smoke-summary.json load-summary.json c test -s "$RESULTS_DIR/$file" done -terraform -chdir="$TF_ROOT" output -json | \ - jq '{aws_region,resource_prefix,ecs_cluster_name,ecs_service_name,rds_identifier,availability_zones}' \ - > "$RESULTS_DIR/final-terraform-outputs.json" +# Infrastructure characteristics and the tested image digest are already +# recorded in environment.json. Do not save live resource IDs or endpoints. terraform -chdir="$TF_ROOT" state list | tee "$RESULTS_DIR/terraform-state-list.txt" terraform -chdir="$TF_ROOT" state list | grep -E 'envs.production|platform_stack' && exit 1 || true diff --git a/benchmarks/underflow-api/RESULTS_TEMPLATE.md b/benchmarks/underflow-api/RESULTS_TEMPLATE.md index 3ab1c75..bb5f1a0 100644 --- a/benchmarks/underflow-api/RESULTS_TEMPLATE.md +++ b/benchmarks/underflow-api/RESULTS_TEMPLATE.md @@ -24,7 +24,7 @@ See `dataset.json` for exact verified counts. ## Thresholds -Measured failure rate `< 1%`, measured request latency p95 `< 200 ms`, p99 `< 500 ms`, and checks pass rate `> 99%`. +Measured failure rate `< 1%` and checks pass rate `> 99%`. For every one-minute CloudWatch datapoint, ALB server-side `TargetResponseTime` p50 `< 30 ms`, p95 `< 100 ms`, and p99 `< 250 ms`. Client-observed latency is retained as diagnostic evidence and is not used for backend acceptance. ## Results diff --git a/benchmarks/underflow-api/k6/common.js b/benchmarks/underflow-api/k6/common.js index 4404e78..1ae48a4 100644 --- a/benchmarks/underflow-api/k6/common.js +++ b/benchmarks/underflow-api/k6/common.js @@ -32,13 +32,7 @@ export const standardThresholds = (failureThreshold = "rate<0.01", abortOnFailur measured_failures: abortOnFailure ? [{ threshold: failureThreshold, abortOnFail: true, delayAbortEval: "30s" }] : [failureThreshold], - measured_duration: ["p(95)<200", "p(99)<500"], checks: ["rate>0.99"], - endpoint_cost_summary_duration: ["p(95)<200", "p(99)<500"], - endpoint_cost_timeseries_duration: ["p(95)<200", "p(99)<500"], - endpoint_cost_by_service_duration: ["p(95)<200", "p(99)<500"], - endpoint_aws_account_list_duration: ["p(95)<200", "p(99)<500"], - endpoint_sync_history_duration: ["p(95)<200", "p(99)<500"], }); export const authenticate = () => { @@ -154,8 +148,10 @@ export const runMeasuredRequest = (data) => { export const writeSummary = (data, defaultName) => { const resultsDirectory = (__ENV.RESULTS_DIR ?? "results").replace(/[\\/]$/, ""); const summaryName = __ENV.SUMMARY_NAME ?? defaultName; + const sanitizedData = { ...data }; + delete sanitizedData.setup_data; return { - [`${resultsDirectory}/${summaryName}`]: JSON.stringify(data, null, 2), + [`${resultsDirectory}/${summaryName}`]: JSON.stringify(sanitizedData, null, 2), stdout: `Saved compact k6 summary to ${resultsDirectory}/${summaryName}\n`, }; }; diff --git a/benchmarks/underflow-api/scripts/check-capacity-result.sh b/benchmarks/underflow-api/scripts/check-capacity-result.sh new file mode 100644 index 0000000..0457e43 --- /dev/null +++ b/benchmarks/underflow-api/scripts/check-capacity-result.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 2 ]]; then + echo "usage: $0 K6_SUMMARY CLOUDWATCH_SUMMARY" >&2 + exit 2 +fi + +K6_SUMMARY="$1" +CLOUDWATCH_SUMMARY="$2" + +for command in jq; do + command -v "$command" >/dev/null 2>&1 || { + echo "$command is required" >&2 + exit 2 + } +done + +[[ -f "$K6_SUMMARY" ]] || { echo "missing k6 summary: $K6_SUMMARY" >&2; exit 2; } +[[ -f "$CLOUDWATCH_SUMMARY" ]] || { echo "missing CloudWatch summary: $CLOUDWATCH_SUMMARY" >&2; exit 2; } + +jq -e ' + .metrics.measured_failures.values.rate < 0.01 and + .metrics.checks.values.rate > 0.99 +' "$K6_SUMMARY" >/dev/null || { + echo "FAIL: functional thresholds were not met" >&2 + exit 1 +} + +jq -e ' + .metrics.alb_target_response.datapoints as $points | + ($points | length) > 0 and + all($points[]; + .ExtendedStatistics.p50 < 0.030 and + .ExtendedStatistics.p95 < 0.100 and + .ExtendedStatistics.p99 < 0.250 + ) +' "$CLOUDWATCH_SUMMARY" >/dev/null || { + echo "FAIL: ALB server-side targets were not met in every one-minute datapoint" >&2 + exit 1 +} + +jq -r ' + .metrics.alb_target_response.datapoints | + "PASS: \(length) datapoints; worst p50=\(map(.ExtendedStatistics.p50) | max * 1000) ms, p95=\(map(.ExtendedStatistics.p95) | max * 1000) ms, p99=\(map(.ExtendedStatistics.p99) | max * 1000) ms" +' "$CLOUDWATCH_SUMMARY" diff --git a/benchmarks/underflow-api/scripts/generate-results.sh b/benchmarks/underflow-api/scripts/generate-results.sh index 473d137..26bc89f 100644 --- a/benchmarks/underflow-api/scripts/generate-results.sh +++ b/benchmarks/underflow-api/scripts/generate-results.sh @@ -25,20 +25,11 @@ import sys directory = pathlib.Path(sys.argv[1]) output = pathlib.Path(sys.argv[2]) -def read(name, required=True): +def read(name): path = directory / name if not path.exists(): - if required: - raise SystemExit(f"missing required evidence: {path}") - return None - return json.loads(path.read_text()) - -environment = read("environment.json") -dataset = read("dataset.json") -smoke = read("smoke-summary.json") -load = read("load-summary.json") -stress = read("stress-summary.json", required=False) -cloudwatch = read("cloudwatch-summary.json") + raise SystemExit(f"missing required evidence: {path}") + return json.loads(path.read_text(encoding="utf-8")) def values(summary, metric): try: @@ -46,18 +37,95 @@ def values(summary, metric): except KeyError as error: raise SystemExit(f"missing metric {metric}: {error}") from error -def metric_row(label, summary): - requests = values(summary, "measured_requests") - duration = values(summary, "measured_duration") - failures = values(summary, "measured_failures") - checks = values(summary, "checks") - return ( - f"| {label} | {requests['count']:.0f} | {requests['rate']:.2f} | " - f"{duration['med']:.2f} | {duration['p(90)']:.2f} | {duration['p(95)']:.2f} | " - f"{duration['p(99)']:.2f} | {duration['max']:.2f} | " - f"{failures['rate'] * 100:.3f}% | {checks['rate'] * 100:.3f}% |" +def points(cloudwatch, metric): + result = cloudwatch["metrics"][metric]["datapoints"] + if not result: + raise SystemExit(f"missing CloudWatch datapoints for {metric}") + return result + +def worst_percentile(cloudwatch, percentile): + return max( + point["ExtendedStatistics"][percentile] * 1000 + for point in points(cloudwatch, "alb_target_response") ) +def maximum_average(cloudwatch, metric): + return max(point["Average"] for point in points(cloudwatch, metric)) + +def minimum_average(cloudwatch, metric): + return min(point["Average"] for point in points(cloudwatch, metric)) + +def count_breaches(cloudwatch, percentile, target_ms): + return sum( + point["ExtendedStatistics"][percentile] * 1000 >= target_ms + for point in points(cloudwatch, "alb_target_response") + ) + +def sum_metric(cloudwatch, metric): + return sum(point.get("Sum", 0) for point in cloudwatch["metrics"][metric]["datapoints"]) + +environment = read("environment.json") +dataset = read("dataset.json") +rollup = read("rollup.json") +smoke = read("smoke-summary.json") +invalid_load = read("load-summary.json") +supported = read("capacity-45vus-post-rollup-soak-summary.json") +supported_cw = read("capacity-45vus-post-rollup-soak-cloudwatch.json") +failed = read("capacity-50vus-post-rollup-soak-summary.json") +failed_cw = read("capacity-50vus-post-rollup-soak-cloudwatch.json") +explain = read("explain-after-rollup.json") + +supported_requests = values(supported, "measured_requests") +supported_failures = values(supported, "measured_failures") +supported_checks = values(supported, "checks") +failed_requests = values(failed, "measured_requests") +failed_failures = values(failed, "measured_failures") +failed_checks = values(failed, "checks") + +supported_p50 = worst_percentile(supported_cw, "p50") +supported_p95 = worst_percentile(supported_cw, "p95") +supported_p99 = worst_percentile(supported_cw, "p99") +failed_p50 = worst_percentile(failed_cw, "p50") +failed_p95 = worst_percentile(failed_cw, "p95") +failed_p99 = worst_percentile(failed_cw, "p99") + +supported_passes = ( + supported_failures["rate"] < 0.01 + and supported_checks["rate"] > 0.99 + and supported_p50 < 30 + and supported_p95 < 100 + and supported_p99 < 250 +) +failed_passes = ( + failed_failures["rate"] < 0.01 + and failed_checks["rate"] > 0.99 + and failed_p50 < 30 + and failed_p95 < 100 + and failed_p99 < 250 +) +if not supported_passes: + raise SystemExit("45-VU soak evidence does not meet the documented targets") +if failed_passes: + raise SystemExit("50-VU soak evidence unexpectedly meets every documented target") +if int(dataset["costRollups"]) != int(rollup["rows"]): + raise SystemExit("dataset and rollup evidence disagree") + +raw_rows = int(dataset["costSnapshots"]) +rollup_rows = int(dataset["costRollups"]) +records_label = f"{raw_rows / 1_000_000:.2f}M" +resume = ( + "Built and benchmarked a multi-tenant AWS cost-monitoring API against " + f"{records_label} cost records, sustaining {supported_requests['rate']:.2f} requests/second " + f"with {supported_p95:.2f} ms worst-minute p95 server-side latency under 45 concurrent virtual users." +) + +plans = explain["plans"] +plan_times = { + name: plans[name][0]["Execution Time"] + for name in ("summary", "timeseries", "byService") +} +stats = {row["relname"]: row for row in explain["stats"]} + endpoint_metrics = [ ("Cost summary", "endpoint_cost_summary_duration"), ("Cost timeseries", "endpoint_cost_timeseries_duration"), @@ -66,95 +134,99 @@ endpoint_metrics = [ ("Sync history", "endpoint_sync_history_duration"), ] -load_duration = values(load, "measured_duration") -load_requests = values(load, "measured_requests") -vus = int(values(load, "vus_max")["max"]) -cost_rows = int(dataset["costSnapshots"]) -records_label = f"{cost_rows / 1_000_000:.2f}M" if cost_rows >= 1_000_000 else f"{cost_rows:,}" -resume = ( - "Built and benchmarked a multi-tenant AWS cost-monitoring API against " - f"{records_label} cost records, sustaining {load_requests['rate']:.2f} requests/second " - f"with {load_duration['p(95)']:.2f} ms p95 latency under {vus} concurrent users." -) - lines = [ "# Underflow API benchmark results", "", "## Objective", "", - "Measure authenticated, PostgreSQL-backed cost-reporting reads on the documented disposable AWS configuration.", + "Measure authenticated, PostgreSQL-backed cost-reporting reads on an isolated, disposable AWS configuration and establish a repeatable server-side capacity boundary.", "", "## Tested commit", "", - f"`{environment['gitSha']}` using image digest `{environment['imageDigest']}`.", + f"API commit `{environment['gitSha']}` using image digest `{environment['imageDigest']}`.", "", "## Infrastructure", "", - "On the documented one-task ECS and db.t4g.micro configuration described in `environment.json`.", + "One ECS Fargate API task with 0.5 vCPU and 1,024 MiB memory, backed by a private, single-AZ `db.t4g.micro` PostgreSQL 16.13 instance with 20 GiB encrypted storage in `us-east-1`.", "", "## Dataset", "", - f"{cost_rows:,} deterministic synthetic cost snapshots across {dataset['workspaces']} workspaces and {dataset['awsAccounts']} synthetic AWS accounts.", + f"{raw_rows:,} deterministic synthetic cost snapshots across {dataset['workspaces']} workspaces and {dataset['awsAccounts']} synthetic AWS accounts. Workspace/date/service reads use {rollup_rows:,} daily rollup rows, a {raw_rows / rollup_rows:.0f}x row-count reduction.", "", "## Workload", "", - "30% summary, 30% timeseries, 25% by service, 10% AWS account list, and 5% sync history, with bearer-token authentication on every measured request.", + "Bearer-authenticated read traffic distributed as 30% cost summary, 30% timeseries, 25% by service, 10% AWS account list, and 5% sync history. Login occurs once during setup and is excluded from measured requests. The supported and failing boundary runs each lasted 15 minutes.", "", - "## Thresholds", + "## Acceptance targets", "", - "Failure rate < 1%, p95 < 200 ms, p99 < 500 ms, and checks pass rate > 99%.", + "Measured failure rate `<1%`, checks pass rate `>99%`, and every one-minute ALB `TargetResponseTime` datapoint p50 `<30 ms`, p95 `<100 ms`, and p99 `<250 ms`. ALB target response time excludes load-generator-to-ALB Internet transit.", "", - "## Results", + "## Capacity results", "", - "| Run | Requests | Requests/s | p50 ms | p90 ms | p95 ms | p99 ms | Max ms | Failures | Checks passed |", - "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", - metric_row("Smoke", smoke), - metric_row("Normal load", load), -] -if stress is not None: - lines.append(metric_row("Stress", stress)) - -lines.extend([ + "| Run | Duration | Requests | Requests/s | Failure rate | Checks | Worst p50 | Worst p95 | Worst p99 | Result |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |", + f"| 45 VUs | 15m | {supported_requests['count']:,} | {supported_requests['rate']:.2f} | {supported_failures['rate'] * 100:.4f}% | {supported_checks['rate'] * 100:.4f}% | {supported_p50:.2f} ms | {supported_p95:.2f} ms | {supported_p99:.2f} ms | Pass |", + f"| 50 VUs | 15m | {failed_requests['count']:,} | {failed_requests['rate']:.2f} | {failed_failures['rate'] * 100:.4f}% | {failed_checks['rate'] * 100:.4f}% | {failed_p50:.2f} ms | {failed_p95:.2f} ms | {failed_p99:.2f} ms | Fail |", "", - "## Per-endpoint results (normal load)", + f"The documented configuration therefore sustained at least **{supported_requests['rate']:.2f} requests/second at 45 continuously active VUs**. The first sustained failing level tested was 50 VUs; this establishes a tested boundary, not a claim that 45 VUs equals 45 registered users or that 45 is the absolute maximum.", "", - "| Endpoint | p95 ms | p99 ms | Max ms |", - "| --- | ---: | ---: | ---: |", -]) + "## Initial smoke and normal-load outcomes", + "", + f"The initial 2-VU smoke run completed {values(smoke, 'measured_requests')['count']:,} measured requests with {values(smoke, 'measured_failures')['rate'] * 100:.3f}% functional failures and {values(smoke, 'checks')['rate'] * 100:.3f}% checks passed. It exposed the pre-rollup cost-query latency bottleneck.", + "", + f"The original ramped normal-load attempt recorded {values(invalid_load, 'measured_failures')['rate'] * 100:.2f}% failures after the load generator's public IP changed and was therefore invalid for capacity claims. It is retained as failure evidence but excluded from the supported result. A separate stress profile was not run after the fixed-load capacity boundary was established.", + "", + "## Database optimization evidence", + "", + f"The raw `cost_snapshots` relation occupied {stats['cost_snapshots']['total_size']}; the rollup relation occupied {stats['workspace_cost_daily_rollups']['total_size']}. Post-rollup full-year `EXPLAIN (ANALYZE, BUFFERS)` execution times were {plan_times['summary']:.3f} ms for summary, {plan_times['timeseries']:.3f} ms for timeseries, and {plan_times['byService']:.3f} ms for by-service. All used index-only scans.", + "", + "## Client-observed endpoint diagnostics", + "", + "These values include Internet transit from the load-generator location and are retained for diagnosis, not backend acceptance.", + "", + "| Endpoint | Average | Median | p90 | p95 | Maximum |", + "| --- | ---: | ---: | ---: | ---: | ---: |", +] for label, metric in endpoint_metrics: - metric_values = values(load, metric) + metric_values = values(supported, metric) lines.append( - f"| {label} | {metric_values['p(95)']:.2f} | {metric_values['p(99)']:.2f} | {metric_values['max']:.2f} |" + f"| {label} | {metric_values['avg']:.2f} ms | {metric_values['med']:.2f} ms | " + f"{metric_values['p(90)']:.2f} ms | {metric_values['p(95)']:.2f} ms | {metric_values['max']:.2f} ms |" ) lines.extend([ "", - "## AWS resource metrics", + "## AWS resource metrics at the supported level", "", - "Captured for the measured interval in `cloudwatch-summary.json`.", + f"Peak one-minute ECS CPU was {maximum_average(supported_cw, 'ecs_cpu'):.2f}% and ECS memory was {maximum_average(supported_cw, 'ecs_memory'):.2f}%. Peak RDS CPU was {maximum_average(supported_cw, 'rds_cpu'):.2f}%, the minimum average freeable memory was {minimum_average(supported_cw, 'rds_free_memory') / 1024 / 1024:.2f} MiB, and database connections peaked at {maximum_average(supported_cw, 'rds_connections'):.0f}. The ALB recorded {sum_metric(supported_cw, 'alb_5xx'):.0f} load-balancer-generated 5xx responses and {sum_metric(supported_cw, 'alb_target_5xx'):.0f} target-generated 5xx responses during {supported_requests['count']:,} measured requests.", "", - "## Observed bottlenecks", + "## Observed bottleneck", "", - "Interpret from the preserved k6 and CloudWatch evidence; do not generalize beyond this configuration.", + f"At 50 VUs, p95 exceeded 100 ms in {count_breaches(failed_cw, 'p95', 100)} of 15 one-minute periods and p99 exceeded 250 ms in {count_breaches(failed_cw, 'p99', 250)} period. Peak one-minute ECS CPU reached {maximum_average(failed_cw, 'ecs_cpu'):.2f}% and RDS CPU reached {maximum_average(failed_cw, 'rds_cpu'):.2f}%. Functional reliability still met its target, so server-side tail latency—not widespread request failure—defined the tested capacity boundary.", "", "## Limitations", "", - "Synthetic data, one AWS region, one Fargate task, one load-generator location, HTTP restricted by CIDR, and a single-AZ burstable RDS instance.", + "Synthetic data; one AWS region; one Fargate task; HTTP restricted to one changing public CIDR; one load-generator location; a single-AZ burstable database; 15-minute confirmation windows; no production AWS integrations; and no post-boundary stress run. Results apply only to the documented configuration and workload.", "", - "## Exact reproduction commands", + "## Exact reproduction", "", - "See `../README.md` and the immutable environment metadata in this directory.", + "See `../../README.md`. The final confirmation command was:", + "", + "```bash", + "CAPACITY_LABEL=post-rollup-soak \\", + " bash benchmarks/underflow-api/scripts/run-capacity-test.sh 45 15m", + "```", "", "## Evidence-backed résumé bullet", "", resume, "", "```latex", - "\\item " + resume.replace("%", "\\%"), + "\\item Built and benchmarked a multi-tenant AWS cost-monitoring API against \\textbf{" + records_label + " cost records}, sustaining \\textbf{" + f"{supported_requests['rate']:.2f} requests/second" + "} with \\textbf{" + f"{supported_p95:.2f} ms worst-minute p95 server-side latency" + "} under \\textbf{45 concurrent virtual users}.", "```", "", ]) output.write_text("\n".join(lines), encoding="utf-8") -print(f"Generated {output} from preserved result values") +print(f"Generated {output} from the preserved 45-VU pass and 50-VU failure evidence") PY diff --git a/benchmarks/underflow-api/scripts/run-capacity-test.sh b/benchmarks/underflow-api/scripts/run-capacity-test.sh new file mode 100644 index 0000000..50b5a69 --- /dev/null +++ b/benchmarks/underflow-api/scripts/run-capacity-test.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +set -euo pipefail + +CAPACITY_VUS="${1:-7}" +CAPACITY_DURATION="${2:-3m}" +CAPACITY_LABEL="${CAPACITY_LABEL:-post-rollup}" +CLOUDWATCH_WAIT_SECONDS="${CLOUDWATCH_WAIT_SECONDS:-120}" + +if [[ ! "$CAPACITY_VUS" =~ ^[1-9][0-9]*$ ]] || (( CAPACITY_VUS > 100 )); then + echo "VUs must be an integer from 1 through 100" >&2 + exit 2 +fi + +if [[ ! "$CAPACITY_LABEL" =~ ^[A-Za-z0-9._-]+$ ]]; then + echo "CAPACITY_LABEL may contain only letters, numbers, dots, underscores, and hyphens" >&2 + exit 2 +fi + +for command in aws curl jq k6 terraform tr; do + command -v "$command" >/dev/null 2>&1 || { + echo "$command is required" >&2 + exit 2 + } +done + +REPO_ROOT="${REPO_ROOT:-$(git rev-parse --show-toplevel)}" +TF_ROOT="${TF_ROOT:-$REPO_ROOT/infra/terraform/envs/api-benchmark}" + +tf_output() { + terraform -chdir="$TF_ROOT" output -raw "$1" | tr -d '\r' +} + +AWS_REGION="$(tf_output aws_region)" +PREFIX="$(tf_output resource_prefix)" +BENCHMARK_ID="${PREFIX#underflow-api-bench-}" +BASE_URL="$(tf_output api_base_url)" + +if [[ -z "${BENCHMARK_RESULTS_DIR:-${RESULTS_DIR:-}}" ]]; then + RESULTS_BASE="$REPO_ROOT/benchmarks/underflow-api/results" + shopt -s nullglob + existing_results=("$RESULTS_BASE"/*-"$BENCHMARK_ID") + shopt -u nullglob + if (( ${#existing_results[@]} > 0 )); then + RESULTS_DIR="${existing_results[${#existing_results[@]} - 1]}" + else + RESULTS_DIR="$RESULTS_BASE/$(date -u +%F)-$BENCHMARK_ID" + fi +else + RESULTS_DIR="${BENCHMARK_RESULTS_DIR:-${RESULTS_DIR:-}}" +fi +mkdir -p "$RESULTS_DIR" + +RUN_NAME="capacity-${CAPACITY_VUS}vus-${CAPACITY_LABEL}" +K6_SUMMARY="$RESULTS_DIR/${RUN_NAME}-summary.json" +CLOUDWATCH_SUMMARY="$RESULTS_DIR/${RUN_NAME}-cloudwatch.json" + +if [[ -e "$K6_SUMMARY" || -e "$CLOUDWATCH_SUMMARY" ]]; then + echo "Refusing to overwrite existing evidence for $RUN_NAME" >&2 + echo "Set CAPACITY_LABEL to a new label before rerunning." >&2 + exit 1 +fi + +bash "$REPO_ROOT/benchmarks/underflow-api/scripts/update-load-test-ip.sh" +curl --fail --silent --show-error "$BASE_URL/api/v1/health" >/dev/null + +TEST_EMAIL="benchmark+01@example.invalid" +WORKSPACE_ID="20000000-0000-4000-8000-000000000001" +TEST_PASSWORD="$(aws secretsmanager get-secret-value \ + --region "$AWS_REGION" \ + --secret-id "$PREFIX-runtime" \ + --query SecretString \ + --output text | jq -r '.BENCHMARK_PASSWORD')" +[[ -n "$TEST_PASSWORD" && "$TEST_PASSWORD" != "null" ]] || { + echo "Benchmark password was missing from Secrets Manager" >&2 + exit 1 +} + +echo "Running $CAPACITY_VUS VUs for $CAPACITY_DURATION" +CAPACITY_START="$(date -u +%FT%TZ)" +set +e +RESULTS_DIR="$RESULTS_DIR" \ +SUMMARY_NAME="${RUN_NAME}-summary.json" \ +BASE_URL="$BASE_URL" \ +TEST_EMAIL="$TEST_EMAIL" \ +TEST_PASSWORD="$TEST_PASSWORD" \ +WORKSPACE_ID="$WORKSPACE_ID" \ +CAPACITY_VUS="$CAPACITY_VUS" \ +CAPACITY_DURATION="$CAPACITY_DURATION" \ + k6 run "$REPO_ROOT/benchmarks/underflow-api/k6/capacity.js" +K6_EXIT=$? +set -e +CAPACITY_END="$(date -u +%FT%TZ)" +unset TEST_PASSWORD + +echo "Waiting ${CLOUDWATCH_WAIT_SECONDS}s for CloudWatch datapoints" +sleep "$CLOUDWATCH_WAIT_SECONDS" + +AWS_REGION="$AWS_REGION" \ +ECS_CLUSTER="$(tf_output ecs_cluster_name)" \ +ECS_SERVICE="$(tf_output ecs_service_name)" \ +RDS_IDENTIFIER="$(tf_output rds_identifier)" \ +ALB_ARN_SUFFIX="$(tf_output load_balancer_arn_suffix)" \ +TARGET_GROUP_ARN_SUFFIX="$(tf_output target_group_arn_suffix)" \ +START_TIME="$CAPACITY_START" \ +END_TIME="$CAPACITY_END" \ +OUTPUT_FILE="$CLOUDWATCH_SUMMARY" \ + bash "$REPO_ROOT/benchmarks/underflow-api/scripts/collect-aws-metadata.sh" + +echo "Functional summary" +jq '{ + iterations: .metrics.iterations.values, + requests: .metrics.measured_requests.values, + failures: .metrics.measured_failures.values, + checks: .metrics.checks.values +}' "$K6_SUMMARY" + +echo "Server and resource summary" +jq '{ + alb_latency: .metrics.alb_target_response.datapoints, + rds_cpu: .metrics.rds_cpu.datapoints, + rds_memory: .metrics.rds_free_memory.datapoints, + ecs_cpu: .metrics.ecs_cpu.datapoints +}' "$CLOUDWATCH_SUMMARY" + +if (( K6_EXIT != 0 )); then + echo "FAIL: k6 exited with status $K6_EXIT" >&2 + exit 1 +fi + +bash "$REPO_ROOT/benchmarks/underflow-api/scripts/check-capacity-result.sh" \ + "$K6_SUMMARY" "$CLOUDWATCH_SUMMARY" + +echo "Evidence saved to:" +echo " $K6_SUMMARY" +echo " $CLOUDWATCH_SUMMARY" From 4e6f32c8d97c7ae05fd7eec4918835545fc6a5ae Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Fri, 11 Sep 2026 04:58:21 +0100 Subject: [PATCH 17/19] docs(benchmark): preserve sanitized capacity evidence --- .../results/2026-09-10-c0cf1f68/RESULTS.md | 85 ++ ...capacity-10vus-post-rollup-cloudwatch.json | 233 ++++ .../capacity-10vus-post-rollup-summary.json | 444 ++++++++ .../capacity-2vus-cloudwatch.json | 233 ++++ .../capacity-2vus-summary.json | 492 +++++++++ .../capacity-3vus-cloudwatch.json | 227 ++++ .../capacity-3vus-summary.json | 492 +++++++++ ...ity-45vus-post-rollup-soak-cloudwatch.json | 859 +++++++++++++++ ...pacity-45vus-post-rollup-soak-summary.json | 444 ++++++++ .../capacity-4vus-cloudwatch.json | 233 ++++ .../capacity-4vus-post-rollup-cloudwatch.json | 233 ++++ .../capacity-4vus-post-rollup-summary.json | 492 +++++++++ .../capacity-4vus-summary.json | 492 +++++++++ ...capacity-50vus-post-rollup-cloudwatch.json | 239 +++++ ...ity-50vus-post-rollup-soak-cloudwatch.json | 843 +++++++++++++++ ...pacity-50vus-post-rollup-soak-summary.json | 444 ++++++++ .../capacity-50vus-post-rollup-summary.json | 444 ++++++++ ...capacity-51vus-post-rollup-cloudwatch.json | 227 ++++ .../capacity-51vus-post-rollup-summary.json | 444 ++++++++ ...capacity-52vus-post-rollup-cloudwatch.json | 227 ++++ .../capacity-52vus-post-rollup-summary.json | 444 ++++++++ ...capacity-55vus-post-rollup-cloudwatch.json | 239 +++++ .../capacity-55vus-post-rollup-summary.json | 444 ++++++++ .../capacity-5vus-cloudwatch.json | 233 ++++ .../capacity-5vus-post-rollup-cloudwatch.json | 227 ++++ .../capacity-5vus-post-rollup-summary.json | 444 ++++++++ .../capacity-5vus-summary.json | 492 +++++++++ .../capacity-7vus-post-rollup-cloudwatch.json | 227 ++++ .../capacity-7vus-post-rollup-summary.json | 444 ++++++++ .../cloudwatch-summary.json | 998 ++++++++++++++++++ .../results/2026-09-10-c0cf1f68/dataset.json | 15 + .../2026-09-10-c0cf1f68/environment.json | 107 ++ .../explain-after-covering-index.json | 606 +++++++++++ .../explain-after-index-cold.json | 171 +++ .../explain-after-index-warm.json | 171 +++ .../explain-after-rollup.json | 386 +++++++ .../explain-all-cost-queries-after-index.json | 638 +++++++++++ .../2026-09-10-c0cf1f68/load-summary.json | 492 +++++++++ .../results/2026-09-10-c0cf1f68/rollup.json | 4 + .../smoke-summary-after-covering-index.json | 492 +++++++++ .../smoke-summary-after-index.json | 436 ++++++++ ...smoke-summary-failed-20260909T201023Z.json | 380 +++++++ .../2026-09-10-c0cf1f68/smoke-summary.json | 436 ++++++++ .../2026-09-10-c0cf1f68/terraform-plan.txt | 995 +++++++++++++++++ .../terraform-state-list.txt | 34 + .../warmup-after-covering-index.json | 492 +++++++++ .../warmup-after-index-summary.json | 311 ++++++ 47 files changed, 18185 insertions(+) create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/RESULTS.md create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-10vus-post-rollup-cloudwatch.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-10vus-post-rollup-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-2vus-cloudwatch.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-2vus-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-3vus-cloudwatch.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-3vus-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-45vus-post-rollup-soak-cloudwatch.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-45vus-post-rollup-soak-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-cloudwatch.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-post-rollup-cloudwatch.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-post-rollup-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-cloudwatch.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-soak-cloudwatch.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-soak-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-51vus-post-rollup-cloudwatch.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-51vus-post-rollup-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-52vus-post-rollup-cloudwatch.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-52vus-post-rollup-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-55vus-post-rollup-cloudwatch.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-55vus-post-rollup-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-cloudwatch.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-post-rollup-cloudwatch.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-post-rollup-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-7vus-post-rollup-cloudwatch.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-7vus-post-rollup-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/cloudwatch-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/dataset.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/environment.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-covering-index.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-index-cold.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-index-warm.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-rollup.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-all-cost-queries-after-index.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/load-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/rollup.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-after-covering-index.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-after-index.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-failed-20260909T201023Z.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/terraform-plan.txt create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/terraform-state-list.txt create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/warmup-after-covering-index.json create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/warmup-after-index-summary.json diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/RESULTS.md b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/RESULTS.md new file mode 100644 index 0000000..3b0c862 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/RESULTS.md @@ -0,0 +1,85 @@ +# Underflow API benchmark results + +## Objective + +Measure authenticated, PostgreSQL-backed cost-reporting reads on an isolated, disposable AWS configuration and establish a repeatable server-side capacity boundary. + +## Tested commit + +API commit `2624a6ccafc427521745b1180764fcde963f90bc` using image digest `sha256:b1eec7b5c7652dfab0fe8259ab66f9b0682e228930da769c8d1688961680e644`. + +## Infrastructure + +One ECS Fargate API task with 0.5 vCPU and 1,024 MiB memory, backed by a private, single-AZ `db.t4g.micro` PostgreSQL 16.13 instance with 20 GiB encrypted storage in `us-east-1`. + +## Dataset + +3,650,000 deterministic synthetic cost snapshots across 10 workspaces and 200 synthetic AWS accounts. Workspace/date/service reads use 182,500 daily rollup rows, a 20x row-count reduction. + +## Workload + +Bearer-authenticated read traffic distributed as 30% cost summary, 30% timeseries, 25% by service, 10% AWS account list, and 5% sync history. Login occurs once during setup and is excluded from measured requests. The supported and failing boundary runs each lasted 15 minutes. + +## Acceptance targets + +Measured failure rate `<1%`, checks pass rate `>99%`, and every one-minute ALB `TargetResponseTime` datapoint p50 `<30 ms`, p95 `<100 ms`, and p99 `<250 ms`. ALB target response time excludes load-generator-to-ALB Internet transit. + +## Capacity results + +| Run | Duration | Requests | Requests/s | Failure rate | Checks | Worst p50 | Worst p95 | Worst p99 | Result | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| 45 VUs | 15m | 153,992 | 170.90 | 0.0325% | 99.9752% | 15.22 ms | 89.89 ms | 226.96 ms | Pass | +| 50 VUs | 15m | 169,377 | 187.92 | 0.0549% | 99.9588% | 16.54 ms | 127.47 ms | 265.47 ms | Fail | + +The documented configuration therefore sustained at least **170.90 requests/second at 45 continuously active VUs**. The first sustained failing level tested was 50 VUs; this establishes a tested boundary, not a claim that 45 VUs equals 45 registered users or that 45 is the absolute maximum. + +## Initial smoke and normal-load outcomes + +The initial 2-VU smoke run completed 66 measured requests with 0.000% functional failures and 100.000% checks passed. It exposed the pre-rollup cost-query latency bottleneck. + +The original ramped normal-load attempt recorded 79.88% failures after the load generator's public IP changed and was therefore invalid for capacity claims. It is retained as failure evidence but excluded from the supported result. A separate stress profile was not run after the fixed-load capacity boundary was established. + +## Database optimization evidence + +The raw `cost_snapshots` relation occupied 1423 MB; the rollup relation occupied 52 MB. Post-rollup full-year `EXPLAIN (ANALYZE, BUFFERS)` execution times were 63.087 ms for summary, 11.025 ms for timeseries, and 8.155 ms for by-service. All used index-only scans. + +## Client-observed endpoint diagnostics + +These values include Internet transit from the load-generator location and are retained for diagnosis, not backend acceptance. + +| Endpoint | Average | Median | p90 | p95 | Maximum | +| --- | ---: | ---: | ---: | ---: | ---: | +| Cost summary | 140.52 ms | 130.19 ms | 162.82 ms | 195.43 ms | 1631.54 ms | +| Cost timeseries | 208.21 ms | 141.46 ms | 276.01 ms | 373.12 ms | 6610.41 ms | +| Cost by service | 144.52 ms | 131.19 ms | 167.54 ms | 202.12 ms | 1823.98 ms | +| AWS account list | 148.64 ms | 125.90 ms | 165.70 ms | 221.99 ms | 3462.79 ms | +| Sync history | 139.22 ms | 123.70 ms | 152.18 ms | 191.61 ms | 1638.74 ms | + +## AWS resource metrics at the supported level + +Peak one-minute ECS CPU was 76.21% and ECS memory was 6.93%. Peak RDS CPU was 66.63%, the minimum average freeable memory was 79.84 MiB, and database connections peaked at 10. The ALB recorded 3 load-balancer-generated 5xx responses and 0 target-generated 5xx responses during 153,992 measured requests. + +## Observed bottleneck + +At 50 VUs, p95 exceeded 100 ms in 5 of 15 one-minute periods and p99 exceeded 250 ms in 1 period. Peak one-minute ECS CPU reached 83.85% and RDS CPU reached 71.19%. Functional reliability still met its target, so server-side tail latency—not widespread request failure—defined the tested capacity boundary. + +## Limitations + +Synthetic data; one AWS region; one Fargate task; HTTP restricted to one changing public CIDR; one load-generator location; a single-AZ burstable database; 15-minute confirmation windows; no production AWS integrations; and no post-boundary stress run. Results apply only to the documented configuration and workload. + +## Exact reproduction + +See `../../README.md`. The final confirmation command was: + +```bash +CAPACITY_LABEL=post-rollup-soak \ + bash benchmarks/underflow-api/scripts/run-capacity-test.sh 45 15m +``` + +## Evidence-backed résumé bullet + +Built and benchmarked a multi-tenant AWS cost-monitoring API against 3.65M cost records, sustaining 170.90 requests/second with 89.89 ms worst-minute p95 server-side latency under 45 concurrent virtual users. + +```latex +\item Built and benchmarked a multi-tenant AWS cost-monitoring API against \textbf{3.65M cost records}, sustaining \textbf{170.90 requests/second} with \textbf{89.89 ms worst-minute p95 server-side latency} under \textbf{45 concurrent virtual users}. +``` diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-10vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-10vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..9dd9028 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-10vus-post-rollup-cloudwatch.json @@ -0,0 +1,233 @@ +{ + "startTime": "2026-09-10T18:18:20Z", + "endTime": "2026-09-10T18:21:22Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.07967098018474315, + "p90": 0.01480722002149371, + "p50": 0.011078576614027212, + "p95": 0.01778017725998972 + } + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.05131022634561018, + "p90": 0.014367543387150064, + "p50": 0.010959019176451491, + "p95": 0.016009033974398738 + } + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.055445969415233325, + "p90": 0.014988910955111562, + "p50": 0.011117015799361602, + "p95": 0.02148186070452684 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 15.721466579901366, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 22.787106196085613, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 22.53944714864095, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 6.396484375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 6.4453125, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 6.510416666666667, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 5.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 3.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 5.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 4.9305393610287505, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 18.993249437453123, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 19.44902754862257, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 135.18083483333334, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 76001280.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 90918912.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 83263488.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 0.002413793103448276, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 0.0012627986348122866, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 0.00034129692832764505, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:18:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:19:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:20:00+01:00", + "Average": 0.0004347826086956522, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-10vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-10vus-post-rollup-summary.json new file mode 100644 index 0000000..ed1a64b --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-10vus-post-rollup-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 390, + "fails": 0 + }, + { + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 390, + "fails": 0 + }, + { + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 390, + "fails": 0, + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid" + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 390, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 2340 + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 2340, + "fails": 0 + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 2340, + "fails": 0 + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 2340, + "fails": 0 + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 780, + "fails": 0 + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 780, + "fails": 0 + }, + { + "passes": 780, + "fails": 0, + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2" + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 780, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 1950 + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 1950, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 1950 + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 1950, + "fails": 0 + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 2340, + "fails": 0 + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 2340, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 2340 + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 2340, + "fails": 0 + } + ], + "name": "", + "path": "" + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "testRunDurationMs": 180996.5683, + "isStdOutTTY": true, + "isStdErrTTY": true + }, + "metrics": { + "http_req_tls_handshaking": { + "contains": "time", + "values": { + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0 + }, + "type": "trend" + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 129.28671610256404, + "min": 114.2435, + "med": 126.31815, + "max": 492.9923, + "p(90)": 132.0206, + "p(95)": 137.0889 + } + }, + "http_req_duration": { + "values": { + "avg": 130.4328539546217, + "min": 108.3892, + "med": 126.6433, + "max": 1177.3111, + "p(90)": 133.2004, + "p(95)": 137.9782 + }, + "type": "trend", + "contains": "time" + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 31200, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "http_req_connecting": { + "contains": "time", + "values": { + "avg": 0.16524775028842456, + "min": 0, + "med": 0, + "max": 122.3256, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend" + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "max": 0.5032, + "p(90)": 0, + "p(95)": 0, + "avg": 6.450455069862837E-05, + "min": 0, + "med": 0 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 124.03695423076923, + "min": 108.3892, + "med": 119.81190000000001, + "max": 803.2904, + "p(90)": 125.4834, + "p(95)": 129.17446499999997 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 145.19138999999996, + "avg": 137.0559506837606, + "min": 116.6628, + "med": 128.9137, + "max": 1177.3111, + "p(90)": 135.39408 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 7801 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 7800 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_reqs": { + "values": { + "count": 7801, + "rate": 43.10026468054311 + }, + "type": "counter", + "contains": "default" + }, + "measured_requests": { + "contains": "default", + "values": { + "count": 7800, + "rate": 43.09473971391313 + }, + "type": "counter" + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 109.4097, + "med": 119.7698, + "max": 688.5023, + "p(90)": 124.95237, + "p(95)": 126.39524, + "avg": 122.81241153846159 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 7800, + "rate": 43.09473971391313 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 10, + "min": 10, + "max": 10 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 230.9306075128205, + "min": 208.6462, + "med": 227.03505, + "max": 1277.5961, + "p(90)": 233.67709, + "p(95)": 238.31685 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 5003592, + "rate": 27644.678830079232 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 124.82565, + "max": 493.9183, + "p(90)": 129.97223, + "p(95)": 133.4991549999998, + "avg": 127.93604846153845, + "min": 113.0858 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "med": 125.6587, + "max": 715.1738, + "p(90)": 132.5007, + "p(95)": 135.7257, + "avg": 128.1199522881682, + "min": 108.3892 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 72206634, + "rate": 398939.2433138192 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 130.4328539546217, + "min": 108.3892, + "med": 126.6433, + "max": 1177.3111, + "p(90)": 133.2004, + "p(95)": 137.9782 + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0.16633584155877448, + "min": 0, + "med": 0, + "max": 122.3256 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 10, + "min": 10, + "max": 10 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 130.36359476923124, + "min": 108.3892, + "med": 126.63905, + "max": 1177.3111, + "p(90)": 133.19968, + "p(95)": 137.96328499999998 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 2.3128371619023183, + "min": 0, + "med": 0.3316, + "max": 848.2044, + "p(90)": 3.271, + "p(95)": 4.8441 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-2vus-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-2vus-cloudwatch.json new file mode 100644 index 0000000..bbbf6c6 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-2vus-cloudwatch.json @@ -0,0 +1,233 @@ +{ + "startTime": "2026-09-10T14:20:37Z", + "endTime": "2026-09-10T14:23:38Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.4469220790572148, + "p90": 0.13038611785583315, + "p50": 0.07453238033386977, + "p95": 0.15222797425470158 + } + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.2615802315178325, + "p90": 0.12640921432900043, + "p50": 0.07695024109803843, + "p95": 0.14644058085164235 + } + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.20681019823844912, + "p90": 0.11525497158473341, + "p50": 0.07536445397883569, + "p95": 0.13858705730247767 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 2.8502402305603027, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 4.901556015014648, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 4.135225772857666, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 0.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 2.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 2.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 3.968584815994397, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 36.016666666666666, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 43.94396228615215, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 117.7651533, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 101449728.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 92250112.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 85606400.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 0.0008333333333333334, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 0.01, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:20:00+01:00", + "Average": 0.0005660377358490566, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:21:00+01:00", + "Average": 0.0007142857142857143, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:22:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-2vus-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-2vus-summary.json new file mode 100644 index 0000000..30e676a --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-2vus-summary.json @@ -0,0 +1,492 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 331, + "fails": 0 + }, + { + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 331, + "fails": 0, + "name": "cost_summary: response is JSON" + }, + { + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 331, + "fails": 0, + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid" + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 331, + "fails": 0 + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 331, + "fails": 0 + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 331, + "fails": 0 + }, + { + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 331, + "fails": 0, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid" + }, + { + "fails": 0, + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 331 + }, + { + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 275, + "fails": 0 + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 275, + "fails": 0 + }, + { + "passes": 275, + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937" + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 275, + "fails": 0 + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 110, + "fails": 0 + }, + { + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 110, + "fails": 0, + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON" + }, + { + "fails": 0, + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 110 + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 110, + "fails": 0 + }, + { + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 55, + "fails": 0, + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200" + }, + { + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 55, + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON" + }, + { + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 55, + "fails": 0 + }, + { + "passes": 55, + "fails": 0, + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d" + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 181055.0222 + }, + "metrics": { + "measured_failures": { + "thresholds": { + "rate<0.01": { + "ok": true + } + }, + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 1102 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 707168, + "rate": 3905.818194972998 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 1103 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 1, + "min": 1, + "max": 2 + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 1102, + "rate": 6.08654754013225 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 1103, + "rate": 6.092070723018033 + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 111.1337, + "med": 192.2361, + "max": 3316.7455, + "p(90)": 258.65104, + "p(95)": 362.6146499999999, + "avg": 226.65930571169542 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0 + } + }, + "http_req_receiving": { + "contains": "time", + "values": { + "p(95)": 8.661789999999984, + "avg": 21.79715593834995, + "min": 0, + "med": 0.5157, + "max": 3024.6642, + "p(90)": 5.43742 + }, + "type": "trend" + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.3155878513145966, + "min": 0, + "med": 0, + "max": 122.0562, + "p(90)": 0, + "p(95)": 0 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 4408, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 1444.6597, + "p(90)": 130.72054, + "p(95)": 167.45368999999974, + "avg": 159.6348818181818, + "min": 117.3265, + "med": 119.4087 + }, + "thresholds": { + "p(99)<500": { + "ok": false + }, + "p(95)<200": { + "ok": true + } + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 169.62410636363634, + "min": 111.1337, + "med": 120.11015, + "max": 3291.5441, + "p(90)": 134.77812000000003, + "p(95)": 175.40819 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": false + } + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 292.69355, + "max": 3416.9556, + "p(90)": 358.76344, + "p(95)": 470.00490000000036, + "avg": 326.8671863883846, + "min": 211.4726 + } + }, + "endpoint_cost_by_service_duration": { + "contains": "time", + "values": { + "avg": 248.13789818181814, + "min": 197.4729, + "med": 214.6382, + "max": 1479.5777, + "p(90)": 269.5189, + "p(95)": 362.23875 + }, + "thresholds": { + "p(99)<500": { + "ok": false + }, + "p(95)<200": { + "ok": false + } + }, + "type": "trend" + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 10207627, + "rate": 56378.590750850766 + } + }, + "measured_duration": { + "thresholds": { + "p(99)<500": { + "ok": false + }, + "p(95)<200": { + "ok": false + } + }, + "type": "trend", + "contains": "time", + "values": { + "p(90)": 258.24661999999995, + "p(95)": 362.03845, + "avg": 226.20558348457357, + "min": 111.1337, + "med": 192.21555, + "max": 3316.7455 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "max": 1477.6435, + "p(90)": 246.01572000000007, + "p(95)": 278.9438899999998, + "avg": 204.86214977334555, + "min": 109.1615, + "med": 189.6664 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 1102, + "rate": 6.08654754013225 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 226.65930571169542, + "min": 111.1337, + "med": 192.2361, + "max": 3316.7455, + "p(90)": 258.65104, + "p(95)": 362.6146499999999 + } + }, + "http_req_blocked": { + "contains": "time", + "values": { + "max": 128.3212, + "p(90)": 0, + "p(95)": 0, + "avg": 0.321267815049864, + "min": 0, + "med": 0 + }, + "type": "trend" + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 170.0405, + "med": 183.3613, + "max": 931.3722, + "p(90)": 227.3473, + "p(95)": 244.928, + "avg": 201.63510211480371 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 262.4194622356497, + "min": 178.3248, + "med": 191.0472, + "max": 3316.7455, + "p(90)": 322.0324, + "p(95)": 454.4524 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-3vus-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-3vus-cloudwatch.json new file mode 100644 index 0000000..c7cab4c --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-3vus-cloudwatch.json @@ -0,0 +1,227 @@ +{ + "startTime": "2026-09-10T14:52:04Z", + "endTime": "2026-09-10T14:55:06Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.3216822121114268, + "p90": 0.15344186149861608, + "p50": 0.08822177870264457, + "p95": 0.178307124267672 + } + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.275133227621956, + "p90": 0.14913721967881835, + "p50": 0.08592872289167662, + "p95": 0.1697540344935103 + } + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.262079332238483, + "p90": 0.14766303987496063, + "p50": 0.0901778833758862, + "p95": 0.17626883014707334 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 4.248351162920396, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 5.800196806589763, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 4.955207506815593, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 3.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 3.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 3.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 22.277434093145022, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 57.641666666666666, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 55.647176411794106, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 89739264.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 90087424.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 87236608.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 0.000273972602739726, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 0.002631578947368421, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T15:52:00+01:00", + "Average": 0.0005263157894736842, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:53:00+01:00", + "Average": 0.008511705685618728, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T15:54:00+01:00", + "Average": 0.000625, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-3vus-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-3vus-summary.json new file mode 100644 index 0000000..4e9acf9 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-3vus-summary.json @@ -0,0 +1,492 @@ +{ + "root_group": { + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 482, + "fails": 0 + }, + { + "passes": 482, + "fails": 0, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c" + }, + { + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 482, + "fails": 0, + "name": "cost_summary: response shape is valid" + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 482, + "fails": 0 + }, + { + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 401, + "fails": 0 + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 401, + "fails": 0 + }, + { + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 401, + "fails": 0 + }, + { + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 401, + "fails": 0, + "name": "cost_by_service: no auth or server error" + }, + { + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 481, + "fails": 0, + "name": "cost_timeseries: status is 200" + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 481, + "fails": 0 + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 481, + "fails": 0 + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 481, + "fails": 0 + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 160, + "fails": 0 + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 160, + "fails": 0 + }, + { + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 160, + "fails": 0, + "name": "aws_account_list: response shape is valid" + }, + { + "fails": 0, + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 160 + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 80, + "fails": 0 + }, + { + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 80, + "fails": 0 + }, + { + "passes": 80, + "fails": 0, + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc" + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 80, + "fails": 0 + } + ], + "name": "" + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdErrTTY": true, + "testRunDurationMs": 180977.8361, + "isStdOutTTY": true + }, + "metrics": { + "measured_duration": { + "thresholds": { + "p(99)<500": { + "ok": false + }, + "p(95)<200": { + "ok": false + } + }, + "type": "trend", + "contains": "time", + "values": { + "max": 1223.4112, + "p(90)": 314.64201, + "p(95)": 376.4216349999997, + "avg": 236.23136197007494, + "min": 111.767, + "med": 213.53379999999999 + } + }, + "measured_failures": { + "thresholds": { + "rate<0.01": { + "ok": true + } + }, + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 1604 + } + }, + "endpoint_sync_history_duration": { + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time", + "values": { + "avg": 131.49302375, + "min": 115.1849, + "med": 122.44595, + "max": 583.3741, + "p(90)": 135.3736, + "p(95)": 148.8091 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 236.5069182554519, + "min": 111.767, + "med": 213.5847, + "max": 1223.4112, + "p(90)": 314.74686, + "p(95)": 377.31175999999994 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.28470255451713394, + "min": 0, + "med": 0, + "max": 114.5268, + "p(90)": 0, + "p(95)": 0 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 113.69002, + "p(95)": 115.72668, + "avg": 25.1699171339564, + "min": 0, + "med": 0.5669, + "max": 1015.2661 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "min": 109.9895, + "med": 204.6345, + "max": 874.686, + "p(90)": 268.07598, + "p(95)": 297.1818399999999, + "avg": 211.33674423676 + } + }, + "data_sent": { + "values": { + "rate": 5686.795809799165, + "count": 1029184 + }, + "type": "counter", + "contains": "data" + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 1605, + "rate": 8.86848928347862 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "rate": 8.86296374498424, + "count": 1604 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.00025688473520249223, + "min": 0, + "med": 0, + "max": 0.4123, + "p(90)": 0, + "p(95)": 0 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "passes": 6416, + "fails": 0, + "rate": 1 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "max": 3, + "value": 3, + "min": 3 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 3, + "min": 3, + "max": 3 + } + }, + "http_req_duration": { + "contains": "time", + "values": { + "p(90)": 314.74686, + "p(95)": 377.31175999999994, + "avg": 236.5069182554519, + "min": 111.767, + "med": 213.5847, + "max": 1223.4112 + }, + "type": "trend" + }, + "http_req_failed": { + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 1605 + }, + "type": "rate" + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 14840765, + "rate": 82003.21829353557 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 291.92862993762975, + "min": 179.9106, + "med": 240.5608, + "max": 1223.4112, + "p(90)": 405.7211, + "p(95)": 590.2395 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 336.9036885286786, + "min": 212.1592, + "med": 313.86969999999997, + "max": 1323.9538, + "p(90)": 415.08569, + "p(95)": 477.61089 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 172.7006, + "med": 205.39325, + "max": 554.7547, + "p(90)": 255.6989, + "p(95)": 276.23161, + "avg": 216.18656639004152 + }, + "thresholds": { + "p(99)<500": { + "ok": true + }, + "p(95)<200": { + "ok": false + } + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0 + } + }, + "endpoint_aws_account_list_duration": { + "values": { + "max": 786.5083, + "p(90)": 139.15415000000002, + "p(95)": 197.9365599999998, + "avg": 141.850551875, + "min": 111.767, + "med": 120.71719999999999 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": false + } + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 237.589, + "max": 1180.7206, + "p(90)": 291.9862, + "p(95)": 333.3501, + "avg": 252.0697715710724, + "min": 203.0169 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "measured_requests": { + "values": { + "count": 1604, + "rate": 8.86296374498424 + }, + "type": "counter", + "contains": "default" + }, + "http_req_blocked": { + "values": { + "avg": 0.29025925233644856, + "min": 0, + "med": 0, + "max": 123.2969, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend", + "contains": "time" + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-45vus-post-rollup-soak-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-45vus-post-rollup-soak-cloudwatch.json new file mode 100644 index 0000000..d48d3e1 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-45vus-post-rollup-soak-cloudwatch.json @@ -0,0 +1,859 @@ +{ + "startTime": "2026-09-10T21:07:35Z", + "endTime": "2026-09-10T21:22:36Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Sum": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Sum": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Sum": 1.0, + "Unit": "Count" + } + ] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.22696422849622452, + "p90": 0.03841808723311325, + "p50": 0.012767480580308825, + "p95": 0.08663686471205145 + } + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.12127545537418484, + "p90": 0.034861958298139366, + "p50": 0.01334601622259427, + "p95": 0.066006978014974 + } + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.15553184370961143, + "p90": 0.046290560453668486, + "p50": 0.014499876365229116, + "p95": 0.08189163656668233 + } + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.14957885435136659, + "p90": 0.055556492335232244, + "p50": 0.014669070692200284, + "p95": 0.0892821045873521 + } + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.13140495969438273, + "p90": 0.0327728163515166, + "p50": 0.013030088230694301, + "p95": 0.06352527315394052 + } + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1394391257218174, + "p90": 0.05100416210263806, + "p50": 0.013983387667723654, + "p95": 0.08133548737421174 + } + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1304723344686426, + "p90": 0.03894339707974348, + "p50": 0.014039776647280941, + "p95": 0.0682588712314812 + } + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1292886727605574, + "p90": 0.0355498927717731, + "p50": 0.01330494667270981, + "p95": 0.06552079636554493 + } + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.12057463938746758, + "p90": 0.05183884785424821, + "p50": 0.014719794287451161, + "p95": 0.08010717484458531 + } + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.12272407084884956, + "p90": 0.02965406917127382, + "p50": 0.01293879627331918, + "p95": 0.05889423773359602 + } + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.11532587931874197, + "p90": 0.04470771790836263, + "p50": 0.01461153690611638, + "p95": 0.07023591080244401 + } + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.16658662201219457, + "p90": 0.05131801693381506, + "p50": 0.015001231002625093, + "p95": 0.08203423009262112 + } + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1429754692803315, + "p90": 0.05179071890944367, + "p50": 0.015222384966804284, + "p95": 0.08382716786850573 + } + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.16332998520837083, + "p90": 0.05782130915813708, + "p50": 0.014950578453640933, + "p95": 0.08989301919448041 + } + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.14521047708988097, + "p90": 0.03558024893885173, + "p50": 0.01367234586144698, + "p95": 0.07367894105208644 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 19.006610927482445, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 71.08097585042317, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 75.06921895345052, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 73.82374827067058, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 66.42612202962239, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 72.2046381632487, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 72.75210571289062, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 67.82080332438152, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 74.06728871663411, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 63.4721425374349, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 75.20867665608723, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 74.73727671305339, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 76.2130126953125, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 75.22903442382812, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 64.43630854288737, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 6.901041666666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 6.884765625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 6.901041666666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 6.884765625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 10.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 3.7185259296314825, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 48.449999999999996, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 60.64164709409825, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 66.40318842029782, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 58.09166666666666, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 58.19166666666666, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 63.633333333333326, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 57.08074430691831, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 64.2393823681446, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 55.93333333333334, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 62.0054967935371, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 64.52311897856451, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 65.89851761684814, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 66.38848633251712, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 66.62944196279751, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 116.5389658, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 111.45983285, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 106.11632803333333, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 100311040.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 94199808.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 91463680.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 87752704.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 83718144.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 94822400.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 93990912.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 91389952.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 86355968.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 84824064.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 102236160.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 95895552.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 94117888.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 88563712.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 87482368.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 0.001, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 0.0025, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 0.0036075949367088606, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 0.0007142857142857143, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 8.064516129032258e-05, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 0.002, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 0.0008333333333333334, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T22:07:00+01:00", + "Average": 0.0063686131386861316, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:08:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:09:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:10:00+01:00", + "Average": 0.00020833333333333335, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:11:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:12:00+01:00", + "Average": 0.009750889679715304, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:13:00+01:00", + "Average": 0.0005, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:14:00+01:00", + "Average": 0.000425531914893617, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:15:00+01:00", + "Average": 0.00036363636363636367, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:16:00+01:00", + "Average": 0.00020833333333333335, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:17:00+01:00", + "Average": 0.007356115107913669, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:18:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:19:00+01:00", + "Average": 0.0004347826086956522, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:20:00+01:00", + "Average": 0.00017857142857142857, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T22:21:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-45vus-post-rollup-soak-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-45vus-post-rollup-soak-summary.json new file mode 100644 index 0000000..1346578 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-45vus-post-rollup-soak-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 38485, + "fails": 13, + "name": "cost_by_service: status is 200" + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 38498, + "fails": 0 + }, + { + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 38485, + "fails": 13 + }, + { + "fails": 13, + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 38485 + }, + { + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 46183, + "fails": 14, + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200" + }, + { + "fails": 1, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 46196 + }, + { + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 46183, + "fails": 14, + "name": "cost_summary: response shape is valid" + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 46183, + "fails": 14 + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 15393, + "fails": 6 + }, + { + "passes": 15398, + "fails": 1, + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516" + }, + { + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 15393, + "fails": 6, + "name": "aws_account_list: response shape is valid" + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 15393, + "fails": 6 + }, + { + "fails": 15, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 46183 + }, + { + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 46197, + "fails": 1, + "name": "cost_timeseries: response is JSON" + }, + { + "passes": 46183, + "fails": 15, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131" + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 46183, + "fails": 15 + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 7698, + "fails": 2 + }, + { + "passes": 7700, + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe" + }, + { + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 7698, + "fails": 2 + }, + { + "passes": 7698, + "fails": 2, + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d" + } + ], + "name": "" + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 901061.1801 + }, + "metrics": { + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 6710.6686, + "p(90)": 344.09765999999996, + "p(95)": 360.70271999999994, + "avg": 263.03108682269135, + "min": 205.1959, + "med": 232.58125 + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.037130415668244675, + "min": 0, + "med": 0, + "max": 137.8727, + "p(90)": 0, + "p(95)": 0 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "passes": 50, + "fails": 153943, + "rate": 0.0003246900833154754 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 195.43402, + "avg": 140.52127147433748, + "min": 110.9487, + "med": 130.1859, + "max": 1631.5384, + "p(90)": 162.82008000000002 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "avg": 9.886228594806257E-05, + "min": 0, + "med": 0, + "max": 0.6337, + "p(90)": 0, + "p(95)": 0 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 98778449, + "rate": 109624.57509160203 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 260.1895499999999, + "avg": 162.57455058379756, + "min": 105.1924, + "med": 132.15019999999998, + "max": 6610.4124, + "p(90)": 243.62148 + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 162.57810323131667, + "min": 105.1924, + "med": 132.1503, + "max": 6610.4124, + "p(90)": 243.62272000000002, + "p(95)": 260.2007199999999 + } + }, + "http_req_waiting": { + "contains": "time", + "values": { + "p(90)": 164.22142000000002, + "p(95)": 198.62101999999993, + "avg": 142.03721965803447, + "min": 105.1924, + "med": 130.9389, + "max": 1742.544 + }, + "type": "trend" + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 141.46480000000003, + "max": 6610.4124, + "p(90)": 276.0128, + "p(95)": 373.121465, + "avg": 208.20745710853464, + "min": 113.3273 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 153993, + "rate": 170.90182487154738 + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "rate": 170.90071506899224, + "count": 153992 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "med": 132.1538, + "max": 6610.4124, + "p(90)": 243.63308000000004, + "p(95)": 260.22270999999995, + "avg": 162.59244396172707, + "min": 105.1924 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 1425079623, + "rate": 1581557.0068636674 + } + }, + "endpoint_aws_account_list_duration": { + "contains": "time", + "values": { + "avg": 148.64191929995468, + "min": 105.1924, + "med": 125.9046, + "max": 3462.7897, + "p(90)": 165.69952000000004, + "p(95)": 221.98865999999987 + }, + "type": "trend" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0.9997516104732713, + "passes": 615815, + "fails": 153 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "iterations": { + "contains": "default", + "values": { + "count": 153992, + "rate": 170.90071506899224 + }, + "type": "counter" + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 110.8367, + "med": 123.70125, + "max": 1638.7445, + "p(90)": 152.17794, + "p(95)": 191.6138749999998, + "avg": 139.21970505194807 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.03706307299682454, + "min": 0, + "med": 0, + "max": 131.9592, + "p(90)": 0, + "p(95)": 0 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0.00032469219180217154, + "passes": 50, + "fails": 153942 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 6469.2012, + "p(90)": 112.06322000000002, + "p(95)": 118.33564, + "avg": 20.5407847109933 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "max": 45, + "value": 16, + "min": 16 + } + }, + "vus_max": { + "values": { + "max": 45, + "value": 45, + "min": 45 + }, + "type": "gauge", + "contains": "default" + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 144.5223717465835, + "min": 112.9334, + "med": 131.18675, + "max": 1823.9798, + "p(90)": 167.54245, + "p(95)": 202.11836000000005 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-cloudwatch.json new file mode 100644 index 0000000..5074f01 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-cloudwatch.json @@ -0,0 +1,233 @@ +{ + "startTime": "2026-09-10T15:05:09Z", + "endTime": "2026-09-10T15:08:11Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.5330995033881941, + "p90": 0.20571109858608008, + "p50": 0.11230603773633399, + "p95": 0.24711965236213634 + } + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.33188158154488734, + "p90": 0.1878762817176436, + "p50": 0.10822247509484144, + "p95": 0.21563325793715646 + } + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.3072971678957348, + "p90": 0.19351794759431754, + "p50": 0.1084316882746557, + "p95": 0.2143750914823625 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 4.707579294840495, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 6.98672358194987, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 6.7127617200215655, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 6.673177083333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 6.689453125, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 6.73828125, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 4.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 3.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 4.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 21.3753562559376, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 64.99808240649648, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 65.79100797921073, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 117.83615691666667, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 94597120.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 95064064.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 77701120.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 0.002, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 0.00013793103448275863, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 0.002, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T16:05:00+01:00", + "Average": 0.00025316455696202533, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T16:06:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T16:07:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..2aa10a0 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-post-rollup-cloudwatch.json @@ -0,0 +1,233 @@ +{ + "startTime": "2026-09-10T16:59:43Z", + "endTime": "2026-09-10T17:02:45Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.10853015813738111, + "p90": 0.014967646517032092, + "p50": 0.011690917322607587, + "p95": 0.017136104420685527 + } + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.15150425628671077, + "p90": 0.016295435084666084, + "p50": 0.011824906477696875, + "p95": 0.024563172947103407 + } + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.04178329584805352, + "p90": 0.015012652418217379, + "p50": 0.011604221704336868, + "p95": 0.01809424324589384 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 5.039510101079941, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 10.869722684224447, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 10.522063573201498, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 4.182942708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 5.696614583333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 6.070963541666667, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 0.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 2.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 2.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 3.4934133733533437, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 8.95, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 9.625, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 127.73084505, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 100794368.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 88588288.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 80076800.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 0.00022988505747126436, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 0.0002786377708978328, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 0.00145985401459854, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T17:59:00+01:00", + "Average": 0.0002, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:00:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:01:00+01:00", + "Average": 0.005596026490066225, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-post-rollup-summary.json new file mode 100644 index 0000000..7960265 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-post-rollup-summary.json @@ -0,0 +1,492 @@ +{ + "root_group": { + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 715, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 715 + }, + { + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 715, + "fails": 0 + }, + { + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 715, + "fails": 0, + "name": "cost_by_service: no auth or server error" + }, + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 859, + "fails": 0 + }, + { + "passes": 859, + "fails": 0, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c" + }, + { + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 859, + "fails": 0, + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid" + }, + { + "passes": 859, + "fails": 0, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f" + }, + { + "passes": 860, + "fails": 0, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa" + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 860, + "fails": 0 + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 860, + "fails": 0 + }, + { + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 860, + "fails": 0, + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error" + }, + { + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 286, + "fails": 0, + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200" + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 286, + "fails": 0 + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 286, + "fails": 0 + }, + { + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 286, + "fails": 0, + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error" + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 144, + "fails": 0 + }, + { + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 144, + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON" + }, + { + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 144, + "fails": 0 + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 144, + "fails": 0 + } + ], + "name": "", + "path": "" + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 181058.8887 + }, + "metrics": { + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 2864, + "rate": 15.818057984136958 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 2864, + "rate": 15.818057984136958 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 248.50261999999992, + "avg": 151.18177898778413, + "min": 109.0623, + "med": 127.4804, + "max": 2717.0338, + "p(90)": 240.85442 + } + }, + "http_req_failed": { + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 2865 + }, + "type": "rate" + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 240.85442, + "p(95)": 248.50261999999992, + "avg": 151.18177898778413, + "min": 109.0623, + "med": 127.4804, + "max": 2717.0338 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 119.0387, + "p(90)": 0, + "p(95)": 0, + "avg": 0.2013464572425829 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "fails": 0, + "rate": 1, + "passes": 11456 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 132.18870181818156, + "min": 114.6623, + "med": 126.7956, + "max": 830.597, + "p(90)": 135.99886, + "p(95)": 142.25488 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "endpoint_cost_summary_duration": { + "contains": "time", + "values": { + "max": 836.0096, + "p(90)": 134.83686, + "p(95)": 143.32411, + "avg": 132.18542409778811, + "min": 113.6895, + "med": 125.277 + }, + "thresholds": { + "p(99)<500": { + "ok": true + }, + "p(95)<200": { + "ok": true + } + }, + "type": "trend" + }, + "http_req_receiving": { + "contains": "time", + "values": { + "avg": 20.356390541012203, + "min": 0, + "med": 0.5418, + "max": 2062.5126, + "p(90)": 112.49145999999999, + "p(95)": 116.43809999999996 + }, + "type": "trend" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 1837389, + "rate": 10148.018764460692 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 350.005825, + "avg": 251.59314270251315, + "min": 209.3998, + "med": 227.97475, + "max": 2817.3962, + "p(90)": 341.48843000000005 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 4, + "min": 4, + "max": 4 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 760.5899, + "p(90)": 126.71690000000001, + "p(95)": 135.55020000000002, + "avg": 131.2874493006994, + "min": 109.0623, + "med": 119.9335 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "avg": 130.82538844677123, + "min": 109.0623, + "med": 126.1781, + "max": 830.0697, + "p(90)": 134.30367999999999, + "p(95)": 141.82551999999998 + } + }, + "endpoint_sync_history_duration": { + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time", + "values": { + "max": 615.5881, + "p(90)": 127.61873000000001, + "p(95)": 134.280035, + "avg": 126.67868611111112, + "min": 115.5742, + "med": 119.7502 + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "max": 125.5202, + "p(90)": 0, + "p(95)": 0, + "avg": 0.2049466317626527, + "min": 0, + "med": 0 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 26529049, + "rate": 146521.66038617687 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 2864 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 150.98240593575477, + "min": 109.0623, + "med": 127.4751, + "max": 2717.0338, + "p(90)": 240.84785, + "p(95)": 248.24059499999998 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 118.2723, + "med": 133.357, + "max": 2717.0338, + "p(90)": 252.21209000000002, + "p(95)": 346.2328099999998, + "avg": 196.00170720930205 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 2865, + "rate": 15.823581049075553 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 1, + "min": 1, + "max": 4 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-summary.json new file mode 100644 index 0000000..3192246 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-4vus-summary.json @@ -0,0 +1,492 @@ +{ + "root_group": { + "groups": [], + "checks": [ + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 567, + "fails": 0 + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 567, + "fails": 0 + }, + { + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 567, + "fails": 0, + "name": "cost_summary: response shape is valid" + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 567, + "fails": 0 + }, + { + "passes": 473, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b" + }, + { + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 473, + "fails": 0, + "name": "cost_by_service: response is JSON" + }, + { + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 473, + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid" + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 473, + "fails": 0 + }, + { + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 568, + "fails": 0, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200" + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 568, + "fails": 0 + }, + { + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 568, + "fails": 0, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid" + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 568, + "fails": 0 + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 189, + "fails": 0 + }, + { + "passes": 189, + "fails": 0, + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516" + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 189, + "fails": 0 + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 189, + "fails": 0 + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 95, + "fails": 0 + }, + { + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 95, + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON" + }, + { + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 95, + "fails": 0 + }, + { + "fails": 0, + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 95 + } + ], + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e" + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 181152.8998 + }, + "metrics": { + "checks": { + "contains": "default", + "values": { + "rate": 1, + "passes": 7568, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate" + }, + "http_reqs": { + "values": { + "count": 1893, + "rate": 10.449736118438883 + }, + "type": "counter", + "contains": "default" + }, + "http_req_duration{expected_response:true}": { + "values": { + "min": 109.7353, + "med": 253.0464, + "max": 1848.1695, + "p(90)": 392.87459999999993, + "p(95)": 498.0584399999997, + "avg": 280.45375324881155 + }, + "type": "trend", + "contains": "time" + }, + "http_req_waiting": { + "contains": "time", + "values": { + "p(90)": 317.15079999999995, + "p(95)": 352.3152999999997, + "avg": 239.33004352879004, + "min": 109.7353, + "med": 228.5054, + "max": 1016.7855 + }, + "type": "trend" + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 1213904, + "rate": 6700.991269475665 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 1892 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 17525377, + "rate": 96743.56314112947 + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 997.2257, + "p(90)": 336.24538, + "p(95)": 396.10353999999984, + "avg": 281.8976847780127, + "min": 202.4459, + "med": 260.235 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 1892, + "rate": 10.444215919749798 + } + }, + "endpoint_cost_summary_duration": { + "values": { + "avg": 242.40121763668418, + "min": 176.2123, + "med": 230.3304, + "max": 748.1477, + "p(90)": 298.06056, + "p(95)": 333.2647599999999 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + }, + "type": "trend", + "contains": "time" + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0.311429265715795, + "min": 0, + "med": 0, + "max": 121.1994, + "p(90)": 0 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 379.31989630281686, + "min": 185.7742, + "med": 333.6333, + "max": 1848.1695, + "p(90)": 496.58567, + "p(95)": 732.1242349999997 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 4, + "min": 4, + "max": 4 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 41.12370972002117, + "min": 0, + "med": 0.8435, + "max": 1542.8107, + "p(90)": 118.36088, + "p(95)": 125.69785999999998 + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 1892, + "rate": 10.444215919749798 + } + }, + "http_req_connecting": { + "contains": "time", + "values": { + "avg": 0.30842921288959324, + "min": 0, + "med": 0, + "max": 119.2845, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend" + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "fails": 1893, + "rate": 0, + "passes": 0 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 214.3145, + "med": 353.30425, + "max": 1948.608, + "p(90)": 493.588, + "p(95)": 598.6581749999995, + "avg": 380.90038319238965 + } + }, + "endpoint_aws_account_list_duration": { + "values": { + "avg": 158.1309185185185, + "min": 109.7353, + "med": 126.4681, + "max": 812.4448, + "p(90)": 147.97592000000003, + "p(95)": 211.26947999999962 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + }, + "type": "trend", + "contains": "time" + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 280.45375324881155, + "min": 109.7353, + "med": 253.0464, + "max": 1848.1695, + "p(90)": 392.87459999999993, + "p(95)": 498.0584399999997 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 4 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 253.0396, + "max": 1848.1695, + "p(90)": 392.0442500000001, + "p(95)": 496.69095499999986, + "avg": 280.2357935517972, + "min": 109.7353 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "endpoint_sync_history_duration": { + "contains": "time", + "values": { + "max": 762.6455, + "p(90)": 144.69596, + "p(95)": 155.57877, + "avg": 148.2798042105263, + "min": 115.0667, + "med": 126.1415 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": false + } + }, + "type": "trend" + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..2b3414e --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-cloudwatch.json @@ -0,0 +1,239 @@ +{ + "startTime": "2026-09-10T18:38:03Z", + "endTime": "2026-09-10T18:41:05Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Sum": 1.0, + "Unit": "Count" + } + ] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.16124677999101966, + "p90": 0.04288549457706691, + "p50": 0.01335367132537653, + "p95": 0.08024803408355827 + } + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.13391357113303803, + "p90": 0.059012790514521074, + "p50": 0.014505140989362704, + "p95": 0.08803598019997966 + } + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1463262010119467, + "p90": 0.049828100975514794, + "p50": 0.013342769202778854, + "p95": 0.08662856869215846 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 53.25306828816732, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 76.05701700846355, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 68.35044860839844, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 6.673177083333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 6.73828125, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 6.803385416666667, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 10.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 22.12736155808641, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 60.85, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 62.600989488764135, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 134.26899898333335, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 71598080.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 73609216.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 86646784.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 0.0010695187165775401, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 0.00033018867924528304, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 0.0001282051282051282, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:38:00+01:00", + "Average": 0.00014705882352941178, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:39:00+01:00", + "Average": 0.004074074074074074, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:40:00+01:00", + "Average": 0.000425531914893617, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-soak-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-soak-cloudwatch.json new file mode 100644 index 0000000..cd06ee8 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-soak-cloudwatch.json @@ -0,0 +1,843 @@ +{ + "startTime": "2026-09-10T20:40:37Z", + "endTime": "2026-09-10T20:55:39Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.26547326186521625, + "p90": 0.09294158986409308, + "p50": 0.016536228178670235, + "p95": 0.12747345451562245 + } + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.17346124005650101, + "p90": 0.07507393920092932, + "p50": 0.016285825444213125, + "p95": 0.10550231332342269 + } + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.14881264126091515, + "p90": 0.05625411187867282, + "p50": 0.01394742836887384, + "p95": 0.08522614579148616 + } + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1619191384153045, + "p90": 0.06312529991127364, + "p50": 0.014618052565306072, + "p95": 0.09654172274626481 + } + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.13082859473711783, + "p90": 0.05421661827612759, + "p50": 0.014845335099832178, + "p95": 0.08558981252585243 + } + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.17898169761226565, + "p90": 0.07728137813055257, + "p50": 0.016322467216005537, + "p95": 0.10429778298826516 + } + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.13536103384583714, + "p90": 0.051090133310175305, + "p50": 0.013822467752679679, + "p95": 0.08097465215572476 + } + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1444790046329604, + "p90": 0.05438658564937058, + "p50": 0.014962453538007492, + "p95": 0.08505956109290651 + } + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1717701376998449, + "p90": 0.07624222907770911, + "p50": 0.015347009431461618, + "p95": 0.10705705545556463 + } + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.19530985855272281, + "p90": 0.06968408185596575, + "p50": 0.015741647860514525, + "p95": 0.09830719195689931 + } + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.17184858624088614, + "p90": 0.06435065183355768, + "p50": 0.01423167976933858, + "p95": 0.09664364288086877 + } + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.16999014648970473, + "p90": 0.07183276565896418, + "p50": 0.015850197658450565, + "p95": 0.10381483079363371 + } + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1350000009605071, + "p90": 0.05838512388694818, + "p50": 0.01518167924987536, + "p95": 0.08763211211987929 + } + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.15774214322854335, + "p90": 0.05221038945771639, + "p50": 0.014080250164527998, + "p95": 0.09186672384761485 + } + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.12448582699691704, + "p90": 0.03866099536963388, + "p50": 0.013520679615344857, + "p95": 0.07188973995583374 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 27.78227763498823, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 82.95702616373698, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 72.49641672770183, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 78.04808044433594, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 75.48256174723308, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 82.08821360270183, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 67.92955780029297, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 74.08062489827473, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 82.41480000813802, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 81.00458526611328, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 76.92803700764973, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 83.85089111328125, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 79.1140874226888, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 79.1009521484375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 68.52549235026042, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 6.868489583333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 6.884765625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 6.917317708333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 6.901041666666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 6.93359375, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 0.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 10.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 3.991666666666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 58.475, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 66.0, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 64.25953689821756, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 68.87610471902617, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 71.18940529735131, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 65.90833333333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 66.54994163748542, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 70.18040678982526, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 65.49832419002517, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 70.44401480049335, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 67.43333333333334, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 67.6, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 69.98167582875227, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 64.27392876785387, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 135.46236903333335, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 129.5748806, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 123.84171361666667, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 106991616.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 81727488.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 85204992.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 85090304.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 93011968.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 88014848.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 85815296.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 84688896.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 100077568.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 94621696.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 91594752.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 89759744.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 88977408.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 92823552.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 90112000.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 0.0002054794520547945, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 0.0029896907216494842, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 0.0009090909090909091, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 0.0006666666666666666, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 0.0002, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:40:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:41:00+01:00", + "Average": 0.0005660377358490566, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:42:00+01:00", + "Average": 0.008998211091234347, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:43:00+01:00", + "Average": 0.0023076923076923075, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:44:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:45:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:46:00+01:00", + "Average": 0.0004166666666666667, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:47:00+01:00", + "Average": 0.006395759717314488, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:48:00+01:00", + "Average": 0.0005555555555555556, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:49:00+01:00", + "Average": 0.00028169014084507044, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:50:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:51:00+01:00", + "Average": 0.0004347826086956522, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:52:00+01:00", + "Average": 0.0035106382978723405, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:53:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:54:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-soak-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-soak-summary.json new file mode 100644 index 0000000..36ca1c1 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-soak-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "groups": [], + "checks": [ + { + "passes": 50785, + "fails": 28, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa" + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 50813, + "fails": 0 + }, + { + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 50785, + "fails": 28, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid" + }, + { + "passes": 50785, + "fails": 28, + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7" + }, + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 50785, + "fails": 29 + }, + { + "fails": 0, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 50814 + }, + { + "fails": 29, + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 50785 + }, + { + "passes": 50785, + "fails": 29, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f" + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 8465, + "fails": 4 + }, + { + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 8469, + "fails": 0, + "name": "sync_history: response is JSON" + }, + { + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 8465, + "fails": 4, + "name": "sync_history: response shape is valid" + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 8465, + "fails": 4 + }, + { + "fails": 9, + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 16928 + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 16937, + "fails": 0 + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 16928, + "fails": 9 + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 16928, + "fails": 9 + }, + { + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 42321, + "fails": 23 + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 42344, + "fails": 0 + }, + { + "passes": 42321, + "fails": 23, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937" + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 42321, + "fails": 23 + } + ], + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e" + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 901347.467 + }, + "metrics": { + "iterations": { + "values": { + "count": 169377, + "rate": 187.91532256006218 + }, + "type": "counter", + "contains": "default" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + } + }, + "endpoint_aws_account_list_duration": { + "contains": "time", + "values": { + "avg": 150.59239438507305, + "min": 95.295, + "med": 122.2539, + "max": 2913.1937, + "p(90)": 185.20208, + "p(95)": 237.59875999999994 + }, + "type": "trend" + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "fails": 169284, + "rate": 0.0005490710072796188, + "passes": 93 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_req_connecting": { + "values": { + "avg": 0.03584236264449928, + "min": 0, + "med": 0, + "max": 123.5881, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend", + "contains": "time" + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 3838.8797, + "p(90)": 242.02646000000001, + "p(95)": 280.06849499999987, + "avg": 165.27676774374635, + "min": 95.295, + "med": 129.62754999999999 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 280.03352, + "avg": 165.28775312402323, + "min": 95.295, + "med": 129.6361, + "max": 3838.8797, + "p(90)": 242.03010000000003 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "rate": 1738643.3704816746, + "count": 1567121798 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 195.3291, + "med": 230.0238, + "max": 3938.9685, + "p(90)": 342.47162, + "p(95)": 380.4692599999996, + "avg": 265.7277010024966 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "fails": 169285, + "rate": 0.0005490677655893918, + "passes": 93 + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 169377, + "rate": 187.91532256006218 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "max": 0.5196, + "p(90)": 0, + "p(95)": 0, + "avg": 5.200025977399663E-05, + "min": 0, + "med": 0 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 217.95698016846194, + "min": 106.7566, + "med": 163.8585, + "max": 3838.8797, + "p(90)": 314.913, + "p(95)": 411.22083999999944 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "med": 127.5148, + "max": 1771.7581, + "p(90)": 177.84153000000003, + "p(95)": 212.58007499999994, + "avg": 141.04151085796562, + "min": 95.295 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 165.2739052244418, + "min": 95.295, + "med": 129.6275, + "max": 3838.8797, + "p(90)": 242.02622, + "p(95)": 280.06283999999994 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0.9995881967445402, + "passes": 677229, + "fails": 279 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 108647176, + "rate": 120538.6157700269 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 8, + "min": 8, + "max": 50 + } + }, + "http_req_blocked": { + "values": { + "med": 0, + "max": 123.5881, + "p(90)": 0, + "p(95)": 0, + "avg": 0.03594038245817049, + "min": 0 + }, + "type": "trend", + "contains": "time" + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 169378, + "rate": 187.91643201012067 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 98.3583, + "med": 121.1113, + "max": 1715.5437, + "p(90)": 170.04915999999997, + "p(95)": 210.82805999999988, + "avg": 141.09344468059976 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 24.235204885522197, + "min": 0, + "med": 0, + "max": 3618.1492, + "p(90)": 106.37881000000002, + "p(95)": 117.50994999999999 + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 215.883825, + "avg": 143.8517163258072, + "min": 104.0605, + "med": 128.3699, + "max": 1772.5991, + "p(90)": 181.64988000000002 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 50, + "min": 50, + "max": 50 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 127.48294999999999, + "max": 1487.4188, + "p(90)": 175.54750000000016, + "p(95)": 209.204255, + "avg": 139.3668943421095, + "min": 101.9511 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-summary.json new file mode 100644 index 0000000..f77a478 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-50vus-post-rollup-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "passes": 9046, + "fails": 0, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa" + }, + { + "passes": 9046, + "fails": 0, + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979" + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 9046, + "fails": 0 + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 9046, + "fails": 0 + }, + { + "fails": 0, + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 3015 + }, + { + "passes": 3015, + "fails": 0, + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516" + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 3015, + "fails": 0 + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 3015, + "fails": 0 + }, + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 9047, + "fails": 0 + }, + { + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 9047, + "fails": 0, + "name": "cost_summary: response is JSON" + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 9047, + "fails": 0 + }, + { + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 9047, + "fails": 0, + "name": "cost_summary: no auth or server error" + }, + { + "passes": 7537, + "fails": 1, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b" + }, + { + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 7537, + "fails": 1, + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON" + }, + { + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 7537, + "fails": 1, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid" + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 7537, + "fails": 1 + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 1508, + "fails": 0 + }, + { + "passes": 1508, + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe" + }, + { + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 1508, + "fails": 0 + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 1508, + "fails": 0 + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 181080.376 + }, + "metrics": { + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "min": 107.9767, + "med": 130.4875, + "max": 1792.5767, + "p(90)": 180.04752000000005, + "p(95)": 218.51911999999996, + "avg": 148.4589563820268 + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 153.54279737330873, + "min": 115.1806, + "med": 130.7686, + "max": 1831.0342, + "p(90)": 185.28279000000003, + "p(95)": 226.4119699999999 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0 + } + }, + "http_req_connecting": { + "values": { + "avg": 0.2033668380036479, + "min": 0, + "med": 0, + "max": 128.3665, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend", + "contains": "time" + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 279126831, + "rate": 1541452.6806593332 + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 30154, + "rate": 166.5227379470429 + } + }, + "http_req_receiving": { + "contains": "time", + "values": { + "avg": 49.614278690101166, + "min": 0, + "med": 0.4015, + "max": 6062.7696, + "p(90)": 118.3341, + "p(95)": 228.39542999999998 + }, + "type": "trend" + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 128.3665, + "p(90)": 0, + "p(95)": 0, + "avg": 0.20368251699552323, + "min": 0 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 3.3161996352180403E-05, + "passes": 1, + "fails": 30154 + } + }, + "endpoint_cost_summary_duration": { + "values": { + "avg": 144.33511214767327, + "min": 114.3393, + "med": 129.8515, + "max": 1211.1026, + "p(90)": 176.2036000000001, + "p(95)": 209.34790999999993 + }, + "type": "trend", + "contains": "time" + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 0.7361, + "p(90)": 0, + "p(95)": 0, + "avg": 7.107279058199305E-05 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 107.9767, + "med": 125.5411, + "max": 2255.2562, + "p(90)": 239.3538, + "p(95)": 552.9870799999978, + "avg": 179.86870663349927 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 30154, + "rate": 166.5227379470429 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 198.0757285368436, + "min": 107.9767, + "med": 133.94335, + "max": 6205.0197, + "p(90)": 275.75085, + "p(95)": 397.13922499999967 + } + }, + "checks": { + "values": { + "rate": 0.9999668369038933, + "passes": 120612, + "fails": 4 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate", + "contains": "default" + }, + "measured_failures": { + "thresholds": { + "rate<0.01": { + "ok": true + } + }, + "type": "rate", + "contains": "default", + "values": { + "fails": 30153, + "rate": 3.316309610665252E-05, + "passes": 1 + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 107.9767, + "med": 133.9433, + "max": 6205.0197, + "p(90)": 275.7474000000001, + "p(95)": 397.12874999999985, + "avg": 198.07330614491732 + } + }, + "iteration_duration": { + "contains": "time", + "values": { + "p(90)": 376.19429, + "p(95)": 500.211154999998, + "avg": 298.6949059660431, + "min": 209.9327, + "med": 234.3594, + "max": 6305.9135 + }, + "type": "trend" + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 10, + "min": 10, + "max": 50 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 152.94021067639255, + "min": 109.4582, + "med": 123.84905, + "max": 2020.6022, + "p(90)": 178.74674999999993, + "p(95)": 237.7946249999996 + } + }, + "http_reqs": { + "values": { + "count": 30155, + "rate": 166.5282603566054 + }, + "type": "counter", + "contains": "default" + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 464.02755, + "p(95)": 782.7374, + "avg": 302.4648887685178, + "min": 114.8763, + "med": 246.51715, + "max": 6205.0197 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 19342569, + "rate": 106817.58800854268 + } + }, + "vus_max": { + "contains": "default", + "values": { + "value": 50, + "min": 50, + "max": 50 + }, + "type": "gauge" + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 198.05799357299142, + "min": 107.9767, + "med": 133.9426, + "max": 6205.0197, + "p(90)": 275.72208, + "p(95)": 397.04061499999995 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-51vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-51vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..42e1643 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-51vus-post-rollup-cloudwatch.json @@ -0,0 +1,227 @@ +{ + "startTime": "2026-09-10T20:17:40Z", + "endTime": "2026-09-10T20:20:42Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.2880415260002257, + "p90": 0.07948940468971274, + "p50": 0.018442464722367446, + "p95": 0.10820152849797136 + } + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1505583141337567, + "p90": 0.0730106115252081, + "p50": 0.018088536052210898, + "p95": 0.10037498725255879 + } + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.14906399169684623, + "p90": 0.06219236034512958, + "p50": 0.016825326654889554, + "p95": 0.08929295067891424 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 22.529106200595077, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 79.64221700032552, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 82.50581868489583, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 0.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 10.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 3.7731762981225745, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 55.76862150039186, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 73.39999999999999, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 104902656.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 78884864.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 78188544.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:17:00+01:00", + "Average": 0.005927272727272727, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:18:00+01:00", + "Average": 0.0013580246913580246, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:19:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-51vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-51vus-post-rollup-summary.json new file mode 100644 index 0000000..f902dbb --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-51vus-post-rollup-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 11233, + "fails": 0, + "name": "cost_summary: status is 200" + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 11233, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 11233 + }, + { + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 11233, + "fails": 0, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error" + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 1873, + "fails": 0 + }, + { + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 1873 + }, + { + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 1873, + "fails": 0, + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid" + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 1873, + "fails": 0 + }, + { + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 9360, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200" + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 9360, + "fails": 0 + }, + { + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 9360, + "fails": 0 + }, + { + "passes": 9360, + "fails": 0, + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2" + }, + { + "passes": 3743, + "fails": 0, + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c" + }, + { + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 3743, + "fails": 0, + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON" + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 3743, + "fails": 0 + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 3743, + "fails": 0 + }, + { + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 11233, + "fails": 0, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200" + }, + { + "passes": 11233, + "fails": 0, + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979" + }, + { + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 11233, + "fails": 0, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid" + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 11233, + "fails": 0 + } + ] + }, + "options": { + "summaryTimeUnit": "", + "noColor": false, + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ] + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 181052.8133 + }, + "metrics": { + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0.16349902518494777, + "min": 0, + "med": 0, + "max": 130.52 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 113.2606, + "med": 131.298, + "max": 548.9019, + "p(90)": 183.78562000000002, + "p(95)": 211.28477999999998, + "avg": 143.49691190243084 + } + }, + "endpoint_cost_by_service_duration": { + "contains": "time", + "values": { + "avg": 145.81175856837612, + "min": 112.9281, + "med": 133.13580000000002, + "max": 584.9632, + "p(90)": 187.61186000000004, + "p(95)": 215.67958499999992 + }, + "type": "trend" + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 37442 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "iterations": { + "values": { + "count": 37442, + "rate": 206.80153662102748 + }, + "type": "counter", + "contains": "default" + }, + "http_req_failed": { + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 37443 + }, + "type": "rate" + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "rate": 206.80705987129778, + "count": 37443 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 245.3787644196366, + "min": 208.3688, + "med": 233.0397, + "max": 1224.5342, + "p(90)": 286.15191, + "p(95)": 313.79029499999996 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 144.80129183537312, + "min": 108.2092, + "med": 132.58065, + "max": 1124.1081, + "p(90)": 185.71535, + "p(95)": 213.46366499999993 + } + }, + "data_sent": { + "contains": "data", + "values": { + "count": 24017470, + "rate": 132654.49766971392 + }, + "type": "counter" + }, + "http_req_blocked": { + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0.1638940175733782, + "min": 0, + "med": 0, + "max": 131.0549 + }, + "type": "trend" + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 149.79524967506393, + "min": 118.8314, + "med": 135.6302, + "max": 1124.1081, + "p(90)": 193.12204000000003, + "p(95)": 221.32451999999998 + } + }, + "http_req_duration": { + "contains": "time", + "values": { + "avg": 144.81406223593302, + "min": 108.2092, + "med": 132.5813, + "max": 1124.1081, + "p(90)": 185.72392000000002, + "p(95)": 213.5090700000001 + }, + "type": "trend" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 12, + "min": 12, + "max": 51 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "max": 1124.1081, + "p(90)": 185.72392000000002, + "p(95)": 213.5090700000001, + "avg": 144.81406223593302, + "min": 108.2092, + "med": 132.5813 + } + }, + "vus_max": { + "contains": "default", + "values": { + "value": 51, + "min": 51, + "max": 51 + }, + "type": "gauge" + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "avg": 5.44667895200705E-05, + "min": 0, + "med": 0, + "max": 0.5154, + "p(90)": 0, + "p(95)": 0 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 346605630, + "rate": 1914389.6395892126 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 149768, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "http_req_receiving": { + "contains": "time", + "values": { + "p(95)": 2.5885500000000006, + "avg": 0.8456026974334306, + "min": 0, + "med": 0, + "max": 974.5422, + "p(90)": 1.5779800000000004 + }, + "type": "trend" + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 37442, + "rate": 206.80153662102748 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "max": 644.1557, + "p(90)": 184.51766, + "p(95)": 212.19169000000008, + "avg": 143.9684050717098, + "min": 108.2092, + "med": 132.1555 + } + }, + "endpoint_aws_account_list_duration": { + "values": { + "max": 818.7096, + "p(90)": 168.78276000000014, + "p(95)": 195.50673999999995, + "avg": 135.9329380443495, + "min": 108.2092, + "med": 126.1689 + }, + "type": "trend", + "contains": "time" + }, + "endpoint_sync_history_duration": { + "values": { + "max": 747.2143, + "p(90)": 171.70332, + "p(95)": 193.89965999999998, + "avg": 135.3465406300053, + "min": 108.6792, + "med": 125.0237 + }, + "type": "trend", + "contains": "time" + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-52vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-52vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..1fbe2d6 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-52vus-post-rollup-cloudwatch.json @@ -0,0 +1,227 @@ +{ + "startTime": "2026-09-10T20:06:18Z", + "endTime": "2026-09-10T20:09:20Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.22347255991764695, + "p90": 0.07572715771518361, + "p50": 0.015252685325852097, + "p95": 0.11130110310856753 + } + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.15085225788166384, + "p90": 0.06767277685608598, + "p50": 0.016104680093595163, + "p95": 0.09582022982669502 + } + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.14168660851949375, + "p90": 0.06248335774006523, + "p50": 0.01614740385775519, + "p95": 0.09204288027000343 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 48.21265488455538, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 80.6585210164388, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 83.10693359375, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 7.877604166666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 10.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 12.99209973665789, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 71.10221777555444, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 74.48651530042811, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 90566656.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 86020096.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 85389312.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 0.0012, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 0.0002666666666666667, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 0.00125, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T21:06:00+01:00", + "Average": 0.00035087719298245617, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:07:00+01:00", + "Average": 0.00834983498349835, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T21:08:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-52vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-52vus-post-rollup-summary.json new file mode 100644 index 0000000..263f42c --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-52vus-post-rollup-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 11189, + "fails": 0, + "name": "cost_summary: status is 200" + }, + { + "passes": 11189, + "fails": 0, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c" + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 11189, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 11189 + }, + { + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 9324 + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 9324, + "fails": 0 + }, + { + "passes": 9324, + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937" + }, + { + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 9324, + "fails": 0, + "name": "cost_by_service: no auth or server error" + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 3730, + "fails": 0 + }, + { + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 3730, + "fails": 0, + "name": "aws_account_list: response is JSON" + }, + { + "passes": 3730, + "fails": 0, + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2" + }, + { + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 3730, + "fails": 0, + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error" + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 1865, + "fails": 0 + }, + { + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 1865, + "fails": 0 + }, + { + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 1865, + "fails": 0 + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 1865, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 11190 + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 11190, + "fails": 0 + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 11190, + "fails": 0 + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 11190, + "fails": 0 + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdErrTTY": true, + "testRunDurationMs": 180969.1261, + "isStdOutTTY": true + }, + "metrics": { + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 37298, + "rate": 206.10145389877084 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 140.69879482573762, + "min": 113.8867, + "med": 125.94675000000001, + "max": 2107.8066, + "p(90)": 172.31829, + "p(95)": 202.19666999999987 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 132.07105, + "max": 3974.3104, + "p(90)": 200.10179000000002, + "p(95)": 241.32360000000003, + "avg": 150.50175418521098, + "min": 111.8465 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 142.67178198230442, + "min": 116.4939, + "med": 131.0257, + "max": 644.4354, + "p(90)": 181.6774600000001, + "p(95)": 211.14314 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 52, + "min": 52, + "max": 52 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 149192, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 345284996, + "rate": 1907977.3629961733 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 1203.7148, + "p(90)": 166.0654000000001, + "p(95)": 196.26023999999998, + "avg": 137.15631957104554, + "min": 111.8465, + "med": 124.3088 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 3698.6537, + "p(90)": 1.2074600000000013, + "p(95)": 2.739839999999999, + "avg": 6.71136261025764 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 130.3903, + "p(90)": 0, + "p(95)": 0, + "avg": 0.17347992707579293 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 37298 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "rate": 206.10145389877084, + "count": 37298 + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 145.47898524238545, + "min": 118.9518, + "med": 131.71305, + "max": 3026.4441, + "p(90)": 186.33369000000002, + "p(95)": 217.41836500000002 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 241.3436699999999, + "avg": 150.5142719643958, + "min": 111.8465, + "med": 132.0713, + "max": 3974.3104, + "p(90)": 200.1154600000001 + } + }, + "http_req_waiting": { + "values": { + "med": 131.5655, + "max": 3025.9005, + "p(90)": 183.85038000000006, + "p(95)": 213.9809, + "avg": 143.80277442022552, + "min": 111.8465 + }, + "type": "trend", + "contains": "time" + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "max": 130.3903, + "p(90)": 0, + "p(95)": 0, + "avg": 0.17381027373388025, + "min": 0, + "med": 0 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 52, + "min": 52, + "max": 52 + } + }, + "iteration_duration": { + "contains": "time", + "values": { + "avg": 251.08215306182612, + "min": 211.8823, + "med": 232.5004, + "max": 4074.3826, + "p(90)": 300.60742000000005, + "p(95)": 341.854545 + }, + "type": "trend" + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 37299, + "rate": 206.10697970320805 + } + }, + "http_req_duration": { + "contains": "time", + "values": { + "max": 3974.3104, + "p(90)": 200.1154600000001, + "p(95)": 241.3436699999999, + "avg": 150.5142719643958, + "min": 111.8465, + "med": 132.0713 + }, + "type": "trend" + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 37299 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0.00013493391243733075, + "min": 0, + "med": 0, + "max": 0.6267, + "p(90)": 0 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 168.0081108042897, + "min": 119.5791, + "med": 135.8696, + "max": 3974.3104, + "p(90)": 244.30289, + "p(95)": 260.6085999999999 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 23925076, + "rate": 132205.2911212019 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-55vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-55vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..f396a77 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-55vus-post-rollup-cloudwatch.json @@ -0,0 +1,239 @@ +{ + "startTime": "2026-09-10T18:55:06Z", + "endTime": "2026-09-10T18:58:08Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Sum": 1.0, + "Unit": "Count" + } + ] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.2361971697087722, + "p90": 0.09394422412950405, + "p50": 0.018468971656289565, + "p95": 0.12256630720052263 + } + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1764701783888151, + "p90": 0.09835390058525331, + "p50": 0.018012820219698068, + "p95": 0.12446378768216686 + } + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.16258226163573955, + "p90": 0.09320919325511794, + "p50": 0.01827358178096868, + "p95": 0.11612541236155473 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 65.79698944091797, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 87.73880004882812, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 89.22252655029297, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 6.819661458333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 6.8359375, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 10.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 27.479211451615594, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 76.36630476476377, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 77.6624631428143, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 134.7748501, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 80838656.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 79757312.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 78917632.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 0.0007086614173228346, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 0.0004761904761904762, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:55:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:56:00+01:00", + "Average": 0.012427843803056027, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:57:00+01:00", + "Average": 0.00024390243902439024, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-55vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-55vus-post-rollup-summary.json new file mode 100644 index 0000000..e06c26c --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-55vus-post-rollup-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 3933, + "fails": 0, + "name": "aws_account_list: status is 200" + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 3933, + "fails": 0 + }, + { + "fails": 0, + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 3933 + }, + { + "passes": 3933, + "fails": 0, + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8" + }, + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 11801, + "fails": 0 + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 11801, + "fails": 0 + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 11801, + "fails": 0 + }, + { + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 11801, + "fails": 0, + "name": "cost_summary: no auth or server error" + }, + { + "passes": 9833, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b" + }, + { + "fails": 0, + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 9833 + }, + { + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 9833, + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid" + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 9833, + "fails": 0 + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 1967, + "fails": 0 + }, + { + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 1967 + }, + { + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 1967, + "fails": 0 + }, + { + "fails": 0, + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 1967 + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 11799, + "fails": 1 + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 11799, + "fails": 1 + }, + { + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 11799, + "fails": 1, + "name": "cost_timeseries: response shape is valid" + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 11799, + "fails": 1 + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "testRunDurationMs": 181170.258, + "isStdOutTTY": true, + "isStdErrTTY": true + }, + "metrics": { + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 1211.7814, + "p(90)": 213.3031, + "p(95)": 239.30273999999997, + "avg": 151.35395052497645, + "min": 109.6584, + "med": 133.701 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 225.61909999999997, + "avg": 143.05961601423465, + "min": 109.9668, + "med": 127.3207, + "max": 784.503, + "p(90)": 201.78064 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 235.709, + "avg": 148.7976235149566, + "min": 113.9269, + "med": 132.4747, + "max": 497.5059, + "p(90)": 210.2992 + } + }, + "http_req_duration{expected_response:true}": { + "contains": "time", + "values": { + "avg": 151.35486167946175, + "min": 109.6584, + "med": 133.70105, + "max": 1211.7814, + "p(90)": 213.30355000000003, + "p(95)": 239.30302999999998 + }, + "type": "trend" + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 26, + "min": 26, + "max": 55 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 1.5909945570102841, + "min": 0, + "med": 0, + "max": 1057.4258, + "p(90)": 1.623119999999999, + "p(95)": 2.646619999999998 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 1211.7814, + "p(90)": 223.59256000000005, + "p(95)": 249.16418999999996, + "avg": 158.53912683050916, + "min": 115.5146, + "med": 137.18075 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 39335, + "rate": 217.11621120504228 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "avg": 149.7628461853296, + "min": 109.6584, + "med": 133.2451, + "max": 719.0125, + "p(90)": 211.53426, + "p(95)": 237.0313 + } + }, + "measured_failures": { + "values": { + "rate": 2.542329791020491E-05, + "passes": 1, + "fails": 39333 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + }, + "type": "rate", + "contains": "default" + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.16881004703190544, + "min": 0, + "med": 0, + "max": 129.1976, + "p(90)": 0, + "p(95)": 0 + } + }, + "data_received": { + "values": { + "count": 364087935, + "rate": 2009645.175865456 + }, + "type": "counter", + "contains": "data" + }, + "measured_duration": { + "values": { + "avg": 151.33989913560657, + "min": 109.6584, + "med": 133.7006, + "max": 1211.7814, + "p(90)": 213.29191000000012, + "p(95)": 239.28955999999997 + }, + "type": "trend", + "contains": "time" + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 251.92874859663345, + "min": 210.1985, + "med": 234.12035, + "max": 1312.2911, + "p(90)": 313.79856000000007, + "p(95)": 339.663715 + } + }, + "checks": { + "contains": "default", + "values": { + "rate": 0.9999745767020898, + "passes": 157332, + "fails": 4 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0 + } + }, + "iterations": { + "contains": "default", + "values": { + "count": 39334, + "rate": 217.1106915352519 + }, + "type": "counter" + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 39334, + "rate": 217.1106915352519 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 196.85150000000004, + "p(95)": 220.77591999999999, + "avg": 142.8809517925248, + "min": 109.6584, + "med": 127.9144, + "max": 925.1758 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 124.2498, + "p(90)": 0, + "p(95)": 0, + "avg": 0.16835549510613954 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 55, + "min": 55, + "max": 55 + } + }, + "http_req_failed": { + "values": { + "rate": 2.542265158256006E-05, + "passes": 1, + "fails": 39334 + }, + "type": "rate", + "contains": "default" + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 112.8023, + "med": 133.5092, + "max": 738.4481, + "p(90)": 214.0598400000001, + "p(95)": 238.10482, + "avg": 150.7914260958001 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.00010978263632896913, + "min": 0, + "med": 0, + "max": 0.5845, + "p(90)": 0, + "p(95)": 0 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 25231080, + "rate": 139267.23005494644 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-cloudwatch.json new file mode 100644 index 0000000..791be12 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-cloudwatch.json @@ -0,0 +1,233 @@ +{ + "startTime": "2026-09-10T13:49:14Z", + "endTime": "2026-09-10T13:52:16Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.38503836010724485, + "p90": 0.24790600143524694, + "p50": 0.13583234992459062, + "p95": 0.29491442946251023 + } + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.4718735570654435, + "p90": 0.2517015409611456, + "p50": 0.1351305709310395, + "p95": 0.2891127463841507 + } + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.4130869283655758, + "p90": 0.24940696483845395, + "p50": 0.13200396833054012, + "p95": 0.280391269297301 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 8.759881378461918, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 9.105795860290527, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 9.782214800516764, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 5.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 5.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 5.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 17.9, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 83.13481817121723, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 83.66523905544354, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 120.18349128333334, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 86781952.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 96403456.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 92573696.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 0.0009090909090909091, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 0.00025, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 0.0009523809523809524, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T14:49:00+01:00", + "Average": 0.00018518518518518518, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T14:50:00+01:00", + "Average": 0.00012987012987012987, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T14:51:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..ae32a06 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-post-rollup-cloudwatch.json @@ -0,0 +1,227 @@ +{ + "startTime": "2026-09-10T17:31:43Z", + "endTime": "2026-09-10T17:34:45Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.1952222781114497, + "p90": 0.014861946841870696, + "p50": 0.011576437432551786, + "p95": 0.016001061270904927 + } + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.04436199798816148, + "p90": 0.015123646812044922, + "p50": 0.011758133175178308, + "p95": 0.016941720825547756 + } + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.04278065045980197, + "p90": 0.014847077393205982, + "p50": 0.01128631323548445, + "p95": 0.01627129332777227 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 4.182211247816062, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 16.581934928894043, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 11.510798454284668, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 6.070963541666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 5.598958333333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 5.989583333333333, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 0.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 2.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 3.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 3.991666666666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 9.153908944010393, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 11.447199479748544, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 82169856.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 83734528.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 89088000.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 0.0004237288135593221, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 0.00013043478260869564, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 0.001858974358974359, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T18:31:00+01:00", + "Average": 0.014932432432432433, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:32:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T18:33:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-post-rollup-summary.json new file mode 100644 index 0000000..3d1aad5 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-post-rollup-summary.json @@ -0,0 +1,444 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 1137, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 1137 + }, + { + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 1137, + "fails": 0, + "name": "cost_summary: response shape is valid" + }, + { + "fails": 0, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 1137 + }, + { + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 947, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200" + }, + { + "passes": 947, + "fails": 0, + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d" + }, + { + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 947, + "fails": 0, + "name": "cost_by_service: response shape is valid" + }, + { + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 947, + "fails": 0, + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error" + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 1137, + "fails": 0 + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 1137, + "fails": 0 + }, + { + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 1137, + "fails": 0, + "name": "cost_timeseries: response shape is valid" + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 1137, + "fails": 0 + }, + { + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 379, + "fails": 0, + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200" + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 379, + "fails": 0 + }, + { + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 379, + "fails": 0, + "name": "aws_account_list: response shape is valid" + }, + { + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 379, + "fails": 0, + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error" + }, + { + "passes": 190, + "fails": 0, + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb" + }, + { + "fails": 0, + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 190 + }, + { + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 190, + "fails": 0, + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid" + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 190, + "fails": 0 + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 180967.6034 + }, + "metrics": { + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 132.9634, + "p(95)": 136.13060000000002, + "avg": 127.81529649169079, + "min": 109.3059, + "med": 125.8793, + "max": 943.6198 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 127.70312102022855, + "min": 115.3949, + "med": 125.0232, + "max": 480.4279, + "p(90)": 130.47238, + "p(95)": 135.9221 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 5, + "min": 5, + "max": 5 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 121.65926473684209, + "min": 111.465, + "med": 120.08824999999999, + "max": 187.5883, + "p(90)": 125.19625, + "p(95)": 127.319445 + } + }, + "http_req_sending": { + "values": { + "max": 0.5284, + "p(90)": 0, + "p(95)": 0, + "avg": 0.0002722236876813506, + "min": 0, + "med": 0 + }, + "type": "trend", + "contains": "time" + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 3790, + "rate": 20.942975034171226 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "passes": 0, + "fails": 3791, + "rate": 0 + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0.18622978106040625, + "min": 0, + "med": 0, + "max": 125.4608, + "p(90)": 0, + "p(95)": 0 + } + }, + "measured_failures": { + "values": { + "rate": 0, + "passes": 0, + "fails": 3790 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + }, + "type": "rate", + "contains": "default" + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 120.1431, + "p(90)": 0, + "p(95)": 0, + "avg": 0.18337327881825374, + "min": 0 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 137.08394701846956, + "min": 109.3059, + "med": 126.63165000000001, + "max": 1203.5025, + "p(90)": 135.13379, + "p(95)": 180.6777899999998 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 2431365, + "rate": 13435.360552495442 + } + }, + "http_req_tls_handshaking": { + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + }, + "type": "trend" + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "med": 0.3258, + "max": 1058.9299, + "p(90)": 4.2874, + "p(95)": 8.092400000000001, + "avg": 9.400878765497245, + "min": 0 + } + }, + "http_req_duration": { + "values": { + "p(90)": 135.1499, + "p(95)": 181.70295, + "avg": 137.21644748087562, + "min": 109.3059, + "med": 126.6335, + "max": 1203.5025 + }, + "type": "trend", + "contains": "time" + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 35086969, + "rate": 193885.36036721367 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 3790, + "rate": 20.942975034171226 + } + }, + "iteration_duration": { + "contains": "time", + "values": { + "avg": 237.67025042216397, + "min": 211.675, + "med": 227.04895, + "max": 1303.9652, + "p(90)": 235.61726000000002, + "p(95)": 281.07308499999965 + }, + "type": "trend" + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 124.33560976253294, + "min": 109.3059, + "med": 120.0058, + "max": 660.801, + "p(90)": 125.74108, + "p(95)": 129.26656999999994 + } + }, + "vus": { + "values": { + "value": 5, + "min": 5, + "max": 5 + }, + "type": "gauge", + "contains": "default" + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 136.37514, + "avg": 129.45352111932436, + "min": 116.6179, + "med": 126.515, + "max": 658.8627, + "p(90)": 132.72353999999999 + } + }, + "checks": { + "contains": "default", + "values": { + "rate": 1, + "passes": 15160, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate" + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 3791, + "rate": 20.948500885103726 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "med": 126.6335, + "max": 1203.5025, + "p(90)": 135.1499, + "p(95)": 181.70295, + "avg": 137.21644748087562, + "min": 109.3059 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 243.0198, + "p(95)": 252.71936, + "avg": 159.6471149516271, + "min": 114.1595, + "med": 129.5399, + "max": 1203.5025 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-summary.json new file mode 100644 index 0000000..0e7b20e --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-5vus-summary.json @@ -0,0 +1,492 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 751, + "fails": 0, + "name": "cost_summary: status is 200" + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 751, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 751 + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 751, + "fails": 0 + }, + { + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 625, + "fails": 0 + }, + { + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 625, + "fails": 0, + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON" + }, + { + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 625, + "fails": 0, + "name": "cost_by_service: response shape is valid" + }, + { + "fails": 0, + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 625 + }, + { + "passes": 751, + "fails": 0, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa" + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 751, + "fails": 0 + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 751, + "fails": 0 + }, + { + "passes": 751, + "fails": 0, + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7" + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 250, + "fails": 0 + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 250, + "fails": 0 + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 250, + "fails": 0 + }, + { + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 250, + "fails": 0, + "name": "aws_account_list: no auth or server error" + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 125, + "fails": 0 + }, + { + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 125, + "fails": 0 + }, + { + "fails": 0, + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 125 + }, + { + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 125, + "fails": 0, + "name": "sync_history: no auth or server error" + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 181246.516 + }, + "metrics": { + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 2503 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0.2873751498202157, + "min": 0, + "med": 0, + "max": 122.9711 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 126.96905000000001, + "max": 210.701, + "p(90)": 141.20443999999998, + "p(95)": 147.67658, + "avg": 129.9706844, + "min": 115.5086 + }, + "thresholds": { + "p(99)<500": { + "ok": true + }, + "p(95)<200": { + "ok": true + } + } + }, + "http_req_blocked": { + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0.2913048341989613, + "min": 0, + "med": 0, + "max": 123.8153 + }, + "type": "trend" + }, + "vus": { + "values": { + "value": 4, + "min": 4, + "max": 5 + }, + "type": "gauge", + "contains": "default" + }, + "iteration_duration": { + "contains": "time", + "values": { + "avg": 360.29667230215784, + "min": 215.999, + "med": 352.35685, + "max": 907.3306, + "p(90)": 468.78731, + "p(95)": 500.6701699999999 + }, + "type": "trend" + }, + "data_sent": { + "values": { + "count": 1605198, + "rate": 8856.435066591845 + }, + "type": "counter", + "contains": "data" + }, + "endpoint_cost_by_service_duration": { + "values": { + "min": 207.9014, + "med": 309.3012, + "max": 735.927, + "p(90)": 389.20082, + "p(95)": 443.75378, + "avg": 318.1573131199999 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 276.96930332889474, + "min": 180.1102, + "med": 253.077, + "max": 806.0354, + "p(90)": 377.8986, + "p(95)": 420.2709 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "max": 806.0354, + "p(90)": 368.35678, + "p(95)": 399.7149600000001, + "avg": 259.8228172992397, + "min": 115.5086, + "med": 252.2266 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 5, + "min": 5, + "max": 5 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 248.8785, + "max": 649.5329, + "p(90)": 325.7928, + "p(95)": 352.39430000000004, + "avg": 257.7986651131828, + "min": 173.3294 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_req_duration": { + "values": { + "avg": 259.8228172992397, + "min": 115.5086, + "med": 252.2266, + "max": 806.0354, + "p(90)": 368.35678, + "p(95)": 399.7149600000001 + }, + "type": "trend", + "contains": "time" + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 2502, + "rate": 13.804403280226362 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 2502, + "rate": 13.804403280226362 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 131.2268, + "max": 188.4988, + "p(90)": 144.42888000000002, + "p(95)": 150.74277999999998, + "avg": 133.3218672, + "min": 119.8368 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 2503, + "rate": 13.809920627660505 + } + }, + "data_received": { + "contains": "data", + "values": { + "rate": 127823.13012847098, + "count": 23167497 + }, + "type": "counter" + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 368.27666, + "p(95)": 398.8659849999998, + "avg": 259.6390765387679, + "min": 115.5086, + "med": 252.17865, + "max": 806.0354 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + } + }, + "http_req_receiving": { + "contains": "time", + "values": { + "med": 0, + "max": 121.1964, + "p(90)": 2.976320000000002, + "p(95)": 5.138820000000006, + "avg": 1.1028236116660006, + "min": 0 + }, + "type": "trend" + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 2502 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 10008, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "http_req_waiting": { + "contains": "time", + "values": { + "avg": 258.7199936875754, + "min": 113.0042, + "med": 251.0832, + "max": 805.5317, + "p(90)": 366.9243, + "p(95)": 397.2793000000001 + }, + "type": "trend" + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-7vus-post-rollup-cloudwatch.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-7vus-post-rollup-cloudwatch.json new file mode 100644 index 0000000..8a7f0d7 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-7vus-post-rollup-cloudwatch.json @@ -0,0 +1,227 @@ +{ + "startTime": "2026-09-10T18:06:08Z", + "endTime": "2026-09-10T18:09:10Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.048875775470276064, + "p90": 0.01470869187793284, + "p50": 0.011105136446586394, + "p95": 0.016513708598530772 + } + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.036291967035372186, + "p90": 0.014591710538548358, + "p50": 0.011155320525666462, + "p95": 0.015682558011621873 + } + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.03823544103971422, + "p90": 0.014827862505333743, + "p50": 0.011094414541070859, + "p95": 0.01750815776761174 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 12.90796184539795, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 17.11275291442871, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 16.957215627034504, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 7.275390625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 6.34765625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 6.331380208333333, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 4.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 2.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 3.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 9.954977488744372, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 15.041666666666666, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 14.291666666666666, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 93843456.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 92295168.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 80420864.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 0.0003275109170305677, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 0.00037037037037037035, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 0.0008823529411764705, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T19:06:00+01:00", + "Average": 0.005150501672240803, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:07:00+01:00", + "Average": 0.00023529411764705883, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T19:08:00+01:00", + "Average": 0.0003846153846153846, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-7vus-post-rollup-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-7vus-post-rollup-summary.json new file mode 100644 index 0000000..74bb09c --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/capacity-7vus-post-rollup-summary.json @@ -0,0 +1,444 @@ +{ + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 181023.486 + }, + "metrics": { + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "rate": 19472.953912731522, + "count": 3525062 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 234.01264, + "p(95)": 236.85278000000005, + "avg": 229.42284242038224, + "min": 212.2446, + "med": 227.2499, + "max": 1432.478 + } + }, + "http_req_waiting": { + "contains": "time", + "values": { + "avg": 127.79697032387158, + "min": 110.5408, + "med": 125.9812, + "max": 987.1489, + "p(90)": 132.99095, + "p(95)": 135.45057500000001 + }, + "type": "trend" + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 1332.3934, + "p(90)": 135.10252, + "p(95)": 139.9969, + "avg": 133.0907663432384, + "min": 118.2188, + "med": 129.1766 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 0.5045, + "p(90)": 0, + "p(95)": 0, + "avg": 9.179403202328966E-05, + "min": 0 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 1, + "min": 1, + "max": 7 + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 5495, + "rate": 30.355177228218885 + } + }, + "data_received": { + "contains": "data", + "values": { + "count": 50878079, + "rate": 281057.88991379825 + }, + "type": "counter" + }, + "endpoint_aws_account_list_duration": { + "values": { + "min": 111.7288, + "med": 120.0673, + "max": 253.2063, + "p(90)": 125.3419, + "p(95)": 126.09172, + "avg": 121.70351748633873 + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 127.39056747572792, + "min": 117.2137, + "med": 125.1144, + "max": 312.1902, + "p(90)": 130.81339, + "p(95)": 134.53433 + } + }, + "http_req_duration{expected_response:true}": { + "contains": "time", + "values": { + "med": 126.86619999999999, + "max": 1332.3934, + "p(90)": 133.58935, + "p(95)": 136.363925, + "avg": 128.97286906841325, + "min": 111.7288 + }, + "type": "trend" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 5496 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 111.7288, + "med": 126.866, + "max": 1332.3934, + "p(90)": 133.58687999999998, + "p(95)": 136.32460000000003, + "avg": 128.87314520473143 + } + }, + "checks": { + "values": { + "rate": 1, + "passes": 21980, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate", + "contains": "default" + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "max": 125.6214, + "p(90)": 0, + "p(95)": 0, + "avg": 0.17479599708879184, + "min": 0, + "med": 0 + } + }, + "measured_failures": { + "contains": "default", + "values": { + "fails": 5495, + "rate": 0, + "passes": 0 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + }, + "type": "rate" + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 133.58935, + "p(95)": 136.363925, + "avg": 128.97286906841325, + "min": 111.7288, + "med": 126.86619999999999, + "max": 1332.3934 + } + }, + "iterations": { + "values": { + "count": 5495, + "rate": 30.355177228218885 + }, + "type": "counter", + "contains": "default" + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "max": 345.2445, + "p(90)": 3.10895, + "p(95)": 4.7539, + "avg": 1.1758069505094617, + "min": 0, + "med": 0.32935000000000003 + } + }, + "endpoint_cost_by_service_duration": { + "values": { + "p(95)": 135.993755, + "avg": 129.82906957787498, + "min": 116.9609, + "med": 126.53725, + "max": 962.727, + "p(90)": 132.00417000000002 + }, + "type": "trend", + "contains": "time" + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 7, + "min": 7, + "max": 7 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 204.954, + "p(90)": 125.98084, + "p(95)": 130.56608, + "avg": 122.00447745454548, + "min": 112.7169, + "med": 120.1279 + } + }, + "http_req_blocked": { + "contains": "time", + "values": { + "max": 125.6214, + "p(90)": 0, + "p(95)": 0, + "avg": 0.17668147743813684, + "min": 0, + "med": 0 + }, + "type": "trend" + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "rate": 30.360701373301364, + "count": 5496 + } + } + }, + "root_group": { + "checks": [ + { + "fails": 0, + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 549 + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 549, + "fails": 0 + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 549, + "fails": 0 + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 549, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 1648 + }, + { + "passes": 1648, + "fails": 0, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c" + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 1648, + "fails": 0 + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 1648, + "fails": 0 + }, + { + "passes": 1374, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b" + }, + { + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 1374, + "fails": 0, + "name": "cost_by_service: response is JSON" + }, + { + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 1374, + "fails": 0 + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 1374, + "fails": 0 + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 1649, + "fails": 0 + }, + { + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 1649, + "fails": 0, + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON" + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 1649, + "fails": 0 + }, + { + "passes": 1649, + "fails": 0, + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7" + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 275, + "fails": 0 + }, + { + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 275, + "fails": 0, + "name": "sync_history: response is JSON" + }, + { + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 275, + "fails": 0, + "name": "sync_history: response shape is valid" + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 275, + "fails": 0 + } + ], + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [] + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/cloudwatch-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/cloudwatch-summary.json new file mode 100644 index 0000000..0dd2eb8 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/cloudwatch-summary.json @@ -0,0 +1,998 @@ +{ + "startTime": "2026-09-10T12:13:57Z", + "endTime": "2026-09-10T12:31:58Z", + "periodSeconds": 60, + "ecsTaskCountsAtCollection": { + "desiredCount": 1, + "runningCount": 1, + "pendingCount": 0 + }, + "metrics": { + "alb_4xx": { + "label": "HTTPCode_ELB_4XX_Count", + "datapoints": [] + }, + "alb_5xx": { + "label": "HTTPCode_ELB_5XX_Count", + "datapoints": [] + }, + "alb_target_5xx": { + "label": "HTTPCode_Target_5XX_Count", + "datapoints": [] + }, + "alb_target_response": { + "label": "TargetResponseTime", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.5005213825997035, + "p90": 0.49081191581736405, + "p50": 0.0016069181341690968, + "p95": 0.4961825739755273 + } + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.5117978895329298, + "p90": 1.1432096936062883, + "p50": 0.5024037003108845, + "p95": 1.2832487783847044 + } + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.8888792236095295, + "p90": 1.5602898584288212, + "p50": 1.1604053865967086, + "p95": 1.6704390030265635 + } + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.8139916951359347, + "p90": 1.530789537650868, + "p50": 1.1505018056846472, + "p95": 1.6206587563758006 + } + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.8450457291684828, + "p90": 1.5547777747106735, + "p50": 1.1348863117813814, + "p95": 1.672208938285089 + } + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.8853114255683752, + "p90": 1.5299423163676742, + "p50": 1.1440282056693523, + "p95": 1.653717772894364 + } + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.771241587019642, + "p90": 1.4986302469710606, + "p50": 1.1469635044910165, + "p95": 1.5969743068030005 + } + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.9176353638449173, + "p90": 1.5645161460898938, + "p50": 1.1572194284423918, + "p95": 1.6994982694971987 + } + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.9035583575561894, + "p90": 1.562309040847082, + "p50": 1.161570698549713, + "p95": 1.6793597591796334 + } + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.8626015031124603, + "p90": 1.5232402809184402, + "p50": 1.1245384911787704, + "p95": 1.6335054497057249 + } + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 1.7890379680264643, + "p90": 1.5100691302246265, + "p50": 1.119703632920338, + "p95": 1.6178605165757192 + } + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 4.242044367626646, + "p90": 3.7482320375318006, + "p50": 1.7648199460882255, + "p95": 3.976938307208984 + } + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 5.683100647998581, + "p90": 5.263568926527421, + "p50": 4.42094810691806, + "p95": 5.4352360097699215 + } + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 6.145181394704302, + "p90": 5.81238796610425, + "p50": 5.244265437348053, + "p95": 5.980980693111135 + } + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 6.0988860981981965, + "p90": 5.772456731265545, + "p50": 5.242209055535328, + "p95": 5.951587426084212 + } + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 6.269539225425209, + "p90": 5.927756138694554, + "p50": 5.267723250809718, + "p95": 6.0471514482254705 + } + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.17863320565535443, + "p90": 0.04408555720637314, + "p50": 0.002900799462199063, + "p95": 0.07173479423210939 + } + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Unit": "Seconds", + "ExtendedStatistics": { + "p99": 0.24798102219877785, + "p90": 0.05944444542334447, + "p50": 0.003232158232002402, + "p95": 0.08606966966592673 + } + } + ] + }, + "ecs_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 0.032596416771411896, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 13.574704806009928, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 12.062845865885416, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 14.789092063903809, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 12.07803757985433, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 11.818715413411459, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 12.702068964640299, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 11.432299931844076, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 12.210658073425293, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 11.095791816711426, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 11.91440455118815, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 11.007564862569174, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 12.858075141906738, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 13.510810534159342, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 13.549819310506185, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 13.187480926513672, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 68.82544199625652, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 77.06313578287761, + "Unit": "Percent" + } + ] + }, + "ecs_memory": { + "label": "MemoryUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 5.46875, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 4.98046875, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 5.826822916666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 6.005859375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 6.087239583333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 6.15234375, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 6.168619791666667, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 6.184895833333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 6.233723958333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 6.25, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 6.25, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 6.25, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 6.34765625, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 6.380208333333333, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 6.4453125, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 6.4453125, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 6.54296875, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 6.640625, + "Unit": "Percent" + } + ] + }, + "ecs_running_tasks": { + "label": "RunningTaskCount", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 1.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 1.0, + "Unit": "Count" + } + ] + }, + "rds_burst_balance": { + "label": "BurstBalance", + "datapoints": [] + }, + "rds_connections": { + "label": "DatabaseConnections", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 0.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 10.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 0.0, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 0.0, + "Unit": "Count" + } + ] + }, + "rds_cpu": { + "label": "CPUUtilization", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 3.7148711456129537, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 35.14198529289156, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 99.52334127764537, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 99.80999683328055, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 99.80833333333334, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 99.6018591014343, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 99.76488636174152, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 99.79177078127601, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 99.69984992496248, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 99.59166666666665, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 99.50833333333334, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 99.66683325004163, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 99.78322494580623, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 99.74846329396479, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 99.77832594419814, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 99.58668733230004, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 61.16870562352078, + "Unit": "Percent" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 4.627313656828414, + "Unit": "Percent" + } + ] + }, + "rds_cpu_credit": { + "label": "CPUCreditBalance", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 136.89453651666668, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 127.89600651666667, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 118.8959455, + "Unit": "Count" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 111.75111935, + "Unit": "Count" + } + ] + }, + "rds_free_memory": { + "label": "FreeableMemory", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 86867968.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 84504576.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 92930048.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 85712896.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 79523840.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 82714624.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 79966208.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 90292224.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 81620992.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 82665472.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 84733952.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 81457152.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 88825856.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 82919424.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 79323136.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 79519744.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 128004096.0, + "Unit": "Bytes" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 113164288.0, + "Unit": "Bytes" + } + ] + }, + "rds_read_latency": { + "label": "ReadLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 0.00012195121951219512, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 0.0003111111111111111, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 0.0021, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 0.0014893617021276594, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 9.900990099009902e-05, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 0.00039999999999999996, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 0.00015267175572519084, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 0.00029761904761904765, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 0.0005952380952380953, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 0.00013157894736842105, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 0.00019047619047619048, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 0.0002631578947368421, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + }, + "rds_write_latency": { + "label": "WriteLatency", + "datapoints": [ + { + "Timestamp": "2026-09-10T13:13:00+01:00", + "Average": 0.008726968174204356, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:14:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:15:00+01:00", + "Average": 0.00022222222222222223, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:16:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:17:00+01:00", + "Average": 0.0003225806451612903, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:18:00+01:00", + "Average": 0.003851351351351351, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:19:00+01:00", + "Average": 0.00125, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:20:00+01:00", + "Average": 0.0004878048780487805, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:21:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:22:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:23:00+01:00", + "Average": 0.004918625678119349, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:24:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:25:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:26:00+01:00", + "Average": 0.0004166666666666667, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:27:00+01:00", + "Average": 0.00020408163265306123, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:28:00+01:00", + "Average": 0.004720720720720721, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:29:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + }, + { + "Timestamp": "2026-09-10T13:30:00+01:00", + "Average": 0.0, + "Unit": "Seconds" + } + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/dataset.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/dataset.json new file mode 100644 index 0000000..a4758ac --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/dataset.json @@ -0,0 +1,15 @@ +{ + "dataset": "underflow-api-benchmark-v1", + "dateRange": { + "from": "2025-01-01", + "to": "2025-12-31" + }, + "servicesPerAccountPerDay": 50, + "users": 10, + "workspaces": 10, + "workspaceMembers": 10, + "awsAccounts": 200, + "costSyncRuns": 200, + "costSnapshots": 3650000, + "costRollups": 182500 +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/environment.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/environment.json new file mode 100644 index 0000000..6e2c956 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/environment.json @@ -0,0 +1,107 @@ +{ + "gitSha": "2624a6ccafc427521745b1180764fcde963f90bc", + "imageDigest": "sha256:b1eec7b5c7652dfab0fe8259ab66f9b0682e228930da769c8d1688961680e644", + "testTimestamp": "2026-09-10T21:07:35Z", + "awsRegion": "us-east-1", + "availabilityZones": [ + "us-east-1a", + "us-east-1b" + ], + "infrastructure": { + "ecs": { + "cpu": 512, + "memoryMiB": 1024, + "desiredCount": 1 + }, + "rds": { + "class": "db.t4g.micro", + "engine": "postgres", + "engineVersion": "16.13", + "storageGiB": 20, + "multiAz": false, + "publiclyAccessible": false + } + }, + "nodeVersion": "v25.9.0", + "k6Version": "k6 v2.2.0 (commit/00a9a1b7f5, go1.26.5, linux/amd64)", + "dataset": "underflow-api-benchmark-v1", + "profiles": { + "smoke": { + "vus": 2, + "duration": "30s" + }, + "supportedCapacity": { + "vus": 45, + "duration": "15m", + "serverSideTargetsMs": { + "p50": 30, + "p95": 100, + "p99": 250 + } + }, + "normalLoad": { + "stages": [ + [ + "1m", + 25 + ], + [ + "3m", + 25 + ], + [ + "1m", + 50 + ], + [ + "5m", + 50 + ], + [ + "2m", + 100 + ], + [ + "5m", + 100 + ], + [ + "1m", + 0 + ] + ] + }, + "stress": { + "stages": [ + [ + "2m", + 100 + ], + [ + "3m", + 100 + ], + [ + "1m", + 150 + ], + [ + "3m", + 150 + ], + [ + "1m", + 200 + ], + [ + "3m", + 200 + ], + [ + "1m", + 0 + ] + ] + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-covering-index.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-covering-index.json new file mode 100644 index 0000000..9d14bea --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-covering-index.json @@ -0,0 +1,606 @@ +{ + "benchmarkDiagnostic": true, + "stats": { + "total_size": "1423 MB", + "table_size": "439 MB", + "indexes_size": "984 MB", + "n_live_tup": "3650010", + "seq_scan": "96", + "idx_scan": "7" + }, + "plans": { + "summary": [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Finalize", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 21816.24, + "Total Cost": 21816.25, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 1662.094, + "Actual Total Time": 1680.351, + "Actual Rows": 1, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 3417, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 1535.725, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 21816.01, + "Total Cost": 21816.22, + "Plan Rows": 2, + "Plan Width": 64, + "Actual Startup Time": 1651.804, + "Actual Total Time": 1680.322, + "Actual Rows": 3, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Single Copy": false, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 3417, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 1535.725, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 20816.01, + "Total Cost": 20816.02, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 1644.889, + "Actual Total Time": 1644.89, + "Actual Rows": 1, + "Actual Loops": 3, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 3417, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 1535.725, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [], + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_cost_snapshots_workspace_date_reporting", + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 0.56, + "Total Cost": 20047.62, + "Plan Rows": 153677, + "Plan Width": 10, + "Actual Startup Time": 2.398, + "Actual Total Time": 1618.116, + "Actual Rows": 121667, + "Actual Loops": 3, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 3417, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 1535.725, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 158, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 5.082, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 16.606, + "Triggers": [], + "Execution Time": 1681.568 + } + ], + "timeseries": [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Sorted", + "Partial Mode": "Finalize", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1000.58, + "Total Cost": 22300.91, + "Plan Rows": 365, + "Plan Width": 68, + "Actual Startup Time": 56.454, + "Actual Total Time": 111, + "Actual Rows": 365, + "Actual Loops": 1, + "Group Key": [ + "usage_date" + ], + "Shared Hit Blocks": 3421, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather Merge", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1000.58, + "Total Cost": 22289.05, + "Plan Rows": 730, + "Plan Width": 68, + "Actual Startup Time": 56.395, + "Actual Total Time": 110.506, + "Actual Rows": 732, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Shared Hit Blocks": 3421, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Sorted", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.56, + "Total Cost": 21204.76, + "Plan Rows": 365, + "Plan Width": 68, + "Actual Startup Time": 0.346, + "Actual Total Time": 46.68, + "Actual Rows": 244, + "Actual Loops": 3, + "Group Key": [ + "usage_date" + ], + "Shared Hit Blocks": 3421, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [], + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_cost_snapshots_workspace_date_reporting", + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 0.56, + "Total Cost": 20047.62, + "Plan Rows": 153677, + "Plan Width": 14, + "Actual Startup Time": 0.034, + "Actual Total Time": 23.068, + "Actual Rows": 121667, + "Actual Loops": 3, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 3421, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 9, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 7.226, + "Triggers": [], + "Execution Time": 112.415 + } + ], + "byService": [ + { + "Plan": { + "Node Type": "Sort", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 22216.96, + "Total Cost": 22217.09, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 105.622, + "Actual Total Time": 108.875, + "Actual Rows": 50, + "Actual Loops": 1, + "Sort Key": [ + "(sum(amount)) DESC" + ], + "Sort Method": "quicksort", + "Sort Space Used": 27, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 3440, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Sorted", + "Partial Mode": "Finalize", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 22202.26, + "Total Cost": 22215.55, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 104.774, + "Actual Total Time": 108.19, + "Actual Rows": 50, + "Actual Loops": 1, + "Group Key": [ + "service_name" + ], + "Shared Hit Blocks": 3437, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather Merge", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 22202.26, + "Total Cost": 22213.93, + "Plan Rows": 100, + "Plan Width": 85, + "Actual Startup Time": 104.756, + "Actual Total Time": 108.068, + "Actual Rows": 150, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Shared Hit Blocks": 3437, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 21202.24, + "Total Cost": 21202.36, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 96.739, + "Actual Total Time": 96.745, + "Actual Rows": 50, + "Actual Loops": 3, + "Sort Key": [ + "service_name" + ], + "Sort Method": "quicksort", + "Sort Space Used": 30, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 3437, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [ + { + "Worker Number": 0, + "Sort Method": "quicksort", + "Sort Space Used": 30, + "Sort Space Type": "Memory" + }, + { + "Worker Number": 1, + "Sort Method": "quicksort", + "Sort Space Used": 30, + "Sort Space Type": "Memory" + } + ], + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Hashed", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 21200.2, + "Total Cost": 21200.82, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 92.664, + "Actual Total Time": 92.682, + "Actual Rows": 50, + "Actual Loops": 3, + "Group Key": [ + "service_name" + ], + "Planned Partitions": 0, + "HashAgg Batches": 1, + "Peak Memory Usage": 48, + "Disk Usage": 0, + "Shared Hit Blocks": 3421, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [ + { + "Worker Number": 0, + "HashAgg Batches": 1, + "Peak Memory Usage": 48, + "Disk Usage": 0 + }, + { + "Worker Number": 1, + "HashAgg Batches": 1, + "Peak Memory Usage": 48, + "Disk Usage": 0 + } + ], + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_cost_snapshots_workspace_date_reporting", + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 0.56, + "Total Cost": 20047.62, + "Plan Rows": 153677, + "Plan Width": 31, + "Actual Startup Time": 0.043, + "Actual Total Time": 47.768, + "Actual Rows": 121667, + "Actual Loops": 3, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 3421, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 26, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 0.9, + "Triggers": [], + "Execution Time": 108.981 + } + ] + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-index-cold.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-index-cold.json new file mode 100644 index 0000000..2c4415e --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-index-cold.json @@ -0,0 +1,171 @@ +{ + "benchmarkDiagnostic": true, + "stats": { + "total_size": "1328 MB", + "table_size": "439 MB", + "indexes_size": "889 MB", + "n_live_tup": "3650010", + "seq_scan": "94", + "idx_scan": "2" + }, + "plan": [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Finalize", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 16626.75, + "Total Cost": 16626.76, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 515.76, + "Actual Total Time": 565.052, + "Actual Rows": 1, + "Actual Loops": 1, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 2203, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 435.312, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 16626.53, + "Total Cost": 16626.74, + "Plan Rows": 2, + "Plan Width": 64, + "Actual Startup Time": 514.577, + "Actual Total Time": 565.03, + "Actual Rows": 3, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Single Copy": false, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 2203, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 435.312, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 15626.53, + "Total Cost": 15626.54, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 501.883, + "Actual Total Time": 501.884, + "Actual Rows": 1, + "Actual Loops": 3, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 2203, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 435.312, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [], + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_cost_snapshots_workspace_date", + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 0.56, + "Total Cost": 14870.31, + "Plan Rows": 151244, + "Plan Width": 10, + "Actual Startup Time": 1.895, + "Actual Total Time": 479.762, + "Actual Rows": 121667, + "Actual Loops": 3, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 6, + "Shared Read Blocks": 2203, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 435.312, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 154, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 2.536, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 10.875, + "Triggers": [], + "Execution Time": 565.241 + } + ] +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-index-warm.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-index-warm.json new file mode 100644 index 0000000..aa57883 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-index-warm.json @@ -0,0 +1,171 @@ +{ + "benchmarkDiagnostic": true, + "stats": { + "total_size": "1328 MB", + "table_size": "439 MB", + "indexes_size": "889 MB", + "n_live_tup": "3650010", + "seq_scan": "94", + "idx_scan": "5" + }, + "plan": [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Finalize", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 16626.75, + "Total Cost": 16626.76, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 300.848, + "Actual Total Time": 304.182, + "Actual Rows": 1, + "Actual Loops": 1, + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 16626.53, + "Total Cost": 16626.74, + "Plan Rows": 2, + "Plan Width": 64, + "Actual Startup Time": 300.834, + "Actual Total Time": 304.17, + "Actual Rows": 3, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Single Copy": false, + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 15626.53, + "Total Cost": 15626.54, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 289.377, + "Actual Total Time": 289.378, + "Actual Rows": 1, + "Actual Loops": 3, + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [], + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_cost_snapshots_workspace_date", + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 0.56, + "Total Cost": 14870.31, + "Plan Rows": 151244, + "Plan Width": 10, + "Actual Startup Time": 1.269, + "Actual Total Time": 267.45, + "Actual Rows": 121667, + "Actual Loops": 3, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 157, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 5.763, + "Triggers": [], + "Execution Time": 304.281 + } + ] +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-rollup.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-rollup.json new file mode 100644 index 0000000..236e045 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-after-rollup.json @@ -0,0 +1,386 @@ +{ + "benchmarkDiagnostic": true, + "stats": [ + { + "relname": "cost_snapshots", + "total_size": "1423 MB", + "table_size": "439 MB", + "indexes_size": "984 MB", + "n_live_tup": "3650010", + "seq_scan": "96", + "idx_scan": "49394" + }, + { + "relname": "workspace_cost_daily_rollups", + "total_size": "52 MB", + "table_size": "16 MB", + "indexes_size": "36 MB", + "n_live_tup": "182500", + "seq_scan": "3", + "idx_scan": "0" + } + ], + "plans": { + "summary": [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Simple", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1709.58, + "Total Cost": 1709.59, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 61.707, + "Actual Total Time": 61.708, + "Actual Rows": 1, + "Actual Loops": 1, + "Shared Hit Blocks": 306, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 2.775, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_workspace_cost_daily_rollups_workspace_service_date", + "Relation Name": "workspace_cost_daily_rollups", + "Alias": "workspace_cost_daily_rollups", + "Startup Cost": 0.42, + "Total Cost": 1618.47, + "Plan Rows": 18222, + "Plan Width": 12, + "Actual Startup Time": 0.103, + "Actual Total Time": 57.74, + "Actual Rows": 18250, + "Actual Loops": 1, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 306, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 2.775, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 149, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 6, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 13.424, + "Triggers": [], + "Execution Time": 63.087 + } + ], + "timeseries": [ + { + "Plan": { + "Node Type": "Sort", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1775.23, + "Total Cost": 1776.14, + "Plan Rows": 365, + "Plan Width": 68, + "Actual Startup Time": 10.673, + "Actual Total Time": 10.702, + "Actual Rows": 365, + "Actual Loops": 1, + "Sort Key": [ + "usage_date" + ], + "Sort Method": "quicksort", + "Sort Space Used": 39, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 312, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Hashed", + "Partial Mode": "Simple", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1755.13, + "Total Cost": 1759.69, + "Plan Rows": 365, + "Plan Width": 68, + "Actual Startup Time": 9.726, + "Actual Total Time": 9.835, + "Actual Rows": 365, + "Actual Loops": 1, + "Group Key": [ + "usage_date" + ], + "Planned Partitions": 0, + "HashAgg Batches": 1, + "Peak Memory Usage": 285, + "Disk Usage": 0, + "Shared Hit Blocks": 309, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_workspace_cost_daily_rollups_workspace_service_date", + "Relation Name": "workspace_cost_daily_rollups", + "Alias": "workspace_cost_daily_rollups", + "Startup Cost": 0.42, + "Total Cost": 1618.47, + "Plan Rows": 18222, + "Plan Width": 16, + "Actual Startup Time": 0.021, + "Actual Total Time": 4.254, + "Actual Rows": 18250, + "Actual Loops": 1, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 309, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 20, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 1.307, + "Triggers": [], + "Execution Time": 11.025 + } + ], + "byService": [ + { + "Plan": { + "Node Type": "Sort", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1757.17, + "Total Cost": 1757.29, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 8.085, + "Actual Total Time": 8.089, + "Actual Rows": 50, + "Actual Loops": 1, + "Sort Key": [ + "(sum(total_amount)) DESC" + ], + "Sort Method": "quicksort", + "Sort Space Used": 27, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 312, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Sorted", + "Partial Mode": "Simple", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 1755.76, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 0.192, + "Actual Total Time": 8.046, + "Actual Rows": 50, + "Actual Loops": 1, + "Group Key": [ + "service_name" + ], + "Shared Hit Blocks": 309, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_workspace_cost_daily_rollups_workspace_service_date", + "Relation Name": "workspace_cost_daily_rollups", + "Alias": "workspace_cost_daily_rollups", + "Startup Cost": 0.42, + "Total Cost": 1618.47, + "Plan Rows": 18222, + "Plan Width": 33, + "Actual Startup Time": 0.018, + "Actual Total Time": 3.916, + "Actual Rows": 18250, + "Actual Loops": 1, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 309, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 20, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 0.18, + "Triggers": [], + "Execution Time": 8.155 + } + ] + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-all-cost-queries-after-index.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-all-cost-queries-after-index.json new file mode 100644 index 0000000..03b00f0 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/explain-all-cost-queries-after-index.json @@ -0,0 +1,638 @@ +{ + "benchmarkDiagnostic": true, + "stats": { + "total_size": "1328 MB", + "table_size": "439 MB", + "indexes_size": "889 MB", + "n_live_tup": "3650010", + "seq_scan": "94", + "idx_scan": "199" + }, + "plans": { + "summary": [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Finalize", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 16626.75, + "Total Cost": 16626.76, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 269.255, + "Actual Total Time": 332.514, + "Actual Rows": 1, + "Actual Loops": 1, + "Shared Hit Blocks": 2206, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 2.112, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 16626.53, + "Total Cost": 16626.74, + "Plan Rows": 2, + "Plan Width": 64, + "Actual Startup Time": 269.239, + "Actual Total Time": 332.5, + "Actual Rows": 3, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Single Copy": false, + "Shared Hit Blocks": 2206, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 2.112, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Plain", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 15626.53, + "Total Cost": 15626.54, + "Plan Rows": 1, + "Plan Width": 64, + "Actual Startup Time": 255.513, + "Actual Total Time": 255.514, + "Actual Rows": 1, + "Actual Loops": 3, + "Shared Hit Blocks": 2206, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 2.112, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [], + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_cost_snapshots_workspace_date", + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 0.56, + "Total Cost": 14870.31, + "Plan Rows": 151244, + "Plan Width": 10, + "Actual Startup Time": 0.872, + "Actual Total Time": 232.371, + "Actual Rows": 121667, + "Actual Loops": 3, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 2206, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 2.112, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 135, + "Shared Read Blocks": 22, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 9.65, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 18.291, + "Triggers": [], + "Execution Time": 333.558 + } + ], + "timeseries": [ + { + "Plan": { + "Node Type": "Aggregate", + "Strategy": "Sorted", + "Partial Mode": "Finalize", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1000.58, + "Total Cost": 17105.34, + "Plan Rows": 365, + "Plan Width": 68, + "Actual Startup Time": 81.49, + "Actual Total Time": 173.273, + "Actual Rows": 365, + "Actual Loops": 1, + "Group Key": [ + "usage_date" + ], + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather Merge", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 1000.58, + "Total Cost": 17093.48, + "Plan Rows": 730, + "Plan Width": 68, + "Actual Startup Time": 81.396, + "Actual Total Time": 172.774, + "Actual Rows": 731, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Sorted", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.56, + "Total Cost": 16009.2, + "Plan Rows": 365, + "Plan Width": 68, + "Actual Startup Time": 0.283, + "Actual Total Time": 44.615, + "Actual Rows": 244, + "Actual Loops": 3, + "Group Key": [ + "usage_date" + ], + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [], + "Plans": [ + { + "Node Type": "Index Only Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "idx_cost_snapshots_workspace_date", + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 0.56, + "Total Cost": 14870.31, + "Plan Rows": 151244, + "Plan Width": 14, + "Actual Startup Time": 0.043, + "Actual Total Time": 20.963, + "Actual Rows": 121667, + "Actual Loops": 3, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Heap Fetches": 0, + "Shared Hit Blocks": 2209, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 9, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 10.911, + "Triggers": [], + "Execution Time": 177.469 + } + ], + "byService": [ + { + "Plan": { + "Node Type": "Sort", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 74400.47, + "Total Cost": 74400.6, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 4752.793, + "Actual Total Time": 4870.886, + "Actual Rows": 50, + "Actual Loops": 1, + "Sort Key": [ + "(sum(amount)) DESC" + ], + "Sort Method": "quicksort", + "Sort Space Used": 27, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 7860, + "Shared Read Blocks": 16821, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 13188.305, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Sorted", + "Partial Mode": "Finalize", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 74385.77, + "Total Cost": 74399.06, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 4750.929, + "Actual Total Time": 4869.189, + "Actual Rows": 50, + "Actual Loops": 1, + "Group Key": [ + "service_name" + ], + "Shared Hit Blocks": 7857, + "Shared Read Blocks": 16821, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 13188.305, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Gather Merge", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 74385.77, + "Total Cost": 74397.44, + "Plan Rows": 100, + "Plan Width": 85, + "Actual Startup Time": 4749.694, + "Actual Total Time": 4867.855, + "Actual Rows": 150, + "Actual Loops": 1, + "Workers Planned": 2, + "Workers Launched": 2, + "Shared Hit Blocks": 7857, + "Shared Read Blocks": 16821, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 13188.305, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 73385.75, + "Total Cost": 73385.87, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 4680.725, + "Actual Total Time": 4681.286, + "Actual Rows": 50, + "Actual Loops": 3, + "Sort Key": [ + "service_name" + ], + "Sort Method": "quicksort", + "Sort Space Used": 30, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 7857, + "Shared Read Blocks": 16821, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 13188.305, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [ + { + "Worker Number": 0, + "Sort Method": "quicksort", + "Sort Space Used": 30, + "Sort Space Type": "Memory" + }, + { + "Worker Number": 1, + "Sort Method": "quicksort", + "Sort Space Used": 30, + "Sort Space Type": "Memory" + } + ], + "Plans": [ + { + "Node Type": "Aggregate", + "Strategy": "Hashed", + "Partial Mode": "Partial", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 73383.71, + "Total Cost": 73384.33, + "Plan Rows": 50, + "Plan Width": 85, + "Actual Startup Time": 4655.295, + "Actual Total Time": 4655.324, + "Actual Rows": 50, + "Actual Loops": 3, + "Group Key": [ + "service_name" + ], + "Planned Partitions": 0, + "HashAgg Batches": 1, + "Peak Memory Usage": 48, + "Disk Usage": 0, + "Shared Hit Blocks": 7841, + "Shared Read Blocks": 16821, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 13188.305, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [ + { + "Worker Number": 0, + "HashAgg Batches": 1, + "Peak Memory Usage": 48, + "Disk Usage": 0 + }, + { + "Worker Number": 1, + "HashAgg Batches": 1, + "Peak Memory Usage": 48, + "Disk Usage": 0 + } + ], + "Plans": [ + { + "Node Type": "Bitmap Heap Scan", + "Parent Relationship": "Outer", + "Parallel Aware": true, + "Async Capable": false, + "Relation Name": "cost_snapshots", + "Alias": "cost_snapshots", + "Startup Cost": 13448.61, + "Total Cost": 72249.38, + "Plan Rows": 151244, + "Plan Width": 31, + "Actual Startup Time": 14.672, + "Actual Total Time": 4583.582, + "Actual Rows": 121667, + "Actual Loops": 3, + "Recheck Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Rows Removed by Index Recheck": 0, + "Exact Heap Blocks": 8323, + "Lossy Heap Blocks": 0, + "Shared Hit Blocks": 7841, + "Shared Read Blocks": 16821, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 13188.305, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [], + "Plans": [ + { + "Node Type": "Bitmap Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Index Name": "idx_cost_snapshots_workspace_date", + "Startup Cost": 0, + "Total Cost": 13357.87, + "Plan Rows": 362985, + "Plan Width": 0, + "Actual Startup Time": 30.291, + "Actual Total Time": 30.291, + "Actual Rows": 365000, + "Actual Loops": 1, + "Index Cond": "((workspace_id = '20000000-0000-4000-8000-[REDACTED_AWS_ACCOUNT_ID]'::uuid) AND (usage_date >= '2025-01-01'::date) AND (usage_date <= '2025-12-31'::date))", + "Shared Hit Blocks": 2202, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0, + "Workers": [] + } + ] + } + ] + } + ] + } + ] + } + ] + } + ] + }, + "Settings": { + "effective_cache_size": "369296kB", + "jit": "off" + }, + "Planning": { + "Shared Hit Blocks": 26, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "I/O Read Time": 0, + "I/O Write Time": 0, + "Temp I/O Read Time": 0, + "Temp I/O Write Time": 0 + }, + "Planning Time": 8.288, + "Triggers": [], + "Execution Time": 4881.756 + } + ] + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/load-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/load-summary.json new file mode 100644 index 0000000..e101bdd --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/load-summary.json @@ -0,0 +1,492 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "passes": 4877, + "fails": 19448, + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b" + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 24177, + "fails": 148 + }, + { + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 4877, + "fails": 19448, + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid" + }, + { + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 5025, + "fails": 19300, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error" + }, + { + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 4896, + "fails": 19429, + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200" + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 24196, + "fails": 129 + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 4896, + "fails": 19429 + }, + { + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 5025, + "fails": 19300, + "name": "cost_timeseries: no auth or server error" + }, + { + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 4088, + "fails": 16183 + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 20172, + "fails": 99 + }, + { + "passes": 4088, + "fails": 16183, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937" + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 4187, + "fails": 16084 + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 1633, + "fails": 6475 + }, + { + "passes": 8067, + "fails": 41, + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516" + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 1633, + "fails": 6475 + }, + { + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 1674, + "fails": 6434, + "name": "aws_account_list: no auth or server error" + }, + { + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 817, + "fails": 3238 + }, + { + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 4034, + "fails": 21 + }, + { + "path": "::sync_history: response shape is valid", + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 817, + "fails": 3238, + "name": "sync_history: response shape is valid" + }, + { + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 838, + "fails": 3217, + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error" + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 1080807.8306 + }, + "metrics": { + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "rate": 173805.80958200176, + "count": 187850680 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 81085, + "rate": 75.02258746125706 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0.4008713186325292, + "passes": 130017, + "fails": 194319 + }, + "thresholds": { + "rate>0.99": { + "ok": false + } + } + }, + "http_req_sending": { + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0.00014903619658383175, + "min": 0, + "med": 0, + "max": 1.4342 + }, + "type": "trend", + "contains": "time" + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0.7988283899611519, + "passes": 64773, + "fails": 16312 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 51730585, + "rate": 47862.888790583864 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 21200.8377, + "p(90)": 1574.81915, + "p(95)": 4862.471445, + "avg": 794.498336821066, + "min": 197.2772, + "med": 220.36725 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 100, + "min": 100, + "max": 100 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 6241.0414, + "p(90)": 1311.5007200000005, + "p(95)": 4267.424819999999, + "avg": 566.8355523247702, + "min": 0, + "med": 119.8733 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 136.7955, + "p(90)": 0, + "p(95)": 0, + "avg": 0.1457691730899674 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 790.09768, + "p(95)": 3839.2237799999994, + "avg": 478.35285334155384, + "min": 0, + "med": 120.1497, + "max": 5705.7234 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 81084, + "rate": 75.02166222739801 + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "max": 6559.6784, + "p(90)": 1635.6005, + "p(95)": 4475.634749999999, + "avg": 615.5000295989349, + "min": 0, + "med": 119.9605 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 580.3618943072064, + "min": 0, + "med": 119.9342, + "max": 6650.735, + "p(90)": 1435.03636, + "p(95)": 4360.035120000001 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "med": 119.7364, + "max": 6559.6784, + "p(90)": 1422.2633, + "p(95)": 4321.152080000003, + "avg": 577.394677929325, + "min": 0 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 1770.1792, + "p(90)": 1.0123, + "p(95)": 2.6079600000000034, + "avg": 2.967067341678489 + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "avg": 2371.683743225847, + "min": 113.3143, + "med": 1430.217, + "max": 6650.735, + "p(90)": 5432.37344, + "p(95)": 5670.554255 + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 4562.528579999998, + "avg": 613.7160610071961, + "min": 0, + "med": 119.9344, + "max": 6650.735, + "p(90)": 1528.07138 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 81084, + "rate": 75.02166222739801 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "fails": 16311, + "rate": 0.7988382418232943, + "passes": 64773 + }, + "thresholds": { + "rate<0.01": { + "ok": false + } + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "max": 136.7955, + "p(90)": 0, + "p(95)": 0, + "avg": 0.14598503422334597, + "min": 0, + "med": 0 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 0, + "max": 100 + } + }, + "measured_duration": { + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + }, + "type": "trend", + "contains": "time", + "values": { + "avg": 580.3614398907285, + "min": 0, + "med": 119.9342, + "max": 6650.735, + "p(90)": 1435.03682, + "p(95)": 4360.062264999995 + } + }, + "endpoint_aws_account_list_duration": { + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + }, + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 119.94300000000001, + "max": 5714.5888, + "p(90)": 827.86022, + "p(95)": 3877.8526899999965, + "avg": 484.0388600764678 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/rollup.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/rollup.json new file mode 100644 index 0000000..0b161d8 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/rollup.json @@ -0,0 +1,4 @@ +{ + "costRollupBackfill": true, + "rows": 182500 +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-after-covering-index.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-after-covering-index.json new file mode 100644 index 0000000..e9b5a7e --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-after-covering-index.json @@ -0,0 +1,492 @@ +{ + "metrics": { + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 200, + "rate": 6.444449128842911 + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 2.0129830845771144, + "min": 0, + "med": 0, + "max": 172.3786, + "p(90)": 0 + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 234.504014, + "min": 204.2331, + "med": 213.54180000000002, + "max": 608.7454, + "p(90)": 261.0743, + "p(95)": 304.49527999999987 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 385.54878999999994, + "avg": 301.729654, + "min": 220.0796, + "med": 293.27065, + "max": 709.2555, + "p(90)": 351.67377 + } + }, + "endpoint_cost_summary_duration": { + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time", + "values": { + "max": 485.3711, + "p(90)": 231.30100000000002, + "p(95)": 243.76201999999995, + "avg": 198.0279983333333, + "min": 175.4515, + "med": 184.13275 + } + }, + "endpoint_cost_timeseries_duration": { + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time", + "values": { + "avg": 211.13826333333336, + "min": 185.184, + "med": 193.88445000000002, + "max": 412.1376, + "p(90)": 268.74838, + "p(95)": 290.67519 + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "avg": 1.7500567164179104, + "min": 0, + "med": 0, + "max": 120.5372, + "p(90)": 0, + "p(95)": 0 + } + }, + "endpoint_sync_history_duration": { + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time", + "values": { + "avg": 123.16491999999998, + "min": 119.9965, + "med": 124.04105, + "max": 127.3809, + "p(90)": 125.15826, + "p(95)": 126.26957999999999 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "passes": 800, + "fails": 0, + "rate": 1 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 1853054, + "rate": 59709.56117999436 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 193.07495, + "max": 608.7454, + "p(90)": 251.50535, + "p(95)": 281.23817499999996, + "avg": 200.1448755, + "min": 119.7259 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + } + }, + "http_req_duration{expected_response:true}": { + "contains": "time", + "values": { + "p(90)": 253.2707, + "p(95)": 284.7907, + "avg": 202.45341592039804, + "min": 119.7259, + "med": 193.1059, + "max": 664.1615 + }, + "type": "trend" + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 2.1177621890547256, + "min": 0, + "med": 0.3143, + "max": 123.5184, + "p(90)": 3.3307, + "p(95)": 6.2356 + } + }, + "vus": { + "values": { + "value": 1, + "min": 1, + "max": 2 + }, + "type": "gauge", + "contains": "default" + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "measured_failures": { + "thresholds": { + "rate<0.01": { + "ok": true + } + }, + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 200 + } + }, + "data_sent": { + "contains": "data", + "values": { + "count": 128572, + "rate": 4142.878566967954 + }, + "type": "counter" + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "rate": 6.444449128842911, + "count": 200 + } + }, + "http_reqs": { + "values": { + "count": 201, + "rate": 6.476671374487125 + }, + "type": "counter", + "contains": "default" + }, + "http_req_waiting": { + "values": { + "avg": 200.33565373134343, + "min": 116.6615, + "med": 191.3569, + "max": 664.1615, + "p(90)": 250.8553, + "p(95)": 265.0447 + }, + "type": "trend", + "contains": "time" + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 201 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 124.6481, + "max": 156.9006, + "p(90)": 128.21906, + "p(95)": 130.64032, + "avg": 126.10747500000002, + "min": 119.7259 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 253.2707, + "p(95)": 284.7907, + "avg": 202.45341592039804, + "min": 119.7259, + "med": 193.1059, + "max": 664.1615 + } + } + }, + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 60, + "fails": 0, + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200" + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 60, + "fails": 0 + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 60, + "fails": 0 + }, + { + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 60, + "fails": 0 + }, + { + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 60, + "fails": 0, + "name": "cost_timeseries: status is 200" + }, + { + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 60, + "fails": 0, + "name": "cost_timeseries: response is JSON" + }, + { + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 60, + "fails": 0, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid" + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 60, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 50 + }, + { + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 50, + "fails": 0 + }, + { + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 50, + "fails": 0 + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 50, + "fails": 0 + }, + { + "fails": 0, + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 20 + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 20, + "fails": 0 + }, + { + "passes": 20, + "fails": 0, + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2" + }, + { + "name": "aws_account_list: no auth or server error", + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 20, + "fails": 0 + }, + { + "passes": 10, + "fails": 0, + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb" + }, + { + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 10, + "fails": 0 + }, + { + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 10, + "fails": 0, + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid" + }, + { + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 10, + "fails": 0 + } + ] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 31034.4602 + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-after-index.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-after-index.json new file mode 100644 index 0000000..90d6216 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-after-index.json @@ -0,0 +1,436 @@ +{ + "root_group": { + "checks": [ + { + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 30, + "fails": 0, + "name": "cost_summary: status is 200" + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 30, + "fails": 0 + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 30, + "fails": 0 + }, + { + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 30, + "fails": 0, + "name": "cost_summary: no auth or server error" + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 30, + "fails": 0 + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 30, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 30 + }, + { + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 30, + "fails": 0, + "name": "cost_timeseries: no auth or server error" + }, + { + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 6, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200" + }, + { + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 6, + "fails": 0, + "name": "cost_by_service: response is JSON" + }, + { + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 6 + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 6, + "fails": 0 + } + ], + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "testRunDurationMs": 32600.2426, + "isStdOutTTY": true, + "isStdErrTTY": true + }, + "metrics": { + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 4.079570149253732, + "min": 0, + "med": 0, + "max": 119.0283, + "p(90)": 2.504759999999999, + "p(95)": 3.777379999999998 + } + }, + "http_req_waiting": { + "values": { + "avg": 851.3495925373134, + "min": 170.7098, + "med": 184.2804, + "max": 7773.1857, + "p(90)": 431.4635799999993, + "p(95)": 7498.600769999999 + }, + "type": "trend", + "contains": "time" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 42894, + "rate": 1315.757079672775 + } + }, + "endpoint_cost_by_service_duration": { + "values": { + "p(90)": 7774.282999999999, + "p(95)": 7774.34245, + "avg": 7530.107083333333, + "min": 7155.4914, + "med": 7569.6728, + "max": 7774.4019 + }, + "thresholds": { + "p(99)<500": { + "ok": false + }, + "p(95)<200": { + "ok": false + } + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 186.0958, + "max": 322.6081, + "p(90)": 209.28229000000002, + "p(95)": 265.0318499999998, + "avg": 196.08340333333334, + "min": 180.212 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 172.9595, + "med": 184.4398, + "max": 7774.4019, + "p(90)": 311.96915, + "p(95)": 7510.936875, + "avg": 857.6684045454547 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 723048, + "rate": 22179.221451560607 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 66 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_req_duration{expected_response:true}": { + "values": { + "p(95)": 7499.357049999999, + "avg": 855.4291626865672, + "min": 172.9595, + "med": 184.5992, + "max": 7774.4019, + "p(90)": 476.62053999999944 + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_summary_duration": { + "contains": "time", + "values": { + "med": 179.01010000000002, + "max": 247.3465, + "p(90)": 201.02833, + "p(95)": 214.19035499999998, + "avg": 184.76567000000003, + "min": 172.9595 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend" + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 855.4291626865672, + "min": 172.9595, + "med": 184.5992, + "max": 7774.4019, + "p(90)": 476.62053999999944, + "p(95)": 7499.357049999999 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 66, + "rate": 2.0245248113583054 + } + }, + "http_req_connecting": { + "values": { + "avg": 5.227479104477612, + "min": 0, + "med": 0, + "max": 119.6825, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend", + "contains": "time" + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 156.021, + "p(90)": 0, + "p(95)": 0, + "avg": 5.8316462686567165 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 449.3528, + "p(95)": 7611.205099999999, + "avg": 961.6354909090909, + "min": 273.3362, + "med": 285.00735, + "max": 7874.4922 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "rate": 2.055199429712219, + "count": 67 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 66, + "rate": 2.0245248113583054 + } + }, + "checks": { + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 264, + "fails": 0 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 67 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-failed-20260909T201023Z.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-failed-20260909T201023Z.json new file mode 100644 index 0000000..184cf27 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary-failed-20260909T201023Z.json @@ -0,0 +1,380 @@ +{ + "root_group": { + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 28, + "fails": 0, + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200" + }, + { + "passes": 28, + "fails": 0, + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c" + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 28, + "fails": 0 + }, + { + "passes": 28, + "fails": 0, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f" + } + ], + "name": "", + "path": "" + }, + "options": { + "noColor": false, + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "" + }, + "state": { + "isStdErrTTY": true, + "testRunDurationMs": 32254.7316, + "isStdOutTTY": true + }, + "metrics": { + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 28, + "rate": 0.8680896913741487 + } + }, + "http_req_duration{expected_response:true}": { + "contains": "time", + "values": { + "avg": 2082.0090206896552, + "min": 720.7265, + "med": 2012.1423, + "max": 3256.4249, + "p(90)": 2636.60236, + "p(95)": 3007.8793399999986 + }, + "type": "trend" + }, + "iterations": { + "values": { + "count": 28, + "rate": 0.8680896913741487 + }, + "type": "counter", + "contains": "default" + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 28 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "avg": 12.376496551724138, + "min": 0, + "med": 0, + "max": 120.3326, + "p(90)": 23.858579999999915, + "p(95)": 119.2929 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 119.2929, + "avg": 14.423544827586209, + "min": 0, + "med": 0, + "max": 179.697, + "p(90)": 23.858579999999915 + } + }, + "endpoint_cost_timeseries_duration": { + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend" + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 2636.60409, + "p(95)": 3038.817934999999, + "avg": 2130.626253571428, + "min": 1794.0838, + "med": 2038.5707, + "max": 3256.4249 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 2130.626253571428, + "min": 1794.0838, + "med": 2038.5707, + "max": 3256.4249, + "p(90)": 2636.60409, + "p(95)": 3038.817934999999 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 29 + } + }, + "data_received": { + "contains": "data", + "values": { + "rate": 501.50781598815104, + "count": 16176 + }, + "type": "counter" + }, + "endpoint_sync_history_duration": { + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 112, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "http_req_receiving": { + "contains": "time", + "values": { + "avg": 0.15027931034482755, + "min": 0, + "med": 0, + "max": 0.7449, + "p(90)": 0.49267999999999995, + "p(95)": 0.55106 + }, + "type": "trend" + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 1894.2972, + "med": 2138.90485, + "max": 3476.0488, + "p(90)": 2736.9842700000004, + "p(95)": 3216.715584999999, + "avg": 2239.4733535714286 + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 2636.60236, + "p(95)": 3007.8793399999986, + "avg": 2082.0090206896552, + "min": 720.7265, + "med": 2012.1423, + "max": 3256.4249 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 18314, + "rate": 567.79266456522 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 29, + "rate": 0.8990928946375111 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "min": 720.7265, + "med": 2011.7252, + "max": 3255.8919, + "p(90)": 2636.60236, + "p(95)": 3007.6159999999986, + "avg": 2081.85874137931 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary.json new file mode 100644 index 0000000..90d6216 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/smoke-summary.json @@ -0,0 +1,436 @@ +{ + "root_group": { + "checks": [ + { + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 30, + "fails": 0, + "name": "cost_summary: status is 200" + }, + { + "name": "cost_summary: response is JSON", + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 30, + "fails": 0 + }, + { + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid", + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 30, + "fails": 0 + }, + { + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 30, + "fails": 0, + "name": "cost_summary: no auth or server error" + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 30, + "fails": 0 + }, + { + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON", + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 30, + "fails": 0 + }, + { + "fails": 0, + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 30 + }, + { + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 30, + "fails": 0, + "name": "cost_timeseries: no auth or server error" + }, + { + "id": "14cee747bc280031a6abb73cb8a32f6b", + "passes": 6, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200" + }, + { + "path": "::cost_by_service: response is JSON", + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 6, + "fails": 0, + "name": "cost_by_service: response is JSON" + }, + { + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid", + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 6 + }, + { + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error", + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 6, + "fails": 0 + } + ], + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "testRunDurationMs": 32600.2426, + "isStdOutTTY": true, + "isStdErrTTY": true + }, + "metrics": { + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 4.079570149253732, + "min": 0, + "med": 0, + "max": 119.0283, + "p(90)": 2.504759999999999, + "p(95)": 3.777379999999998 + } + }, + "http_req_waiting": { + "values": { + "avg": 851.3495925373134, + "min": 170.7098, + "med": 184.2804, + "max": 7773.1857, + "p(90)": 431.4635799999993, + "p(95)": 7498.600769999999 + }, + "type": "trend", + "contains": "time" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 42894, + "rate": 1315.757079672775 + } + }, + "endpoint_cost_by_service_duration": { + "values": { + "p(90)": 7774.282999999999, + "p(95)": 7774.34245, + "avg": 7530.107083333333, + "min": 7155.4914, + "med": 7569.6728, + "max": 7774.4019 + }, + "thresholds": { + "p(99)<500": { + "ok": false + }, + "p(95)<200": { + "ok": false + } + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 186.0958, + "max": 322.6081, + "p(90)": 209.28229000000002, + "p(95)": 265.0318499999998, + "avg": 196.08340333333334, + "min": 180.212 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 172.9595, + "med": 184.4398, + "max": 7774.4019, + "p(90)": 311.96915, + "p(95)": 7510.936875, + "avg": 857.6684045454547 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": false + } + } + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 723048, + "rate": 22179.221451560607 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 66 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_req_duration{expected_response:true}": { + "values": { + "p(95)": 7499.357049999999, + "avg": 855.4291626865672, + "min": 172.9595, + "med": 184.5992, + "max": 7774.4019, + "p(90)": 476.62053999999944 + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_summary_duration": { + "contains": "time", + "values": { + "med": 179.01010000000002, + "max": 247.3465, + "p(90)": 201.02833, + "p(95)": 214.19035499999998, + "avg": 184.76567000000003, + "min": 172.9595 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend" + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 855.4291626865672, + "min": 172.9595, + "med": 184.5992, + "max": 7774.4019, + "p(90)": 476.62053999999944, + "p(95)": 7499.357049999999 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 66, + "rate": 2.0245248113583054 + } + }, + "http_req_connecting": { + "values": { + "avg": 5.227479104477612, + "min": 0, + "med": 0, + "max": 119.6825, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend", + "contains": "time" + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 156.021, + "p(90)": 0, + "p(95)": 0, + "avg": 5.8316462686567165 + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "iteration_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 449.3528, + "p(95)": 7611.205099999999, + "avg": 961.6354909090909, + "min": 273.3362, + "med": 285.00735, + "max": 7874.4922 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "rate": 2.055199429712219, + "count": 67 + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 66, + "rate": 2.0245248113583054 + } + }, + "checks": { + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate", + "contains": "default", + "values": { + "rate": 1, + "passes": 264, + "fails": 0 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 67 + } + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/terraform-plan.txt b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/terraform-plan.txt new file mode 100644 index 0000000..e15c86e --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/terraform-plan.txt @@ -0,0 +1,995 @@ + +Terraform used the selected providers to generate the following execution +plan. Resource actions are indicated with the following symbols: + + create + <= read (data resources) + +Terraform will perform the following actions: + + # data.aws_iam_policy_document.task_execution_secrets will be read during apply + # (config refers to values not yet known) + <= data "aws_iam_policy_document" "task_execution_secrets" { + + id = (known after apply) + + json = (known after apply) + + minified_json = (known after apply) + + + statement { + + actions = [ + + "secretsmanager:GetSecretValue", + ] + + resources = [ + + (known after apply), + ] + + sid = "ReadBenchmarkRuntimeSecret" + } + } + + # aws_cloudwatch_log_group.api will be created + + resource "aws_cloudwatch_log_group" "api" { + + arn = (known after apply) + + deletion_protection_enabled = (known after apply) + + id = (known after apply) + + log_group_class = (known after apply) + + name = "/ecs/underflow-api-bench-c0cf1f68/api" + + name_prefix = (known after apply) + + region = "us-east-1" + + retention_in_days = 3 + + skip_destroy = false + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-api-logs" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-api-logs" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + } + + # aws_db_instance.postgres will be created + + resource "aws_db_instance" "postgres" { + + address = (known after apply) + + allocated_storage = 20 + + apply_immediately = true + + arn = (known after apply) + + auto_minor_version_upgrade = true + + availability_zone = (known after apply) + + backup_retention_period = 0 + + backup_target = (known after apply) + + backup_window = (known after apply) + + ca_cert_identifier = (known after apply) + + character_set_name = (known after apply) + + copy_tags_to_snapshot = false + + database_insights_mode = (known after apply) + + db_name = "underflow_benchmark" + + db_subnet_group_name = "underflow-api-bench-c0cf1f68-db-subnets" + + dedicated_log_volume = false + + delete_automated_backups = true + + deletion_protection = false + + domain_fqdn = (known after apply) + + endpoint = (known after apply) + + engine = "postgres" + + engine_lifecycle_support = (known after apply) + + engine_version = "16.13" + + engine_version_actual = (known after apply) + + hosted_zone_id = (known after apply) + + id = (known after apply) + + identifier = "underflow-api-bench-c0cf1f68-postgres" + + identifier_prefix = (known after apply) + + instance_class = "db.t4g.micro" + + iops = (known after apply) + + kms_key_id = (known after apply) + + latest_restorable_time = (known after apply) + + license_model = (known after apply) + + listener_endpoint = (known after apply) + + maintenance_window = (known after apply) + + master_user_secret = (known after apply) + + master_user_secret_kms_key_id = (known after apply) + + monitoring_interval = 0 + + monitoring_role_arn = (known after apply) + + multi_az = false + + nchar_character_set_name = (known after apply) + + network_type = (known after apply) + + option_group_name = (known after apply) + + parameter_group_name = (known after apply) + + password = (sensitive value) + + password_wo = (write-only attribute) + + performance_insights_enabled = false + + performance_insights_kms_key_id = (known after apply) + + performance_insights_retention_period = (known after apply) + + port = 5432 + + publicly_accessible = false + + region = "us-east-1" + + replica_mode = (known after apply) + + replicas = (known after apply) + + resource_id = (known after apply) + + skip_final_snapshot = true + + snapshot_identifier = (known after apply) + + status = (known after apply) + + storage_encrypted = true + + storage_throughput = (known after apply) + + storage_type = "gp3" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-postgres" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-postgres" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + timezone = (known after apply) + + upgrade_rollout_order = (known after apply) + + username = "underflow_benchmark" + + vpc_security_group_ids = (known after apply) + } + + # aws_db_subnet_group.benchmark will be created + + resource "aws_db_subnet_group" "benchmark" { + + arn = (known after apply) + + description = "Managed by Terraform" + + id = (known after apply) + + name = "underflow-api-bench-c0cf1f68-db-subnets" + + name_prefix = (known after apply) + + region = "us-east-1" + + subnet_ids = (known after apply) + + supported_network_types = (known after apply) + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-db-subnets" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-db-subnets" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_ecr_repository.api will be created + + resource "aws_ecr_repository" "api" { + + arn = (known after apply) + + force_delete = true + + id = (known after apply) + + image_tag_mutability = "IMMUTABLE" + + name = "underflow-api-bench-c0cf1f68-api" + + region = "us-east-1" + + registry_id = (known after apply) + + repository_url = (known after apply) + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-api" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-api" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + + image_scanning_configuration { + + scan_on_push = true + } + } + + # aws_ecs_cluster.benchmark will be created + + resource "aws_ecs_cluster" "benchmark" { + + arn = (known after apply) + + id = (known after apply) + + name = "underflow-api-bench-c0cf1f68-cluster" + + region = "us-east-1" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-cluster" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-cluster" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + + setting { + + name = "containerInsights" + + value = "enabled" + } + } + + # aws_ecs_service.api will be created + + resource "aws_ecs_service" "api" { + + arn = (known after apply) + + availability_zone_rebalancing = (known after apply) + + cluster = (known after apply) + + deployment_maximum_percent = 200 + + deployment_minimum_healthy_percent = 100 + + desired_count = 1 + + enable_ecs_managed_tags = false + + enable_execute_command = false + + health_check_grace_period_seconds = 90 + + iam_role = (known after apply) + + id = (known after apply) + + launch_type = "FARGATE" + + name = "underflow-api-bench-c0cf1f68-api" + + platform_version = (known after apply) + + region = "us-east-1" + + scheduling_strategy = "REPLICA" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-api" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-api" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + task_definition = (known after apply) + + triggers = (known after apply) + + wait_for_steady_state = true + + + deployment_circuit_breaker { + + enable = true + + rollback = true + } + + + deployment_configuration (known after apply) + + + load_balancer { + + container_name = "api" + + container_port = 3080 + + target_group_arn = (known after apply) + # (1 unchanged attribute hidden) + } + + + network_configuration { + + assign_public_ip = true + + security_groups = (known after apply) + + subnets = (known after apply) + } + } + + # aws_ecs_task_definition.api will be created + + resource "aws_ecs_task_definition" "api" { + + arn = (known after apply) + + arn_without_revision = (known after apply) + + container_definitions = (known after apply) + + cpu = "512" + + enable_fault_injection = (known after apply) + + execution_role_arn = (known after apply) + + family = "underflow-api-bench-c0cf1f68-api" + + id = (known after apply) + + memory = "1024" + + network_mode = "awsvpc" + + region = "us-east-1" + + requires_compatibilities = [ + + "FARGATE", + ] + + revision = (known after apply) + + skip_destroy = false + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-api" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-api" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + task_role_arn = (known after apply) + + track_latest = false + + + runtime_platform { + + cpu_architecture = "X86_64" + + operating_system_family = "LINUX" + } + } + + # aws_iam_role.task_execution will be created + + resource "aws_iam_role" "task_execution" { + + arn = (known after apply) + + assume_role_policy = jsonencode( + { + + Statement = [ + + { + + Action = "sts:AssumeRole" + + Effect = "Allow" + + Principal = { + + Service = "ecs-tasks.amazonaws.com" + } + }, + ] + + Version = "2012-10-17" + } + ) + + create_date = (known after apply) + + force_detach_policies = false + + id = (known after apply) + + managed_policy_arns = (known after apply) + + max_session_duration = 3600 + + name = "underflow-api-bench-c0cf1f68-task-execution" + + name_prefix = (known after apply) + + path = "/" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-task-execution" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-task-execution" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + unique_id = (known after apply) + + + inline_policy (known after apply) + } + + # aws_iam_role.task_runtime will be created + + resource "aws_iam_role" "task_runtime" { + + arn = (known after apply) + + assume_role_policy = jsonencode( + { + + Statement = [ + + { + + Action = "sts:AssumeRole" + + Effect = "Allow" + + Principal = { + + Service = "ecs-tasks.amazonaws.com" + } + }, + ] + + Version = "2012-10-17" + } + ) + + create_date = (known after apply) + + force_detach_policies = false + + id = (known after apply) + + managed_policy_arns = (known after apply) + + max_session_duration = 3600 + + name = "underflow-api-bench-c0cf1f68-task-runtime" + + name_prefix = (known after apply) + + path = "/" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-task-runtime" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-task-runtime" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + unique_id = (known after apply) + + + inline_policy (known after apply) + } + + # aws_iam_role_policy.task_execution_secrets will be created + + resource "aws_iam_role_policy" "task_execution_secrets" { + + id = (known after apply) + + name = "underflow-api-bench-c0cf1f68-read-runtime-secret" + + name_prefix = (known after apply) + + policy = (known after apply) + + role = (known after apply) + } + + # aws_iam_role_policy_attachment.task_execution will be created + + resource "aws_iam_role_policy_attachment" "task_execution" { + + id = (known after apply) + + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy" + + role = "underflow-api-bench-c0cf1f68-task-execution" + } + + # aws_internet_gateway.benchmark will be created + + resource "aws_internet_gateway" "benchmark" { + + arn = (known after apply) + + id = (known after apply) + + owner_id = (known after apply) + + region = "us-east-1" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-igw" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-igw" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_lb.api will be created + + resource "aws_lb" "api" { + + arn = (known after apply) + + arn_suffix = (known after apply) + + client_keep_alive = 3600 + + desync_mitigation_mode = "defensive" + + dns_name = (known after apply) + + drop_invalid_header_fields = false + + enable_deletion_protection = false + + enable_http2 = true + + enable_prefix_for_ipv6_source_nat = (known after apply) + + enable_tls_version_and_cipher_suite_headers = false + + enable_waf_fail_open = false + + enable_xff_client_port = false + + enable_zonal_shift = false + + enforce_security_group_inbound_rules_on_private_link_traffic = (known after apply) + + id = (known after apply) + + idle_timeout = 60 + + internal = false + + ip_address_type = (known after apply) + + load_balancer_type = "application" + + name = "underflow-api-bench-c0cf1f68-alb" + + name_prefix = (known after apply) + + preserve_host_header = false + + region = "us-east-1" + + secondary_ips_auto_assigned_per_subnet = (known after apply) + + security_groups = (known after apply) + + subnets = (known after apply) + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-alb" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-alb" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + + xff_header_processing_mode = "append" + + zone_id = (known after apply) + + + subnet_mapping (known after apply) + } + + # aws_lb_listener.http will be created + + resource "aws_lb_listener" "http" { + + arn = (known after apply) + + id = (known after apply) + + load_balancer_arn = (known after apply) + + port = 80 + + protocol = "HTTP" + + region = "us-east-1" + + routing_http_request_x_amzn_mtls_clientcert_header_name = (known after apply) + + routing_http_request_x_amzn_mtls_clientcert_issuer_header_name = (known after apply) + + routing_http_request_x_amzn_mtls_clientcert_leaf_header_name = (known after apply) + + routing_http_request_x_amzn_mtls_clientcert_serial_number_header_name = (known after apply) + + routing_http_request_x_amzn_mtls_clientcert_subject_header_name = (known after apply) + + routing_http_request_x_amzn_mtls_clientcert_validity_header_name = (known after apply) + + routing_http_request_x_amzn_tls_cipher_suite_header_name = (known after apply) + + routing_http_request_x_amzn_tls_version_header_name = (known after apply) + + routing_http_response_access_control_allow_credentials_header_value = (known after apply) + + routing_http_response_access_control_allow_headers_header_value = (known after apply) + + routing_http_response_access_control_allow_methods_header_value = (known after apply) + + routing_http_response_access_control_allow_origin_header_value = (known after apply) + + routing_http_response_access_control_expose_headers_header_value = (known after apply) + + routing_http_response_access_control_max_age_header_value = (known after apply) + + routing_http_response_content_security_policy_header_value = (known after apply) + + routing_http_response_server_enabled = (known after apply) + + routing_http_response_strict_transport_security_header_value = (known after apply) + + routing_http_response_x_content_type_options_header_value = (known after apply) + + routing_http_response_x_frame_options_header_value = (known after apply) + + ssl_policy = (known after apply) + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-http" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-http" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + tcp_idle_timeout_seconds = (known after apply) + + + default_action { + + order = (known after apply) + + target_group_arn = (known after apply) + + type = "forward" + } + + + mutual_authentication (known after apply) + } + + # aws_lb_target_group.api will be created + + resource "aws_lb_target_group" "api" { + + arn = (known after apply) + + arn_suffix = (known after apply) + + connection_termination = (known after apply) + + deregistration_delay = "300" + + id = (known after apply) + + ip_address_type = (known after apply) + + lambda_multi_value_headers_enabled = false + + load_balancer_arns = (known after apply) + + load_balancing_algorithm_type = (known after apply) + + load_balancing_anomaly_mitigation = (known after apply) + + load_balancing_cross_zone_enabled = (known after apply) + + name = "underflow-api-bench-c0cf1f68-tg" + + name_prefix = (known after apply) + + port = 3080 + + preserve_client_ip = (known after apply) + + protocol = "HTTP" + + protocol_version = (known after apply) + + proxy_protocol_v2 = false + + region = "us-east-1" + + slow_start = 0 + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-tg" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-tg" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + target_type = "ip" + + vpc_id = (known after apply) + + + health_check { + + enabled = true + + healthy_threshold = 2 + + interval = 30 + + matcher = "200" + + path = "/api/v1/health" + + port = "traffic-port" + + protocol = "HTTP" + + timeout = 5 + + unhealthy_threshold = 3 + } + + + stickiness (known after apply) + + + target_failover (known after apply) + + + target_group_health (known after apply) + + + target_health_state (known after apply) + } + + # aws_route_table.public will be created + + resource "aws_route_table" "public" { + + arn = (known after apply) + + id = (known after apply) + + owner_id = (known after apply) + + propagating_vgws = (known after apply) + + region = "us-east-1" + + route = [ + + { + + cidr_block = "[REDACTED_IPV4]/0" + + gateway_id = (known after apply) + # (12 unchanged attributes hidden) + }, + ] + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-public" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-public" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_route_table_association.public[0] will be created + + resource "aws_route_table_association" "public" { + + id = (known after apply) + + region = "us-east-1" + + route_table_id = (known after apply) + + subnet_id = (known after apply) + } + + # aws_route_table_association.public[1] will be created + + resource "aws_route_table_association" "public" { + + id = (known after apply) + + region = "us-east-1" + + route_table_id = (known after apply) + + subnet_id = (known after apply) + } + + # aws_secretsmanager_secret.runtime will be created + + resource "aws_secretsmanager_secret" "runtime" { + + arn = (known after apply) + + description = "Disposable API benchmark runtime secrets." + + force_overwrite_replica_secret = false + + id = (known after apply) + + name = "underflow-api-bench-c0cf1f68-runtime" + + name_prefix = (known after apply) + + policy = (known after apply) + + recovery_window_in_days = 0 + + region = "us-east-1" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-runtime" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-runtime" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + + replica (known after apply) + } + + # aws_secretsmanager_secret_version.runtime will be created + + resource "aws_secretsmanager_secret_version" "runtime" { + + arn = (known after apply) + + has_secret_string_wo = (known after apply) + + id = (known after apply) + + region = "us-east-1" + + secret_arn = (known after apply) + + secret_id = (known after apply) + + secret_string = (sensitive value) + + secret_string_wo = (write-only attribute) + + version_id = (known after apply) + + version_stages = (known after apply) + } + + # aws_security_group.alb will be created + + resource "aws_security_group" "alb" { + + arn = (known after apply) + + description = "Benchmark load generator ingress only." + + egress = [ + + { + + cidr_blocks = [ + + "[REDACTED_IPV4]/0", + ] + + from_port = 0 + + ipv6_cidr_blocks = [] + + prefix_list_ids = [] + + protocol = "-1" + + security_groups = [] + + self = false + + to_port = 0 + # (1 unchanged attribute hidden) + }, + ] + + id = (known after apply) + + ingress = [ + + { + + cidr_blocks = [ + + "[REDACTED_IPV4]/32", + ] + + description = "HTTP from the explicitly configured load generator" + + from_port = 80 + + ipv6_cidr_blocks = [] + + prefix_list_ids = [] + + protocol = "tcp" + + security_groups = [] + + self = false + + to_port = 80 + }, + ] + + name = "underflow-api-bench-c0cf1f68-alb" + + name_prefix = (known after apply) + + owner_id = (known after apply) + + region = "us-east-1" + + revoke_rules_on_delete = false + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-alb" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-alb" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_security_group.ecs will be created + + resource "aws_security_group" "ecs" { + + arn = (known after apply) + + description = "Benchmark API tasks; no direct public ingress." + + egress = [ + + { + + cidr_blocks = [ + + "[REDACTED_IPV4]/0", + ] + + from_port = 0 + + ipv6_cidr_blocks = [] + + prefix_list_ids = [] + + protocol = "-1" + + security_groups = [] + + self = false + + to_port = 0 + # (1 unchanged attribute hidden) + }, + ] + + id = (known after apply) + + ingress = [ + + { + + cidr_blocks = [] + + description = "API traffic from the benchmark ALB only" + + from_port = 3080 + + ipv6_cidr_blocks = [] + + prefix_list_ids = [] + + protocol = "tcp" + + security_groups = (known after apply) + + self = false + + to_port = 3080 + }, + ] + + name = "underflow-api-bench-c0cf1f68-ecs" + + name_prefix = (known after apply) + + owner_id = (known after apply) + + region = "us-east-1" + + revoke_rules_on_delete = false + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-ecs" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-ecs" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_security_group.rds will be created + + resource "aws_security_group" "rds" { + + arn = (known after apply) + + description = "Benchmark PostgreSQL ingress from API tasks only." + + egress = [ + + { + + cidr_blocks = [ + + "[REDACTED_IPV4]/0", + ] + + from_port = 0 + + ipv6_cidr_blocks = [] + + prefix_list_ids = [] + + protocol = "-1" + + security_groups = [] + + self = false + + to_port = 0 + # (1 unchanged attribute hidden) + }, + ] + + id = (known after apply) + + ingress = [ + + { + + cidr_blocks = [] + + description = "PostgreSQL from benchmark API tasks only" + + from_port = 5432 + + ipv6_cidr_blocks = [] + + prefix_list_ids = [] + + protocol = "tcp" + + security_groups = (known after apply) + + self = false + + to_port = 5432 + }, + ] + + name = "underflow-api-bench-c0cf1f68-rds" + + name_prefix = (known after apply) + + owner_id = (known after apply) + + region = "us-east-1" + + revoke_rules_on_delete = false + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-rds" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-rds" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_subnet.public[0] will be created + + resource "aws_subnet" "public" { + + arn = (known after apply) + + assign_ipv6_address_on_creation = false + + availability_zone = "us-east-1a" + + availability_zone_id = (known after apply) + + cidr_block = "[REDACTED_IPV4]/24" + + enable_dns64 = false + + enable_resource_name_dns_a_record_on_launch = false + + enable_resource_name_dns_aaaa_record_on_launch = false + + id = (known after apply) + + ipv6_cidr_block = (known after apply) + + ipv6_cidr_block_association_id = (known after apply) + + ipv6_native = false + + map_public_ip_on_launch = true + + owner_id = (known after apply) + + private_dns_hostname_type_on_launch = (known after apply) + + region = "us-east-1" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-public-1" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-public-1" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_subnet.public[1] will be created + + resource "aws_subnet" "public" { + + arn = (known after apply) + + assign_ipv6_address_on_creation = false + + availability_zone = "us-east-1b" + + availability_zone_id = (known after apply) + + cidr_block = "[REDACTED_IPV4]/24" + + enable_dns64 = false + + enable_resource_name_dns_a_record_on_launch = false + + enable_resource_name_dns_aaaa_record_on_launch = false + + id = (known after apply) + + ipv6_cidr_block = (known after apply) + + ipv6_cidr_block_association_id = (known after apply) + + ipv6_native = false + + map_public_ip_on_launch = true + + owner_id = (known after apply) + + private_dns_hostname_type_on_launch = (known after apply) + + region = "us-east-1" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-public-2" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-public-2" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + + vpc_id = (known after apply) + } + + # aws_vpc.benchmark will be created + + resource "aws_vpc" "benchmark" { + + arn = (known after apply) + + cidr_block = "[REDACTED_IPV4]/16" + + default_network_acl_id = (known after apply) + + default_route_table_id = (known after apply) + + default_security_group_id = (known after apply) + + dhcp_options_id = (known after apply) + + enable_dns_hostnames = true + + enable_dns_support = true + + enable_network_address_usage_metrics = (known after apply) + + id = (known after apply) + + instance_tenancy = "default" + + ipv6_association_id = (known after apply) + + ipv6_cidr_block = (known after apply) + + ipv6_cidr_block_network_border_group = (known after apply) + + main_route_table_id = (known after apply) + + owner_id = (known after apply) + + region = "us-east-1" + + tags = { + + "Name" = "underflow-api-bench-c0cf1f68-vpc" + } + + tags_all = { + + "Name" = "underflow-api-bench-c0cf1f68-vpc" + + "environment" = "api-benchmark" + + "managed" = "terraform" + + "project" = "underflow" + + "purpose" = "disposable-load-test" + } + } + + # random_password.benchmark will be created + + resource "random_password" "benchmark" { + + bcrypt_hash = (sensitive value) + + id = (known after apply) + + length = 24 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + numeric = true + + result = (sensitive value) + + special = false + + upper = true + } + + # random_password.csrf will be created + + resource "random_password" "csrf" { + + bcrypt_hash = (sensitive value) + + id = (known after apply) + + length = 48 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + numeric = true + + result = (sensitive value) + + special = false + + upper = true + } + + # random_password.database will be created + + resource "random_password" "database" { + + bcrypt_hash = (sensitive value) + + id = (known after apply) + + length = 32 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + numeric = true + + override_special = "!#$%&*+-.:=?@_" + + result = (sensitive value) + + special = true + + upper = true + } + + # random_password.jwt_access will be created + + resource "random_password" "jwt_access" { + + bcrypt_hash = (sensitive value) + + id = (known after apply) + + length = 64 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + numeric = true + + result = (sensitive value) + + special = false + + upper = true + } + + # random_password.jwt_refresh will be created + + resource "random_password" "jwt_refresh" { + + bcrypt_hash = (sensitive value) + + id = (known after apply) + + length = 64 + + lower = true + + min_lower = 0 + + min_numeric = 0 + + min_special = 0 + + min_upper = 0 + + number = true + + numeric = true + + result = (sensitive value) + + special = false + + upper = true + } + +Plan: 31 to add, 0 to change, 0 to destroy. + +Changes to Outputs: + + alb_dns_name = (known after apply) + + api_base_url = (known after apply) + + availability_zones = [ + + "us-east-1a", + + "us-east-1b", + ] + + aws_region = "us-east-1" + + ecr_repository_url = (known after apply) + + ecs_cluster_name = "underflow-api-bench-c0cf1f68-cluster" + + ecs_security_group_id = (known after apply) + + ecs_service_name = "underflow-api-bench-c0cf1f68-api" + + load_balancer_arn_suffix = (known after apply) + + rds_identifier = "underflow-api-bench-c0cf1f68-postgres" + + resource_prefix = "underflow-api-bench-c0cf1f68" + + runtime_secret_arn = (sensitive value) + + subnet_ids = [ + + (known after apply), + + (known after apply), + ] + + target_group_arn_suffix = (known after apply) + + task_definition_arn = (known after apply) diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/terraform-state-list.txt b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/terraform-state-list.txt new file mode 100644 index 0000000..71a0fc2 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/terraform-state-list.txt @@ -0,0 +1,34 @@ +data.aws_availability_zones.available +data.aws_iam_policy_document.ecs_tasks_assume_role +data.aws_iam_policy_document.task_execution_secrets +aws_cloudwatch_log_group.api +aws_db_instance.postgres +aws_db_subnet_group.benchmark +aws_ecr_repository.api +aws_ecs_cluster.benchmark +aws_ecs_service.api +aws_ecs_task_definition.api +aws_iam_role.task_execution +aws_iam_role.task_runtime +aws_iam_role_policy.task_execution_secrets +aws_iam_role_policy_attachment.task_execution +aws_internet_gateway.benchmark +aws_lb.api +aws_lb_listener.http +aws_lb_target_group.api +aws_route_table.public +aws_route_table_association.public[0] +aws_route_table_association.public[1] +aws_secretsmanager_secret.runtime +aws_secretsmanager_secret_version.runtime +aws_security_group.alb +aws_security_group.ecs +aws_security_group.rds +aws_subnet.public[0] +aws_subnet.public[1] +aws_vpc.benchmark +random_password.benchmark +random_password.csrf +random_password.database +random_password.jwt_access +random_password.jwt_refresh diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/warmup-after-covering-index.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/warmup-after-covering-index.json new file mode 100644 index 0000000..4b93a52 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/warmup-after-covering-index.json @@ -0,0 +1,492 @@ +{ + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [ + { + "name": "cost_summary: status is 200", + "path": "::cost_summary: status is 200", + "id": "27ace42e0838171e90274ce32cd6290b", + "passes": 120, + "fails": 0 + }, + { + "path": "::cost_summary: response is JSON", + "id": "5c98da1347202513e59aea4d725ece0c", + "passes": 120, + "fails": 0, + "name": "cost_summary: response is JSON" + }, + { + "id": "7e51481ff86a92d413a2e54328bd5585", + "passes": 120, + "fails": 0, + "name": "cost_summary: response shape is valid", + "path": "::cost_summary: response shape is valid" + }, + { + "fails": 0, + "name": "cost_summary: no auth or server error", + "path": "::cost_summary: no auth or server error", + "id": "52c2c984dbda937f4b960b19288e383f", + "passes": 120 + }, + { + "name": "cost_timeseries: status is 200", + "path": "::cost_timeseries: status is 200", + "id": "10cbb58adda88d83d01e1781d8b897aa", + "passes": 120, + "fails": 0 + }, + { + "id": "31828de59a3a5cd044ecaca2017c8979", + "passes": 120, + "fails": 0, + "name": "cost_timeseries: response is JSON", + "path": "::cost_timeseries: response is JSON" + }, + { + "name": "cost_timeseries: response shape is valid", + "path": "::cost_timeseries: response shape is valid", + "id": "4f849d0c12b65a21be4c90c5f9132131", + "passes": 120, + "fails": 0 + }, + { + "name": "cost_timeseries: no auth or server error", + "path": "::cost_timeseries: no auth or server error", + "id": "dfcadc5ebdac709bc5fc6776e3b473f7", + "passes": 120, + "fails": 0 + }, + { + "passes": 100, + "fails": 0, + "name": "cost_by_service: status is 200", + "path": "::cost_by_service: status is 200", + "id": "14cee747bc280031a6abb73cb8a32f6b" + }, + { + "id": "aa7f737b89a366e668e00f7bb4be1b9d", + "passes": 100, + "fails": 0, + "name": "cost_by_service: response is JSON", + "path": "::cost_by_service: response is JSON" + }, + { + "id": "424b793ff7dc5af08b0bc8cb8679d937", + "passes": 100, + "fails": 0, + "name": "cost_by_service: response shape is valid", + "path": "::cost_by_service: response shape is valid" + }, + { + "id": "59a586bb19bed0d458e09f66a5cc6be2", + "passes": 100, + "fails": 0, + "name": "cost_by_service: no auth or server error", + "path": "::cost_by_service: no auth or server error" + }, + { + "name": "aws_account_list: status is 200", + "path": "::aws_account_list: status is 200", + "id": "2d94d8e00e13c682d675579272418e2c", + "passes": 40, + "fails": 0 + }, + { + "name": "aws_account_list: response is JSON", + "path": "::aws_account_list: response is JSON", + "id": "5dcfc003a1b44986b53d1776ddc85516", + "passes": 40, + "fails": 0 + }, + { + "name": "aws_account_list: response shape is valid", + "path": "::aws_account_list: response shape is valid", + "id": "955e13b396fc3ef5a9711cdab920d1b2", + "passes": 40, + "fails": 0 + }, + { + "path": "::aws_account_list: no auth or server error", + "id": "556b2ff8f62e4a746ceb9342c8dcd3a8", + "passes": 40, + "fails": 0, + "name": "aws_account_list: no auth or server error" + }, + { + "fails": 0, + "name": "sync_history: status is 200", + "path": "::sync_history: status is 200", + "id": "d620da2a1a1b5f344b27753879e622fb", + "passes": 20 + }, + { + "name": "sync_history: response is JSON", + "path": "::sync_history: response is JSON", + "id": "219e8a6ca86ab825972b2a10e12b1bbe", + "passes": 20, + "fails": 0 + }, + { + "id": "2868204372a6dd88299dc61219c010bc", + "passes": 20, + "fails": 0, + "name": "sync_history: response shape is valid", + "path": "::sync_history: response shape is valid" + }, + { + "fails": 0, + "name": "sync_history: no auth or server error", + "path": "::sync_history: no auth or server error", + "id": "c449aa17fa63a421de42d93de335151d", + "passes": 20 + } + ] + }, + "options": { + "noColor": false, + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "" + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 61152.0884 + }, + "metrics": { + "vus_max": { + "values": { + "value": 2, + "min": 2, + "max": 2 + }, + "type": "gauge", + "contains": "default" + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "rate": 6.557421185308203, + "count": 401 + } + }, + "data_sent": { + "type": "counter", + "contains": "data", + "values": { + "count": 256862, + "rate": 4200.379851622532 + } + }, + "vus": { + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + }, + "type": "gauge" + }, + "endpoint_sync_history_duration": { + "values": { + "max": 130.7253, + "p(90)": 129.31742, + "p(95)": 130.30882, + "avg": 124.50813999999998, + "min": 119.9934, + "med": 125.0402 + }, + "thresholds": { + "p(99)<500": { + "ok": true + }, + "p(95)<200": { + "ok": true + } + }, + "type": "trend", + "contains": "time" + }, + "endpoint_cost_timeseries_duration": { + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time", + "values": { + "max": 434.0696, + "p(90)": 231.75937000000005, + "p(95)": 306.80621499999995, + "avg": 208.06491833333328, + "min": 179.4055, + "med": 194.57015 + } + }, + "endpoint_cost_summary_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 203.2400591666666, + "min": 176.923, + "med": 187.61725, + "max": 379.5257, + "p(90)": 241.04909000000004, + "p(95)": 270.70661 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "min": 203.0791, + "med": 217.70815, + "max": 358.0709, + "p(90)": 265.58596000000006, + "p(95)": 302.73309499999993, + "avg": 231.28093599999988 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_req_connecting": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 118.434, + "p(90)": 0, + "p(95)": 0, + "avg": 0.8739768079800498 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + } + }, + "http_req_failed": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 401 + } + }, + "data_received": { + "contains": "data", + "values": { + "count": 3704464, + "rate": 60577.88207933059 + }, + "type": "counter" + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 400 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "http_req_duration{expected_response:true}": { + "type": "trend", + "contains": "time", + "values": { + "min": 114.2297, + "med": 195.4294, + "max": 724.7384, + "p(90)": 246.434, + "p(95)": 274.0703, + "avg": 201.48684488778053 + } + }, + "http_req_duration": { + "values": { + "p(95)": 274.0703, + "avg": 201.48684488778053, + "min": 114.2297, + "med": 195.4294, + "max": 724.7384, + "p(90)": 246.434 + }, + "type": "trend", + "contains": "time" + }, + "http_req_blocked": { + "contains": "time", + "values": { + "avg": 0.896937157107232, + "min": 0, + "med": 0, + "max": 127.4808, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend" + }, + "measured_requests": { + "type": "counter", + "contains": "default", + "values": { + "count": 400, + "rate": 6.541068514023145 + } + }, + "iterations": { + "type": "counter", + "contains": "default", + "values": { + "count": 400, + "rate": 6.541068514023145 + } + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 246.21125, + "p(95)": 271.78782999999993, + "avg": 200.17871599999998, + "min": 114.2297, + "med": 195.31595, + "max": 434.0696 + }, + "thresholds": { + "p(95)<200": { + "ok": false + }, + "p(99)<500": { + "ok": true + } + } + }, + "endpoint_aws_account_list_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 125.58075, + "max": 209.3966, + "p(90)": 134.1675, + "p(95)": 135.64968000000002, + "avg": 127.41581749999997, + "min": 114.2297 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "checks": { + "contains": "default", + "values": { + "rate": 1, + "passes": 1600, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + }, + "type": "rate" + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + } + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "avg": 1.702756608478804, + "min": 0, + "med": 0.3967, + "max": 119.1295, + "p(90)": 3.8914, + "p(95)": 5.5716 + } + }, + "http_req_waiting": { + "contains": "time", + "values": { + "avg": 199.7840882793018, + "min": 114.2297, + "med": 194.3901, + "max": 723.9436, + "p(90)": 243.9218, + "p(95)": 271.6677 + }, + "type": "trend" + }, + "iteration_duration": { + "contains": "time", + "values": { + "avg": 301.2783107499998, + "min": 214.3706, + "med": 296.28955, + "max": 644.4486, + "p(90)": 346.88457, + "p(95)": 372.1879999999999 + }, + "type": "trend" + } + } +} diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/warmup-after-index-summary.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/warmup-after-index-summary.json new file mode 100644 index 0000000..d2248b9 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/warmup-after-index-summary.json @@ -0,0 +1,311 @@ +{ + "metrics": { + "data_sent": { + "values": { + "count": 0, + "rate": 0 + }, + "type": "counter", + "contains": "data" + }, + "data_received": { + "type": "counter", + "contains": "data", + "values": { + "count": 0, + "rate": 0 + } + }, + "endpoint_sync_history_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "checks": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 0 + }, + "thresholds": { + "rate>0.99": { + "ok": true + } + } + }, + "endpoint_cost_timeseries_duration": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "http_req_connecting": { + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "type": "trend", + "contains": "time" + }, + "http_req_receiving": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + } + }, + "http_reqs": { + "type": "counter", + "contains": "default", + "values": { + "count": 1, + "rate": 0.047426040346869126 + } + }, + "http_req_tls_handshaking": { + "type": "trend", + "contains": "time", + "values": { + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0 + } + }, + "vus": { + "type": "gauge", + "contains": "default", + "values": { + "value": 0, + "min": 0, + "max": 0 + } + }, + "http_req_waiting": { + "type": "trend", + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + } + }, + "endpoint_aws_account_list_duration": { + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend", + "contains": "time" + }, + "measured_failures": { + "type": "rate", + "contains": "default", + "values": { + "rate": 0, + "passes": 0, + "fails": 0 + }, + "thresholds": { + "rate<0.01": { + "ok": true + } + } + }, + "endpoint_cost_by_service_duration": { + "type": "trend", + "contains": "time", + "values": { + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0, + "max": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "vus_max": { + "type": "gauge", + "contains": "default", + "values": { + "value": 2, + "min": 2, + "max": 2 + } + }, + "http_req_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0 + } + }, + "http_req_failed": { + "contains": "default", + "values": { + "rate": 1, + "passes": 1, + "fails": 0 + }, + "type": "rate" + }, + "measured_duration": { + "type": "trend", + "contains": "time", + "values": { + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + } + }, + "endpoint_cost_summary_duration": { + "contains": "time", + "values": { + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0, + "min": 0, + "med": 0 + }, + "thresholds": { + "p(95)<200": { + "ok": true + }, + "p(99)<500": { + "ok": true + } + }, + "type": "trend" + }, + "http_req_blocked": { + "type": "trend", + "contains": "time", + "values": { + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0, + "avg": 0 + } + }, + "http_req_sending": { + "type": "trend", + "contains": "time", + "values": { + "avg": 0, + "min": 0, + "med": 0, + "max": 0, + "p(90)": 0, + "p(95)": 0 + } + } + }, + "root_group": { + "name": "", + "path": "", + "id": "d41d8cd98f00b204e9800998ecf8427e", + "groups": [], + "checks": [] + }, + "options": { + "summaryTrendStats": [ + "avg", + "min", + "med", + "max", + "p(90)", + "p(95)" + ], + "summaryTimeUnit": "", + "noColor": false + }, + "state": { + "isStdOutTTY": true, + "isStdErrTTY": true, + "testRunDurationMs": 21085.4626 + } +} \ No newline at end of file From c496f8ed790de04a4603d8ae994cd8995dcea65f Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Fri, 11 Sep 2026 05:15:43 +0100 Subject: [PATCH 18/19] docs(benchmark): record sanitized image provenance --- benchmarks/underflow-api/results/2026-09-10-c0cf1f68/RESULTS.md | 2 +- .../underflow-api/results/2026-09-10-c0cf1f68/environment.json | 1 + benchmarks/underflow-api/scripts/generate-results.sh | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/RESULTS.md b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/RESULTS.md index 3b0c862..d19abba 100644 --- a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/RESULTS.md +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/RESULTS.md @@ -6,7 +6,7 @@ Measure authenticated, PostgreSQL-backed cost-reporting reads on an isolated, di ## Tested commit -API commit `2624a6ccafc427521745b1180764fcde963f90bc` using image digest `sha256:b1eec7b5c7652dfab0fe8259ab66f9b0682e228930da769c8d1688961680e644`. +The deployed image was built from local commit `2624a6ccafc427521745b1180764fcde963f90bc` using image digest `sha256:b1eec7b5c7652dfab0fe8259ab66f9b0682e228930da769c8d1688961680e644`. After an evidence-only history rewrite removed a bearer token from an earlier commit, the code-equivalent repository commit is `f6c356d5e38409b5dfcd44908af68cc968335631`. ## Infrastructure diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/environment.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/environment.json index 6e2c956..99d5b6e 100644 --- a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/environment.json +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/environment.json @@ -1,5 +1,6 @@ { "gitSha": "2624a6ccafc427521745b1180764fcde963f90bc", + "repositoryEquivalentSha": "f6c356d5e38409b5dfcd44908af68cc968335631", "imageDigest": "sha256:b1eec7b5c7652dfab0fe8259ab66f9b0682e228930da769c8d1688961680e644", "testTimestamp": "2026-09-10T21:07:35Z", "awsRegion": "us-east-1", diff --git a/benchmarks/underflow-api/scripts/generate-results.sh b/benchmarks/underflow-api/scripts/generate-results.sh index 26bc89f..406b15f 100644 --- a/benchmarks/underflow-api/scripts/generate-results.sh +++ b/benchmarks/underflow-api/scripts/generate-results.sh @@ -143,7 +143,7 @@ lines = [ "", "## Tested commit", "", - f"API commit `{environment['gitSha']}` using image digest `{environment['imageDigest']}`.", + f"The deployed image was built from local commit `{environment['gitSha']}` using image digest `{environment['imageDigest']}`. After an evidence-only history rewrite removed a bearer token from an earlier commit, the code-equivalent repository commit is `{environment['repositoryEquivalentSha']}`.", "", "## Infrastructure", "", From 8964efbed6e7fe49e43512ba9851e72c14462168 Mon Sep 17 00:00:00 2001 From: md-dev970 Date: Fri, 11 Sep 2026 08:19:16 +0100 Subject: [PATCH 19/19] docs(benchmark): record verified AWS teardown --- benchmarks/underflow-api/README.md | 2 ++ .../teardown-verification.json | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 benchmarks/underflow-api/results/2026-09-10-c0cf1f68/teardown-verification.json diff --git a/benchmarks/underflow-api/README.md b/benchmarks/underflow-api/README.md index ab8c91e..d9d7698 100644 --- a/benchmarks/underflow-api/README.md +++ b/benchmarks/underflow-api/README.md @@ -8,6 +8,8 @@ No result in this directory should be treated as measured evidence until all com The completed benchmark sustained **170.90 requests/second for 15 minutes at 45 continuously active VUs** with a 0.0325% functional failure rate. Across every one-minute ALB server-side datapoint, the worst p50 was 15.22 ms, p95 was 89.89 ms, and p99 was 226.96 ms. The 50-VU soak was the first sustained failing level because its p95 and p99 targets were breached. See [`results/2026-09-10-c0cf1f68/RESULTS.md`](results/2026-09-10-c0cf1f68/RESULTS.md) for the generated report and limitations; VUs are continuously active workers, not registered-user counts. +The disposable stack was destroyed on 2026-09-11. Terraform state and independent AWS inventory checks both reported zero remaining benchmark resources; the sanitized verification record is stored with the result evidence. + ## Safety boundary The only authorized Terraform root is `infra/terraform/envs/api-benchmark`. It has local, isolated state and does not reference production state or modules. Every named resource begins with `underflow-api-bench-` and AWS provider default tags apply: diff --git a/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/teardown-verification.json b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/teardown-verification.json new file mode 100644 index 0000000..36f0e81 --- /dev/null +++ b/benchmarks/underflow-api/results/2026-09-10-c0cf1f68/teardown-verification.json @@ -0,0 +1,19 @@ +{ + "verifiedAt": "2026-09-11T07:18:38Z", + "terraformApplyDestroyed": 31, + "terraformStateResources": 0, + "prefixScopedInventory": { + "ecsClusters": 0, + "activeTaskDefinitions": 0, + "rdsInstances": 0, + "ecrRepositories": 0, + "loadBalancers": 0, + "targetGroups": 0, + "logGroups": 0, + "vpcs": 0, + "securityGroups": 0, + "iamRoles": 0, + "secretsIncludingPendingDeletion": 0 + }, + "localSensitiveArtifactsRemoved": true +}