Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
*.sh text eol=lf
*.tftpl text eol=lf
*.yml text eol=lf
*.yaml text eol=lf
43 changes: 43 additions & 0 deletions .github/workflows/load-test-run.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,19 @@ on:
required: true
default: "15m"
type: string
test_mode:
description: "k6 test mode"
required: true
default: "bruno-all-apis"
type: choice
options:
- bruno-all-apis
- whole-user-flow
api_docs_ref:
description: "api-docs git ref used when test_mode is bruno-all-apis"
required: false
default: "main"
type: string
target_base_url:
description: "Target base URL. Empty uses Terraform default."
required: false
Expand Down Expand Up @@ -60,6 +73,15 @@ jobs:
token: ${{ secrets.GH_PAT }}
persist-credentials: false

- uses: actions/checkout@v4
if: inputs.test_mode == 'bruno-all-apis'
with:
repository: solid-connection/api-docs
ref: ${{ inputs.api_docs_ref }}
path: api-docs
token: ${{ secrets.GH_PAT }}
persist-credentials: false

- uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{ vars.AWS_LOAD_TEST_ROLE_ARN }}
Expand All @@ -75,11 +97,15 @@ jobs:
sudo apt-get update
sudo apt-get install -y jq

- name: Test Bruno k6 generator
run: python -m unittest scripts.load_test.tests.test_generate_bruno_k6

- name: Run k6 on load generator
env:
VUS: ${{ inputs.vus }}
ITERATIONS: ${{ inputs.iterations }}
MAX_DURATION: ${{ inputs.max_duration }}
TEST_MODE: ${{ inputs.test_mode }}
TARGET_BASE_URL: ${{ inputs.target_base_url }}
PROMETHEUS_REMOTE_WRITE_URL: ${{ inputs.prometheus_remote_write_url }}
run: |
Expand All @@ -89,6 +115,23 @@ jobs:
--max-duration "$MAX_DURATION"
)

case "$TEST_MODE" in
bruno-all-apis)
args+=(
--script bruno-all-apis.js
--generate-bruno-script
--bruno-collection-dir "api-docs/Solid Connection"
)
;;
whole-user-flow)
args+=(--script whole-user-flow.js)
;;
*)
echo "::error::Invalid test_mode: $TEST_MODE"
exit 1
;;
esac

if [ -n "$TARGET_BASE_URL" ]; then
args+=(--target-base-url "$TARGET_BASE_URL")
fi
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/load-test-stop.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ on:
required: true
default: true
type: boolean
destroy_rds:
destroy_infra:
description: "Destroy load test Terraform stack"
required: true
default: true
Expand Down Expand Up @@ -58,7 +58,7 @@ jobs:
args+=(--restore-stage-dev)
fi

if [ "${{ inputs.destroy_rds }}" != "true" ]; then
if [ "${{ inputs.destroy_infra }}" != "true" ]; then
args+=(--skip-terraform-destroy)
fi

Expand Down
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
*.tfvars
.terraform.lock.hcl
modules/shared_resources/dist/*.zip
config/load-test/k6/bruno-all-apis.js
__pycache__/

# --- Secrets (보안상 절대 커밋 금지) ---
*.pem
Expand All @@ -17,4 +19,4 @@ modules/shared_resources/dist/*.zip
.DS_Store

# --- AGENTS ---
AGENTS.local.md
AGENTS.local.md
189 changes: 132 additions & 57 deletions environment/load_test/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,39 @@ data "aws_instance" "stage_api" {
}
}

data "aws_instance" "prod_db" {
filter {
name = "tag:Name"
values = [var.prod_db_instance_name]
}

filter {
name = "instance-state-name"
values = ["running"]
}
}

data "aws_subnet" "stage_api" {
id = data.aws_instance.stage_api.subnet_id
}

data "aws_subnets" "target" {
filter {
name = "vpc-id"
values = [data.aws_subnet.stage_api.vpc_id]
}
locals {
load_test_db_subnet_id = var.load_test_db_subnet_id != null ? var.load_test_db_subnet_id : data.aws_instance.prod_db.subnet_id
load_test_db_ami_id = var.load_test_db_ami_id != null ? var.load_test_db_ami_id : data.aws_instance.prod_db.ami
load_test_db_ssm_endpoint_services = toset([
"ec2messages",
"ssm",
"ssmmessages",
])

source_security_group_ids = setunion(
data.aws_instance.prod_api.vpc_security_group_ids,
data.aws_instance.stage_api.vpc_security_group_ids
)
}

data "aws_subnet" "load_test_db" {
id = local.load_test_db_subnet_id
}

data "aws_ami" "ubuntu" {
Expand All @@ -48,27 +72,10 @@ data "aws_ami" "ubuntu" {
}
}

data "aws_db_instance" "prod" {
db_instance_identifier = var.prod_rds_identifier
}

data "aws_db_snapshot" "latest_prod" {
db_instance_identifier = var.prod_rds_identifier
most_recent = true
snapshot_type = "automated"
}

locals {
source_security_group_ids = setunion(
data.aws_instance.prod_api.vpc_security_group_ids,
data.aws_instance.stage_api.vpc_security_group_ids
)
}

resource "aws_security_group" "load_test_db" {
name = "sc-load-test-db-sg"
description = "Security group for load test RDS"
vpc_id = data.aws_subnet.stage_api.vpc_id
description = "Security group for load test MySQL EC2"
vpc_id = data.aws_subnet.load_test_db.vpc_id
Comment thread
coderabbitai[bot] marked this conversation as resolved.

egress {
from_port = 0
Expand All @@ -87,13 +94,111 @@ resource "aws_security_group_rule" "load_test_db_mysql" {

type = "ingress"
description = "MySQL from prod/stage API server"
from_port = 3306
to_port = 3306
from_port = var.load_test_db_port
to_port = var.load_test_db_port
protocol = "tcp"
security_group_id = aws_security_group.load_test_db.id
source_security_group_id = each.value
}

resource "aws_security_group" "load_test_db_ssm_endpoint" {
name = "sc-load-test-db-ssm-endpoint-sg"
description = "Security group for load test DB SSM interface endpoints"
vpc_id = data.aws_subnet.load_test_db.vpc_id

ingress {
from_port = 443
to_port = 443
protocol = "tcp"
security_groups = [aws_security_group.load_test_db.id]
}

egress {
from_port = 0
to_port = 0
protocol = "-1"
cidr_blocks = ["0.0.0.0/0"]
}

tags = {
Name = "solid-connection-load-test-db-ssm-endpoint-sg"
}
}

resource "aws_vpc_endpoint" "load_test_db_ssm" {
for_each = local.load_test_db_ssm_endpoint_services

vpc_id = data.aws_subnet.load_test_db.vpc_id
service_name = "com.amazonaws.ap-northeast-2.${each.key}"
vpc_endpoint_type = "Interface"
subnet_ids = [local.load_test_db_subnet_id]
security_group_ids = [aws_security_group.load_test_db_ssm_endpoint.id]
private_dns_enabled = false

tags = {
Name = "solid-connection-load-test-db-${each.key}-endpoint"
}
}

resource "aws_ebs_volume" "load_test_db_data" {
availability_zone = data.aws_subnet.load_test_db.availability_zone
size = var.allocated_storage
type = "gp3"
encrypted = true

tags = {
Name = "${var.load_test_db_instance_name}-data"
}
}

resource "aws_instance" "load_test_db" {
ami = local.load_test_db_ami_id
instance_type = var.load_test_db_instance_type
subnet_id = local.load_test_db_subnet_id
vpc_security_group_ids = [aws_security_group.load_test_db.id]
associate_public_ip_address = var.load_test_db_associate_public_ip
iam_instance_profile = var.load_test_db_instance_profile_name

metadata_options {
http_endpoint = "enabled"
http_tokens = "required"
http_put_response_hop_limit = 1
}

root_block_device {
volume_size = 8
volume_type = "gp3"
encrypted = true
delete_on_termination = true
}

user_data = templatefile("${path.module}/templates/load_test_mysql_setup.sh.tftpl", {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Wait for the database restore before switching stage

In the inspected Load Test Start flow (load-test-start.ymlstart.sh), Terraform returns once the EC2 instance is running, not when this user-data script has downloaded and restored the S3 dump. start.sh then immediately restarts stage against the new private IP and reports the environment ready, while the database may still be unavailable; user-data failures are likewise never surfaced. Poll cloud-init status --wait or the generated ready marker through SSM before switching stage or completing the workflow.

Useful? React with 👍 / 👎.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

반영했습니다. scripts/load_test/start.sh에서 Terraform apply 이후 load-test DB 인스턴스의 SSM online 상태를 먼저 기다리고, cloud-init status --wait --long/opt/solid-connection/load-test-db-ready marker 존재 확인이 성공한 뒤에만 stage 전환을 진행하도록 했습니다. 실패 시 cloud-final 로그를 출력하고 workflow가 실패하도록 처리했습니다.

aws_region = "ap-northeast-2"
data_volume_id = aws_ebs_volume.load_test_db_data.id
db_name = var.db_name
db_port = var.load_test_db_port
ec2messages_endpoint_host = aws_vpc_endpoint.load_test_db_ssm["ec2messages"].dns_entry[0].dns_name
load_test_parameter_prefix = var.load_test_parameter_prefix
mysql_backup_bucket_name = var.mysql_backup_bucket_name
mysql_config_content = file("${path.module}/../../modules/app_stack/templates/mysql_tuning.cnf")
ssm_endpoint_host = aws_vpc_endpoint.load_test_db_ssm["ssm"].dns_entry[0].dns_name
ssmmessages_endpoint_host = aws_vpc_endpoint.load_test_db_ssm["ssmmessages"].dns_entry[0].dns_name
})

user_data_replace_on_change = true
Comment thread
coderabbitai[bot] marked this conversation as resolved.

tags = {
Name = var.load_test_db_instance_name
}
}

resource "aws_volume_attachment" "load_test_db_data" {
device_name = "/dev/sdf"
volume_id = aws_ebs_volume.load_test_db_data.id
instance_id = aws_instance.load_test_db.id
stop_instance_before_detaching = true
}

resource "aws_security_group" "load_generator" {
count = var.create_load_generator ? 1 : 0

Expand Down Expand Up @@ -152,39 +257,9 @@ resource "aws_instance" "load_generator" {
}
}

resource "aws_db_subnet_group" "load_test" {
name = "sc-load-test-db-subnet-group"
subnet_ids = data.aws_subnets.target.ids

tags = {
Name = "solid-connection-load-test-db-subnet-group"
}
}

resource "aws_db_instance" "load_test" {
identifier = var.rds_identifier
instance_class = var.db_instance_class
parameter_group_name = var.db_parameter_group_name
snapshot_identifier = data.aws_db_snapshot.latest_prod.id
db_subnet_group_name = aws_db_subnet_group.load_test.name
vpc_security_group_ids = [aws_security_group.load_test_db.id]
publicly_accessible = false
skip_final_snapshot = true
copy_tags_to_snapshot = true
deletion_protection = false
backup_retention_period = 0
apply_immediately = true
storage_encrypted = true
kms_key_id = var.kms_key_arn

tags = {
Name = var.rds_identifier
}
}

resource "aws_ssm_parameter" "load_test_datasource_url" {
name = "${var.load_test_parameter_prefix}/spring.datasource.url"
type = "String"
value = "jdbc:mysql://${aws_db_instance.load_test.address}:${aws_db_instance.load_test.port}/${var.db_name}?serverTimezone=Asia/Seoul&characterEncoding=UTF-8"
value = "jdbc:mysql://${aws_instance.load_test_db.private_ip}:${var.load_test_db_port}/${var.db_name}?serverTimezone=Asia/Seoul&characterEncoding=UTF-8"
overwrite = true
}
Loading
Loading