refactor(ami): share provisioning steps between qemu and ami builds

ansible/files/adminapi.service.j2

@@ -14,9 +14,9 @@ Restart=always
 RestartSec=3
 TimeoutStopSec=10
 Environment="AWS_USE_DUALSTACK_ENDPOINT=true"
-{% if qemu %}
-Environment="AWS_SDK_LOAD_CONFIG=true"
-{% endif %}
+{% for env in adminapi_extra_env %}
+Environment="{{ env }}"
+{% endfor %}
 StandardOutput=journal
 StandardError=journal
 

ansible/files/gotrue.service.j2

@@ -90,15 +90,10 @@ Environment=GOTRUE_RELOADING_POLLER_ENABLED=false
 # ensures only 1 reload operation occurs during a burst of config updates.
 Environment=GOTRUE_RELOADING_GRACE_PERIOD_INTERVAL=2s
 
-{% if qemu %}
-# v3 does not use filesystem notifications for config reloads.
-Environment=GOTRUE_RELOADING_NOTIFY_ENABLED=false
-{% else %}
-# v2 currently relies on notify support, so we will enable it until both v2 / v3
-# have migrated to strictly use signals across all projects. The default is true
-# in gotrue but we will set it defensively here.
-Environment=GOTRUE_RELOADING_NOTIFY_ENABLED=true
-{% endif %}
+{% for line in gotrue_reloading_notify_comment %}
+# {{ line }}
+{% endfor %}
+Environment=GOTRUE_RELOADING_NOTIFY_ENABLED={{ gotrue_reloading_notify_enabled | lower }}
 
 Slice=services.slice
 

ansible/files/postgres_exporter.service.j2

@@ -6,8 +6,7 @@ StartLimitBurst=0
 
 [Service]
 Type=simple
-ExecStart=/opt/postgres_exporter/postgres_exporter --disable-settings-metrics --extend.query-path="/opt/postgres_exporter/queries.yml" --disable-default-metrics --no-collector.locks --no-collector.replication --no-collector.replication_slot --no-collector.stat_bgwriter --no-collector.stat_database --no-collector.stat_user_tables --no-collector.statio_user_tables --no-collector.wal --collector.database_wraparound {% if qemu %}--no-collector.database {% endif %}
-
+ExecStart=/opt/postgres_exporter/postgres_exporter --disable-settings-metrics --extend.query-path="/opt/postgres_exporter/queries.yml" --disable-default-metrics --no-collector.locks --no-collector.replication --no-collector.replication_slot --no-collector.stat_bgwriter --no-collector.stat_database --no-collector.stat_user_tables --no-collector.statio_user_tables --no-collector.wal --collector.database_wraparound{% for flag in postgres_exporter_extra_flags %} {{ flag }}{% endfor %} 
 User=postgres
 Group=postgres
 Restart=always

ansible/files/supabase_admin_agent_config/supabase-admin-agent_salt.service.j2

@@ -6,15 +6,9 @@ Requires=local-fs.target
 
 [Service]
 Type=oneshot
-{% if qemu %}
-ExecStart=/opt/supabase-admin-agent/supabase-admin-agent --config /opt/supabase-admin-agent/config.yaml salt --apply --store-result --targets-key  targetsv3-main.yaml
-User=root
-Group=root
-{% else %}
-ExecStart=/opt/supabase-admin-agent/supabase-admin-agent --config /opt/supabase-admin-agent/config.yaml salt --apply --store-result
-User=supabase-admin-agent
-Group=supabase-admin-agent
-{% endif %}
+ExecStart=/opt/supabase-admin-agent/supabase-admin-agent --config /opt/supabase-admin-agent/config.yaml {{ supabase_admin_agent_salt.args }}
+User={{ supabase_admin_agent_salt.user }}
+Group={{ supabase_admin_agent_salt.group }}
 StandardOutput=journal
 StandardError=journal
 StateDirectory=supabase-admin-agent

ansible/files/supabase_admin_agent_config/supabase-admin-agent_salt.timer.j2

@@ -3,19 +3,13 @@ Description=Run Supabase supabase-admin-agent salt on a schedule
 Requires=supabase-admin-agent_salt.service
 
 [Timer]
-{# We're using a significantly lower frequency for triggering this agent on qemu images for the moment. Once we've performed additional validations re: the aggregate impact of running it more frequently, the frequency can be increased. #}
-{% if qemu %}
-OnCalendar=*-*-* 0,6,12,18:00:00
+OnCalendar={{ supabase_admin_agent_timer.on_calendar }}
 RandomizedDelaySec={{ supabase_admin_agent_splay }}
-AccuracySec=1h
-OnBootSec=5m
-Persistent=true
-{% else %}
-OnCalendar=*:0/10
-RandomizedDelaySec={{ supabase_admin_agent_splay }}
-AccuracySec=1s
-Persistent=true
+AccuracySec={{ supabase_admin_agent_timer.accuracy_sec }}
+{% if supabase_admin_agent_timer.on_boot_sec %}
+OnBootSec={{ supabase_admin_agent_timer.on_boot_sec }}
 {% endif %}
+Persistent=true
 
 [Install]
 WantedBy=timers.target

ansible/playbook.yml

@@ -6,6 +6,7 @@
 
   vars_files:
     - ./vars.yml
+    - "./vars/{{ 'qemu' if qemu else 'aws' }}.yml"
 
   # Set facts only in here, never in any tasks.
   # This avoids redefining/duplicates or different values (buggy)
@@ -186,7 +187,7 @@
     - name: Run supascan baseline validation
       when: stage2
       shell: >-
-        baseline={{ 'ami' if not qemu else 'qemu' }}-build-{{ platform }}
+        baseline={{ supascan_baseline_prefix }}-build-{{ platform }}
         nix shell {{ nix_store_paths['supascan'] }} --command
         /tmp/ansible-playbook/ansible/files/supascan_ami.sh
         /tmp/ansible-playbook/audit-specs/baselines/$baseline

ansible/tasks/internal/admin-api.yml

@@ -7,7 +7,7 @@
 - name: adminapi - system user
   ansible.builtin.user:
     name: adminapi
-    uid: "{{ 1006 if not qemu else 1007 }}"
+    uid: "{{ uids.adminapi }}"
     group: adminapi
     groups: admin,envoy,gotrue,kong,pgbouncer,postgres,postgrest,root,systemd-journal,vector,wal-g
 

ansible/tasks/internal/supabase-admin-agent.yml

@@ -1,7 +1,7 @@
 - name: supabase-admin-agent - system group
   ansible.builtin.group:
     name: supabase-admin-agent
-    gid: "{{ 987 if not qemu else 988 }}"
+    gid: "{{ gids.supabase_admin_agent }}"
     system: yes
 
 - name: supabase-admin-agent - system user

ansible/tasks/setup-envoy.yml

@@ -7,7 +7,7 @@
 - name: Envoy - user
   ansible.builtin.user:
     name: envoy
-    uid: "{{ 1003 if not qemu else 1004 }}"
+    uid: "{{ uids.envoy }}"
     group: envoy
     state: present
 

ansible/tasks/setup-gotrue.yml

@@ -13,7 +13,7 @@
 - name: Gotrue - system user
   ansible.builtin.user:
     name: gotrue
-    uid: "{{ 1001 if not qemu else 1002 }}"
+    uid: "{{ uids.gotrue }}"
     group: gotrue
     state: present
 

ansible/tasks/setup-kong.yml

@@ -7,7 +7,7 @@
 - name: Kong - user
   ansible.builtin.user:
     name: kong
-    uid: "{{ 1004 if not qemu else 1005 }}"
+    uid: "{{ uids.kong }}"
     group: kong
     state: present
 

ansible/tasks/setup-pgbouncer.yml

@@ -8,7 +8,7 @@
 - name: Create pgbouncer user
   ansible.builtin.user:
     name: 'pgbouncer'
-    uid: "{{ 1000 if not qemu else 1001 }}"
+    uid: "{{ uids.pgbouncer }}"
     comment: 'PgBouncer user'
     group: pgbouncer
     groups: 'postgres,ssl-cert'

ansible/tasks/setup-postgrest.yml

@@ -7,7 +7,7 @@
 - name: PostgREST - user
   ansible.builtin.user:
     name: postgrest
-    uid: "{{ 1002 if not qemu else 1003 }}"
+    uid: "{{ uids.postgrest }}"
     group: postgrest
     state: present
 

ansible/tasks/setup-supabase-internal.yml

@@ -23,7 +23,7 @@
 - name: add vector system group
   ansible.builtin.group:
     name: vector
-    gid: "{{ 989 if not qemu else 990 }}"
+    gid: "{{ gids.vector }}"
     system: yes
 
 - name: add vector system user

ansible/tasks/setup-system.yml

@@ -4,7 +4,7 @@
     - name: Create ubuntu primary group
       ansible.builtin.group:
         name: ubuntu
-        gid: "{{ 1014 if not qemu else 1000 }}"
+        gid: "{{ gids.ubuntu }}"
         state: present
 
     - name: Create ubuntu user supplementary groups # noqa: explicit-identity-id
@@ -19,7 +19,7 @@
     - name: Create ubuntu user
       ansible.builtin.user:
         name: ubuntu
-        uid: "{{ 1007 if not qemu else 1000 }}"
+        uid: "{{ uids.ubuntu }}"
         group: ubuntu
         groups:
           - adm

ansible/vars/aws.yml

@@ -0,0 +1,34 @@
+---
+uids:
+  pgbouncer: 1000
+  gotrue: 1001
+  postgrest: 1002
+  envoy: 1003
+  kong: 1004
+  adminapi: 1006
+  ubuntu: 1007
+gids:
+  ubuntu: 1014
+  vector: 989
+  supabase_admin_agent: 987
+
+supascan_baseline_prefix: ami
+
+adminapi_extra_env: []
+
+gotrue_reloading_notify_enabled: true
+gotrue_reloading_notify_comment:
+  - v2 currently relies on notify support, so we will enable it until both v2 / v3
+  - have migrated to strictly use signals across all projects. The default is true
+  - in gotrue but we will set it defensively here.
+
+postgres_exporter_extra_flags: []
+
+supabase_admin_agent_salt:
+  args: salt --apply --store-result
+  user: supabase-admin-agent
+  group: supabase-admin-agent
+supabase_admin_agent_timer:
+  on_calendar: "*:0/10"
+  accuracy_sec: 1s
+  on_boot_sec: ""

ansible/vars/qemu.yml

@@ -0,0 +1,37 @@
+---
+uids:
+  pgbouncer: 1001
+  gotrue: 1002
+  postgrest: 1003
+  envoy: 1004
+  kong: 1005
+  adminapi: 1007
+  ubuntu: 1000
+gids:
+  ubuntu: 1000
+  vector: 990
+  supabase_admin_agent: 988
+
+supascan_baseline_prefix: qemu
+
+adminapi_extra_env:
+  - AWS_SDK_LOAD_CONFIG=true
+
+gotrue_reloading_notify_enabled: false
+gotrue_reloading_notify_comment:
+  - v3 does not use filesystem notifications for config reloads.
+
+postgres_exporter_extra_flags:
+  - --no-collector.database
+
+supabase_admin_agent_salt:
+  args: salt --apply --store-result --targets-key  targetsv3-main.yaml
+  user: root
+  group: root
+# We're using a significantly lower frequency for triggering this agent on qemu
+# images for the moment. Once we've performed additional validations re: the
+# aggregate impact of running it more frequently, the frequency can be increased.
+supabase_admin_agent_timer:
+  on_calendar: "*-*-* 0,6,12,18:00:00"
+  accuracy_sec: 1h
+  on_boot_sec: 5m

packer/qemu.pkr.hcl

@@ -104,6 +104,11 @@ build {
     destination = "/tmp/cleanup-qemu.sh"
   }
 
+  provisioner "file" {
+    source      = "packer/scripts/provision-common.sh"
+    destination = "/tmp/provision-common.sh"
+  }
+
   provisioner "shell" {
     environment_vars = [
       "NIX_STORE_PATHS=${jsonencode({ nix_store_paths = var.nix_store_paths })}",

packer/scripts/chroot-bootstrap-nix.sh

@@ -8,20 +8,14 @@ set -o errexit
 set -o pipefail
 set -o xtrace
 
-function setup_apt {
-	export DEBIAN_FRONTEND=noninteractive
-}
+# shellcheck source=/dev/null
+source /tmp/provision-common.sh
 
-function cleanup_apt {
+function clean_apt_cache {
 	apt-get clean
 	apt-get autoremove --purge --yes
 }
 
-function update_and_upgrade_apt {
-	apt-get update --yes
-	apt-get dist-upgrade --yes
-}
-
 function install_initial_packages {
 	local packages=(
 		e2fsprogs
@@ -92,31 +86,6 @@ function install_initial_packages {
 	fi
 }
 
-function setup_locale {
-	cat >>/etc/locale.gen <<-EOF
-		en_US.UTF-8 UTF-8
-	EOF
-	cat >/etc/default/locale <<-EOF
-		LANG="C.UTF-8"
-		LC_CTYPE="C.UTF-8"
-	EOF
-	locale-gen en_US.UTF-8
-}
-
-function setup_postgesql_env {
-	# Create the directory if it doesn't exist
-	mkdir -p /etc/environment.d
-
-	# Define the contents of the PostgreSQL environment file
-	tee /etc/environment.d/postgresql.env >/dev/null <<-EOF
-		LOCALE_ARCHIVE=/usr/lib/locale/locale-archive
-		LANG="en_US.UTF-8"
-		LANGUAGE="en_US.UTF-8"
-		LC_ALL="en_US.UTF-8"
-		LC_CTYPE="en_US.UTF-8"
-	EOF
-}
-
 function setup_apparmor {
 	# Copy apparmor profiles
 	cp -rv /tmp/apparmor_profiles/* /etc/apparmor.d/
@@ -176,13 +145,6 @@ function setup_eth0_interface {
 	chmod 600 /etc/netplan/eth0.yaml
 }
 
-function disable_sshd_passwd_auth {
-	sed -i -E \
-		-e 's/^#?\s*PasswordAuthentication\s+(yes|no)\s*$/PasswordAuthentication no/g' \
-		-e 's/^#?\s*ChallengeResponseAuthentication\s+(yes|no)\s*$/ChallengeResponseAuthentication no/g' \
-		/etc/ssh/sshd_config
-}
-
 function create_admin_account {
 	groupadd admin
 }
@@ -224,5 +186,5 @@ set_default_target
 setup_eth0_interface
 disable_sshd_passwd_auth
 disable_fsck
-cleanup_apt
+clean_apt_cache
 enable_services