Add ALTS deployment roles and playbooks

.gitignore

@@ -4,4 +4,6 @@
 !inventory/group_vars/all.yml.tmpl
 vars.yml
 roles/ezamriy.fail2ban
+roles/geerlingguy.docker
+roles/geerlingguy.redis
 .vscode/*

README.md

@@ -159,6 +159,22 @@ rabbitmq_vhost:
 pulp_password:
 ```
 
+## ALTS (production-like deployment)
+Standalone ALTS services (RabbitMQ broker, scheduler with Flower, Celery
+workers) are deployed by separate playbooks using the `inventories/alts`
+inventory. Copy it, put your hosts into `hosts.yml` and adjust
+`group_vars/*.yml` (passwords, JWT secret, Pulp credentials, etc.).
+
+TLS certificates and keys for RabbitMQ and Celery are stored encrypted
+with Ansible Vault in `playbooks/files/alts/{celery,rabbitmq}/`, so the
+playbooks need the vault password:
+```bash
+ansible-galaxy install -r requirements.yml
+ansible-playbook -i inventories/alts --ask-vault-pass playbooks/alts_rabbitmq.yml
+ansible-playbook -i inventories/alts --ask-vault-pass playbooks/alts_scheduler.yml -e flower_password=<password>
+ansible-playbook -i inventories/alts --ask-vault-pass playbooks/alts_celery.yml
+```
+
 ## Contributing to albs-deploy
 Any question? Found a bug? File an [issue](https://github.com/AlmaLinux/build-system/issues).
 Do you want to contribute with source code?

inventories/alts/group_vars/celery.yml

@@ -0,0 +1,47 @@
+---
+
+broker_type: rabbitmq
+logs_uploader_type: pulp
+terraform_plugin_cache_dir: /home/{{ test_system_user }}/.terraform.d/plugins-cache
+test_system_user: test-system
+test_system_vhost: test_system
+test_system_password: some-another-secret-password
+test_system_config_dir: /home/{{ test_system_user }}/.config/alts
+
+rabbitmq_host: localhost
+rabbitmq_port: 5672
+rabbitmq_ssl_port: 5671
+
+celery_ssl: true
+celery_ssl_src_certificate: "files/alts/celery/client_certificate.pem"
+celery_ssl_src_key: "files/alts/celery/client_key.pem"
+celery_src_cacert: "files/alts/celery/ca_certificate.pem"
+celery_ssl_dir: "{{ test_system_config_dir }}/ssl"
+celery_ssl_certificate: "{{ celery_ssl_dir }}/{{ celery_ssl_src_certificate | basename }}"
+celery_ssl_key: "{{ celery_ssl_dir }}/{{ celery_ssl_src_key | basename }}"
+celery_cacert: "{{ celery_ssl_dir }}/{{ celery_src_cacert | basename }}"
+
+result_backend_name: local
+celery_result_folder: /srv/celery_results
+celery_pool_type: prefork
+
+worker_prefetch_multiplier: 1
+result_expires: 3600
+tests_exec_timeout: 3600
+broker_pool: 30
+
+# Optional overrides for environment provisioning
+# cleanup_yum_conf_docs_settings: false
+# ignore_ansible_install_errors: true
+# docker_image_overrides:
+#   - dist_name: almalinux
+#     dist_versions: ['8', '9']
+#     platform: linux/386
+#     image: quay.io/almalinuxautobot/almalinux
+
+celery_log_file_dir: /var/log/celery
+celery_log_file: "{{ celery_log_file_dir }}/celery.log"
+celery_concurrency: 10
+celery_loglevel: INFO
+celery_queues:
+  - "docker-x86_64-0"

inventories/alts/group_vars/rabbitmq.yml

@@ -0,0 +1,19 @@
+---
+
+broker_type: rabbitmq
+logs_uploader_type: pulp
+rabbitmq_ssl: true
+rabbitmq_copy_ssl_files: true
+rabbitmq_admin_user: admin
+rabbitmq_admin_password: some-secret-password
+rabbitmq_ssl_folder: "/etc/rabbitmq/ssl/"
+rabbitmq_cacert: "files/alts/rabbitmq/ca_certificate.pem"
+rabbitmq_server_key: "files/alts/rabbitmq/server_key.pem"
+rabbitmq_server_cert: "files/alts/rabbitmq/server_certificate.pem"
+rabbitmq_conf_ssl_options_cacertfile: "{{ rabbitmq_ssl_folder }}/{{ rabbitmq_cacert | basename }}"
+rabbitmq_conf_ssl_options_keyfile: "{{ rabbitmq_ssl_folder }}/{{ rabbitmq_server_key | basename }}"
+rabbitmq_conf_ssl_options_certfile: "{{ rabbitmq_ssl_folder }}/{{ rabbitmq_server_cert | basename }}"
+rabbitmq_ssl_verify: "verify_peer"
+rabbitmq_ssl_fail_if_no_peer_cert: true
+rabbitmq_test_system_password: some-another-secret-password
+rabbitmq_test_system_vhost: /test-system

inventories/alts/group_vars/scheduler.yml

@@ -0,0 +1,57 @@
+---
+
+# Firewall management (set to false for cloud instances using security groups)
+manage_firewall: false
+
+broker_type: rabbitmq
+test_system_user: albs
+test_system_vhost: alts
+test_system_password: some-another-secret-password
+test_system_config_dir: ~/.config/alts
+
+rabbitmq_host: localhost
+rabbitmq_port: 5672
+rabbitmq_ssl_port: 5671
+
+celery_ssl: true
+celery_ssl_src_certificate: "files/alts/celery/client_certificate.pem"
+celery_ssl_src_key: "files/alts/celery/client_key.pem"
+celery_src_cacert: "files/alts/celery/ca_certificate.pem"
+celery_ssl_dir: "{{ test_system_config_dir }}/ssl"
+celery_ssl_certificate: "{{ celery_ssl_dir }}/{{ celery_ssl_src_certificate | basename }}"
+celery_ssl_key: "{{ celery_ssl_dir }}/{{ celery_ssl_src_key | basename }}"
+celery_cacert: "{{ celery_ssl_dir }}/{{ celery_src_cacert | basename }}"
+
+# Redis configuration for result backend (geerlingguy.redis role variables)
+redis_bind_interface: 0.0.0.0
+redis_port: 6379
+# redis_requirepass: ""  # Uncomment and set if you need password authentication
+# redis_maxmemory: 512mb
+# redis_maxmemory_policy: allkeys-lru
+
+# Celery result backend configuration
+result_backend_name: redis
+alts_redis_host: localhost
+alts_redis_port: 6379
+alts_redis_results_db: 1
+
+# Legacy local result backend (not used with Redis)
+# result_backend_name: local
+# celery_result_folder: /srv/celery_results
+
+celery_pid_file_dir: /var/run/celery
+celery_pid_file: "{{ celery_pid_file_dir }}/worker.pid"
+celery_concurrency: 10
+celery_loglevel: INFO
+celery_queues:
+  - "docker-x86_64-0"
+
+scheduler_working_directory: /srv/alts/scheduler
+
+# Flower monitoring dashboard (deployed with the scheduler)
+flower_port: 5555
+# flower_password: ""  # Required, pass via --extra-vars or a vaulted vars file
+jwt_secret: very-secret-on5
+broker_pool: 30
+worker_prefetch_multiplier: 1
+result_expires: 3600

inventories/alts/hosts.yml

@@ -0,0 +1,29 @@
+---
+all:
+  children:
+    rabbitmq:
+      hosts:
+        rabbitmq01:
+    scheduler:
+      hosts:
+        scheduler01:
+    celery:
+      hosts:
+        x86_64-node01:
+          celery_queues:
+            - "docker-x86_64-0"
+        aarch64-node01:
+          terraform_arch: arm
+          celery_queues:
+            - "docker-aarch64-0"
+          build_terraform: true
+          node_arch: aarch64
+          terraform_build_ver: v1.11.4
+          docker_build_ver: 3.6.1
+        s390x-node01:
+          celery_queues:
+            - "docker-s390x-0"
+          build_terraform: true
+          node_arch: s390x
+          terraform_build_ver: v1.11.4
+          docker_build_ver: 3.6.1

playbooks/alts_celery.yml

@@ -0,0 +1,17 @@
+---
+- name: Install ALTS Celery worker
+  hosts: celery
+  vars:
+    docker_users:
+      - "{{ test_system_user }}"
+  roles:
+    - alts_base
+    # TODO: add role for building docker for s390x
+    - role: geerlingguy.docker
+      tags:
+        - install_docker
+    - alts_docker_customization
+    - role: alts_terraform
+      tags:
+        - terraform
+    - alts_celery

playbooks/alts_rabbitmq.yml

@@ -0,0 +1,6 @@
+---
+- name: Install RabbitMQ for ALTS and setup TLS
+  hosts: rabbitmq
+  roles:
+    - alts_base
+    - alts_rabbitmq_server

playbooks/alts_scheduler.yml

@@ -0,0 +1,8 @@
+---
+- name: Configure ALTS scheduler
+  hosts: scheduler
+  roles:
+    - alts_base
+    - geerlingguy.redis
+    - alts_scheduler
+    - { role: alts_flower, tags: ["flower-setup"] }

playbooks/files/alts/celery/ca_certificate.pem

@@ -0,0 +1,66 @@
+$ANSIBLE_VAULT;1.1;AES256
+30613663326363633063656562663732383763633563356364666436306261653935626163363365
+3238366436326361396337373463343435636264626366370a306165376661386161333961663739
+62363262373262316262393531646364303438636330336461653131326165373034393131363564
+6661656137636661650a343233373037653139373635356630626561663136653835323261643533
+32633937336232623239663537613737626237373963656433306461363362396164376234616431
+66313265653631313836353337336232316661323761666465663733366239373333623931316436
+32323339323264346633613137366136333832636638653464363134663439333738333464333532
+35353438376432363132613230643130353465363236613531666538653135373562303733313463
+35346564306363383862613361656432336632653234626364623133346439333635376166333434
+36373931336633616631393363656235353539393831373761666262666462636139366632616236
+32346562623037376138313737613232303261633066376265353664323062616263326263316161
+65306531313236383661316337353333313239626535396332326165303965663933326339656431
+61623533393365643436363162353136333732323763643331303161313735336337636365643431
+34626436343665326563303666613265353935323262363232316233346664663265653437653966
+65366566386436383237363033663339386636373938343232336166356565396564613832663133
+65636336383535663334373831356162316134393439366563373232333663336239623366363038
+64386132616164323662386436336264666363363463626530646630336137333763383763303238
+62623837396561366139343763636437343762323034326139623438336263313636353465326265
+61363130373366356231323738353333636137616137303339363832353332383932343364356664
+66346366366362393439303564653730393964663330313762613337616361616431366435303138
+63313437303837653034653234393864653731396631643961346666616536373533633231616533
+63656663373037323330616266343131613133376432343535613136623634313737353032636163
+35373663346536613362643838386262643934353638303762303862363436633530373238626264
+38616334363766373065613834303161623338613334353766353836616230613235613065656435
+61383330613836326665393934323039323764656237373464333838326161383863623064626637
+37366230303731306636313936343165623664613331343133663263663938643964396663356665
+36666631616161393639326463316166316136376231383332353861636539633965303364346666
+34373238356664306332303235316134333262333931663335356466663431373337393239656164
+38333062313036336334636563346633323234316230336464366538373638663834363065356133
+62303564613438663265623135336266323666633265653035653634313733646235346362376432
+32643335316461313762323734363737303936373735633437366162366532656435393633323934
+61373736316563326163303330323733336532306366613835653236643533353463386362323531
+30643339636566363564386466303336363132376534376662333437396232613864353262633966
+33613463353036356664306439316565316131336334306533613661373265363332653034626366
+30656232666432386639313338643834653262653663663532353836376633303935626239633364
+35376436623838353733336630653130393161366639626331653061306264613762656366656233
+61323361313565333530616662656632353430393062646361386564393664633461356438316336
+38333565646161663637323461313336386139333561333838666161336566323034633365343033
+33363439633331383035373034353966636662626661636432613030376435333239666666356263
+37353961656365353030323330376564633433613664346132353965663932333039363436626635
+32393461646562333633323664663937363630306139303638333866393464613162303639666565
+65366463656463323632373065626138323831613564343034363534626266343337383261666638
+35343532373136366566306532366531633562356565366235306635646535313662393765623936
+66346663353032316466333061623535393163663134396564663835316133356636613330386235
+32613562333134653233623938636138616332663264363165626433343633333365313235363662
+64393663383630353038386336316130333265663066383063313636613663653136313264373164
+34653766666536613266613666366136373137356338313135323964333631376632363336343363
+65373235333232663031613639366235343962376137326332376361333264333839653733333631
+36373138343735373239643437313430326665633966346566623930646334326238306136623461
+33303161353037343835666662396634323337316362393139363636303435366465626430613165
+31333739633330346136633965623065396137633733316330393566643363313061663037636234
+33306134316462343062373564303039343463653639323436306331343866646262613361353537
+31386463393561303864316231626234376465313663346566656361303862663439383536346339
+30386462613066666438373163393731303039663233313363646530363064613965356434646637
+33356236323634303735636136383130363566666363633864383437393436633633316364663161
+38613932333363633162383133626639316133386638666563656166653430353539353961363666
+63643462616330633435636236393132636238636662326639366630613439336437623232663964
+39663131343032373430326234303361373965393637346137356632616434346339623962373036
+64623834323765343966656636623635366430323938363334343933643835373264393038366564
+37326563303534663531303038366165633034333765366234663163636365333832643938633961
+66313764383065626231643164633232616564373564346536306661393136313462353535626263
+63633338316534346564613330666461366434343330353632383936616166373932336365643430
+36316361383663663131636537346633393066616638396232613963663634343361643465653136
+32333930313762653363366230333136643538356362653336376263646636326631363139333239
+3731

playbooks/files/alts/celery/client_certificate.pem

@@ -0,0 +1,65 @@
+$ANSIBLE_VAULT;1.1;AES256
+65613762663337663661643562316666656564336438663061303234636363326532303334333835
+3536393861383930636231636262343639646663366161370a383366396335393165346265386432
+65316233313638396466373261323334303439313865636263373461333234613632613461383135
+3238643636643731640a386535663136323761646566666432656434346435646132626635323239
+63366234636435386338333335306633373034363337326564363435643464636131663362643862
+66383466323138353634326538306536303635653230623936363835616464303338353537366161
+64613833356334633935326262623938633433383234393465653362616665323561643337303735
+65333961316233373739383364306133376432396439393935623363346233396361316563633231
+62343635393465343536636334653636656134636362653536353339613162333131636438346465
+39376663313832306138383934623838333062356264643764653638653437626262336531363962
+64656539393539643532616262383230383839353938663833366364646162633133313861633539
+30663137353565323231623630363639376561636366633363623731373933333462393963333232
+32666466323163643063373633313134386336383032353434613163316531626531666665323861
+62323734623139326130343936636234663463613931363439646332323732623437313761313933
+64316266323264383365616438393766313038343963643363306532663535616462313763623361
+38393138363962646331356333383165343739323064646235646535306235663538363938663663
+31363338376135316137343738323339313638313831303832393639636661623833393331386136
+61316532303361643263616132646565366438636639326339353031383037323333643066343833
+39306364306133663465313230626164333831636335363630633437636535623336623334653534
+66363337393839373934393866316366633762303637613263396465323737633964383538313765
+39633132303638626336653939333261366534656332373662306532333736613438313662396163
+65633431626136623838616165313936633264376639336366396362393035376335626330376162
+39343732343839303361653939373463303839663465386366363062636264393235333762353738
+39653439633533393138616437613034666463636433326337343661616536643664373837646636
+30306134326336373661306562376437613433656437626563666134663339353133626430623337
+30633364326134343462333063653965313136303765656338313964333863663166326361363430
+30333838656431356464316663303236393462303066306436313732326366643031343064323466
+34373563303737303863646465373436383239643564323935663235633635383333633662323035
+61333033323237653538643664373162663033646538373930353163336465333761613830626534
+36373732663264663531313337383931313833646163393931323731653362386264366661663835
+66343634333039303365383338643763323735353634663362633631343239373866373336366366
+36326364663763356338343465393665326265306162646638353536663864383934623632626439
+30653634663661386632303632646163373036323462393339666663336633383136616165663739
+39623164313733396337626266663462366337636537343134356134663461313265643033313963
+32326464383136356130623739656237353666633935363930316563383566396237383630393366
+66333935643761333337353132353537663063333836613136643635646261343932643134363638
+34643266333837613930366535633433363931363137656630393862653763303331373764653039
+36373537663536333230393031623261356134313665373732303661623664613661656631663530
+63623139366135343563623732636635636561616535313934646662633062626531646364303164
+65393161396264353132613436636265373330376463646562656539326431663833396265386137
+62336166636165303566613837653166383834343235373466656136623134646665333230353537
+37666663366235653665666338313236353562323161613231303364343931336139393737393836
+31313735653339356339646236336234613762396365613939373930326532623263363234316535
+38393264353233663066326535663235336133396433373931303963643131643063343064363765
+65633164303062343765343966333433393661656264363732623938626464663837336165643662
+33636132653266656665363333366637363961616232643466663330393632643063643065303830
+61333731626636383362376261636664303534393834326263373330303636396338333262306636
+38653132623566323132306163666666383765383831343132373135646131633633353736396539
+64366161326462366136316165616138353561363764306466613431666439613964393936386538
+31633965643465646137313365396438616433373135613339386330393138626165333736646235
+35663563353733646339653635323730353439366138666438353265316230633264393233373661
+38643766323731643336363663316439383032373662353236646137613739306264343565643135
+62366132303335613231653034663062316133326632356439303265386461633061656463363233
+36303436306436653763396562633265666465646364323764323565396638396562623066366133
+36623263326530663638393530633633373835626531346365343939303936326363383563313138
+62356435626537353839633563623633666433383061626433303932633536346563376164646638
+35313235363436303339336362396361373330353031396333636130376435623130663333353637
+30373938316539326133373733313363663338393735323839663738393039353135303237383165
+62366633646163373032373364613362323239616535393464373937306662333534373536303330
+32343763393066383462653036636533663131356464666130336333643032666465323862306438
+36343030373533326466393339316433306466366161663633303262393431353430636665663362
+32656430373032623837393732326331366464376634333138643766363233633162336339643535
+31303935613463373530383965633137633434643237646335643465373132633862303864356263
+35353435663061316133