server-tests-postgres.yml

name: Server (Postgres)

on:
  schedule:
    # 03:00 AM IST daily (21:30 UTC the previous day)
    - cron: "30 21 * * *"
  pull_request:
    # 'labeled' so adding the 'postgres' label to an already-open PR re-triggers the run.
    types: [opened, reopened, synchronize, labeled]
    paths-ignore:
      - '**.js'
      - '**.md'
      - '**.html'
      - 'crowdin.yml'
      - '.coderabbit.yml'
      - '.mergify.yml'
  workflow_dispatch:

concurrency:
  group: server-postgres-develop-${{ github.event_name }}-${{ github.event.number || github.event_name == 'workflow_dispatch' && github.run_id || '' }}${{ github.event.action == 'labeled' && github.event.label.name != 'postgres' && format('-{0}', github.run_id) || '' }}
  cancel-in-progress: true

permissions:
  contents: read

# Postgres CI stays on GitHub-hosted (free, full-speed VM per shard) but follows the same fan-out
# we built for MariaDB: build the bench + reinstall ONCE in the setup job, bake the PostgreSQL
# PGDATA into the artifact, and have 4 test shards start Postgres on that datadir — no per-shard
# clone/build/reinstall/restore. Shards reuse the setup job's exact Python so the venv transplants.
env:
  TZ: 'Asia/Kolkata'
  NODE_ENV: "production"
  PYTHON_VERSION: '3.14'

jobs:
  setup:
    name: Build & reinstall (setup)
    runs-on: ubuntu-latest
    # Runs on the daily schedule (and workflow_dispatch). On PRs it runs ONLY when the PR carries
    # the 'postgres' label — the test job needs setup, so it's skipped too when this is.
    if: >-
      (github.event.action != 'labeled' || github.event.label.name == 'postgres')
      && (github.event_name != 'pull_request'
      || contains(github.event.pull_request.labels.*.name, 'postgres'))
    timeout-minutes: 40
    outputs:
      python-version: ${{ steps.setup-python.outputs.python-version }}
    steps:
      - name: Clone
        uses: actions/checkout@v6

      - name: Setup Python
        id: setup-python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Check for valid Python & Merge Conflicts
        run: |
          python -m compileall -fq "${GITHUB_WORKSPACE}"
          if grep -lr --exclude-dir=node_modules "^<<<<<<< " "${GITHUB_WORKSPACE}"
              then echo "Found merge conflicts"
              exit 1
          fi

      - name: Setup Node
        uses: actions/setup-node@v6
        with:
          node-version: 24
          check-latest: true

      - name: Add to Hosts
        run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts

      - name: Cache deps (uv/pip/npm/yarn)
        uses: actions/cache@v4
        with:
          path: |
            ~/.cache/uv
            ~/.cache/pip
            ~/.npm
            ~/.cache/yarn
          key: ${{ runner.os }}-deps-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/yarn.lock') }}
          restore-keys: ${{ runner.os }}-deps-

      # Warm-bench cache (the big one): install.sh saves the built base bench — frappe + env +
      # node_modules + assets — here as frappe-bench-base-*.tar.zst. Later runs restore it and only
      # fast-forward to the live develop SHA + rebuild the delta, so the bench BUILD is near-free and
      # only the test_site reinstall (per-run DB, uncacheable) stays slow — matching the self-hosted
      # box. The first run after a deps change populates it; every run after that is fast.
      - name: Cache warm bench (base build)
        uses: actions/cache@v4
        with:
          path: ~/bench-cache
          key: ${{ runner.os }}-warmbench-v2-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/yarn.lock') }}
          restore-keys: ${{ runner.os }}-warmbench-v2-

      # Postgres runs in-runner on a PGDATA OUTSIDE the bench (install.sh wipes ~/frappe-bench);
      # after the reinstall it's moved into the bench so it ships in the artifact.
      - name: Start DB
        run: bash ${GITHUB_WORKSPACE}/.github/helper/start-db.sh
        env:
          DB: postgres
          CI_DB_DATADIR: /home/runner/pgdata

      - name: Install
        run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
        env:
          DB: postgres
          TYPE: server
          FRAPPE_BRANCH: develop
          # Anonymous git to github.com gets throttled to a 401; authenticate the clones.
          CI_GITHUB_TOKEN: ${{ github.token }}
          BENCH_CACHE_DIR: /home/runner/bench-cache

      - name: Warm up test data
        run: |
          cd ~/frappe-bench/
          bench --site test_site run-tests --lightmode --module erpnext.tests.bootstrap_test_data

      - name: Stop DB and stage datadir
        run: |
          PG_BIN=$(ls -d /usr/lib/postgresql/*/bin | sort -V | tail -1)
          # Clean shutdown so the baked datadir is consistent. Do NOT swallow a failed stop with
          # `|| true`: moving and tarring a still-running cluster ships a torn datadir the shards
          # cannot crash-recover (full_page_writes is off). Fail the job instead — mirrors the
          # MariaDB sister's "don't bake a dirty datadir" guard.
          "$PG_BIN/pg_ctl" -D /home/runner/pgdata -m fast -w stop
          mv /home/runner/pgdata /home/runner/frappe-bench/pgdata

      - name: Package bench for test shards
        run: |
          cp "${GITHUB_WORKSPACE}/.github/helper/hydrate.sh" /home/runner/frappe-bench/hydrate.sh
          cp "${GITHUB_WORKSPACE}/.github/helper/start-db.sh" /home/runner/frappe-bench/start-db.sh
          tar czpf "${GITHUB_WORKSPACE}/bench.tar.gz" -C /home/runner \
            --exclude='.git' --exclude='node_modules' frappe-bench
          ls -lh "${GITHUB_WORKSPACE}/bench.tar.gz"

      - name: Upload bench artifact
        uses: actions/upload-artifact@v4
        with:
          name: bench-pg
          path: bench.tar.gz
          retention-days: 1
          compression-level: 0

  test:
    name: Python Unit Tests
    needs: setup
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:
        container: [1, 2, 3, 4]
    steps:
      - name: Download bench artifact
        uses: actions/download-artifact@v4
        with:
          name: bench-pg

      - name: Setup Python
        uses: actions/setup-python@v6
        with:
          python-version: ${{ needs.setup.outputs.python-version }}

      - name: Add to Hosts
        run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts

      # The bench CLI (frappe-bench) and redis are global/system tools — not in the bench tarball.
      # The setup runner got them via install.sh; the MariaDB shards get them from the arc5 image.
      # GitHub-hosted PG shards install them here (cheap vs the build+reinstall that setup did once).
      - name: Install shard runtime (bench CLI + redis + wkhtmltopdf)
        run: |
          pip install frappe-bench
          command -v redis-server >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq redis-server; }
          # wkhtmltopdf (patched-qt build) for print-format / PDF tests — same .deb install.sh uses.
          if ! command -v wkhtmltopdf >/dev/null; then
            wget -qO /tmp/wkhtmltox.deb https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox_0.12.6.1-2.jammy_amd64.deb
            sudo apt-get install -y -qq /tmp/wkhtmltox.deb
          fi

      - name: Untar bench
        run: |
          tar xzpf "${GITHUB_WORKSPACE}/bench.tar.gz" -C /home/runner
          ls -ld /home/runner/frappe-bench

      - name: Hydrate (start Postgres on the baked datadir)
        run: bash /home/runner/frappe-bench/hydrate.sh
        env:
          DB: postgres
          DB_HOST: 127.0.0.1

      - name: Run Tests
        run: |
          cd ~/frappe-bench/
          # print-format / PDF tests are engine-independent (they exercise wkhtmltopdf rendering,
          # not postgres SQL — the MariaDB CI already covers them). They only fetch the static asset
          # bundles from http://test_site:8000/assets/..., so a plain static file server over sites/
          # satisfies wkhtmltopdf without the frappe web server (which never bound on a bare runner).
          ( cd ~/frappe-bench/sites && nohup python3 -m http.server 8000 --bind 127.0.0.1 > ~/frappe-bench/web.log 2>&1 & )
          for _ in $(seq 1 15); do (exec 3<>/dev/tcp/127.0.0.1/8000) 2>/dev/null && { exec 3>&- 3<&-; break; }; sleep 1; done
          bench --site test_site run-parallel-tests --lightmode --app erpnext \
            --total-builds ${{ strategy.job-total }} --build-number ${{ matrix.container }}
        env:
          TYPE: server
          FRAPPE_PRELOAD_DATABASE_DRIVERS: postgres