From 52a27a71e14abaf719fae1fc68a6718b7ad9a73e Mon Sep 17 00:00:00 2001 From: Ritwik G Date: Fri, 6 Nov 2026 14:28:51 +0530 Subject: [PATCH 1/5] fix: stop publishing the x2text service's port on the host x2text-service is an internal service: the backend and workers reach it by service name on the compose network. Keep a commented-out loopback mapping for local development, and publish it on loopback in the e2e overlay, whose readiness probe calls it from the host. Co-Authored-By: Claude Opus 5.5 --- docker/docker-compose.yaml | 7 +++++-- docs/local-dev-setup-executor-migration.md | 4 ++-- tests/compose/docker-compose.test.yaml | 6 ++++++ workers/sample.env | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index f280571df9..6ed190aee9 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -134,8 +134,11 @@ services: image: unstract/x2text-service:${VERSION} container_name: unstract-x2text-service restart: unless-stopped - ports: - - "3004:3004" + # Internal service: not published on the host. Other services reach it on the + # compose network as http://unstract-x2text-service:3004. + # For local development only, uncomment to expose it on loopback: + # ports: + # - "127.0.0.1:3004:3004" env_file: - ../x2text-service/.env depends_on: diff --git a/docs/local-dev-setup-executor-migration.md b/docs/local-dev-setup-executor-migration.md index 96cb8a8a6a..e864fd2690 100644 --- a/docs/local-dev-setup-executor-migration.md +++ b/docs/local-dev-setup-executor-migration.md @@ -210,7 +210,7 @@ cp sample.env .env Then edit `workers/.env` — change all Docker hostnames to `localhost`: -> The runner's port 5002 is not published on the host by default. Uncomment the loopback `ports` entry for `runner` in `docker/docker-compose.yaml` and recreate it (`docker compose up -d runner` from `docker/`), or workers on the host cannot reach it. +> The runner's port 5002 and the x2text service's port 3004 are not published on the host by default. Uncomment the loopback `ports` entries for `runner` and `x2text-service` in `docker/docker-compose.yaml` and recreate them (`docker compose up -d runner x2text-service` from `docker/`), or workers on the host cannot reach them. ```ini # === CRITICAL CHANGES FOR LOCAL DEV === @@ -375,7 +375,7 @@ cd workers && ./run-worker.sh all | Backend API | 8000 | http://localhost:8000/api/v1/ | | Frontend | 3000 | http://frontend.unstract.localhost | | Platform Service | 3001 | http://localhost:3001 | -| X2Text Service | 3004 | http://localhost:3004 | +| X2Text Service | 3004 | http://localhost:3004 (not published by default; uncomment the loopback `ports` entry for `x2text-service` in `docker/docker-compose.yaml`) | | Runner | 5002 | http://localhost:5002 (not published by default; uncomment the loopback `ports` entry for `runner` in `docker/docker-compose.yaml`) | | Celery Flower (optional) | 5555 | http://localhost:5555 | diff --git a/tests/compose/docker-compose.test.yaml b/tests/compose/docker-compose.test.yaml index f56528ba7d..fe8e73dac7 100644 --- a/tests/compose/docker-compose.test.yaml +++ b/tests/compose/docker-compose.test.yaml @@ -33,6 +33,12 @@ services: platform-service: image: unstract/platform-service:${UNSTRACT_TEST_VERSION:-latest} + # Not published by docker/docker-compose.yaml; the rig's readiness probe and + # e2e smoke test reach its health endpoint from the host (UNSTRACT_X2TEXT_URL). + x2text-service: + ports: + - "127.0.0.1:3004:3004" + runner: image: unstract/runner:${UNSTRACT_TEST_VERSION:-latest} diff --git a/workers/sample.env b/workers/sample.env index a8f531a304..235307101a 100644 --- a/workers/sample.env +++ b/workers/sample.env @@ -533,7 +533,7 @@ GOOGLE_OAUTH2_SECRET= # REDIS_HOST=localhost # CACHE_REDIS_HOST=localhost # PLATFORM_SERVICE_HOST=http://localhost -# X2TEXT_HOST=http://localhost +# X2TEXT_HOST=http://localhost # also uncomment the x2text-service loopback port in docker/docker-compose.yaml # UNSTRACT_RUNNER_HOST=http://localhost # also uncomment the runner's loopback port in docker/docker-compose.yaml # WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS={"provider": "minio", "credentials": {"endpoint_url": "http://localhost:9000", "key": "minio", "secret": "minio123"}} # API_FILE_STORAGE_CREDENTIALS={"provider": "minio", "credentials": {"endpoint_url": "http://localhost:9000", "key": "minio", "secret": "minio123"}} From 5e5f3f6cdf709aaa4a962ace1e2b75ac88f2a830 Mon Sep 17 00:00:00 2001 From: Ritwik G Date: Fri, 6 Nov 2026 14:37:26 +0530 Subject: [PATCH 2/5] docs: note the x2text port opt-in for the rig's local runtime Co-Authored-By: Claude Opus 5.5 --- tests/README.md | 2 +- tests/rig/runtime.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/README.md b/tests/README.md index 9b6f679dcd..8c8a40d921 100644 --- a/tests/README.md +++ b/tests/README.md @@ -201,7 +201,7 @@ Three modes behind one protocol, chosen by `--runtime` or `UNSTRACT_E2E_RUNTIME` |---|---|---| | `compose` | CI; testing the prod image. | `docker compose -f docker/docker-compose.yaml -f tests/compose/docker-compose.test.yaml up -d --wait`, then HTTP. Teardown wipes volumes. | | `testcontainers` | Local iteration on infra-only groups. | Stands up Postgres/Redis/RabbitMQ/MinIO via testcontainers and exposes their handles on `PlatformEndpoints.infra`. **Stub today**: does NOT auto-launch backend/prompt-service/etc. as subprocesses — full-platform e2e on testcontainers will need that wiring added. Use `compose` for now if you need the whole stack. | -| `local` | After `./run-platform.sh`. | Assume a developer-managed stack; read URLs from env. | +| `local` | After `./run-platform.sh`. | Assume a developer-managed stack; read URLs from env. The smoke gate probes x2text-service at `localhost:3004`, which `docker/docker-compose.yaml` does not publish: uncomment its loopback `ports` entry, or start the stack with `-f tests/compose/docker-compose.test.yaml` as well. | The rig brings the platform up **once** per `run` invocation (if any selected group has `requires_platform: true`) and exports its URLs via env vars (`UNSTRACT_BACKEND_URL`, `UNSTRACT_PROMPT_SERVICE_URL`, etc.). The rig uses `env.setdefault(...)` so a pre-set value (e.g. from `local` runtime or a developer override) wins over the runtime's default — useful when iterating against a custom stack, but means stale shell env can mask wiring bugs. The smoke test asserts the fixture's URL matches the env var to catch this. diff --git a/tests/rig/runtime.py b/tests/rig/runtime.py index 52a2840c22..e3c7e400bb 100644 --- a/tests/rig/runtime.py +++ b/tests/rig/runtime.py @@ -315,6 +315,9 @@ class TestcontainersRuntime: For full platform e2e against testcontainers, use ``ComposeRuntime`` for now or set ``UNSTRACT_E2E_RUNTIME=local`` after running ``run-platform.sh``. + With ``local``, x2text-service must be reachable at ``UNSTRACT_X2TEXT_URL`` + (default ``localhost:3004``), which ``docker/docker-compose.yaml`` does not + publish: uncomment its loopback ``ports`` entry or add the e2e overlay. """ name: ClassVar[str] = "testcontainers" From 39e34d9539acb05623f0a04428a6cf90bc618610 Mon Sep 17 00:00:00 2001 From: Ritwik G Date: Fri, 6 Nov 2026 14:45:52 +0530 Subject: [PATCH 3/5] docs: keep only the loopback-port remedy for the rig's local runtime Co-Authored-By: Claude Opus 5.5 --- tests/README.md | 2 +- tests/rig/runtime.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/README.md b/tests/README.md index 8c8a40d921..3ec3190156 100644 --- a/tests/README.md +++ b/tests/README.md @@ -201,7 +201,7 @@ Three modes behind one protocol, chosen by `--runtime` or `UNSTRACT_E2E_RUNTIME` |---|---|---| | `compose` | CI; testing the prod image. | `docker compose -f docker/docker-compose.yaml -f tests/compose/docker-compose.test.yaml up -d --wait`, then HTTP. Teardown wipes volumes. | | `testcontainers` | Local iteration on infra-only groups. | Stands up Postgres/Redis/RabbitMQ/MinIO via testcontainers and exposes their handles on `PlatformEndpoints.infra`. **Stub today**: does NOT auto-launch backend/prompt-service/etc. as subprocesses — full-platform e2e on testcontainers will need that wiring added. Use `compose` for now if you need the whole stack. | -| `local` | After `./run-platform.sh`. | Assume a developer-managed stack; read URLs from env. The smoke gate probes x2text-service at `localhost:3004`, which `docker/docker-compose.yaml` does not publish: uncomment its loopback `ports` entry, or start the stack with `-f tests/compose/docker-compose.test.yaml` as well. | +| `local` | After `./run-platform.sh`. | Assume a developer-managed stack; read URLs from env. The smoke gate probes x2text-service at `localhost:3004`, which `docker/docker-compose.yaml` does not publish: uncomment its loopback `ports` entry there and recreate the container (`docker compose up -d x2text-service` from `docker/`). | The rig brings the platform up **once** per `run` invocation (if any selected group has `requires_platform: true`) and exports its URLs via env vars (`UNSTRACT_BACKEND_URL`, `UNSTRACT_PROMPT_SERVICE_URL`, etc.). The rig uses `env.setdefault(...)` so a pre-set value (e.g. from `local` runtime or a developer override) wins over the runtime's default — useful when iterating against a custom stack, but means stale shell env can mask wiring bugs. The smoke test asserts the fixture's URL matches the env var to catch this. diff --git a/tests/rig/runtime.py b/tests/rig/runtime.py index e3c7e400bb..84661a533e 100644 --- a/tests/rig/runtime.py +++ b/tests/rig/runtime.py @@ -137,7 +137,9 @@ def from_env(cls, *, infra: InfraEndpoints | None = None) -> PlatformEndpoints: """Build a ``PlatformEndpoints`` from ``UNSTRACT_*`` env vars. Used by every runtime so the lookup logic stays single-sourced. - Defaults match the dev compose stack at ``docker/docker-compose.yaml``. + Defaults are the localhost ports of the dev compose stack at + ``docker/docker-compose.yaml``. The runner and x2text-service are not + published there by default; see the loopback ``ports`` entries. """ return cls( backend_url=os.environ.get("UNSTRACT_BACKEND_URL", "http://localhost:8000"), @@ -317,7 +319,7 @@ class TestcontainersRuntime: or set ``UNSTRACT_E2E_RUNTIME=local`` after running ``run-platform.sh``. With ``local``, x2text-service must be reachable at ``UNSTRACT_X2TEXT_URL`` (default ``localhost:3004``), which ``docker/docker-compose.yaml`` does not - publish: uncomment its loopback ``ports`` entry or add the e2e overlay. + publish: uncomment its loopback ``ports`` entry there. """ name: ClassVar[str] = "testcontainers" From 281a022e707a9b6f7dc0ace3934637270c63766d Mon Sep 17 00:00:00 2001 From: Ritwik G Date: Fri, 6 Nov 2026 14:55:51 +0530 Subject: [PATCH 4/5] docs: re-run run-platform.sh to recreate containers after enabling a port Co-Authored-By: Claude Opus 5.5 --- docs/local-dev-setup-executor-migration.md | 2 +- tests/README.md | 2 +- tests/rig/runtime.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/local-dev-setup-executor-migration.md b/docs/local-dev-setup-executor-migration.md index e864fd2690..f928679011 100644 --- a/docs/local-dev-setup-executor-migration.md +++ b/docs/local-dev-setup-executor-migration.md @@ -210,7 +210,7 @@ cp sample.env .env Then edit `workers/.env` — change all Docker hostnames to `localhost`: -> The runner's port 5002 and the x2text service's port 3004 are not published on the host by default. Uncomment the loopback `ports` entries for `runner` and `x2text-service` in `docker/docker-compose.yaml` and recreate them (`docker compose up -d runner x2text-service` from `docker/`), or workers on the host cannot reach them. +> The runner's port 5002 and the x2text service's port 3004 are not published on the host by default. Uncomment the loopback `ports` entries for `runner` and `x2text-service` in `docker/docker-compose.yaml` then re-run `./run-platform.sh` so the containers are recreated with them, or workers on the host cannot reach them. ```ini # === CRITICAL CHANGES FOR LOCAL DEV === diff --git a/tests/README.md b/tests/README.md index 3ec3190156..7547b6a0e5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -201,7 +201,7 @@ Three modes behind one protocol, chosen by `--runtime` or `UNSTRACT_E2E_RUNTIME` |---|---|---| | `compose` | CI; testing the prod image. | `docker compose -f docker/docker-compose.yaml -f tests/compose/docker-compose.test.yaml up -d --wait`, then HTTP. Teardown wipes volumes. | | `testcontainers` | Local iteration on infra-only groups. | Stands up Postgres/Redis/RabbitMQ/MinIO via testcontainers and exposes their handles on `PlatformEndpoints.infra`. **Stub today**: does NOT auto-launch backend/prompt-service/etc. as subprocesses — full-platform e2e on testcontainers will need that wiring added. Use `compose` for now if you need the whole stack. | -| `local` | After `./run-platform.sh`. | Assume a developer-managed stack; read URLs from env. The smoke gate probes x2text-service at `localhost:3004`, which `docker/docker-compose.yaml` does not publish: uncomment its loopback `ports` entry there and recreate the container (`docker compose up -d x2text-service` from `docker/`). | +| `local` | After `./run-platform.sh`. | Assume a developer-managed stack; read URLs from env. The smoke gate probes x2text-service at `localhost:3004`, which `docker/docker-compose.yaml` does not publish: uncomment its loopback `ports` entry there, then re-run `./run-platform.sh` so the container is recreated with it. | The rig brings the platform up **once** per `run` invocation (if any selected group has `requires_platform: true`) and exports its URLs via env vars (`UNSTRACT_BACKEND_URL`, `UNSTRACT_PROMPT_SERVICE_URL`, etc.). The rig uses `env.setdefault(...)` so a pre-set value (e.g. from `local` runtime or a developer override) wins over the runtime's default — useful when iterating against a custom stack, but means stale shell env can mask wiring bugs. The smoke test asserts the fixture's URL matches the env var to catch this. diff --git a/tests/rig/runtime.py b/tests/rig/runtime.py index 84661a533e..f93c65f021 100644 --- a/tests/rig/runtime.py +++ b/tests/rig/runtime.py @@ -137,9 +137,9 @@ def from_env(cls, *, infra: InfraEndpoints | None = None) -> PlatformEndpoints: """Build a ``PlatformEndpoints`` from ``UNSTRACT_*`` env vars. Used by every runtime so the lookup logic stays single-sourced. - Defaults are the localhost ports of the dev compose stack at - ``docker/docker-compose.yaml``. The runner and x2text-service are not - published there by default; see the loopback ``ports`` entries. + Defaults are localhost URLs for a developer-managed stack. Not every + one is published by ``docker/docker-compose.yaml`` as shipped; see the + commented-out loopback ``ports`` entries there. """ return cls( backend_url=os.environ.get("UNSTRACT_BACKEND_URL", "http://localhost:8000"), From 7edb91ad31e72d1eef87543ffb132e9ed23357ba Mon Sep 17 00:00:00 2001 From: Ritwik G Date: Fri, 6 Nov 2026 15:03:09 +0530 Subject: [PATCH 5/5] docs: give each recreate instruction the command its reader started with Co-Authored-By: Claude Opus 5.5 --- docs/local-dev-setup-executor-migration.md | 2 +- tests/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/local-dev-setup-executor-migration.md b/docs/local-dev-setup-executor-migration.md index f928679011..fb24d25539 100644 --- a/docs/local-dev-setup-executor-migration.md +++ b/docs/local-dev-setup-executor-migration.md @@ -210,7 +210,7 @@ cp sample.env .env Then edit `workers/.env` — change all Docker hostnames to `localhost`: -> The runner's port 5002 and the x2text service's port 3004 are not published on the host by default. Uncomment the loopback `ports` entries for `runner` and `x2text-service` in `docker/docker-compose.yaml` then re-run `./run-platform.sh` so the containers are recreated with them, or workers on the host cannot reach them. +> The runner's port 5002 and the x2text service's port 3004 are not published on the host by default. Uncomment the loopback `ports` entries for `runner` and `x2text-service` in `docker/docker-compose.yaml` then recreate them from `docker/` with the same tag you started the stack with (`VERSION= docker compose up -d runner x2text-service`), or workers on the host cannot reach them. ```ini # === CRITICAL CHANGES FOR LOCAL DEV === diff --git a/tests/README.md b/tests/README.md index 7547b6a0e5..22029f2358 100644 --- a/tests/README.md +++ b/tests/README.md @@ -201,7 +201,7 @@ Three modes behind one protocol, chosen by `--runtime` or `UNSTRACT_E2E_RUNTIME` |---|---|---| | `compose` | CI; testing the prod image. | `docker compose -f docker/docker-compose.yaml -f tests/compose/docker-compose.test.yaml up -d --wait`, then HTTP. Teardown wipes volumes. | | `testcontainers` | Local iteration on infra-only groups. | Stands up Postgres/Redis/RabbitMQ/MinIO via testcontainers and exposes their handles on `PlatformEndpoints.infra`. **Stub today**: does NOT auto-launch backend/prompt-service/etc. as subprocesses — full-platform e2e on testcontainers will need that wiring added. Use `compose` for now if you need the whole stack. | -| `local` | After `./run-platform.sh`. | Assume a developer-managed stack; read URLs from env. The smoke gate probes x2text-service at `localhost:3004`, which `docker/docker-compose.yaml` does not publish: uncomment its loopback `ports` entry there, then re-run `./run-platform.sh` so the container is recreated with it. | +| `local` | After `./run-platform.sh`. | Assume a developer-managed stack; read URLs from env. The smoke gate probes x2text-service at `localhost:3004`, which `docker/docker-compose.yaml` does not publish: uncomment its loopback `ports` entry there, then re-run `./run-platform.sh` with the options you started it with, so the container is recreated with it. | The rig brings the platform up **once** per `run` invocation (if any selected group has `requires_platform: true`) and exports its URLs via env vars (`UNSTRACT_BACKEND_URL`, `UNSTRACT_PROMPT_SERVICE_URL`, etc.). The rig uses `env.setdefault(...)` so a pre-set value (e.g. from `local` runtime or a developer override) wins over the runtime's default — useful when iterating against a custom stack, but means stale shell env can mask wiring bugs. The smoke test asserts the fixture's URL matches the env var to catch this.