From 43356be7bea13eceb13ac3eb53f93c7f30618d7f Mon Sep 17 00:00:00 2001 From: Matteo Morando Date: Thu, 5 Nov 2026 17:02:53 +0400 Subject: [PATCH 1/3] Add integration test suite against a real Redis server Run TaskManager instances against a real Redis (REDIS_URL, default redis://localhost:6379/15) with a random key prefix per test and cleanup limited to that prefix. Tests are marked `integration`, deselected by default and skipped when Redis does not answer (failed when REDIS_REQUIRED is set). - Safety-net scenarios: single execution, cron rescheduling, leader election and failover, followers never scheduling, allow_parallel, consumer group recovery after a flush. - Known bugs (FTM-01, 02, 03, 04, 06, 07, 11, 12) as strict xfail tests that assert the correct behaviour. - New Integration workflow: Redis service, Python 3.10 and 3.14, and a lowest-direct dependency job. - Treat warnings as errors and fix the statistics pipeline mock that emitted RuntimeWarnings. --- .github/workflows/integration.yml | 98 +++++++ CONTRIBUTING.md | 23 ++ pyproject.toml | 17 ++ tests/integration/__init__.py | 0 tests/integration/conftest.py | 262 ++++++++++++++++++ tests/integration/test_known_bugs.py | 396 +++++++++++++++++++++++++++ tests/integration/test_scheduling.py | 305 +++++++++++++++++++++ tests/test_statistics.py | 4 +- 8 files changed, 1104 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/integration.yml create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/test_known_bugs.py create mode 100644 tests/integration/test_scheduling.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 0000000..d314303 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,98 @@ +name: Integration + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +jobs: + # Integration suite against a real Redis server (tests marked "integration"). + integration: + runs-on: ubuntu-latest + # A hung task would otherwise block stop() (no shutdown timeout) for hours. + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + # Oldest and newest supported Python versions. + python-version: ["3.10", "3.14"] + + services: + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 2s + --health-timeout 3s + --health-retries 15 + + env: + REDIS_URL: redis://localhost:6379/15 + # Fail instead of skipping when the Redis service is not reachable. + REDIS_REQUIRED: "1" + + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + + - name: Set up Python ${{ matrix.python-version }} + run: uv python install ${{ matrix.python-version }} + + - name: Install dependencies + run: uv sync --python ${{ matrix.python-version }} + + - name: Run integration tests + run: uv run --python ${{ matrix.python-version }} pytest -m integration -v + + # Unit and integration suites with the lowest versions allowed by the + # declared dependency ranges, to catch lower bounds that are too loose. + lowest-direct: + runs-on: ubuntu-latest + timeout-minutes: 15 + + services: + redis: + image: redis:7-alpine + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 2s + --health-timeout 3s + --health-retries 15 + + env: + REDIS_URL: redis://localhost:6379/15 + REDIS_REQUIRED: "1" + UV_RESOLUTION: lowest-direct + + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + + - name: Set up Python 3.10 + run: uv python install 3.10 + + - name: Install lowest direct dependencies + # Re-resolves the lock with the lowest allowed versions (not committed). + run: uv sync --python 3.10 + + - name: Run unit tests + run: uv run --python 3.10 pytest + + - name: Run integration tests + run: uv run --python 3.10 pytest -m integration diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6bd856e..096cc9e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,6 +46,29 @@ With coverage: uv run coverage run -m pytest && uv run coverage report ``` +The default run uses mocked Redis only and does not need a Redis server. Any warning raised during the tests fails the run. + +### Running Integration Tests + +The tests in `tests/integration/` run one or more `TaskManager` instances against a real Redis server. They carry the `integration` marker and are deselected by default. + +1. Start a throwaway Redis server on a free port, without persistence: + ```bash + redis-server --port 6399 --save "" --appendonly no --daemonize yes + ``` +2. Point the tests at it with `REDIS_URL` (default: `redis://localhost:6379/15`) and select the marker: + ```bash + REDIS_URL=redis://localhost:6399/0 uv run pytest -m integration + ``` +3. Stop the server when you are done: + ```bash + redis-cli -p 6399 shutdown nosave + ``` + +If Redis does not answer, the integration tests are skipped (set `REDIS_REQUIRED=1` to make them fail instead, as CI does). Each test uses its own random key prefix and deletes only its own keys at the end: the database is never flushed. + +Tests marked `xfail(strict=True)` in `tests/integration/test_known_bugs.py` describe known bugs: they assert the correct behaviour and fail today. When a fix makes one pass, pytest reports it as `XPASS(strict)` and the run fails: remove the marker in the same change, so the scenario becomes a regression test. + ### Code Quality The project uses pre-commit hooks that run automatically on `git commit`. You can also run them manually: diff --git a/pyproject.toml b/pyproject.toml index 758ecce..a43880c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -152,3 +152,20 @@ max-args = 8 [tool.pytest.ini_options] asyncio_mode = "auto" +# Integration tests need a real Redis server: they are deselected by default and +# run with `pytest -m integration` (the last -m on the command line wins). +addopts = "-m 'not integration'" +markers = [ + "integration: tests that run against a real Redis server (REDIS_URL, default redis://localhost:6379/15)", +] +# Any warning raised during the tests is an error. +filterwarnings = [ + "error", + # Python 3.14 deprecates asyncio.iscoroutinefunction, still used by the + # stream consumer: remove this entry once it switches to inspect.iscoroutinefunction. + "ignore:'asyncio.iscoroutinefunction' is deprecated:DeprecationWarning", +] + +[tool.coverage.run] +# The integration suite is deselected in the default (coverage) run. +omit = ["tests/integration/*"] diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..589f0d2 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,262 @@ +"""Fixtures for the integration suite, which runs against a real Redis server. + +The Redis server is selected with the ``REDIS_URL`` environment variable +(default ``redis://localhost:6379/15``). When the server does not answer, every +integration test is skipped instead of failing, so the suite stays usable on +machines without Redis. + +Isolation rules: +- every test gets its own random ``redis_key_prefix``, so tests never see each + other's keys and can share a Redis database with other users; +- at teardown only the keys under that prefix are deleted (``SCAN`` + ``DEL``), + the database is never flushed. +""" + +import asyncio +import os +import time +import uuid +from collections.abc import AsyncIterator, Awaitable, Callable +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any +from urllib.parse import unquote, urlparse + +import pytest +from fastapi import FastAPI +from redis import Redis as SyncRedis +from redis.asyncio import Redis + +from fastapi_task_manager import Config, TaskGroup, TaskManager +from fastapi_task_manager.redis_keys import RedisKeyBuilder + +DEFAULT_REDIS_URL = "redis://localhost:6379/15" + +# Timings scaled down from the defaults so that every scenario completes in a +# few seconds. Leader lock TTL = ceil(0.5 * 3) = 2 s; running key TTL = +# ceil(0.5 * 3) = 2 s; stuck-message threshold = ceil(0.5 * 9) = 5 s. +# The Reconciler is kept out of the test window by default: a trigger stores +# next_run = 0, which the Reconciler can see as overdue and republish while the +# Coordinator is publishing the same run, so it would run twice (a race in the +# library, not in the tests). Scenarios about reconciliation enable it explicitly. +FAST_CONFIG: dict[str, Any] = { + "poll_interval": 0.05, + "stream_block_ms": 100, + "leader_heartbeat_interval": 0.5, + "leader_retry_interval": 0.2, + "reconciliation_interval": 3600, + "running_heartbeat_interval": 0.5, + "retry_backoff": 1.0, + "retry_backoff_max": 1.0, +} + +# A cron expression that never fires during a test run (midnight, January 1st). +# Used together with a pre-seeded next_run so that tasks run only when a test +# triggers them explicitly. +YEARLY_CRON = "0 0 1 1 *" + + +@dataclass(frozen=True) +class RedisTarget: + """Connection parameters parsed from ``REDIS_URL``.""" + + url: str + host: str + port: int + password: str | None + db: int + + +def _parse_redis_url(url: str) -> RedisTarget: + """Split a ``redis://[:password@]host[:port][/db]`` URL into Config fields.""" + parsed = urlparse(url) + db_path = parsed.path.lstrip("/") + return RedisTarget( + url=url, + host=parsed.hostname or "localhost", + port=parsed.port or 6379, + password=unquote(parsed.password) if parsed.password else None, + db=int(db_path) if db_path else 0, + ) + + +@pytest.fixture(scope="session") +def redis_target() -> RedisTarget: + """Return the Redis server to test against, or skip when it does not answer.""" + target = _parse_redis_url(os.environ.get("REDIS_URL", DEFAULT_REDIS_URL)) + client = SyncRedis( + host=target.host, + port=target.port, + password=target.password, + db=target.db, + socket_connect_timeout=1, + socket_timeout=1, + ) + try: + client.ping() + except Exception as exc: # any connection problem means "no Redis available" + message = f"Redis not reachable at {target.url}: {exc!r}" + # In CI Redis is a provided service: a missing server must fail the run, + # not turn the whole suite into silent skips. + if os.environ.get("REDIS_REQUIRED"): + pytest.fail(message) + pytest.skip(message) + finally: + client.close() + return target + + +@pytest.fixture +def key_prefix() -> str: + """Return a Redis key prefix unique to the current test.""" + return f"ftm-it-{uuid.uuid4().hex[:12]}" + + +@pytest.fixture +def keys(key_prefix: str) -> RedisKeyBuilder: + """Return a key builder bound to the test prefix.""" + return RedisKeyBuilder(key_prefix) + + +@pytest.fixture +async def redis(redis_target: RedisTarget, key_prefix: str) -> AsyncIterator[Redis]: + """Return an async Redis client for assertions; delete the test's keys afterwards.""" + client = Redis( + host=redis_target.host, + port=redis_target.port, + password=redis_target.password, + db=redis_target.db, + decode_responses=True, + ) + try: + yield client + finally: + # Only the keys owned by this test are removed: never FLUSHDB/FLUSHALL. + stale_keys = [key async for key in client.scan_iter(match=f"{key_prefix}*", count=500)] + if stale_keys: + await client.delete(*stale_keys) + await client.aclose() + + +@pytest.fixture +def release() -> asyncio.Event: + """Event that long-running probe tasks wait on; set at teardown to let them finish.""" + return asyncio.Event() + + +ManagerFactory = Callable[..., Awaitable[TaskManager]] + + +@pytest.fixture +async def make_manager( + redis_target: RedisTarget, + key_prefix: str, + redis: Redis, # noqa: ARG001 - requested so that key cleanup runs after the managers stop + release: asyncio.Event, +) -> AsyncIterator[ManagerFactory]: + """Factory that builds (and by default starts) TaskManager instances. + + Every instance shares the test prefix and Redis database, which is how + several application replicas see each other in production. All instances + are stopped at teardown. + """ + created: list[TaskManager] = [] + + async def factory(groups: list[TaskGroup], *, start: bool = True, **overrides: Any) -> TaskManager: + config = Config( + redis_host=redis_target.host, + redis_port=redis_target.port, + redis_password=redis_target.password, + redis_db=redis_target.db, + redis_key_prefix=key_prefix, + **{**FAST_CONFIG, **overrides}, + ) + manager = TaskManager(app=FastAPI(), config=config) + for group in groups: + manager.add_task_group(group) + created.append(manager) + if start: + await manager.start() + return manager + + yield factory + + # Let any probe still blocked on the event finish, otherwise stop() would + # wait for it (the consumer waits for running tasks without a timeout). + release.set() + # Stop every instance even if one of them fails to stop, then report the first error. + errors: list[BaseException] = [] + for manager in reversed(created): + if manager.runner is not None: + try: + await manager.stop() + except Exception as exc: + errors.append(exc) + if errors: + raise errors[0] + + +# --------------------------------------------------------------------------- +# Helpers shared by the scenario modules +# --------------------------------------------------------------------------- + + +async def wait_until(predicate: Callable[[], Any], timeout: float, interval: float = 0.05) -> bool: + """Poll ``predicate`` (sync or async) until it is truthy or ``timeout`` elapses. + + Returns True as soon as the condition holds, False on timeout. Used both + to wait for an expected event and, with a bounded timeout, to give an + unwanted event the chance to happen. + """ + deadline = time.monotonic() + timeout + while True: + result = predicate() + if asyncio.iscoroutine(result): + result = await result + if result: + return True + if time.monotonic() >= deadline: + return False + await asyncio.sleep(interval) + + +class PreconditionError(Exception): + """A scenario could not reach the state it needs before its real assertion. + + Deliberately not an AssertionError: the known-bug tests are + ``xfail(raises=AssertionError)``, and a broken setup must surface as a real + failure instead of being mistaken for the documented bug. + """ + + +def require(condition: object, message: str) -> None: + """Raise PreconditionError with ``message`` when ``condition`` is falsy.""" + if not condition: + raise PreconditionError(message) + + +async def wait_for_leader(managers: list[TaskManager], timeout: float = 5.0) -> TaskManager: + """Wait until exactly one of ``managers`` is leader and return it.""" + + def single_leader() -> bool: + return sum(1 for m in managers if m.runner is not None and m.runner.is_leader) == 1 + + require(await wait_until(single_leader, timeout), "no single leader was elected in time") + return next(m for m in managers if m.runner is not None and m.runner.is_leader) + + +async def seed_next_run_far_future(redis: Redis, keys: RedisKeyBuilder, group: str, task: str) -> None: + """Store a next_run one year ahead, so the task only runs when a test triggers it. + + Without this, a task without next_run is scheduled immediately at startup + (see FTM-07), which would make run counts depend on that behaviour. + """ + one_year_ahead = datetime.now(timezone.utc).timestamp() + 365 * 24 * 3600 + await redis.set(keys.next_run_key(group, task), one_year_ahead) + + +def task_name(group: TaskGroup, prefix: str) -> str: + """Return the internal name of the only task in ``group`` whose name starts with ``prefix``.""" + matches = [t.name for t in group.tasks if t.name.startswith(prefix)] + require(len(matches) == 1, f"expected one task named {prefix}*, found {matches}") + return matches[0] diff --git a/tests/integration/test_known_bugs.py b/tests/integration/test_known_bugs.py new file mode 100644 index 0000000..20724a8 --- /dev/null +++ b/tests/integration/test_known_bugs.py @@ -0,0 +1,396 @@ +"""Integration tests that document known bugs (one test per finding). + +Every test here describes the CORRECT behaviour and currently fails because +of the bug named in its ``xfail`` reason. The markers are strict and only +accept an ``AssertionError``: when a fix lands, the test starts passing, +pytest reports it as XPASS(strict) and the build fails until the marker is +removed, which turns the scenario into a regression test. Any other error +(Redis down, a fixture failure, a crash) is reported as a real failure. +""" + +import asyncio +import time + +import pytest +from fastapi import HTTPException +from redis.asyncio import Redis + +from fastapi_task_manager import TaskGroup +from fastapi_task_manager.redis_keys import RedisKeyBuilder +from fastapi_task_manager.schema.task import CreateDynamicTaskRequest +from fastapi_task_manager.task_router_services import ( + create_dynamic_task, + delete_dynamic_task, + disable_tasks, + get_tasks, + trigger_tasks, +) +from tests.integration.conftest import ( + YEARLY_CRON, + ManagerFactory, + require, + seed_next_run_far_future, + task_name, + wait_for_leader, + wait_until, +) + +pytestmark = pytest.mark.integration + + +def known_bug(reason: str) -> pytest.MarkDecorator: + """Strict xfail that only accepts the assertion describing the bug.""" + return pytest.mark.xfail(strict=True, raises=AssertionError, reason=reason) + + +def _long_task_group(starts: list[float], release: asyncio.Event) -> TaskGroup: + """Group "g" with a yearly, non-parallel task "long" that runs until ``release`` is set.""" + group = TaskGroup(name="g") + + @group.add_task(YEARLY_CRON, name="long", allow_parallel=False) + async def long() -> None: + starts.append(time.monotonic()) + # Bounded wait as a safety net: the fixture sets the event at teardown. + await asyncio.wait_for(release.wait(), timeout=15) + + return group + + +# Stuck-message threshold = ceil(0.2 * 9) = 2 s, running key TTL = 1 s, +# renewed every 0.2 s: the running key never expires while the task runs. +# Reconciliation every second (it is off by default in the integration config). +LONG_TASK_TIMINGS = {"running_heartbeat_interval": 0.2, "reconciliation_interval": 1} + + +# --------------------------------------------------------------------------- +# FTM-01: live long-running tasks are executed again +# --------------------------------------------------------------------------- + + +@known_bug( + "FTM-01: the Reconciler requeues a pending message whose task is still running " + "(it ignores the running key), so a long task starts again in parallel", +) +async def test_reconciler_does_not_restart_a_task_that_is_still_running( + make_manager: ManagerFactory, + redis: Redis, + keys: RedisKeyBuilder, + release: asyncio.Event, +) -> None: + starts: list[float] = [] + group = _long_task_group(starts, release) + name = task_name(group, "long") + await seed_next_run_far_future(redis, keys, "g", name) + manager = await make_manager([group], **LONG_TASK_TIMINGS) + await wait_for_leader([manager]) + + await trigger_tasks(manager, task_group_name="g", task_name=name) + require(await wait_until(lambda: len(starts) >= 1, timeout=5), "task never started") + + # The task stays alive (heartbeat renewed) well past the 2 s stuck threshold + # and several 1 s reconciliation cycles. + await wait_until(lambda: len(starts) > 1, timeout=5) + assert len(starts) == 1, ( + f"FTM-01: a running task with allow_parallel=False was started {len(starts)} times, " + f"at {[round(s - starts[0], 1) for s in starts]} s" + ) + + +@known_bug( + "FTM-01: a starting instance claims (XAUTOCLAIM) the pending message of a task that is " + "still running on another live instance and runs it again", +) +async def test_starting_instance_does_not_steal_a_running_task( + make_manager: ManagerFactory, + redis: Redis, + keys: RedisKeyBuilder, + release: asyncio.Event, +) -> None: + starts: list[float] = [] + # Reconciliation is pushed out of the test window: only startup recovery is exercised. + timings = {**LONG_TASK_TIMINGS, "reconciliation_interval": 3600} + group_a = _long_task_group(starts, release) + name = task_name(group_a, "long") + await seed_next_run_far_future(redis, keys, "g", name) + first = await make_manager([group_a], **timings) + await wait_for_leader([first]) + + await trigger_tasks(first, task_group_name="g", task_name=name) + require(await wait_until(lambda: len(starts) >= 1, timeout=5), "task never started") + + # Wait until the pending message is older than the 2 s stuck threshold. + async def pending_message_idle() -> bool: + pending = await redis.xpending_range( + keys.task_stream_low_key(), + keys.consumer_group_name(), + min="-", + max="+", + count=10, + ) + return any(entry["time_since_delivered"] > 2_100 for entry in pending) + + require(await wait_until(pending_message_idle, timeout=5), "message never became idle") + + # A new replica starts (rolling deploy / autoscaling) while the task is alive. + await make_manager([_long_task_group(starts, release)], **timings) + + await wait_until(lambda: len(starts) > 1, timeout=2) + assert len(starts) == 1, ( + f"FTM-01: a task still running on a live instance was started {len(starts)} times " + "after another instance started" + ) + + +# --------------------------------------------------------------------------- +# FTM-02: dynamic tasks are not shared between instances +# --------------------------------------------------------------------------- + + +def _dynamic_group(calls: list[str], label: str) -> TaskGroup: + """Group "g" exposing a registered function "work" for dynamic tasks.""" + group = TaskGroup(name="g") + + @group.register_function("work") + async def work() -> None: + calls.append(label) + + return group + + +@known_bug( + "FTM-02: a dynamic task created through a follower is only known to that follower; the leader never schedules it", +) +async def test_dynamic_task_created_on_follower_runs(make_manager: ManagerFactory) -> None: + calls: list[str] = [] + leader = await make_manager([_dynamic_group(calls, "A")]) + await wait_for_leader([leader]) + follower = await make_manager([_dynamic_group(calls, "B")]) + await wait_for_leader([leader, follower]) + + await create_dynamic_task( + follower, + CreateDynamicTaskRequest(task_group_name="g", function_name="work", cron_expression=YEARLY_CRON, name="dyn"), + ) + await trigger_tasks(follower, task_group_name="g", task_name="dyn") + + ran = await wait_until(lambda: len(calls) >= 1, timeout=4) + assert ran, "FTM-02: dynamic task created and triggered on a follower never ran" + + +@known_bug( + "FTM-02: deleting a dynamic task through a follower leaves it registered on the leader, which keeps scheduling it", +) +async def test_dynamic_task_deleted_on_follower_is_removed_everywhere(make_manager: ManagerFactory) -> None: + calls: list[str] = [] + leader = await make_manager([_dynamic_group(calls, "A")]) + await wait_for_leader([leader]) + await create_dynamic_task( + leader, + CreateDynamicTaskRequest(task_group_name="g", function_name="work", cron_expression=YEARLY_CRON, name="dyn"), + ) + # The follower starts after the creation, so it loads the task from Redis. + follower = await make_manager([_dynamic_group(calls, "B")]) + require(any(t.name == "dyn" for t in follower.task_groups[0].tasks), "follower did not load the dynamic task") + + await delete_dynamic_task(follower, task_group_name="g", task_name="dyn") + + def leader_dropped_task() -> bool: + return all(t.name != "dyn" for t in leader.task_groups[0].tasks) + + dropped = await wait_until(leader_dropped_task, timeout=3) + assert dropped, "FTM-02: dynamic task deleted on a follower is still registered (and scheduled) on the leader" + + +# --------------------------------------------------------------------------- +# FTM-03: messages for unknown tasks are acknowledged and lost +# --------------------------------------------------------------------------- + + +@known_bug( + "FTM-03: an instance that does not know a task ACKs and deletes its message, " + "so the run is lost (rolling deploy with mixed versions)", +) +async def test_message_for_unknown_task_survives_until_a_capable_instance_runs_it( + make_manager: ManagerFactory, + redis: Redis, + keys: RedisKeyBuilder, +) -> None: + calls: list[str] = [] + + # New version of the application: it adds the task "g/new_task". + new_group = TaskGroup(name="g") + + @new_group.add_task(YEARLY_CRON, name="new_task") + async def new_task() -> None: + calls.append("new") + + name = task_name(new_group, "new_task") + await seed_next_run_far_future(redis, keys, "g", name) + + # Old version, already running: same group, without the new task. Short + # heartbeat so that a fix relying on pending-message reclaim (2 s threshold) + # fits in the test window. + old = await make_manager([TaskGroup(name="g")], **LONG_TASK_TIMINGS) + await wait_for_leader([old]) + + # The message the new version's leader would publish for its new task + # (same fields as Coordinator._publish_task), delivered to the old instance. + message_id = await redis.xadd( + keys.task_stream_low_key(), + {"group": "g", "task": name, "scheduled_at": str(time.time())}, + ) + + async def delivered() -> bool: + groups = await redis.xinfo_groups(keys.task_stream_low_key()) + return any(g["last-delivered-id"] == message_id for g in groups) + + require(await wait_until(delivered, timeout=5), "message was never delivered to the old instance") + + # The new version comes up; the run must eventually happen there. + await make_manager([new_group], **LONG_TASK_TIMINGS) + + ran = await wait_until(lambda: len(calls) >= 1, timeout=8) + assert ran, ( + "FTM-03: the message for a task unknown to the consuming instance was acknowledged " + f"and dropped (stream length now {await redis.xlen(keys.task_stream_low_key())}); " + "the instance that knows the task never ran it" + ) + + +# --------------------------------------------------------------------------- +# FTM-04: retry with backoff does not retry +# --------------------------------------------------------------------------- + + +@known_bug( + "FTM-04: after a failure only retry_after is written; next_run already points to the " + "next cron occurrence, so the task is not retried after the backoff", +) +async def test_failed_task_is_retried_after_backoff( + make_manager: ManagerFactory, + redis: Redis, + keys: RedisKeyBuilder, +) -> None: + attempts: list[float] = [] + group = TaskGroup(name="g") + + @group.add_task(YEARLY_CRON, name="flaky") + async def flaky() -> None: + attempts.append(time.monotonic()) + if len(attempts) == 1: + msg = "first attempt fails" + raise RuntimeError(msg) + + name = task_name(group, "flaky") + await seed_next_run_far_future(redis, keys, "g", name) + manager = await make_manager([group], retry_backoff=1.0) + await wait_for_leader([manager]) + + await trigger_tasks(manager, task_group_name="g", task_name=name) + require(await wait_until(lambda: len(attempts) >= 1, timeout=5), "task never ran") + + # retry_backoff is 1 s: a retry is expected well within 4 s. + retried = await wait_until(lambda: len(attempts) >= 2, timeout=4) + assert retried, "FTM-04: a failed task was not retried after retry_backoff=1s (waits for the next cron run)" + + +# --------------------------------------------------------------------------- +# FTM-06: the disabled flag expires +# --------------------------------------------------------------------------- + + +@known_bug("FTM-06: disable sets the flag with ex=statistics_redis_expiration, so it re-enables itself after 5 days") +async def test_disabled_flag_does_not_expire( + make_manager: ManagerFactory, + redis: Redis, + keys: RedisKeyBuilder, +) -> None: + group = TaskGroup(name="g") + + @group.add_task(YEARLY_CRON, name="probe") + async def probe() -> None: + return None + + name = task_name(group, "probe") + await seed_next_run_far_future(redis, keys, "g", name) + manager = await make_manager([group]) + + await disable_tasks(manager, task_group_name="g", task_name=name) + + require(await redis.exists(keys.disabled_key("g", name)), "disable_tasks did not set the flag") + ttl = await redis.ttl(keys.disabled_key("g", name)) + assert ttl == -1, f"FTM-06: the disabled flag expires on its own (TTL {ttl} s)" + + +# --------------------------------------------------------------------------- +# FTM-07: every task runs at first startup +# --------------------------------------------------------------------------- + + +@known_bug( + "FTM-07: a task without next_run is considered due (by the Coordinator and by the Reconciler), " + "so it runs at first startup whatever its cron", +) +async def test_yearly_task_does_not_run_at_first_startup(make_manager: ManagerFactory) -> None: + calls: list[str] = [] + group = TaskGroup(name="g") + + @group.add_task(YEARLY_CRON, name="yearly") + async def yearly() -> None: + calls.append("run") + + # Fresh prefix: no next_run stored, as on a first deploy. Reconciliation is + # on, because the Reconciler also republishes tasks without next_run. + manager = await make_manager([group], reconciliation_interval=1) + await wait_for_leader([manager]) + + ran = await wait_until(lambda: len(calls) >= 1, timeout=2) + assert not ran, f"FTM-07: a task with cron {YEARLY_CRON!r} ran right at first startup" + + +# --------------------------------------------------------------------------- +# FTM-11 / FTM-12: tag handling +# --------------------------------------------------------------------------- + + +@known_bug("FTM-11: the tag filter of the bulk API endpoints also matches tasks without tags") +async def test_tag_filter_does_not_match_untagged_tasks( + make_manager: ManagerFactory, + redis: Redis, + keys: RedisKeyBuilder, +) -> None: + group = TaskGroup(name="g") + + @group.add_task(YEARLY_CRON, name="untagged") + async def untagged() -> None: + return None + + name = task_name(group, "untagged") + await seed_next_run_far_future(redis, keys, "g", name) + manager = await make_manager([group]) + + # No task carries this tag: the endpoint must answer 404 and touch nothing. + matched: list[str] = [] + try: + result = await disable_tasks(manager, tag="does-not-exist") + matched = [a.task for a in result.affected_tasks] + except HTTPException as exc: + require(exc.status_code == 404, f"unexpected HTTP status {exc.status_code}") + assert matched == [], f"FTM-11: disable with tag='does-not-exist' matched untagged tasks: {matched}" + assert not await redis.exists(keys.disabled_key("g", name)) + + +@known_bug("FTM-12: operator precedence in TaskGroup.add_task drops the task tags when the group has tags") +async def test_task_keeps_its_tags_when_the_group_has_tags(make_manager: ManagerFactory) -> None: + group = TaskGroup(name="g", tags=["group"]) + + @group.add_task(YEARLY_CRON, name="tagged", tags=["task-tag"]) + async def tagged() -> None: + return None + + manager = await make_manager([group]) + + tasks = await get_tasks(manager, tag="task-tag") + assert [t.tags for t in tasks] == [["group", "task-tag"]], ( + f"FTM-12: filtering by the task's own tag found {[t.tags for t in tasks]}" + ) diff --git a/tests/integration/test_scheduling.py b/tests/integration/test_scheduling.py new file mode 100644 index 0000000..eebc7b1 --- /dev/null +++ b/tests/integration/test_scheduling.py @@ -0,0 +1,305 @@ +"""Integration tests for behaviours that work today (safety net). + +Each test runs one or more TaskManager instances against a real Redis server +and checks an end-to-end guarantee: a triggered task runs once, cron +scheduling advances next_run, a single leader is elected, leadership fails +over, and the consumer survives the loss of its consumer group. +""" + +import asyncio +import time +from datetime import datetime, timezone + +import pytest +from cronsim import CronSim +from redis.asyncio import Redis + +from fastapi_task_manager import TaskGroup +from fastapi_task_manager.redis_keys import RedisKeyBuilder +from fastapi_task_manager.task_router_services import trigger_tasks +from tests.integration.conftest import ( + YEARLY_CRON, + ManagerFactory, + seed_next_run_far_future, + task_name, + wait_for_leader, + wait_until, +) + +pytestmark = pytest.mark.integration + + +def _counting_group(calls: list[str], label: str, expr: str = YEARLY_CRON, **task_options) -> TaskGroup: + """Build a group "g" with one task "probe" that appends ``label`` to ``calls`` when it runs. + + Each TaskManager instance needs its own TaskGroup object; building them + with the same group name, task name and cron gives every instance the + same internal task name, exactly like replicas of the same application. + """ + group = TaskGroup(name="g") + + @group.add_task(expr, name="probe", **task_options) + async def probe() -> None: + calls.append(label) + + return group + + +async def test_triggered_task_runs_exactly_once( + make_manager: ManagerFactory, + redis: Redis, + keys: RedisKeyBuilder, +) -> None: + """A triggered task runs once, records statistics and clears its runtime keys.""" + calls: list[str] = [] + group = _counting_group(calls, "A") + name = task_name(group, "probe") + await seed_next_run_far_future(redis, keys, "g", name) + + manager = await make_manager([group]) + await wait_for_leader([manager]) + await trigger_tasks(manager, task_group_name="g", task_name=name) + + assert await wait_until(lambda: len(calls) >= 1, timeout=5), "triggered task never ran" + # Several coordinator cycles: a duplicate publish would show up here. + await wait_until(lambda: len(calls) > 1, timeout=1) + assert calls == ["A"] + + # Success path side effects: one stats entry, no running key, not tracked as scheduled. + # The consumer records stats, deletes the running key, then removes the + # scheduled-set member, in this order, once the task returns. + stats_key = keys.stats_stream_key("g", name) + + async def tracking_cleared() -> bool: + return ( + await redis.xlen(stats_key) >= 1 + and not await redis.sismember(keys.scheduled_set_key(), f"g:{name}") # ty: ignore[invalid-await] + ) + + assert await wait_until(tracking_cleared, timeout=2), "success path did not record stats and clear tracking" + assert await redis.xlen(stats_key) == 1 + # Checked right away: the running key also has a TTL (2 s), so waiting + # for it to disappear would pass even if the consumer never deleted it. + assert not await redis.exists(keys.running_task_key("g", name)), "running key left behind after success" + + +async def test_cron_task_runs_when_due_and_schedules_next_occurrence( + make_manager: ManagerFactory, + redis: Redis, + keys: RedisKeyBuilder, +) -> None: + """A task whose next_run comes due runs once and next_run moves to the next cron occurrence.""" + calls: list[str] = [] + # Yearly cron: the next occurrence can never come due inside the test window + # (a per-minute cron would legitimately run again when the test crosses a minute). + group = _counting_group(calls, "A") + name = task_name(group, "probe") + next_run_key = keys.next_run_key("g", name) + # Due one second from now. + seeded = time.time() + 1 + await redis.set(next_run_key, seeded) + + manager = await make_manager([group]) + await wait_for_leader([manager]) + + assert await wait_until(lambda: len(calls) >= 1, timeout=5), "due task never ran" + await wait_until(lambda: len(calls) > 1, timeout=1) + assert calls == ["A"] + + # The coordinator stores the next cron occurrence as the new next_run (just after publishing). + async def next_run_updated() -> bool: + return float(await redis.get(next_run_key) or 0) != seeded + + assert await wait_until(next_run_updated, timeout=2), "next_run was never updated" + next_run = float(await redis.get(next_run_key)) + expected = next(CronSim(YEARLY_CRON, datetime.now(timezone.utc))).timestamp() + assert next_run == expected + + +async def test_only_one_instance_becomes_leader( + make_manager: ManagerFactory, + redis: Redis, + keys: RedisKeyBuilder, +) -> None: + """With two instances on the same Redis only one holds the leader lock.""" + first = await make_manager([TaskGroup(name="g")]) + leader = await wait_for_leader([first]) + second = await make_manager([TaskGroup(name="g")]) + + # Give the second instance several acquisition attempts (retry interval 0.2 s). + both_leaders = await wait_until( + lambda: ( + first.runner is not None + and second.runner is not None + and first.runner.is_leader + and second.runner.is_leader + ), + timeout=1.5, + ) + assert not both_leaders, "two instances believe they are leader at the same time" + assert leader is first + assert second.runner is not None + assert not second.runner.is_leader + + # The lock is held by the leader's worker id. + lock_owner = await redis.get(keys.leader_lock_key()) + assert lock_owner is not None + assert first.runner is not None + assert first.runner.worker_id.split("-")[-1] in lock_owner + + +async def test_instance_that_is_not_leader_never_schedules( + make_manager: ManagerFactory, + redis: Redis, + keys: RedisKeyBuilder, +) -> None: + """A follower leaves due tasks alone: only the lock holder publishes them.""" + # Another (fake) worker holds the leader lock for the whole test. + await redis.set(keys.leader_lock_key(), "another-worker", ex=60) + + calls: list[str] = [] + group = _counting_group(calls, "A") + name = task_name(group, "probe") + next_run_key = keys.next_run_key("g", name) + # The task is overdue: a leader would publish it on its next cycle. + await redis.set(next_run_key, "0") + + manager = await make_manager([group]) + + # Many coordinator and acquisition cycles (retry interval 0.2 s). + published = await wait_until(lambda: _next_run_changed(redis, next_run_key), timeout=1.5) + assert not published, "an instance that is not leader rescheduled the task" + assert calls == [] + assert manager.runner is not None + assert not manager.runner.is_leader + + +async def _next_run_changed(redis: Redis, next_run_key: str) -> bool: + """True when next_run no longer holds the "0" seeded by the test.""" + return await redis.get(next_run_key) != "0" + + +async def test_leadership_fails_over_when_leader_stops(make_manager: ManagerFactory) -> None: + """When the leader stops, the remaining instance takes over.""" + first = await make_manager([TaskGroup(name="g")]) + await wait_for_leader([first]) + second = await make_manager([TaskGroup(name="g")]) + + await first.stop() + + assert await wait_until( + lambda: second.runner is not None and second.runner.is_leader, + timeout=5, + ), "the surviving instance never became leader" + + +async def test_triggered_task_runs_once_across_instances( + make_manager: ManagerFactory, + redis: Redis, + keys: RedisKeyBuilder, +) -> None: + """With two instances, a triggered task runs on exactly one of them, once.""" + calls: list[str] = [] + group_a = _counting_group(calls, "A") + group_b = _counting_group(calls, "B") + name = task_name(group_a, "probe") + await seed_next_run_far_future(redis, keys, "g", name) + + first = await make_manager([group_a]) + await wait_for_leader([first]) + second = await make_manager([group_b]) + await wait_for_leader([first, second]) + + await trigger_tasks(first, task_group_name="g", task_name=name) + + assert await wait_until(lambda: len(calls) >= 1, timeout=5), "triggered task never ran" + await wait_until(lambda: len(calls) > 1, timeout=1.5) + assert len(calls) == 1, f"task ran {len(calls)} times: {calls}" + + +async def test_allow_parallel_false_waits_for_running_execution( + make_manager: ManagerFactory, + redis: Redis, + keys: RedisKeyBuilder, + release: asyncio.Event, +) -> None: + """With allow_parallel=False a second trigger waits until the running execution ends.""" + starts: list[float] = [] + running = 0 + max_running = 0 + group = TaskGroup(name="g") + + @group.add_task(YEARLY_CRON, name="probe", allow_parallel=False) + async def probe() -> None: + nonlocal running, max_running + starts.append(time.monotonic()) + running += 1 + max_running = max(max_running, running) + try: + # Stay busy until the test releases the task (bounded as a safety net). + await asyncio.wait_for(release.wait(), timeout=10) + finally: + running -= 1 + + name = task_name(group, "probe") + await seed_next_run_far_future(redis, keys, "g", name) + manager = await make_manager([group]) + await wait_for_leader([manager]) + + await trigger_tasks(manager, task_group_name="g", task_name=name) + assert await wait_until(lambda: len(starts) == 1, timeout=5), "first execution never started" + + # Second trigger while the first execution is still running. + await trigger_tasks(manager, task_group_name="g", task_name=name) + started_again = await wait_until(lambda: len(starts) > 1, timeout=1) + assert not started_again, "a second execution started while the first was running" + + # Once the first execution ends, the pending trigger is honoured. + release.set() + assert await wait_until(lambda: len(starts) == 2, timeout=5), "pending trigger never ran" + assert max_running == 1 + + +async def test_consumer_recovers_after_redis_keys_are_flushed( + make_manager: ManagerFactory, + redis: Redis, + keys: RedisKeyBuilder, +) -> None: + """Deleting every key of the app (as FLUSHALL does) must not stop task execution. + + Only the keys under the test prefix are deleted, which removes the streams + and their consumer groups exactly like a flush, without touching other + data on the server. + """ + calls: list[str] = [] + group = _counting_group(calls, "A") + name = task_name(group, "probe") + await seed_next_run_far_future(redis, keys, "g", name) + manager = await make_manager([group]) + await wait_for_leader([manager]) + + # Simulate the flush. + flushed = [key async for key in redis.scan_iter(match=f"{keys.prefix}*")] + assert flushed + await redis.delete(*flushed) + + # The consumer recreates its consumer group after hitting NOGROUP. + async def group_exists() -> bool: + try: + groups = await redis.xinfo_groups(keys.task_stream_low_key()) + except Exception: + return False + return any(g["name"] == keys.consumer_group_name() for g in groups) + + assert await wait_until(group_exists, timeout=5), "consumer group was never recreated" + assert await wait_until(lambda: manager.runner is not None and manager.runner.is_leader, timeout=5) + + # Right after the flush the coordinator may publish the task once on its own + # (next_run is gone). Push next_run away again and count from here, so the + # assertion below needs a message published after the consumer recovered. + await seed_next_run_far_future(redis, keys, "g", name) + runs_before_trigger = len(calls) + + # A task triggered after the flush still runs. + await trigger_tasks(manager, task_group_name="g", task_name=name) + assert await wait_until(lambda: len(calls) > runs_before_trigger, timeout=5), "task never ran after the flush" diff --git a/tests/test_statistics.py b/tests/test_statistics.py index 4224e9f..a4005da 100644 --- a/tests/test_statistics.py +++ b/tests/test_statistics.py @@ -12,7 +12,9 @@ def _make_mock_redis(): so we use MagicMock for the redis client and configure pipeline accordingly. """ redis = MagicMock() - pipe = AsyncMock() + # Pipeline command methods (xadd, expire, ...) only queue commands and are + # sync in redis-py; only execute() is a coroutine. + pipe = MagicMock() pipe.execute = AsyncMock(return_value=[]) # pipeline() is sync but returns an async context manager ctx = MagicMock() From 32db10d9fc11b9a2618826439d449ab2aa52ef9a Mon Sep 17 00:00:00 2001 From: Matteo Morando Date: Thu, 5 Nov 2026 17:26:54 +0400 Subject: [PATCH 2/3] Format integration test so both ruff versions agree --- tests/integration/test_scheduling.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_scheduling.py b/tests/integration/test_scheduling.py index eebc7b1..4cb3b69 100644 --- a/tests/integration/test_scheduling.py +++ b/tests/integration/test_scheduling.py @@ -71,10 +71,10 @@ async def test_triggered_task_runs_exactly_once( stats_key = keys.stats_stream_key("g", name) async def tracking_cleared() -> bool: - return ( - await redis.xlen(stats_key) >= 1 - and not await redis.sismember(keys.scheduled_set_key(), f"g:{name}") # ty: ignore[invalid-await] - ) + if await redis.xlen(stats_key) < 1: + return False + still_scheduled = await redis.sismember(keys.scheduled_set_key(), f"g:{name}") # ty: ignore[invalid-await] + return not still_scheduled assert await wait_until(tracking_cleared, timeout=2), "success path did not record stats and clear tracking" assert await redis.xlen(stats_key) == 1 From 8638155a376e3f0691b16a73ed5e3d3b2447cffc Mon Sep 17 00:00:00 2001 From: Matteo Morando Date: Thu, 5 Nov 2026 18:07:41 +0400 Subject: [PATCH 3/3] Assert integration facts directly and harden waits under load - Every fact polled inside a wait_until helper is asserted again after the wait, so a vacuous helper can no longer hide a regression. The success-path test now waits for the message ACK and then checks statistics, scheduled-set cleanup and running key directly. - Waits for something that must happen use a 15 s upper bound. - Known-bug windows for events that must happen on their own start only once the bug's preconditions hold, avoiding false XPASS on slow machines. --- tests/integration/conftest.py | 8 +++- tests/integration/test_known_bugs.py | 58 ++++++++++++++++-------- tests/integration/test_scheduling.py | 68 +++++++++++++++++----------- 3 files changed, 87 insertions(+), 47 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 589f0d2..eb2a775 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -50,6 +50,12 @@ "retry_backoff_max": 1.0, } +# Upper bound for waits on something that MUST happen. wait_until returns as +# soon as the condition holds, so a generous bound costs nothing on a passing +# run and keeps the suite stable on a loaded machine. Windows in which something +# must NOT happen, and the windows of the known-bug tests, stay short on purpose. +EVENTUALLY = 15.0 + # A cron expression that never fires during a test run (midnight, January 1st). # Used together with a pre-seeded next_run so that tasks run only when a test # triggers them explicitly. @@ -235,7 +241,7 @@ def require(condition: object, message: str) -> None: raise PreconditionError(message) -async def wait_for_leader(managers: list[TaskManager], timeout: float = 5.0) -> TaskManager: +async def wait_for_leader(managers: list[TaskManager], timeout: float = EVENTUALLY) -> TaskManager: """Wait until exactly one of ``managers`` is leader and return it.""" def single_leader() -> bool: diff --git a/tests/integration/test_known_bugs.py b/tests/integration/test_known_bugs.py index 20724a8..1ab7403 100644 --- a/tests/integration/test_known_bugs.py +++ b/tests/integration/test_known_bugs.py @@ -26,6 +26,7 @@ trigger_tasks, ) from tests.integration.conftest import ( + EVENTUALLY, YEARLY_CRON, ManagerFactory, require, @@ -61,6 +62,24 @@ async def long() -> None: # Reconciliation every second (it is off by default in the integration config). LONG_TASK_TIMINGS = {"running_heartbeat_interval": 0.2, "reconciliation_interval": 1} +# Window in which a buggy behaviour that happens on its own (a duplicate start, +# a run at startup) is given the chance to show up. It starts only once the +# preconditions of the bug hold, so a slow machine cannot make the bug "vanish" +# and report a false XPASS. +BUG_WINDOW = 5.0 + + +async def _pending_message_idle(redis: Redis, keys: RedisKeyBuilder) -> bool: + """True when a pending message of the low priority stream is older than the 2 s stuck threshold.""" + pending = await redis.xpending_range( + keys.task_stream_low_key(), + keys.consumer_group_name(), + min="-", + max="+", + count=10, + ) + return any(entry["time_since_delivered"] > 2_100 for entry in pending) + # --------------------------------------------------------------------------- # FTM-01: live long-running tasks are executed again @@ -85,11 +104,16 @@ async def test_reconciler_does_not_restart_a_task_that_is_still_running( await wait_for_leader([manager]) await trigger_tasks(manager, task_group_name="g", task_name=name) - require(await wait_until(lambda: len(starts) >= 1, timeout=5), "task never started") + require(await wait_until(lambda: len(starts) >= 1, timeout=EVENTUALLY), "task never started") + + # The task stays alive (heartbeat renewed). Wait until its pending message is + # past the 2 s stuck threshold (or the Reconciler already acted on it), then + # leave several 1 s reconciliation cycles for a duplicate start to appear. + async def message_stuck_or_restarted() -> bool: + return len(starts) > 1 or await _pending_message_idle(redis, keys) - # The task stays alive (heartbeat renewed) well past the 2 s stuck threshold - # and several 1 s reconciliation cycles. - await wait_until(lambda: len(starts) > 1, timeout=5) + require(await wait_until(message_stuck_or_restarted, timeout=EVENTUALLY), "message never became idle") + await wait_until(lambda: len(starts) > 1, timeout=BUG_WINDOW) assert len(starts) == 1, ( f"FTM-01: a running task with allow_parallel=False was started {len(starts)} times, " f"at {[round(s - starts[0], 1) for s in starts]} s" @@ -116,25 +140,18 @@ async def test_starting_instance_does_not_steal_a_running_task( await wait_for_leader([first]) await trigger_tasks(first, task_group_name="g", task_name=name) - require(await wait_until(lambda: len(starts) >= 1, timeout=5), "task never started") + require(await wait_until(lambda: len(starts) >= 1, timeout=EVENTUALLY), "task never started") # Wait until the pending message is older than the 2 s stuck threshold. - async def pending_message_idle() -> bool: - pending = await redis.xpending_range( - keys.task_stream_low_key(), - keys.consumer_group_name(), - min="-", - max="+", - count=10, - ) - return any(entry["time_since_delivered"] > 2_100 for entry in pending) - - require(await wait_until(pending_message_idle, timeout=5), "message never became idle") + require( + await wait_until(lambda: _pending_message_idle(redis, keys), timeout=EVENTUALLY), + "message never became idle", + ) # A new replica starts (rolling deploy / autoscaling) while the task is alive. await make_manager([_long_task_group(starts, release)], **timings) - await wait_until(lambda: len(starts) > 1, timeout=2) + await wait_until(lambda: len(starts) > 1, timeout=BUG_WINDOW) assert len(starts) == 1, ( f"FTM-01: a task still running on a live instance was started {len(starts)} times " "after another instance started" @@ -244,7 +261,7 @@ async def delivered() -> bool: groups = await redis.xinfo_groups(keys.task_stream_low_key()) return any(g["last-delivered-id"] == message_id for g in groups) - require(await wait_until(delivered, timeout=5), "message was never delivered to the old instance") + require(await wait_until(delivered, timeout=EVENTUALLY), "message was never delivered to the old instance") # The new version comes up; the run must eventually happen there. await make_manager([new_group], **LONG_TASK_TIMINGS) @@ -287,7 +304,7 @@ async def flaky() -> None: await wait_for_leader([manager]) await trigger_tasks(manager, task_group_name="g", task_name=name) - require(await wait_until(lambda: len(attempts) >= 1, timeout=5), "task never ran") + require(await wait_until(lambda: len(attempts) >= 1, timeout=EVENTUALLY), "task never ran") # retry_backoff is 1 s: a retry is expected well within 4 s. retried = await wait_until(lambda: len(attempts) >= 2, timeout=4) @@ -344,7 +361,8 @@ async def yearly() -> None: manager = await make_manager([group], reconciliation_interval=1) await wait_for_leader([manager]) - ran = await wait_until(lambda: len(calls) >= 1, timeout=2) + # Covers several Coordinator cycles and Reconciler cycles (1 s). + ran = await wait_until(lambda: len(calls) >= 1, timeout=BUG_WINDOW) assert not ran, f"FTM-07: a task with cron {YEARLY_CRON!r} ran right at first startup" diff --git a/tests/integration/test_scheduling.py b/tests/integration/test_scheduling.py index 4cb3b69..b655ee1 100644 --- a/tests/integration/test_scheduling.py +++ b/tests/integration/test_scheduling.py @@ -18,6 +18,7 @@ from fastapi_task_manager.redis_keys import RedisKeyBuilder from fastapi_task_manager.task_router_services import trigger_tasks from tests.integration.conftest import ( + EVENTUALLY, YEARLY_CRON, ManagerFactory, seed_next_run_far_future, @@ -60,26 +61,28 @@ async def test_triggered_task_runs_exactly_once( await wait_for_leader([manager]) await trigger_tasks(manager, task_group_name="g", task_name=name) - assert await wait_until(lambda: len(calls) >= 1, timeout=5), "triggered task never ran" + assert await wait_until(lambda: len(calls) >= 1, timeout=EVENTUALLY), "triggered task never ran" # Several coordinator cycles: a duplicate publish would show up here. await wait_until(lambda: len(calls) > 1, timeout=1) assert calls == ["A"] - # Success path side effects: one stats entry, no running key, not tracked as scheduled. - # The consumer records stats, deletes the running key, then removes the - # scheduled-set member, in this order, once the task returns. - stats_key = keys.stats_stream_key("g", name) - - async def tracking_cleared() -> bool: - if await redis.xlen(stats_key) < 1: - return False - still_scheduled = await redis.sismember(keys.scheduled_set_key(), f"g:{name}") # ty: ignore[invalid-await] - return not still_scheduled - - assert await wait_until(tracking_cleared, timeout=2), "success path did not record stats and clear tracking" - assert await redis.xlen(stats_key) == 1 - # Checked right away: the running key also has a TTL (2 s), so waiting - # for it to disappear would pass even if the consumer never deleted it. + # Success path side effects: one stats entry, no running key, not tracked as + # scheduled. Once the task returns, the consumer records stats, deletes the + # running key, removes the scheduled-set member and finally ACKs and deletes + # the stream message. The test waits only for that last step, then asserts + # every side effect directly. + stream_key = keys.task_stream_low_key() + + async def message_acknowledged() -> bool: + # The message was published before the task ran: an empty stream means XDEL happened. + return await redis.xlen(stream_key) == 0 + + assert await wait_until(message_acknowledged, timeout=EVENTUALLY), "the task message was never acknowledged" + assert await redis.xlen(keys.stats_stream_key("g", name)) == 1, "success was not recorded in the statistics" + scheduled = await redis.sismember(keys.scheduled_set_key(), f"g:{name}") # ty: ignore[invalid-await] + assert not scheduled, "task still tracked as scheduled after success" + # The running key also has a TTL (2 s): checked right after the ACK, before + # it could expire on its own, so a missing delete is still caught. assert not await redis.exists(keys.running_task_key("g", name)), "running key left behind after success" @@ -102,7 +105,7 @@ async def test_cron_task_runs_when_due_and_schedules_next_occurrence( manager = await make_manager([group]) await wait_for_leader([manager]) - assert await wait_until(lambda: len(calls) >= 1, timeout=5), "due task never ran" + assert await wait_until(lambda: len(calls) >= 1, timeout=EVENTUALLY), "due task never ran" await wait_until(lambda: len(calls) > 1, timeout=1) assert calls == ["A"] @@ -110,7 +113,7 @@ async def test_cron_task_runs_when_due_and_schedules_next_occurrence( async def next_run_updated() -> bool: return float(await redis.get(next_run_key) or 0) != seeded - assert await wait_until(next_run_updated, timeout=2), "next_run was never updated" + assert await wait_until(next_run_updated, timeout=EVENTUALLY), "next_run was never updated" next_run = float(await redis.get(next_run_key)) expected = next(CronSim(YEARLY_CRON, datetime.now(timezone.utc))).timestamp() assert next_run == expected @@ -169,6 +172,8 @@ async def test_instance_that_is_not_leader_never_schedules( # Many coordinator and acquisition cycles (retry interval 0.2 s). published = await wait_until(lambda: _next_run_changed(redis, next_run_key), timeout=1.5) assert not published, "an instance that is not leader rescheduled the task" + # Direct checks, independent of the polling helper above. + assert await redis.get(next_run_key) == "0", "an instance that is not leader rescheduled the task" assert calls == [] assert manager.runner is not None assert not manager.runner.is_leader @@ -189,8 +194,10 @@ async def test_leadership_fails_over_when_leader_stops(make_manager: ManagerFact assert await wait_until( lambda: second.runner is not None and second.runner.is_leader, - timeout=5, + timeout=EVENTUALLY, ), "the surviving instance never became leader" + assert second.runner is not None + assert second.runner.is_leader, "the surviving instance never became leader" async def test_triggered_task_runs_once_across_instances( @@ -212,7 +219,7 @@ async def test_triggered_task_runs_once_across_instances( await trigger_tasks(first, task_group_name="g", task_name=name) - assert await wait_until(lambda: len(calls) >= 1, timeout=5), "triggered task never ran" + assert await wait_until(lambda: len(calls) >= 1, timeout=EVENTUALLY), "triggered task never ran" await wait_until(lambda: len(calls) > 1, timeout=1.5) assert len(calls) == 1, f"task ran {len(calls)} times: {calls}" @@ -247,17 +254,19 @@ async def probe() -> None: await wait_for_leader([manager]) await trigger_tasks(manager, task_group_name="g", task_name=name) - assert await wait_until(lambda: len(starts) == 1, timeout=5), "first execution never started" + assert await wait_until(lambda: len(starts) == 1, timeout=EVENTUALLY), "first execution never started" # Second trigger while the first execution is still running. await trigger_tasks(manager, task_group_name="g", task_name=name) started_again = await wait_until(lambda: len(starts) > 1, timeout=1) assert not started_again, "a second execution started while the first was running" + assert len(starts) == 1, "a second execution started while the first was running" # Once the first execution ends, the pending trigger is honoured. release.set() - assert await wait_until(lambda: len(starts) == 2, timeout=5), "pending trigger never ran" - assert max_running == 1 + assert await wait_until(lambda: len(starts) == 2, timeout=EVENTUALLY), "pending trigger never ran" + assert len(starts) == 2, "pending trigger never ran" + assert max_running == 1, "two executions overlapped" async def test_consumer_recovers_after_redis_keys_are_flushed( @@ -291,8 +300,13 @@ async def group_exists() -> bool: return False return any(g["name"] == keys.consumer_group_name() for g in groups) - assert await wait_until(group_exists, timeout=5), "consumer group was never recreated" - assert await wait_until(lambda: manager.runner is not None and manager.runner.is_leader, timeout=5) + assert await wait_until(group_exists, timeout=EVENTUALLY), "consumer group was never recreated" + # Direct check, independent of the polling helper above. + groups = await redis.xinfo_groups(keys.task_stream_low_key()) + assert keys.consumer_group_name() in [g["name"] for g in groups], "consumer group was never recreated" + assert await wait_until(lambda: manager.runner is not None and manager.runner.is_leader, timeout=EVENTUALLY) + assert manager.runner is not None + assert manager.runner.is_leader, "leadership was not reacquired after the flush" # Right after the flush the coordinator may publish the task once on its own # (next_run is gone). Push next_run away again and count from here, so the @@ -302,4 +316,6 @@ async def group_exists() -> bool: # A task triggered after the flush still runs. await trigger_tasks(manager, task_group_name="g", task_name=name) - assert await wait_until(lambda: len(calls) > runs_before_trigger, timeout=5), "task never ran after the flush" + ran_again = await wait_until(lambda: len(calls) > runs_before_trigger, timeout=EVENTUALLY) + assert ran_again, "task never ran after the flush" + assert len(calls) > runs_before_trigger, "task never ran after the flush"