From cd79480517659ad2386df3c4cb151cdf88f1ac6f Mon Sep 17 00:00:00 2001 From: Paulo Date: Mon, 17 Aug 2026 19:45:53 +0200 Subject: [PATCH 1/2] The app engine pool is sized for the run queue, not a lone request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Twenty concurrent scout runs exhausted the application engine: it carried SQLAlchemy's default QueuePool (5 + 10 overflow) while both DBOS engines are tuned to twenty, and one engine serves every run's steps plus request handling. The sixteenth waiter timed out at five seconds and failed its run. The pool is now explicit — twenty plus ten overflow — matching the width the queue actually runs at. pool_timeout stays low: a checkout wait blocks the event loop, so the fix is capacity, not patience. The queue itself stays uncapped deliberately: a parked run holds its queue slot, so a concurrency cap could starve the appliance behind a handful of runs waiting on humans. --- backend/druks/database.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/backend/druks/database.py b/backend/druks/database.py index 0c4bbc7d..0d900e99 100644 --- a/backend/druks/database.py +++ b/backend/druks/database.py @@ -81,8 +81,16 @@ def create_engine_from_url(database_url: str): # at the lifecycle boundary (the API session dependency, the worker session # wrapper) so a failed unit of work rolls back instead of leaving partial # writes. Model methods ``flush()``; the boundary commits. Low pool_timeout - # because a checkout wait blocks the event loop. - return create_engine(database_url, pool_pre_ping=True, pool_timeout=5) + # because a checkout wait blocks the event loop. The pool serves every + # concurrent run's steps plus request handling at once, so it is sized for + # the run queue running wide, not for a lone request. + return create_engine( + database_url, + pool_pre_ping=True, + pool_timeout=5, + pool_size=20, + max_overflow=10, + ) def get_session(engine) -> Session: From 3614c054ca99de18b8033baddcd862404dc18fd3 Mon Sep 17 00:00:00 2001 From: Paulo Date: Mon, 17 Aug 2026 19:50:45 +0200 Subject: [PATCH 2/2] Overflow does the burst work: ceiling 50 inside the default budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Overflow connections open on demand and close on return, so a high ceiling costs nothing at idle. Fifty keeps the appliance — with both DBOS engines at twenty — inside Postgres's default hundred. --- backend/druks/database.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/druks/database.py b/backend/druks/database.py index 0d900e99..aa37929d 100644 --- a/backend/druks/database.py +++ b/backend/druks/database.py @@ -82,14 +82,17 @@ def create_engine_from_url(database_url: str): # wrapper) so a failed unit of work rolls back instead of leaving partial # writes. Model methods ``flush()``; the boundary commits. Low pool_timeout # because a checkout wait blocks the event loop. The pool serves every - # concurrent run's steps plus request handling at once, so it is sized for - # the run queue running wide, not for a lone request. + # concurrent run's steps plus request handling at once: a modest steady + # pool, with overflow doing the burst work — overflow connections open on + # demand and close on return, so the ceiling is high while idle cost is + # not. Ceiling 50 keeps the appliance (with DBOS's two engines at 20 each) + # inside Postgres's default 100 connections. return create_engine( database_url, pool_pre_ping=True, pool_timeout=5, pool_size=20, - max_overflow=10, + max_overflow=30, )