From 89f8da4442a395b383de3070a8252f9b611bfc93 Mon Sep 17 00:00:00 2001 From: reshke Date: Tue, 20 Oct 2026 18:58:12 +0000 Subject: [PATCH] Advance oldestXmin when replaying DISTRIBUTEDLOG_TRUNCATE WAL records DistributedLog truncate runs only on primaries and removes distributed xact log segments older than the cutoff page, emitting a TRUNCATE WAL record. Its redo branch in DistributedLog_redo() however only performed the SimpleLruTruncate() and never advanced DistributedLogShared->oldestXmin, so on a mirror/standby replaying the record the in-memory horizon stayed stale while the pre-truncation segments were already deleted. without fix, standby query fails with ``` @@ -57,19 +57,8 @@ -- above. It must return the rows (or defer cleanly), never fail on the -- removed distributed log segment. -1S: select * from hs_dlog_trunc order by a; - a | msg -----+-------- - 1 | row 1 - 2 | row 2 - 3 | row 3 - 4 | row 4 - 5 | row 5 - 6 | row 6 - 7 | row 7 - 8 | row 8 - 9 | row 9 - 10 | row 10 -(10 rows) +DETAIL: Could not open file "pg_distributedlog/0000": No such file or directory. +ERROR: could not access status of transaction 3 -1Sq: ... 1: drop table hs_dlog_trunc; ``` A reader resolving a pre-truncation gxid then fell through the stale-horizon guard in DistributedLog_CommittedCheck() (and DistributedLog_GetDistributedXid(), which had no guard at all) into an SLRU read of a removed segment, producing a hard distributed-log read error instead of a soft not-found that takes the standby in-doubt path. To fix, in the DISTRIBUTEDLOG_TRUNCATE redo branch, CAS-advance the local oldestXmin to the first xid of the cutoff page (never moving it backwards), and add a horizon guard to DistributedLog_GetDistributedXid() so trimmed xids return not-found cleanly. --- src/backend/access/transam/distributedlog.c | 50 +++++++++++++ .../expected/hot_standby/dlog_truncation.out | 74 +++++++++++++++++++ src/test/isolation2/hot_standby_schedule | 1 + .../sql/hot_standby/dlog_truncation.sql | 48 ++++++++++++ 4 files changed, 173 insertions(+) create mode 100644 src/test/isolation2/expected/hot_standby/dlog_truncation.out create mode 100644 src/test/isolation2/sql/hot_standby/dlog_truncation.sql diff --git a/src/backend/access/transam/distributedlog.c b/src/backend/access/transam/distributedlog.c index 0453031775f..387607b3857 100644 --- a/src/backend/access/transam/distributedlog.c +++ b/src/backend/access/transam/distributedlog.c @@ -474,9 +474,24 @@ DistributedLog_GetDistributedXid( int entryno = TransactionIdToEntry(localXid); int slotno; DistributedLogEntry *ptr; + TransactionId oldestXmin; Assert(!IS_QUERY_DISPATCHER()); + /* + * If the requested xid is below the truncation horizon, its entry + * has already been trimmed: return "not found" cleanly instead of + * attempting an SLRU read of a removed page. + */ + oldestXmin = (TransactionId) pg_atomic_read_u32( + (pg_atomic_uint32 *) &DistributedLogShared->oldestXmin); + if (oldestXmin != InvalidTransactionId && + TransactionIdPrecedes(localXid, oldestXmin)) + { + *distribXid = 0; + return; + } + slotno = SimpleLruReadPage_ReadOnly(DistributedLogCtl, page, localXid); ptr = (DistributedLogEntry *) DistributedLogCtl->shared->page_buffer[slotno]; ptr += entryno; @@ -1068,9 +1083,44 @@ DistributedLog_redo(XLogReaderState *record) else if (info == DISTRIBUTEDLOG_TRUNCATE) { int page; + TransactionId cutoffXid; + TransactionId curOldestXmin; memcpy(&page, XLogRecGetData(record), sizeof(int)); + /* + * The primary already removed all DistributedLog pages older + * than the cutoff page. Advance our oldestXmin to the first + * xid on the cutoff page, so that standby/mirror readers + * resolving pre-truncation gxids take the soft "below + * oldestXmin => not found" path in + * DistributedLog_CommittedCheck() / + * DistributedLog_GetDistributedXid() instead of erroring out + * when the SLRU read fails on a truncated segment. + * + * The loop also covers a replay that runs before the first + * DistributedLog_AdvanceOldestXmin() call, where the shared + * oldestXmin still holds its InvalidTransactionId initial + * value: nothing better has been written yet, and the cutoff + * is a valid lower bound, so CAS from 0 installs it. + */ + cutoffXid = ((TransactionId) page) * ENTRIES_PER_PAGE; + if (cutoffXid < FirstNormalTransactionId) + cutoffXid = FirstNormalTransactionId; + + curOldestXmin = (TransactionId) pg_atomic_read_u32( + (pg_atomic_uint32 *) &DistributedLogShared->oldestXmin); + while (TransactionIdPrecedes(curOldestXmin, cutoffXid)) + { + uint32 expected = (uint32) curOldestXmin; + + if (pg_atomic_compare_exchange_u32( + (pg_atomic_uint32 *) &DistributedLogShared->oldestXmin, + &expected, (uint32) cutoffXid)) + break; + curOldestXmin = expected; + } + elog((Debug_print_full_dtm ? LOG : DEBUG5), "Redo DISTRIBUTEDLOG_TRUNCATE page %d", page); diff --git a/src/test/isolation2/expected/hot_standby/dlog_truncation.out b/src/test/isolation2/expected/hot_standby/dlog_truncation.out new file mode 100644 index 00000000000..c811fd54c6c --- /dev/null +++ b/src/test/isolation2/expected/hot_standby/dlog_truncation.out @@ -0,0 +1,74 @@ +-- Distributed log truncation and hot standby reads +-- +-- DistributedLog_Truncate() runs on the primaries and removes distributed log +-- segments below the cutoff, emitting a TRUNCATE WAL record. Its redo branch +-- must advance the mirror's in-memory oldestXmin horizon along with the +-- segment removal; otherwise a standby reader resolving a pre-truncation gxid +-- on a mirror first passes the stale-horizon guard and then fails with a hard +-- "Could not open file pg_distributedlog/NNNN" error on the removed segment. +-- With the horizon advanced, those gxids resolve to not-found and take the +-- soft in-doubt path instead. +-- +-- start_matchsubs +-- +-- m/(seg\d+ slice\d+ [\d.]+:\d+ pid \d+)/ +-- s/(seg\d+ slice\d+ [\d.]+:\d+ pid \d+)/seg slice
pid / +-- +-- m/(seg\d+ [\d.]+:\d+ pid \d+)/ +-- s/(seg\d+ [\d.]+:\d+ pid \d+)/seg
pid / +-- +-- end_matchsubs + +1: create table hs_dlog_trunc(a int, msg text); +CREATE +1: insert into hs_dlog_trunc select g, 'row ' || g from generate_series(1, 10) g; +INSERT 10 + +-- Burn more than one distributed-log segment worth of segment-local xids on +-- every primary (32 SLRU pages * 4096 entries per page = 131072 xids per +-- segment), so that the next horizon advance truncates the pages backing the +-- rows inserted above. txid_current() allocates an xid without touching any +-- table, so the burn leaves no distributed state behind. +!\retcode sh -ec 'export PGDATABASE=postgres; F=$(mktemp); trap "rm -f $F" EXIT; printf "SELECT txid_current();\n" > $F; for p in $(psql -Atqc "select port from gp_segment_configuration where role = chr(112) and content >= 0 order by content"); do PGOPTIONS="-c gp_role=utility" pgbench -n -p $p -c 2 -j 2 -t 70000 -f $F > /dev/null; done'; +-- start_ignore + +-- end_ignore +(exited with code 0) + +-- Run a few dispatched statements so the primaries advance the distributed +-- log horizon and emit the TRUNCATE record. +1: create table hs_dlog_dummy(a int); +CREATE +1: drop table hs_dlog_dummy; +DROP + +-- Wait until every mirror has replayed the truncation (the first remaining +-- segment moves past 0000). Without this the read could race the replay and +-- the test would not exercise the removed-segment path deterministically. +!\retcode sh -ec 'export PGDATABASE=postgres; for i in $(seq 1 120); do gone=1; for d in $(psql -Atqc "select datadir from gp_segment_configuration where role = chr(109) and content >= 0"); do f=$(ls $d/pg_distributedlog 2>/dev/null | sort | head -1); [ "$f" = "0000" ] && gone=0; done; [ $gone -eq 1 ] && exit 0; sleep 1; done; exit 1'; +-- start_ignore + +-- end_ignore +(exited with code 0) + +-- The standby reader resolves the pre-truncation gxids of the rows committed +-- above. It must return the rows (or defer cleanly), never fail on the +-- removed distributed log segment. +-1S: select * from hs_dlog_trunc order by a; + a | msg +----+-------- + 1 | row 1 + 2 | row 2 + 3 | row 3 + 4 | row 4 + 5 | row 5 + 6 | row 6 + 7 | row 7 + 8 | row 8 + 9 | row 9 + 10 | row 10 +(10 rows) +-1Sq: ... + +1: drop table hs_dlog_trunc; +DROP diff --git a/src/test/isolation2/hot_standby_schedule b/src/test/isolation2/hot_standby_schedule index 73e0f71a84c..7e24db7f3c2 100644 --- a/src/test/isolation2/hot_standby_schedule +++ b/src/test/isolation2/hot_standby_schedule @@ -3,4 +3,5 @@ test: hot_standby/basic test: hot_standby/transaction_isolation test: hot_standby/query_conflict test: hot_standby/faults +test: hot_standby/dlog_truncation test: hot_standby/teardown diff --git a/src/test/isolation2/sql/hot_standby/dlog_truncation.sql b/src/test/isolation2/sql/hot_standby/dlog_truncation.sql new file mode 100644 index 00000000000..d3c6ce29bce --- /dev/null +++ b/src/test/isolation2/sql/hot_standby/dlog_truncation.sql @@ -0,0 +1,48 @@ +-- Distributed log truncation and hot standby reads +-- +-- DistributedLog_Truncate() runs on the primaries and removes distributed log +-- segments below the cutoff, emitting a TRUNCATE WAL record. Its redo branch +-- must advance the mirror's in-memory oldestXmin horizon along with the +-- segment removal; otherwise a standby reader resolving a pre-truncation gxid +-- on a mirror first passes the stale-horizon guard and then fails with a hard +-- "Could not open file pg_distributedlog/NNNN" error on the removed segment. +-- With the horizon advanced, those gxids resolve to not-found and take the +-- soft in-doubt path instead. +-- +-- start_matchsubs +-- +-- m/(seg\d+ slice\d+ [\d.]+:\d+ pid \d+)/ +-- s/(seg\d+ slice\d+ [\d.]+:\d+ pid \d+)/seg slice
pid / +-- +-- m/(seg\d+ [\d.]+:\d+ pid \d+)/ +-- s/(seg\d+ [\d.]+:\d+ pid \d+)/seg
pid / +-- +-- end_matchsubs + +1: create table hs_dlog_trunc(a int, msg text); +1: insert into hs_dlog_trunc select g, 'row ' || g from generate_series(1, 10) g; + +-- Burn more than one distributed-log segment worth of segment-local xids on +-- every primary (32 SLRU pages * 4096 entries per page = 131072 xids per +-- segment), so that the next horizon advance truncates the pages backing the +-- rows inserted above. txid_current() allocates an xid without touching any +-- table, so the burn leaves no distributed state behind. +!\retcode sh -ec 'export PGDATABASE=postgres; F=$(mktemp); trap "rm -f $F" EXIT; printf "SELECT txid_current();\n" > $F; for p in $(psql -Atqc "select port from gp_segment_configuration where role = chr(112) and content >= 0 order by content"); do PGOPTIONS="-c gp_role=utility" pgbench -n -p $p -c 2 -j 2 -t 70000 -f $F > /dev/null; done'; + +-- Run a few dispatched statements so the primaries advance the distributed +-- log horizon and emit the TRUNCATE record. +1: create table hs_dlog_dummy(a int); +1: drop table hs_dlog_dummy; + +-- Wait until every mirror has replayed the truncation (the first remaining +-- segment moves past 0000). Without this the read could race the replay and +-- the test would not exercise the removed-segment path deterministically. +!\retcode sh -ec 'export PGDATABASE=postgres; for i in $(seq 1 120); do gone=1; for d in $(psql -Atqc "select datadir from gp_segment_configuration where role = chr(109) and content >= 0"); do f=$(ls $d/pg_distributedlog 2>/dev/null | sort | head -1); [ "$f" = "0000" ] && gone=0; done; [ $gone -eq 1 ] && exit 0; sleep 1; done; exit 1'; + +-- The standby reader resolves the pre-truncation gxids of the rows committed +-- above. It must return the rows (or defer cleanly), never fail on the +-- removed distributed log segment. +-1S: select * from hs_dlog_trunc order by a; +-1Sq: + +1: drop table hs_dlog_trunc;