diff --git a/src/backend/access/transam/distributedlog.c b/src/backend/access/transam/distributedlog.c index 0453031775f..531d6087c43 100644 --- a/src/backend/access/transam/distributedlog.c +++ b/src/backend/access/transam/distributedlog.c @@ -474,9 +474,25 @@ DistributedLog_GetDistributedXid( int entryno = TransactionIdToEntry(localXid); int slotno; DistributedLogEntry *ptr; + TransactionId oldestXmin; Assert(!IS_QUERY_DISPATCHER()); + /* + * If the requested xid is below the truncation horizon, its entry + * has already been trimmed: return "not found" cleanly instead of + * attempting an SLRU read of a removed page. + */ + oldestXmin = (TransactionId)pg_atomic_read_u32((pg_atomic_uint32 *)&DistributedLogShared->oldestXmin); + if (oldestXmin == InvalidTransactionId) + elog(PANIC, "DistributedLog's OldestXmin not initialized yet"); + + if (TransactionIdPrecedes(localXid, oldestXmin)) + { + *distribXid = 0; + return; + } + slotno = SimpleLruReadPage_ReadOnly(DistributedLogCtl, page, localXid); ptr = (DistributedLogEntry *) DistributedLogCtl->shared->page_buffer[slotno]; ptr += entryno; @@ -1068,9 +1084,44 @@ DistributedLog_redo(XLogReaderState *record) else if (info == DISTRIBUTEDLOG_TRUNCATE) { int page; + TransactionId cutoffXid; + TransactionId curOldestXmin; memcpy(&page, XLogRecGetData(record), sizeof(int)); + /* + * The primary already removed all DistributedLog pages older + * than the cutoff page. Advance our oldestXmin to the first + * xid on the cutoff page, so that standby/mirror readers + * resolving pre-truncation gxids take the soft "below + * oldestXmin => not found" path in + * DistributedLog_CommittedCheck() / + * DistributedLog_GetDistributedXid() instead of erroring out + * when the SLRU read fails on a truncated segment. + * + * The loop also covers a replay that runs before the first + * DistributedLog_AdvanceOldestXmin() call, where the shared + * oldestXmin still holds its InvalidTransactionId initial + * value: nothing better has been written yet, and the cutoff + * is a valid lower bound, so CAS from 0 installs it. + */ + cutoffXid = ((TransactionId) page) * ENTRIES_PER_PAGE; + if (cutoffXid < FirstNormalTransactionId) + cutoffXid = FirstNormalTransactionId; + + curOldestXmin = (TransactionId) pg_atomic_read_u32( + (pg_atomic_uint32 *) &DistributedLogShared->oldestXmin); + while (TransactionIdPrecedes(curOldestXmin, cutoffXid)) + { + uint32 expected = (uint32) curOldestXmin; + + if (pg_atomic_compare_exchange_u32( + (pg_atomic_uint32 *) &DistributedLogShared->oldestXmin, + &expected, (uint32) cutoffXid)) + break; + curOldestXmin = expected; + } + elog((Debug_print_full_dtm ? LOG : DEBUG5), "Redo DISTRIBUTEDLOG_TRUNCATE page %d", page); diff --git a/src/test/isolation2/expected/hot_standby/dlog_truncation.out b/src/test/isolation2/expected/hot_standby/dlog_truncation.out new file mode 100644 index 00000000000..c811fd54c6c --- /dev/null +++ b/src/test/isolation2/expected/hot_standby/dlog_truncation.out @@ -0,0 +1,74 @@ +-- Distributed log truncation and hot standby reads +-- +-- DistributedLog_Truncate() runs on the primaries and removes distributed log +-- segments below the cutoff, emitting a TRUNCATE WAL record. Its redo branch +-- must advance the mirror's in-memory oldestXmin horizon along with the +-- segment removal; otherwise a standby reader resolving a pre-truncation gxid +-- on a mirror first passes the stale-horizon guard and then fails with a hard +-- "Could not open file pg_distributedlog/NNNN" error on the removed segment. +-- With the horizon advanced, those gxids resolve to not-found and take the +-- soft in-doubt path instead. +-- +-- start_matchsubs +-- +-- m/(seg\d+ slice\d+ [\d.]+:\d+ pid \d+)/ +-- s/(seg\d+ slice\d+ [\d.]+:\d+ pid \d+)/seg slice
pid / +-- +-- m/(seg\d+ [\d.]+:\d+ pid \d+)/ +-- s/(seg\d+ [\d.]+:\d+ pid \d+)/seg
pid / +-- +-- end_matchsubs + +1: create table hs_dlog_trunc(a int, msg text); +CREATE +1: insert into hs_dlog_trunc select g, 'row ' || g from generate_series(1, 10) g; +INSERT 10 + +-- Burn more than one distributed-log segment worth of segment-local xids on +-- every primary (32 SLRU pages * 4096 entries per page = 131072 xids per +-- segment), so that the next horizon advance truncates the pages backing the +-- rows inserted above. txid_current() allocates an xid without touching any +-- table, so the burn leaves no distributed state behind. +!\retcode sh -ec 'export PGDATABASE=postgres; F=$(mktemp); trap "rm -f $F" EXIT; printf "SELECT txid_current();\n" > $F; for p in $(psql -Atqc "select port from gp_segment_configuration where role = chr(112) and content >= 0 order by content"); do PGOPTIONS="-c gp_role=utility" pgbench -n -p $p -c 2 -j 2 -t 70000 -f $F > /dev/null; done'; +-- start_ignore + +-- end_ignore +(exited with code 0) + +-- Run a few dispatched statements so the primaries advance the distributed +-- log horizon and emit the TRUNCATE record. +1: create table hs_dlog_dummy(a int); +CREATE +1: drop table hs_dlog_dummy; +DROP + +-- Wait until every mirror has replayed the truncation (the first remaining +-- segment moves past 0000). Without this the read could race the replay and +-- the test would not exercise the removed-segment path deterministically. +!\retcode sh -ec 'export PGDATABASE=postgres; for i in $(seq 1 120); do gone=1; for d in $(psql -Atqc "select datadir from gp_segment_configuration where role = chr(109) and content >= 0"); do f=$(ls $d/pg_distributedlog 2>/dev/null | sort | head -1); [ "$f" = "0000" ] && gone=0; done; [ $gone -eq 1 ] && exit 0; sleep 1; done; exit 1'; +-- start_ignore + +-- end_ignore +(exited with code 0) + +-- The standby reader resolves the pre-truncation gxids of the rows committed +-- above. It must return the rows (or defer cleanly), never fail on the +-- removed distributed log segment. +-1S: select * from hs_dlog_trunc order by a; + a | msg +----+-------- + 1 | row 1 + 2 | row 2 + 3 | row 3 + 4 | row 4 + 5 | row 5 + 6 | row 6 + 7 | row 7 + 8 | row 8 + 9 | row 9 + 10 | row 10 +(10 rows) +-1Sq: ... + +1: drop table hs_dlog_trunc; +DROP diff --git a/src/test/isolation2/hot_standby_schedule b/src/test/isolation2/hot_standby_schedule index 73e0f71a84c..7e24db7f3c2 100644 --- a/src/test/isolation2/hot_standby_schedule +++ b/src/test/isolation2/hot_standby_schedule @@ -3,4 +3,5 @@ test: hot_standby/basic test: hot_standby/transaction_isolation test: hot_standby/query_conflict test: hot_standby/faults +test: hot_standby/dlog_truncation test: hot_standby/teardown diff --git a/src/test/isolation2/sql/hot_standby/dlog_truncation.sql b/src/test/isolation2/sql/hot_standby/dlog_truncation.sql new file mode 100644 index 00000000000..d3c6ce29bce --- /dev/null +++ b/src/test/isolation2/sql/hot_standby/dlog_truncation.sql @@ -0,0 +1,48 @@ +-- Distributed log truncation and hot standby reads +-- +-- DistributedLog_Truncate() runs on the primaries and removes distributed log +-- segments below the cutoff, emitting a TRUNCATE WAL record. Its redo branch +-- must advance the mirror's in-memory oldestXmin horizon along with the +-- segment removal; otherwise a standby reader resolving a pre-truncation gxid +-- on a mirror first passes the stale-horizon guard and then fails with a hard +-- "Could not open file pg_distributedlog/NNNN" error on the removed segment. +-- With the horizon advanced, those gxids resolve to not-found and take the +-- soft in-doubt path instead. +-- +-- start_matchsubs +-- +-- m/(seg\d+ slice\d+ [\d.]+:\d+ pid \d+)/ +-- s/(seg\d+ slice\d+ [\d.]+:\d+ pid \d+)/seg slice
pid / +-- +-- m/(seg\d+ [\d.]+:\d+ pid \d+)/ +-- s/(seg\d+ [\d.]+:\d+ pid \d+)/seg
pid / +-- +-- end_matchsubs + +1: create table hs_dlog_trunc(a int, msg text); +1: insert into hs_dlog_trunc select g, 'row ' || g from generate_series(1, 10) g; + +-- Burn more than one distributed-log segment worth of segment-local xids on +-- every primary (32 SLRU pages * 4096 entries per page = 131072 xids per +-- segment), so that the next horizon advance truncates the pages backing the +-- rows inserted above. txid_current() allocates an xid without touching any +-- table, so the burn leaves no distributed state behind. +!\retcode sh -ec 'export PGDATABASE=postgres; F=$(mktemp); trap "rm -f $F" EXIT; printf "SELECT txid_current();\n" > $F; for p in $(psql -Atqc "select port from gp_segment_configuration where role = chr(112) and content >= 0 order by content"); do PGOPTIONS="-c gp_role=utility" pgbench -n -p $p -c 2 -j 2 -t 70000 -f $F > /dev/null; done'; + +-- Run a few dispatched statements so the primaries advance the distributed +-- log horizon and emit the TRUNCATE record. +1: create table hs_dlog_dummy(a int); +1: drop table hs_dlog_dummy; + +-- Wait until every mirror has replayed the truncation (the first remaining +-- segment moves past 0000). Without this the read could race the replay and +-- the test would not exercise the removed-segment path deterministically. +!\retcode sh -ec 'export PGDATABASE=postgres; for i in $(seq 1 120); do gone=1; for d in $(psql -Atqc "select datadir from gp_segment_configuration where role = chr(109) and content >= 0"); do f=$(ls $d/pg_distributedlog 2>/dev/null | sort | head -1); [ "$f" = "0000" ] && gone=0; done; [ $gone -eq 1 ] && exit 0; sleep 1; done; exit 1'; + +-- The standby reader resolves the pre-truncation gxids of the rows committed +-- above. It must return the rows (or defer cleanly), never fail on the +-- removed distributed log segment. +-1S: select * from hs_dlog_trunc order by a; +-1Sq: + +1: drop table hs_dlog_trunc;