Advance oldestXmin when replaying DISTRIBUTEDLOG_TRUNCATE WAL records

src/backend/access/transam/distributedlog.c

@@ -474,9 +474,24 @@ DistributedLog_GetDistributedXid(
 	int			entryno = TransactionIdToEntry(localXid);
 	int			slotno;
 	DistributedLogEntry *ptr;
+	TransactionId oldestXmin;
 
 	Assert(!IS_QUERY_DISPATCHER());
 
+	/*
+	 * If the requested xid is below the truncation horizon, its entry
+	 * has already been trimmed: return "not found" cleanly instead of
+	 * attempting an SLRU read of a removed page.
+	 */
+	oldestXmin = (TransactionId) pg_atomic_read_u32(
+		(pg_atomic_uint32 *) &DistributedLogShared->oldestXmin);
+	if (oldestXmin != InvalidTransactionId &&
+		TransactionIdPrecedes(localXid, oldestXmin))
+	{
+		*distribXid = 0;
+		return;
+	}
+
 	slotno = SimpleLruReadPage_ReadOnly(DistributedLogCtl, page, localXid);
 	ptr = (DistributedLogEntry *) DistributedLogCtl->shared->page_buffer[slotno];
 	ptr += entryno;
@@ -1068,9 +1083,44 @@ DistributedLog_redo(XLogReaderState *record)
 	else if (info == DISTRIBUTEDLOG_TRUNCATE)
 	{
 		int			page;
+		TransactionId cutoffXid;
+		TransactionId curOldestXmin;
 
 		memcpy(&page, XLogRecGetData(record), sizeof(int));
 
+		/*
+		 * The primary already removed all DistributedLog pages older
+		 * than the cutoff page.  Advance our oldestXmin to the first
+		 * xid on the cutoff page, so that standby/mirror readers
+		 * resolving pre-truncation gxids take the soft "below
+		 * oldestXmin => not found" path in
+		 * DistributedLog_CommittedCheck() /
+		 * DistributedLog_GetDistributedXid() instead of erroring out
+		 * when the SLRU read fails on a truncated segment.
+		 *
+		 * The loop also covers a replay that runs before the first
+		 * DistributedLog_AdvanceOldestXmin() call, where the shared
+		 * oldestXmin still holds its InvalidTransactionId initial
+		 * value: nothing better has been written yet, and the cutoff
+		 * is a valid lower bound, so CAS from 0 installs it.
+		 */
+		cutoffXid = ((TransactionId) page) * ENTRIES_PER_PAGE;
+		if (cutoffXid < FirstNormalTransactionId)
+			cutoffXid = FirstNormalTransactionId;
+
+		curOldestXmin = (TransactionId) pg_atomic_read_u32(
+			(pg_atomic_uint32 *) &DistributedLogShared->oldestXmin);
+		while (TransactionIdPrecedes(curOldestXmin, cutoffXid))
+		{
+			uint32		expected = (uint32) curOldestXmin;
+
+			if (pg_atomic_compare_exchange_u32(
+					(pg_atomic_uint32 *) &DistributedLogShared->oldestXmin,
+					&expected, (uint32) cutoffXid))
+				break;
+			curOldestXmin = expected;
+		}
+
 		elog((Debug_print_full_dtm ? LOG : DEBUG5),
 			 "Redo DISTRIBUTEDLOG_TRUNCATE page %d",
 			 page);

src/test/isolation2/expected/hot_standby/dlog_truncation.out

@@ -0,0 +1,74 @@
+-- Distributed log truncation and hot standby reads
+--
+-- DistributedLog_Truncate() runs on the primaries and removes distributed log
+-- segments below the cutoff, emitting a TRUNCATE WAL record.  Its redo branch
+-- must advance the mirror's in-memory oldestXmin horizon along with the
+-- segment removal; otherwise a standby reader resolving a pre-truncation gxid
+-- on a mirror first passes the stale-horizon guard and then fails with a hard
+-- "Could not open file pg_distributedlog/NNNN" error on the removed segment.
+-- With the horizon advanced, those gxids resolve to not-found and take the
+-- soft in-doubt path instead.
+--
+-- start_matchsubs
+--
+-- m/(seg\d+ slice\d+ [\d.]+:\d+ pid \d+)/
+-- s/(seg\d+ slice\d+ [\d.]+:\d+ pid \d+)/seg<ID> slice<SID> <ADDRESS> pid <PID>/
+--
+-- m/(seg\d+ [\d.]+:\d+ pid \d+)/
+-- s/(seg\d+ [\d.]+:\d+ pid \d+)/seg<ID> <ADDRESS> pid <PID>/
+--
+-- end_matchsubs
+
+1: create table hs_dlog_trunc(a int, msg text);
+CREATE
+1: insert into hs_dlog_trunc select g, 'row ' || g from generate_series(1, 10) g;
+INSERT 10
+
+-- Burn more than one distributed-log segment worth of segment-local xids on
+-- every primary (32 SLRU pages * 4096 entries per page = 131072 xids per
+-- segment), so that the next horizon advance truncates the pages backing the
+-- rows inserted above.  txid_current() allocates an xid without touching any
+-- table, so the burn leaves no distributed state behind.
+!\retcode sh -ec 'export PGDATABASE=postgres; F=$(mktemp); trap "rm -f $F" EXIT; printf "SELECT txid_current();\n" > $F; for p in $(psql -Atqc "select port from gp_segment_configuration where role = chr(112) and content >= 0 order by content"); do PGOPTIONS="-c gp_role=utility" pgbench -n -p $p -c 2 -j 2 -t 70000 -f $F > /dev/null; done';
+-- start_ignore
+
+-- end_ignore
+(exited with code 0)
+
+-- Run a few dispatched statements so the primaries advance the distributed
+-- log horizon and emit the TRUNCATE record.
+1: create table hs_dlog_dummy(a int);
+CREATE
+1: drop table hs_dlog_dummy;
+DROP
+
+-- Wait until every mirror has replayed the truncation (the first remaining
+-- segment moves past 0000).  Without this the read could race the replay and
+-- the test would not exercise the removed-segment path deterministically.
+!\retcode sh -ec 'export PGDATABASE=postgres; for i in $(seq 1 120); do gone=1; for d in $(psql -Atqc "select datadir from gp_segment_configuration where role = chr(109) and content >= 0"); do f=$(ls $d/pg_distributedlog 2>/dev/null | sort | head -1); [ "$f" = "0000" ] && gone=0; done; [ $gone -eq 1 ] && exit 0; sleep 1; done; exit 1';
+-- start_ignore
+
+-- end_ignore
+(exited with code 0)
+
+-- The standby reader resolves the pre-truncation gxids of the rows committed
+-- above.  It must return the rows (or defer cleanly), never fail on the
+-- removed distributed log segment.
+-1S: select * from hs_dlog_trunc order by a;
+ a  | msg    
+----+--------
+ 1  | row 1  
+ 2  | row 2  
+ 3  | row 3  
+ 4  | row 4  
+ 5  | row 5  
+ 6  | row 6  
+ 7  | row 7  
+ 8  | row 8  
+ 9  | row 9  
+ 10 | row 10 
+(10 rows)
+-1Sq: ... <quitting>
+
+1: drop table hs_dlog_trunc;
+DROP

src/test/isolation2/hot_standby_schedule

@@ -3,4 +3,5 @@ test: hot_standby/basic
 test: hot_standby/transaction_isolation
 test: hot_standby/query_conflict
 test: hot_standby/faults
+test: hot_standby/dlog_truncation
 test: hot_standby/teardown

src/test/isolation2/sql/hot_standby/dlog_truncation.sql

@@ -0,0 +1,48 @@
+-- Distributed log truncation and hot standby reads
+--
+-- DistributedLog_Truncate() runs on the primaries and removes distributed log
+-- segments below the cutoff, emitting a TRUNCATE WAL record.  Its redo branch
+-- must advance the mirror's in-memory oldestXmin horizon along with the
+-- segment removal; otherwise a standby reader resolving a pre-truncation gxid
+-- on a mirror first passes the stale-horizon guard and then fails with a hard
+-- "Could not open file pg_distributedlog/NNNN" error on the removed segment.
+-- With the horizon advanced, those gxids resolve to not-found and take the
+-- soft in-doubt path instead.
+--
+-- start_matchsubs
+--
+-- m/(seg\d+ slice\d+ [\d.]+:\d+ pid \d+)/
+-- s/(seg\d+ slice\d+ [\d.]+:\d+ pid \d+)/seg<ID> slice<SID> <ADDRESS> pid <PID>/
+--
+-- m/(seg\d+ [\d.]+:\d+ pid \d+)/
+-- s/(seg\d+ [\d.]+:\d+ pid \d+)/seg<ID> <ADDRESS> pid <PID>/
+--
+-- end_matchsubs
+
+1: create table hs_dlog_trunc(a int, msg text);
+1: insert into hs_dlog_trunc select g, 'row ' || g from generate_series(1, 10) g;
+
+-- Burn more than one distributed-log segment worth of segment-local xids on
+-- every primary (32 SLRU pages * 4096 entries per page = 131072 xids per
+-- segment), so that the next horizon advance truncates the pages backing the
+-- rows inserted above.  txid_current() allocates an xid without touching any
+-- table, so the burn leaves no distributed state behind.
+!\retcode sh -ec 'export PGDATABASE=postgres; F=$(mktemp); trap "rm -f $F" EXIT; printf "SELECT txid_current();\n" > $F; for p in $(psql -Atqc "select port from gp_segment_configuration where role = chr(112) and content >= 0 order by content"); do PGOPTIONS="-c gp_role=utility" pgbench -n -p $p -c 2 -j 2 -t 70000 -f $F > /dev/null; done';
+
+-- Run a few dispatched statements so the primaries advance the distributed
+-- log horizon and emit the TRUNCATE record.
+1: create table hs_dlog_dummy(a int);
+1: drop table hs_dlog_dummy;
+
+-- Wait until every mirror has replayed the truncation (the first remaining
+-- segment moves past 0000).  Without this the read could race the replay and
+-- the test would not exercise the removed-segment path deterministically.
+!\retcode sh -ec 'export PGDATABASE=postgres; for i in $(seq 1 120); do gone=1; for d in $(psql -Atqc "select datadir from gp_segment_configuration where role = chr(109) and content >= 0"); do f=$(ls $d/pg_distributedlog 2>/dev/null | sort | head -1); [ "$f" = "0000" ] && gone=0; done; [ $gone -eq 1 ] && exit 0; sleep 1; done; exit 1';
+
+-- The standby reader resolves the pre-truncation gxids of the rows committed
+-- above.  It must return the rows (or defer cleanly), never fail on the
+-- removed distributed log segment.
+-1S: select * from hs_dlog_trunc order by a;
+-1Sq:
+
+1: drop table hs_dlog_trunc;